tests: new modules and tests

Author: gsmolk

tests/__init__.py

@@ -6,7 +6,7 @@
     ptrack_move_to_tablespace, ptrack_recovery, ptrack_vacuum, \
     ptrack_vacuum_bits_frozen, ptrack_vacuum_bits_visibility, \
     ptrack_vacuum_full, ptrack_vacuum_truncate, pgpro560, pgpro589, \
-    false_positive, replica
+    false_positive, replica, compression, page, ptrack, archive
 
 
 def load_tests(loader, tests, pattern):
@@ -32,5 +32,24 @@ def load_tests(loader, tests, pattern):
     suite.addTests(loader.loadTestsFromModule(pgpro560))
     suite.addTests(loader.loadTestsFromModule(pgpro589))
     suite.addTests(loader.loadTestsFromModule(false_positive))
+    suite.addTests(loader.loadTestsFromModule(compression))
+    suite.addTests(loader.loadTestsFromModule(page))
+    suite.addTests(loader.loadTestsFromModule(ptrack))
+    suite.addTests(loader.loadTestsFromModule(archive))
 
     return suite
+
+# ToDo:
+#  archive:
+#    discrepancy of instance`s PGDATA and node`s PGDATA should lead to archive-push refusal to work
+#    discrepancy of instance`s SYSTEMID and node`s SYSTEMID should lead to archive-push refusal to work
+#  replica:
+#    backup should exit with correct error message if some master* option is missing
+#    --master* options shoukd not work when backuping master
+#  logging:
+#     https://jira.postgrespro.ru/browse/PGPRO-584
+#     https://jira.postgrespro.ru/secure/attachment/20420/20420_doc_logging.md
+#  ptrack:
+#      ptrack backup on replica should work correctly
+#
+#  function to compare original instance and restored

tests/archive.py

@@ -0,0 +1,308 @@
+import os
+import unittest
+from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, archive_script
+from datetime import datetime, timedelta
+import subprocess
+from sys import exit
+from time import sleep
+
+
+module_name = 'archive'
+
+
+class ArchiveTest(ProbackupTest, unittest.TestCase):
+
+    # @unittest.expectedFailure
+    # @unittest.skip("skip")
+    def test_pgpro434_1(self):
+        """Description in jira issue PGPRO-434"""
+        fname = self.id().split('.')[3]
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+        node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
+            set_replication=True,
+            initdb_params=['--data-checksums'],
+            pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s'}
+            )
+        self.init_pb(backup_dir)
+        self.add_instance(backup_dir, 'node', node)
+        self.set_archiving(backup_dir, 'node', node)
+        # force more frequent wal switch
+        node.append_conf('postgresql.auto.conf', 'archive_timeout  = 30')
+        node.start()
+
+        node.safe_psql(
+            "postgres",
+            "create table t_heap as select 1 as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,100) i")
+
+        result = node.safe_psql("postgres", "SELECT * FROM t_heap")
+        self.backup_node(backup_dir, 'node', node)
+        node.cleanup()
+
+        self.restore_node(backup_dir, 'node', node)
+        node.start()
+
+        # Recreate backup calagoue
+        self.init_pb(backup_dir)
+        self.add_instance(backup_dir, 'node', node)
+
+        # Make backup
+        self.backup_node(backup_dir, 'node', node)
+        node.cleanup()
+
+        # Restore Database
+        self.restore_node(backup_dir, 'node', node)
+        node.start()
+
+        self.assertEqual(result, node.safe_psql("postgres", "SELECT * FROM t_heap"),
+            'data after restore not equal to original data')
+        # Clean after yourself
+        self.del_test_dir(module_name, fname)
+
+    # @unittest.skip("skip")
+    def test_pgpro434_2(self):
+        """Check that timelines are correct"""
+        fname = self.id().split('.')[3]
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+        node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
+            set_replication=True,
+            initdb_params=['--data-checksums'],
+            pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s'}
+            )
+        self.init_pb(backup_dir)
+        self.add_instance(backup_dir, 'node', node)
+        self.set_archiving(backup_dir, 'node', node)
+        node.start()
+
+        # FIRST TIMELINE
+        node.safe_psql(
+            "postgres",
+            "create table t_heap as select 1 as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,100) i")
+        backup_id = self.backup_node(backup_dir, 'node', node)
+        recovery_time = self.show_pb(backup_dir, 'node', backup_id)["recovery-time"]
+        node.safe_psql(
+            "postgres",
+            "insert into t_heap select 100501 as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i")
+
+        # SECOND TIMELIN
+        node.cleanup()
+        self.restore_node(backup_dir, 'node', node, options=["--time={0}".format(recovery_time)])
+        node.start()
+        if self.verbose:
+            print('Second timeline')
+            print(node.safe_psql("postgres","select redo_wal_file from pg_control_checkpoint()"))
+        node.safe_psql(
+            "postgres",
+            "insert into t_heap select 2 as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(100,200) i")
+        backup_id = self.backup_node(backup_dir, 'node', node)
+        recovery_time = self.show_pb(backup_dir, 'node', backup_id)["recovery-time"]
+        node.safe_psql(
+            "postgres",
+            "insert into t_heap select 100502 as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i")
+
+        # THIRD TIMELINE
+        node.cleanup()
+        self.restore_node(backup_dir, 'node', node, options=["--time={0}".format(recovery_time)])
+        node.start()
+        if self.verbose:
+            print('third timeline')
+            print(node.safe_psql("postgres","select redo_wal_file from pg_control_checkpoint()"))
+        node.safe_psql(
+            "postgres",
+            "insert into t_heap select 3 as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(200,300) i")
+        backup_id = self.backup_node(backup_dir, 'node', node)
+        recovery_time = self.show_pb(backup_dir, 'node', backup_id)["recovery-time"]
+        result = node.safe_psql("postgres", "SELECT * FROM t_heap")
+        node.safe_psql(
+            "postgres",
+            "insert into t_heap select 100503 as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i")
+
+        # FOURTH TIMELINE
+        node.cleanup()
+        self.restore_node(backup_dir, 'node', node, options=["--time={0}".format(recovery_time)])
+        node.start()
+        if self.verbose:
+            print('Fourth timeline')
+            print(node.safe_psql("postgres","select redo_wal_file from pg_control_checkpoint()"))
+
+        # FIFTH TIMELINE
+        node.cleanup()
+        self.restore_node(backup_dir, 'node', node, options=["--time={0}".format(recovery_time)])
+        node.start()
+        if self.verbose:
+            print('Fifth timeline')
+            print(node.safe_psql("postgres","select redo_wal_file from pg_control_checkpoint()"))
+
+        # SIXTH TIMELINE
+        node.cleanup()
+        self.restore_node(backup_dir, 'node', node, options=["--time={0}".format(recovery_time)])
+        node.start()
+        if self.verbose:
+            print('Sixth timeline')
+            print(node.safe_psql("postgres","select redo_wal_file from pg_control_checkpoint()"))
+
+        self.assertFalse(node.execute("postgres","select exists(select 1 from t_heap where id > 100500)")[0][0],
+            'data after restore not equal to original data')
+
+        self.assertEqual(result, node.safe_psql("postgres", "SELECT * FROM t_heap"),
+            'data after restore not equal to original data')
+
+        # Clean after yourself
+        # self.del_test_dir(module_name, fname)
+
+    # @unittest.skip("skip")
+    def test_pgpro434_3(self):
+        """Check pg_stop_backup_timeout"""
+        fname = self.id().split('.')[3]
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+        node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
+            set_replication=True,
+            initdb_params=['--data-checksums'],
+            pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s'}
+            )
+        self.init_pb(backup_dir)
+        self.add_instance(backup_dir, 'node', node)
+        self.set_archiving(backup_dir, 'node', node)
+
+        archive_script = os.path.join(backup_dir,'archive_script.sh')
+        with open(archive_script, 'w+') as f:
+            f.write(archive_script.format(backup_dir=backup_dir, node_name='node', count_limit=2))
+
+        st = os.stat(archive_script)
+        os.chmod(archive_script, st.st_mode | 0o111)
+        node.append_conf('postgresql.auto.conf', "archive_command  = '{0} %p %f'".format(archive_script))
+        node.start()
+        try:
+            self.backup_node(backup_dir, 'node', node, options=["--stream"])
+            # we should die here because exception is what we expect to happen
+            self.assertEqual(1, 0, "Expecting Error because pg_stop_backup failed to answer.\n Output: {0} \n CMD: {1}".format(
+                repr(self.output), self.cmd))
+        except ProbackupException as e:
+            self.assertTrue("ERROR: pg_stop_backup doesn't answer" in e.message
+                and "cancel it" in e.message,
+                '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd))
+
+        # Clean after yourself
+        self.del_test_dir(module_name, fname)
+
+    # @unittest.skip("skip")
+    def test_arhive_push_file_exists(self):
+        """Archive-push if file exists"""
+        fname = self.id().split('.')[3]
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+        node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname),
+            set_replication=True,
+            initdb_params=['--data-checksums'],
+            pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s', 'archive_timeout': '1'}
+            )
+        self.init_pb(backup_dir)
+        self.add_instance(backup_dir, 'node', node)
+        self.set_archiving(backup_dir, 'node', node)
+
+        wals_dir = os.path.join(backup_dir, 'wal', 'node')
+        file = os.path.join(wals_dir, '000000010000000000000001')
+        with open(file, 'a') as f:
+            pass
+        node.start()
+        node.safe_psql(
+            "postgres",
+            "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,100500) i")
+        log_file = os.path.join(node.logs_dir, 'postgresql.log')
+        with open(log_file, 'r') as f:
+            log_content = f.read()
+            self.assertTrue('LOG:  archive command failed with exit code 1' in log_content
+             and 'DETAIL:  The failed archive command was:' in log_content
+             and 'INFO: pg_probackup archive-push from' in log_content
+             and "ERROR: file '{0}', already exists.".format(file) in log_content,
+             'Expecting error messages about failed archive_command'
+            )
+            self.assertFalse('pg_probackup archive-push completed successfully' in log_content)
+
+        os.remove(file)
+        sleep(5)
+        node.safe_psql('postgres', 'select pg_switch_xlog()')
+
+        with open(log_file, 'r') as f:
+            log_content = f.read()
+            self.assertTrue('pg_probackup archive-push completed successfully' in log_content,
+             'Expecting messages about successfull execution archive_command')
+
+        # Clean after yourself
+        self.del_test_dir(module_name, fname)
+
+    @unittest.expectedFailure
+    def test_replica_archive(self):
+        """make node withput archiving, take stream backup and turn it into replica, set replica with archiving, make archive backup from replica"""
+        fname = self.id().split('.')[3]
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+        master = self.make_simple_node(base_dir="{0}/{1}/master".format(module_name, fname),
+            set_replication=True,
+            initdb_params=['--data-checksums'],
+            pg_options={'wal_level': 'replica', 'max_wal_senders': '2', 'checkpoint_timeout': '30s'}
+            )
+        self.init_pb(backup_dir)
+        self.add_instance(backup_dir, 'master', master)
+        # force more frequent wal switch
+        master.start()
+
+        replica = self.make_simple_node(base_dir="{0}/{1}/replica".format(module_name, fname))
+        replica.cleanup()
+
+        master.psql(
+            "postgres",
+            "create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i")
+
+        self.backup_node(backup_dir, 'master', master, options=['--stream'])
+        before = master.safe_psql("postgres", "SELECT * FROM t_heap")
+
+        # Settings for Replica
+        self.restore_node(backup_dir, 'master', replica)
+        self.set_replica(master, replica)
+        self.set_archiving(backup_dir, 'replica', replica, replica=True)
+        replica.start({"-t": "600"})
+
+        # Check data correctness on replica
+        after = replica.safe_psql("postgres", "SELECT * FROM t_heap")
+        self.assertEqual(before, after)
+
+        # Change data on master, take FULL backup from replica, restore taken backup and check that restored data equal to original data
+        master.psql(
+            "postgres",
+            "insert into t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(256,512) i")
+        before = master.safe_psql("postgres", "SELECT * FROM t_heap")
+        self.add_instance(backup_dir, 'replica', replica)
+        backup_id = self.backup_node(backup_dir, 'replica', replica, options=['--archive-timeout=30',
+            '--master-host=localhost', '--master-db=postgres','--master-port={0}'.format(master.port)])
+        self.validate_pb(backup_dir, 'replica')
+        self.assertEqual('OK', self.show_pb(backup_dir, 'replica', backup_id)['status'])
+
+        # RESTORE FULL BACKUP TAKEN FROM replica
+        node = self.make_simple_node(base_dir="{0}/{1}/node".format(module_name, fname))
+        node.cleanup()
+        self.restore_node(backup_dir, 'replica', data_dir=node.data_dir)
+        node.append_conf('postgresql.auto.conf', 'port = {0}'.format(node.port))
+        node.start({"-t": "600"})
+        # CHECK DATA CORRECTNESS
+        after = node.safe_psql("postgres", "SELECT * FROM t_heap")
+        self.assertEqual(before, after)
+
+        # Change data on master, make PAGE backup from replica, restore taken backup and check that restored data equal to original data
+        master.psql(
+            "postgres",
+            "insert into t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(512,768) i")
+        before = master.safe_psql("postgres", "SELECT * FROM t_heap")
+        backup_id = self.backup_node(backup_dir, 'replica', replica, backup_type='page', options=['--archive-timeout=30',
+            '--master-host=localhost', '--master-db=postgres','--master-port={0}'.format(master.port)])
+        self.validate_pb(backup_dir, 'replica')
+        self.assertEqual('OK', self.show_pb(backup_dir, 'replica', backup_id)['status'])
+
+        # RESTORE PAGE BACKUP TAKEN FROM replica
+        node.cleanup()
+        self.restore_node(backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id)
+        node.append_conf('postgresql.auto.conf', 'port = {0}'.format(node.port))
+        node.start({"-t": "600"})
+        # CHECK DATA CORRECTNESS
+        after = node.safe_psql("postgres", "SELECT * FROM t_heap")
+        self.assertEqual(before, after)
+
+        # Clean after yourself
+        self.del_test_dir(module_name, fname)