diff --git a/Makefile b/Makefile index f24246528f..ab4ee1df38 100644 --- a/Makefile +++ b/Makefile @@ -34,6 +34,8 @@ REGRESS = security rum rum_validate rum_hash ruminv timestamp orderby orderby_ha TAP_TESTS = 1 +EXTRA_CLEAN = pglist_tmp + ifdef USE_PGXS PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) diff --git a/README.md b/README.md index 5f3e6887cf..b6fb08420c 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,26 @@ Typical installation procedure may look like this: > **Important:** Don't forget to set the `PG_CONFIG` variable in case you want to test `RUM` on a custom build of PostgreSQL. Read more [here](https://wiki.postgresql.org/wiki/Building_and_Installing_PostgreSQL_Extension_Modules). +## Tests + +$ make check + +This command runs: +- regression tests; +- isolation tests; +- tap tests. + + One of the tap tests downloads a 1GB archive and then unpacks it + into a file weighing almost 3GB. It is disabled by default. + + To run this test, you need to set an environment variable: + + $ export PG_TEST_EXTRA=big_values + + The way to turn it off again: + + $ export -n PG_TEST_EXTRA + ## Common operators and functions The **rum** module provides next operators. diff --git a/t/002_pglist.pl b/t/002_pglist.pl new file mode 100644 index 0000000000..7b2d76c058 --- /dev/null +++ b/t/002_pglist.pl @@ -0,0 +1,207 @@ +# Test RUM index with big base 'pglist'. +use strict; +use warnings; +use Config; +use Test::More; + +plan skip_all => 'This test requires downloading a 1GB archive. ' . + 'The unpacked file weighs almost 3GB. ' . + 'Perform only if the big_values is enabled in PG_TEST_EXTRA' + unless $ENV{PG_TEST_EXTRA} && $ENV{PG_TEST_EXTRA} =~ /\bbig_values\b/; + +plan tests => 4; + +my $node; + +# Utility function + +sub file_exists +{ + my ($file) = @_; + return -e $file; +} + +# Check the existence of the test base, install if necessary + +sub install_pglist +{ + my $dir = Cwd->getcwd; #current directory + + my %config = ( + #directory with pglist dump must be inside the current directory + pglist_tmp_dir => $dir . '/pglist_tmp/', + dump_name => 'pglist-28-04-16.dump', + dump_url => 'http://www.sai.msu.su/~megera/postgres/files/pglist-28-04-16.dump.gz', + pglist_archive => $dir . '/pglist_tmp/' . 'pglist-28-04-16.dump.gz', + ); + + my $path_to_dump = $config{pglist_tmp_dir} . $config{dump_name}; + + if (file_exists($path_to_dump)) + { + note($config{dump_name} . ' already installed'); + } + else + { + # Create folder /contrib/rum/pglist_tmp if not already exists + mkdir($config{pglist_tmp_dir}, 0700) + unless file_exists($config{pglist_tmp_dir}); + + # Download archive pglist-28-04-16.dump.gz if not already exists + unless (file_exists($config{pglist_archive})) + { + note('Downloading pglist dump in ' . $config{pglist_archive}); + + # Flag "-nv" allows us to avoid frequent messages + # about the download status in the log. + # But it can be enabled for debugging purposes. + system("wget -P $config{pglist_tmp_dir} -nv $config{dump_url}") == 0 + or die "Couldn't get archive by link: $?"; + } + + # Unzip the dump. Delete archive to save memory + system("gzip -d $config{pglist_archive}") == 0 + or die "Couldn't extract archive: $?"; + + file_exists($path_to_dump) + or die "Failed to get " . $config{dump_name}; + + note($config{dump_name} . ' is ready to use'); + } + + $node->psql("postgres", "CREATE DATABASE pglist"); + $node->psql("postgres", "CREATE ROLE oleg"); + my $command = "'" . $path_to_dump . "'"; + my $result = $node->psql("pglist", '\i ' . $command); +} + +# Tests SELECT constructions to 'pglist' base + +sub test_select +{ + note("Creating index 'rumidx_orderby_sent'"); + + $node->safe_psql("pglist", "CREATE INDEX rumidx_orderby_sent ON pglist " . + "USING rum (fts rum_tsvector_timestamp_ops, sent) " . + "WITH (attach=sent, to=fts, order_by_attach=t)"); + + note("Test ORDER BY timestamp"); + + my $result1 = $node->safe_psql("pglist", + "SELECT sent, subject FROM pglist WHERE fts @@ " . + "to_tsquery('english', 'backend <-> crushed') " . + "ORDER BY sent <=| '2016-01-01 00:01' LIMIT 5"); + + is($result1, '1999-06-02 11:52:46|Re: [HACKERS] PID of backend'); + + note("Test tsvector filter"); + + my $result2 = $node->safe_psql("pglist", + "SELECT count(*) FROM pglist " . + "WHERE fts @@ to_tsquery('english', 'tom & lane')"); + + is($result2, '222813'); + + $node->safe_psql("pglist", "DROP INDEX rumidx_orderby_sent"); +} + +sub test_order_by +{ + note("Creating index 'pglist_rum_idx'"); + + $node->safe_psql("pglist", + "CREATE INDEX pglist_rum_idx ON pglist " . + "USING rum (fts rum_tsvector_ops)"); + + note("Test ORDER BY tsvector"); + + my $result3 = $node->safe_psql("pglist", + "SELECT id FROM pglist " . + "WHERE fts @@ to_tsquery('english', 'postgres:*') " . + "ORDER BY fts <=> " . + "to_tsquery('english', 'postgres:*') LIMIT 9"); + + is((split(" ", $result3))[0], '816114'); + + # Autovacuum after large update, with active RUM index crashes postgres + note("Test Issue #19"); + + my $stderr; + $node->safe_psql("pglist", "DELETE FROM pglist WHERE id < 100000"); + $node->safe_psql("pglist", "vacuum", stderr => \$stderr); + + is($stderr, undef); + + $node->safe_psql("pglist", "DROP INDEX pglist_rum_idx"); +} + +# Start backend + +my $pg_15_modules; + +BEGIN +{ + $pg_15_modules = eval + { + require PostgreSQL::Test::Cluster; + require PostgreSQL::Test::Utils; + return 1; + }; + + unless (defined $pg_15_modules) + { + $pg_15_modules = 0; + + require PostgresNode; + require TestLib; + } +} + +note('PostgreSQL 15 modules are used: ' . ($pg_15_modules ? 'yes' : 'no')); + +if ($pg_15_modules) +{ + $node = PostgreSQL::Test::Cluster->new("master"); +} +else +{ + $node = PostgresNode::get_new_node("master"); +} + +$node->init(allows_streaming => 1); +$node->append_conf("postgresql.conf", "shared_buffers='4GB'\n" . + "maintenance_work_mem='2GB'\n" . + "max_wal_size='2GB'\n" . + "work_mem='50MB'"); +$node->start; + +# Check the existence of the pglist base + +note('Check the existence of the pglist base...'); +my $check_pglist = $node->psql('postgres', "SELECT count(*) FROM pg_database " . + "WHERE datistemplate = false AND " . + "datname = 'pglist'"); +if ($check_pglist == 1) +{ + note("pglist already exists"); +} +else +{ + note("Create pglist database"); + install_pglist(); +} + +$node->psql("pglist", "CREATE EXTENSION rum"); +note('Setup is completed successfully'); + +eval +{ + test_select(); + test_order_by(); + $node->stop(); + done_testing(); + 1; +} or do { + note('Something went wrong: $@\n'); +}; + diff --git a/tests/README.md b/tests/README.md deleted file mode 100644 index 77b7d78e88..0000000000 --- a/tests/README.md +++ /dev/null @@ -1,14 +0,0 @@ -## Running tests - -Install testgres: - -``` -pip install testgres==0.4.0 -``` - -Run command: - -``` -python -m unittest pglist_tests -``` - diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/pglist_tests.py b/tests/pglist_tests.py deleted file mode 100644 index 7b8e4558e0..0000000000 --- a/tests/pglist_tests.py +++ /dev/null @@ -1,157 +0,0 @@ -# coding: utf-8 -""" - Test RUM index with big base 'pglist' - - Copyright (c) 2015-2021, Postgres Professional -""" -import unittest -import os -import sys -import gzip -import testgres as tg - -if sys.version_info[0] < 3: - import urllib as request -else: - import urllib.request as request - -from os.path import expanduser - - -class PglistTests(unittest.TestCase): - - def setUp(self): - current_dir = os.path.dirname(os.path.abspath(__file__)) - - self.node = tg.get_new_node("pglist", - os.path.join(current_dir, "tmp_install")) - try: - self.node.init() - self.node.append_conf("postgresql.conf", - "shared_buffers='4GB'\n" - "maintenance_work_mem='2GB'\n" - "max_wal_size='2GB'\n" - "work_mem='50MB'") - self.node.start() - except Exception as e: - self.printlog(os.path.join(self.node.logs_dir, "postgresql.log")) - raise e - - def tearDown(self): - tg.stop_all() - - def init_pglist_data(self, node): - # Check if 'pglist' base exists - bases = node.execute("postgres", - "SELECT count(*) FROM pg_database " - "WHERE datistemplate = false AND " - " datname = 'pglist'") - if bases[0][0] != 0: - return - - # Check if 'pglist' dump exists - home = expanduser("~") - pglist_dump = os.path.join(home, "pglist-28-04-16.dump") - if not os.path.isfile(pglist_dump): - pglist_dumpgz = pglist_dump + ".gz" - if not os.path.isfile(pglist_dumpgz): - print("Downloading: {0}".format(pglist_dumpgz)) - request.urlretrieve( - "http://www.sai.msu.su/~megera/postgres/files/pglist-28-04-16.dump.gz", - pglist_dumpgz) - - print("Decompressing: {0}".format(pglist_dumpgz)) - gz = gzip.open(pglist_dumpgz, 'rb') - with open(pglist_dump, 'wb') as f: - f.write(gz.read()) - - os.remove(pglist_dumpgz) - - # Restore dump file - print("Restoring 'pglist'") - node.safe_psql("postgres", "CREATE DATABASE pglist") - node.psql("pglist", filename=pglist_dump) - - node.safe_psql("pglist", "CREATE EXTENSION rum") - - def printlog(self, logfile): - with open(logfile, 'r') as log: - for line in log.readlines(): - print(line) - - def test_order_by(self): - """Tests SELECT constructions to 'pglist' base""" - try: - self.init_pglist_data(self.node) - - print("Creating index 'rumidx_orderby_sent'") - - self.node.safe_psql( - "pglist", - "CREATE INDEX rumidx_orderby_sent ON pglist USING rum (" - " fts rum_tsvector_timestamp_ops, sent) " - " WITH (attach=sent, to=fts, order_by_attach=t)") - - print("Running tests") - - self.assertEqual( - self.node.safe_psql( - "pglist", - "SELECT sent, subject " - " FROM pglist " - " WHERE fts @@ " - " to_tsquery('english', 'backend <-> crushed') " - " ORDER BY sent <=| '2016-01-01 00:01' LIMIT 5" - ), - b'1999-06-02 11:52:46|Re: [HACKERS] PID of backend\n' - ) - - self.assertEqual( - self.node.safe_psql( - "pglist", - "SELECT count(*) FROM pglist " - "WHERE fts @@ to_tsquery('english', 'tom & lane')" - ), - b'222813\n' - ) - - self.node.safe_psql("pglist", "DROP INDEX rumidx_orderby_sent"); - - print("Creating index 'pglist_rum_idx'") - - self.node.safe_psql( - "pglist", - "CREATE INDEX pglist_rum_idx ON pglist USING rum (" - " fts rum_tsvector_ops)") - - print("Running tests") - - self.assertEqual( - self.node.execute( - "pglist", - "SELECT id FROM pglist " - "WHERE fts @@ to_tsquery('english', 'postgres:*') " - "ORDER BY fts <=> to_tsquery('english', 'postgres:*') " - "LIMIT 9" - )[0][0], - 816114 - ) - - # Autovacuum after large update, with active RUM index crashes postgres - print("Test Issue #19") - - self.node.safe_psql( - "pglist", - "DELETE FROM pglist WHERE id < 100000") - self.node.safe_psql( - "pglist", - "vacuum") - - self.node.safe_psql("pglist", "DROP INDEX pglist_rum_idx"); - - except Exception as e: - self.printlog(os.path.join(self.node.logs_dir, "postgresql.log")) - raise e - -if __name__ == "__main__": - unittest.main() diff --git a/travis/run_tests.sh b/travis/run_tests.sh index aaebf3f200..37bba84d64 100644 --- a/travis/run_tests.sh +++ b/travis/run_tests.sh @@ -38,6 +38,7 @@ if [ "$LEVEL" = "hardcore" ]; then ./configure \ CFLAGS='-fno-omit-frame-pointer' \ --enable-cassert \ + --enable-tap-tests \ --prefix=$CUSTOM_PG_BIN \ --quiet