Skip to content

Commit 172259a

Browse files
alvherrealvherre
committed
Verify roundtrip dump/restore of regression database
Add a test to pg_upgrade's test suite that verifies that dump-restore-dump of regression database produces equivalent output to dumping it directly. This was already being tested by running pg_upgrade itself, but non-binary-upgrade mode was not being covered. The regression database has accrued, over time, a sufficient collection of interesting objects to ensure good coverage, but there hasn't been a concerted effort to be completely exhaustive, so it is likely still possible to have more. This'd belong more naturally in the pg_dump test suite, but we chose to put it in src/bin/pg_upgrade/t/002_pg_upgrade.pl because we need a run of the regression tests which is already done here, so this has less total test runtime impact. Also, experiments have shown that using parallel dump/restore is slightly faster, so we use --format=directory -j2. This test has already reported pg_dump bugs, as fixed in fd41ba9, 74563f6, d611f8b, 4694aed. Author: Ashutosh Bapat <[email protected]> Reviewed-by: Michael Paquier <[email protected]> Reviewed-by: Daniel Gustafsson <[email protected]> Reviewed-by: Tom Lane <[email protected]> Reviewed-by: Álvaro Herrera <[email protected]> Discussion: https://www.postgresql.org/message-id/CAExHW5uF5V=Cjecx3_Z=7xfh4rg2Wf61PT+hfquzjBqouRzQJQ@mail.gmail.com
1 parent 764d501 commit 172259a

File tree

4 files changed

+254
-3
lines changed

4 files changed

+254
-3
lines changed

‎src/bin/pg_upgrade/t/002_pg_upgrade.pl

+94-3
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,15 @@
1111

1212
use PostgreSQL::Test::Cluster;
1313
use PostgreSQL::Test::Utils;
14+
use PostgreSQL::Test::AdjustDump;
1415
use PostgreSQL::Test::AdjustUpgrade;
1516
use Test::More;
1617

1718
# Can be changed to test the other modes.
1819
my $mode = $ENV{PG_TEST_PG_UPGRADE_MODE} || '--copy';
1920

21+
my $tempdir = PostgreSQL::Test::Utils::tempdir;
22+
2023
# Generate a database with a name made of a range of ASCII characters.
2124
sub generate_db
2225
{
@@ -35,8 +38,8 @@ sub generate_db
3538
"created database with ASCII characters from $from_char to $to_char");
3639
}
3740

38-
# Filter the contents of a dump before its use in a content comparison.
39-
# This returns the path to the filtered dump.
41+
# Filter the contents of a dump before its use in a content comparison for
42+
# upgrade testing. This returns the path to the filtered dump.
4043
sub filter_dump
4144
{
4245
my ($is_old, $old_version, $dump_file) = @_;
@@ -60,6 +63,41 @@ sub filter_dump
6063
return $dump_file_filtered;
6164
}
6265

66+
# Dump database db from the given node in plain format and adjust it for
67+
# comparing dumps from the original and the restored database.
68+
#
69+
# file_prefix is used to create unique names for all dump files so that they
70+
# remain available for debugging in case the test fails.
71+
#
72+
# adjust_child_columns is passed to adjust_regress_dumpfile() which actually
73+
# adjusts the dump output.
74+
#
75+
# The name of the file containting adjusted dump is returned.
76+
sub get_dump_for_comparison
77+
{
78+
my ($node, $db, $file_prefix, $adjust_child_columns) = @_;
79+
80+
my $dumpfile = $tempdir . '/' . $file_prefix . '.sql';
81+
my $dump_adjusted = "${dumpfile}_adjusted";
82+
83+
open(my $dh, '>', $dump_adjusted)
84+
|| die "could not open $dump_adjusted for writing $!";
85+
86+
# Don't dump statistics, because there are still some bugs.
87+
$node->run_log(
88+
[
89+
'pg_dump', '--no-sync', '--no-statistics',
90+
'-d' => $node->connstr($db),
91+
'-f' => $dumpfile
92+
]);
93+
94+
print $dh adjust_regress_dumpfile(slurp_file($dumpfile),
95+
$adjust_child_columns);
96+
close($dh);
97+
98+
return $dump_adjusted;
99+
}
100+
63101
# The test of pg_upgrade requires two clusters, an old one and a new one
64102
# that gets upgraded. Before running the upgrade, a logical dump of the
65103
# old cluster is taken, and a second logical dump of the new one is taken
@@ -80,7 +118,6 @@ sub filter_dump
80118
}
81119

82120
# Paths to the dumps taken during the tests.
83-
my $tempdir = PostgreSQL::Test::Utils::tempdir;
84121
my $dump1_file = "$tempdir/dump1.sql";
85122
my $dump2_file = "$tempdir/dump2.sql";
86123

@@ -264,6 +301,60 @@ sub filter_dump
264301
is($rc, 0, 'regression tests pass');
265302
}
266303

304+
# Test that dump/restore of the regression database roundtrips cleanly. This
305+
# doesn't work well when the nodes are different versions, so skip it in that
306+
# case. Note that this isn't a pg_restore test, but it's convenient to do it
307+
# here because we've gone to the trouble of creating the regression database.
308+
#
309+
# Do this while the old cluster is running before it is shut down by the
310+
# upgrade test.
311+
SKIP:
312+
{
313+
my $dstnode = PostgreSQL::Test::Cluster->new('dst_node');
314+
315+
skip "different Postgres versions"
316+
if ($oldnode->pg_version != $dstnode->pg_version);
317+
skip "source node not using default install"
318+
if (defined $oldnode->install_path);
319+
320+
# Dump the original database for comparison later.
321+
my $src_dump =
322+
get_dump_for_comparison($oldnode, 'regression', 'src_dump', 1);
323+
324+
# Setup destination database cluster
325+
$dstnode->init(%node_params);
326+
# Stabilize stats for comparison.
327+
$dstnode->append_conf('postgresql.conf', 'autovacuum = off');
328+
$dstnode->start;
329+
330+
my $dump_file = "$tempdir/regression.dump";
331+
332+
# Use --create in dump and restore commands so that the restored
333+
# database has the same configurable variable settings as the original
334+
# database so that the dumps taken from both databases taken do not
335+
# differ because of locale changes. Additionally this provides test
336+
# coverage for --create option.
337+
#
338+
# Use directory format so that we can use parallel dump/restore.
339+
$oldnode->command_ok(
340+
[
341+
'pg_dump', '-Fd', '-j2', '--no-sync',
342+
'-d' => $oldnode->connstr('regression'),
343+
'--create', '-f' => $dump_file
344+
],
345+
'pg_dump on source instance');
346+
347+
$dstnode->command_ok(
348+
[ 'pg_restore', '--create', '-j2', '-d' => 'postgres', $dump_file ],
349+
'pg_restore to destination instance');
350+
351+
my $dst_dump =
352+
get_dump_for_comparison($dstnode, 'regression', 'dest_dump', 0);
353+
354+
compare_files($src_dump, $dst_dump,
355+
'dump outputs from original and restored regression databases match');
356+
}
357+
267358
# Initialize a new node for the upgrade.
268359
my $newnode = PostgreSQL::Test::Cluster->new('new_node');
269360

‎src/test/perl/Makefile

+2
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ install: all installdirs
2525
$(INSTALL_DATA) $(srcdir)/PostgreSQL/Test/Kerberos.pm '$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Test/Kerberos.pm'
2626
$(INSTALL_DATA) $(srcdir)/PostgreSQL/Test/Cluster.pm '$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Test/Cluster.pm'
2727
$(INSTALL_DATA) $(srcdir)/PostgreSQL/Test/BackgroundPsql.pm '$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Test/BackgroundPsql.pm'
28+
$(INSTALL_DATA) $(srcdir)/PostgreSQL/Test/AdjustDump.pm '$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Test/AdjustDump.pm'
2829
$(INSTALL_DATA) $(srcdir)/PostgreSQL/Test/AdjustUpgrade.pm '$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Test/AdjustUpgrade.pm'
2930
$(INSTALL_DATA) $(srcdir)/PostgreSQL/Version.pm '$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Version.pm'
3031

@@ -35,6 +36,7 @@ uninstall:
3536
rm -f '$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Test/Kerberos.pm'
3637
rm -f '$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Test/Cluster.pm'
3738
rm -f '$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Test/BackgroundPsql.pm'
39+
rm -f '$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Test/AdjustDump.pm'
3840
rm -f '$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Test/AdjustUpgrade.pm'
3941
rm -f '$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Version.pm'
4042

‎src/test/perl/PostgreSQL/Test/AdjustDump.pm

+157
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
# Copyright (c) 2024-2025, PostgreSQL Global Development Group
2+
3+
=pod
4+
5+
=head1 NAME
6+
7+
PostgreSQL::Test::AdjustDump - helper module for dump/restore tests
8+
9+
=head1 SYNOPSIS
10+
11+
use PostgreSQL::Test::AdjustDump;
12+
13+
# Adjust contents of dump output file so that dump output from original
14+
# regression database and that from the restored regression database match
15+
$dump = adjust_regress_dumpfile($dump, $adjust_child_columns);
16+
17+
=head1 DESCRIPTION
18+
19+
C<PostgreSQL::Test::AdjustDump> encapsulates various hacks needed to
20+
compare the results of dump/restore tests.
21+
22+
=cut
23+
24+
package PostgreSQL::Test::AdjustDump;
25+
26+
use strict;
27+
use warnings FATAL => 'all';
28+
29+
use Exporter 'import';
30+
use Test::More;
31+
32+
our @EXPORT = qw(
33+
adjust_regress_dumpfile
34+
);
35+
36+
=pod
37+
38+
=head1 ROUTINES
39+
40+
=over
41+
42+
=item $dump = adjust_regress_dumpfile($dump, $adjust_child_columns)
43+
44+
Edit a dump output file, taken from the source regression database,
45+
to remove the known differences to a dump taken after restoring the
46+
same database.
47+
48+
Arguments:
49+
50+
=over
51+
52+
=item C<dump>: Contents of dump file
53+
54+
=item C<adjust_child_columns>: 1 indicates that the given dump file requires
55+
adjusting columns in the child tables; usually when the dump is from original
56+
database. 0 indicates no such adjustment is needed; usually when the dump is
57+
from restored database.
58+
59+
=back
60+
61+
Returns the adjusted dump text.
62+
63+
Adjustments Applied:
64+
65+
=over
66+
67+
=item Column reordering on child table creation
68+
69+
This rearranges the column declarations in the C<CREATE TABLE... INHERITS>
70+
statements in the dump file from original database so that they match those
71+
from the restored database.
72+
73+
Only executed if C<adjust_child_columns> is true.
74+
75+
Context: some regression tests purposefully create child tables in such a way
76+
that the order of their inherited columns differ from column orders of their
77+
respective parents. In the restored database, however, the order of their
78+
inherited columns are same as that of their respective parents. Thus the column
79+
orders of these child tables in the original database and those in the restored
80+
database differ, causing difference in the dump outputs. See
81+
C<MergeAttributes()> and C<dumpTableSchema()> for details.
82+
83+
=item Removal of problematic C<COPY> statements
84+
85+
Remove COPY statements to abnormal children tables.
86+
87+
Context: This difference is caused because of columns that are added to parent
88+
tables that already have children; because recreating the children tables puts
89+
the columns from the parent ahead of columns declared locally in children,
90+
these extra columns are in earlier position compared to the original database.
91+
Reordering columns on the entire C<COPY> data is impractical, so we just remove
92+
them.
93+
94+
=item Newline adjustment
95+
96+
Windows-style newlines are changed to Unix-style. Empty lines are trimmed.
97+
98+
=back
99+
100+
=cut
101+
102+
sub adjust_regress_dumpfile
103+
{
104+
my ($dump, $adjust_child_columns) = @_;
105+
106+
# use Unix newlines
107+
$dump =~ s/\r\n/\n/g;
108+
109+
# Adjust the CREATE TABLE ... INHERITS statements.
110+
if ($adjust_child_columns)
111+
{
112+
$dump =~ s/(^CREATE\sTABLE\sgenerated_stored_tests\.gtestxx_4\s\()
113+
(\n\s+b\sinteger),
114+
(\n\s+a\sinteger\sNOT\sNULL)/$1$3,$2/mgx;
115+
116+
$dump =~ s/(^CREATE\sTABLE\sgenerated_virtual_tests\.gtestxx_4\s\()
117+
(\n\s+b\sinteger),
118+
(\n\s+a\sinteger\sNOT\sNULL)/$1$3,$2/mgx;
119+
120+
$dump =~ s/(^CREATE\sTABLE\spublic\.test_type_diff2_c1\s\()
121+
(\n\s+int_four\sbigint),
122+
(\n\s+int_eight\sbigint),
123+
(\n\s+int_two\ssmallint)/$1$4,$2,$3/mgx;
124+
125+
$dump =~ s/(^CREATE\sTABLE\spublic\.test_type_diff2_c2\s\()
126+
(\n\s+int_eight\sbigint),
127+
(\n\s+int_two\ssmallint),
128+
(\n\s+int_four\sbigint)/$1$3,$4,$2/mgx;
129+
}
130+
131+
# Remove COPY statements with differing column order
132+
for my $table (
133+
'public\.b_star', 'public\.c_star',
134+
'public\.cc2', 'public\.d_star',
135+
'public\.e_star', 'public\.f_star',
136+
'public\.renamecolumnanother', 'public\.renamecolumnchild',
137+
'public\.test_type_diff2_c1', 'public\.test_type_diff2_c2',
138+
'public\.test_type_diff_c')
139+
{
140+
# This multiline pattern matches the whole COPY, up to the
141+
# terminating "\."
142+
$dump =~ s/^COPY $table \(.+?^\\\.$//sm;
143+
}
144+
145+
# Suppress blank lines, as some places in pg_dump emit more or fewer.
146+
$dump =~ s/\n\n+/\n/g;
147+
148+
return $dump;
149+
}
150+
151+
=pod
152+
153+
=back
154+
155+
=cut
156+
157+
1;

‎src/test/perl/meson.build

+1
Original file line numberDiff line numberDiff line change
@@ -13,5 +13,6 @@ install_data(
1313
'PostgreSQL/Test/Kerberos.pm',
1414
'PostgreSQL/Test/Cluster.pm',
1515
'PostgreSQL/Test/BackgroundPsql.pm',
16+
'PostgreSQL/Test/AdjustDump.pm',
1617
'PostgreSQL/Test/AdjustUpgrade.pm',
1718
install_dir: dir_pgxs / 'src/test/perl/PostgreSQL/Test')

0 commit comments

Comments
 (0)