From ca9112a424ff68ec4f2ef67b47122f7d61412964 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 15 Aug 2016 13:49:49 -0400 Subject: Stamp HEAD as 10devel. This is a good bit more complicated than the average new-version stamping commit, because it includes various adjustments in pursuit of changing from three-part to two-part version numbers. It's likely some further work will be needed around that change; but this is enough to get through the regression tests, at least in Unix builds. Peter Eisentraut and Tom Lane --- doc/src/sgml/runtime.sgml | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/runtime.sgml b/doc/src/sgml/runtime.sgml index 8ba95e1b84..66fbe441ac 100644 --- a/doc/src/sgml/runtime.sgml +++ b/doc/src/sgml/runtime.sgml @@ -1601,17 +1601,26 @@ $ kill -INT `head -1 /usr/local/pgsql/data/postmaster.pid` - PostgreSQL major versions are represented by the - first two digit groups of the version number, e.g., 8.4. - PostgreSQL minor versions are represented by the - third group of version digits, e.g., 8.4.2 is the second minor - release of 8.4. Minor releases never change the internal storage - format and are always compatible with earlier and later minor - releases of the same major version number, e.g., 8.4.2 is compatible - with 8.4, 8.4.1 and 8.4.6. To update between compatible versions, - you simply replace the executables while the server is down and - restart the server. The data directory remains unchanged — - minor upgrades are that simple. + Current PostgreSQL version numbers consist of a + major and a minor version number. For example, in the version number 10.1, + the 10 is the major version number and the 1 is the minor version number, + meaning this would be the first minor release of the major release 10. For + releases before PostgreSQL version 10.0, version + numbers consist of three numbers, for example, 9.5.3. In those cases, the + major version consists of the first two digit groups of the version number, + e.g., 9.5, and the minor version is the third number, e.g., 3, meaning this + would be the third minor release of the major release 9.5. + + + + Minor releases never change the internal storage format and are always + compatible with earlier and later minor releases of the same major version + number. For example, version 10.1 is compatible with version 10.0 and + version 10.6. Similarly, for example, 9.5.3 is compatible with 9.5.0, + 9.5.1, and 9.5.6. To update between compatible versions, you simply + replace the executables while the server is down and restart the server. + The data directory remains unchanged — minor upgrades are that + simple. -- cgit v1.2.3 From 8fc571b7dd9fa1659536a26bb085584b50a65a51 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 16 Aug 2016 10:59:14 -0400 Subject: Doc: remove out-of-date claim that pg_am rows must be inserted by hand. Commit 473b93287 added a sentence about that, but neglected to remove the adjacent sentence it had falsified. Per Alexander Law. --- doc/src/sgml/indexam.sgml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/indexam.sgml b/doc/src/sgml/indexam.sgml index b59cd0363a..40f201b11b 100644 --- a/doc/src/sgml/indexam.sgml +++ b/doc/src/sgml/indexam.sgml @@ -51,16 +51,9 @@ pg_am system catalog. The pg_am entry specifies a name and a handler function for the access - method. There is not currently any special support - for creating or deleting pg_am entries; - anyone able to write a new access method is expected to be competent - to insert an appropriate row for themselves. - - - - Index access methods can be defined and dropped using + method. These entries can be created and deleted using the and - SQL commands respectively. + SQL commands. -- cgit v1.2.3 From 9b002cc9fec557fcfe17d67f55b53804447230e5 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 16 Aug 2016 11:35:36 -0400 Subject: Doc: copy-editing in create_access_method.sgml. Improve shaky English grammar. And markup. --- doc/src/sgml/ref/create_access_method.sgml | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/create_access_method.sgml b/doc/src/sgml/ref/create_access_method.sgml index 3c091f8021..0a30e6ea3c 100644 --- a/doc/src/sgml/ref/create_access_method.sgml +++ b/doc/src/sgml/ref/create_access_method.sgml @@ -57,29 +57,28 @@ CREATE ACCESS METHOD name - access_method_type + access_method_type - This clause specifies type of access method to define. + This clause specifies the type of access method to define. Only INDEX is supported at present. - HANDLER handler_function + handler_function - handler_function is the - name of a previously registered function that will be called to - retrieve the struct which contains required parameters and functions - of access method to the core. The handler function must take single - argument of type internal, and its return type depends on the - type of access method; for INDEX access methods, it - must be index_am_handler. - - - See for index access methods API. + handler_function is the + name (possibly schema-qualified) of a previously registered function + that represents the access method. The handler function must be + declared to take a single argument of type internal, + and its return type depends on the type of access method; + for INDEX access methods, it must + be index_am_handler. The C-level API that the handler + function must implement varies depending on the type of access method. + The index access method API is described in . @@ -90,7 +89,7 @@ CREATE ACCESS METHOD name Examples - Create an access method heptree with + Create an index access method heptree with handler function heptree_handler: CREATE ACCESS METHOD heptree TYPE INDEX HANDLER heptree_handler; -- cgit v1.2.3 From f0fe1c8f70bacb65513f1cbaea14eb384d346ee8 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 16 Aug 2016 12:00:00 -0400 Subject: Fix typos From: Alexander Law --- doc/src/sgml/release-9.6.sgml | 2 +- doc/src/sgml/runtime.sgml | 2 +- src/backend/access/transam/multixact.c | 2 +- src/backend/utils/adt/tsquery.c | 2 +- src/test/regress/expected/privileges.out | 2 +- src/test/regress/sql/privileges.sql | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/release-9.6.sgml b/doc/src/sgml/release-9.6.sgml index cc886fa2bb..9003b1f6e4 100644 --- a/doc/src/sgml/release-9.6.sgml +++ b/doc/src/sgml/release-9.6.sgml @@ -1026,7 +1026,7 @@ This commit is also listed under libpq and psql This view exposes the same information available from - the the pg_config comand-line utility, + the pg_config comand-line utility, namely assorted compile-time configuration information for PostgreSQL. diff --git a/doc/src/sgml/runtime.sgml b/doc/src/sgml/runtime.sgml index 66fbe441ac..60a06590fe 100644 --- a/doc/src/sgml/runtime.sgml +++ b/doc/src/sgml/runtime.sgml @@ -184,7 +184,7 @@ postgres$ initdb -D /usr/local/pgsql/data - Non-C and and non-POSIX locales rely on the + Non-C and non-POSIX locales rely on the operating system's collation library for character set ordering. This controls the ordering of keys stored in indexes. For this reason, a cluster cannot switch to an incompatible collation library version, diff --git a/src/backend/access/transam/multixact.c b/src/backend/access/transam/multixact.c index c2e4fa377d..0c8c17af33 100644 --- a/src/backend/access/transam/multixact.c +++ b/src/backend/access/transam/multixact.c @@ -2802,7 +2802,7 @@ ReadMultiXactCounts(uint32 *multixacts, MultiXactOffset *members) * more aggressive in clamping this value. That not only causes autovacuum * to ramp up, but also makes any manual vacuums the user issues more * aggressive. This happens because vacuum_set_xid_limits() clamps the - * freeze table and and the minimum freeze age based on the effective + * freeze table and the minimum freeze age based on the effective * autovacuum_multixact_freeze_max_age this function returns. In the worst * case, we'll claim the freeze_max_age to zero, and every vacuum of any * table will try to freeze every multixact. diff --git a/src/backend/utils/adt/tsquery.c b/src/backend/utils/adt/tsquery.c index c0a4a0606b..3d11a1c208 100644 --- a/src/backend/utils/adt/tsquery.c +++ b/src/backend/utils/adt/tsquery.c @@ -691,7 +691,7 @@ parse_tsquery(char *buf, findoprnd(ptr, query->size, &needcleanup); /* - * QI_VALSTOP nodes should be cleaned and and OP_PHRASE should be pushed + * QI_VALSTOP nodes should be cleaned and OP_PHRASE should be pushed * down */ if (needcleanup) diff --git a/src/test/regress/expected/privileges.out b/src/test/regress/expected/privileges.out index 996ebcdca2..f66b4432a1 100644 --- a/src/test/regress/expected/privileges.out +++ b/src/test/regress/expected/privileges.out @@ -390,7 +390,7 @@ INSERT INTO atest5(two) VALUES (6) ON CONFLICT (two) DO UPDATE set one = 8; -- f ERROR: permission denied for relation atest5 INSERT INTO atest5(three) VALUES (4) ON CONFLICT (two) DO UPDATE set three = 10; -- fails (due to INSERT) ERROR: permission denied for relation atest5 --- Check that the the columns in the inference require select privileges +-- Check that the columns in the inference require select privileges -- Error. No privs on four INSERT INTO atest5(three) VALUES (4) ON CONFLICT (four) DO UPDATE set three = 10; ERROR: permission denied for relation atest5 diff --git a/src/test/regress/sql/privileges.sql b/src/test/regress/sql/privileges.sql index 0aa9c672d5..00dc7bd4ab 100644 --- a/src/test/regress/sql/privileges.sql +++ b/src/test/regress/sql/privileges.sql @@ -259,7 +259,7 @@ INSERT INTO atest5(two) VALUES (6) ON CONFLICT (two) DO UPDATE set three = EXCLU INSERT INTO atest5(two) VALUES (6) ON CONFLICT (two) DO UPDATE set three = EXCLUDED.three; INSERT INTO atest5(two) VALUES (6) ON CONFLICT (two) DO UPDATE set one = 8; -- fails (due to UPDATE) INSERT INTO atest5(three) VALUES (4) ON CONFLICT (two) DO UPDATE set three = 10; -- fails (due to INSERT) --- Check that the the columns in the inference require select privileges +-- Check that the columns in the inference require select privileges -- Error. No privs on four INSERT INTO atest5(three) VALUES (4) ON CONFLICT (four) DO UPDATE set three = 10; -- cgit v1.2.3 From 639166641102871e09f9c4aebc71df57566a0a4a Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 16 Aug 2016 12:00:00 -0400 Subject: doc: Remove some confusion from pg_archivecleanup doc From: Jeff Janes --- doc/src/sgml/ref/pgarchivecleanup.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/pgarchivecleanup.sgml b/doc/src/sgml/ref/pgarchivecleanup.sgml index 60a7fc4e6b..abe01bef4f 100644 --- a/doc/src/sgml/ref/pgarchivecleanup.sgml +++ b/doc/src/sgml/ref/pgarchivecleanup.sgml @@ -122,7 +122,7 @@ pg_archivecleanup: removing file "archive/00000001000000370000000E" extension - When using the program as a standalone utility, provide an extension + Provide an extension that will be stripped from all file names before deciding if they should be deleted. This is typically useful for cleaning up archives that have been compressed during storage, and therefore have had an -- cgit v1.2.3 From d125d25790901683d5ad16bfc96e9de4ccc9a581 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Tue, 16 Aug 2016 23:04:50 -0400 Subject: docs: my third pass over the 9.6 release notes Backpatch-through: 9.6 --- doc/src/sgml/release-9.6.sgml | 628 ++++++++++++++++++++++-------------------- 1 file changed, 332 insertions(+), 296 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/release-9.6.sgml b/doc/src/sgml/release-9.6.sgml index 9003b1f6e4..8d7356e27f 100644 --- a/doc/src/sgml/release-9.6.sgml +++ b/doc/src/sgml/release-9.6.sgml @@ -401,7 +401,7 @@ This commit is also listed under libpq and psql Allow GIN index builds to make effective use of - settings larger than 1GB (Robert Abraham, Teodor Sigaev) + settings larger than 1 GB (Robert Abraham, Teodor Sigaev) @@ -467,55 +467,130 @@ This commit is also listed under libpq and psql - General Performance + Sorting - Avoid re-vacuuming pages containing only frozen tuples (Masahiko - Sawada, Robert Haas, Andres Freund) + Improve sorting performance by using quicksort, not replacement + selection sort, when performing external sort steps (Peter + Geoghegan) - Formerly, anti-wraparound vacuum had to visit every page of - a table, even pages where there was nothing to do. Now, pages - containing only already-frozen tuples are identified in the table's - visibility map, and can be skipped by vacuum even when doing - transaction wraparound prevention. This should greatly reduce the - cost of maintaining large tables containing mostly-unchanged data. + The new approach makes better use of the CPU cache + for typical cache sizes and data volumes. Where necessary, + the behavior can be adjusted via the new configuration parameter + . + + + - If necessary, vacuum can be forced to process all-frozen - pages using the new DISABLE_PAGE_SKIPPING option. - Normally, this should never be needed but it might help in - recovering from visibility-map corruption. + Speed up text sorts where the same string occurs multiple times + (Peter Geoghegan) - Avoid useless heap-truncation attempts during VACUUM - (Jeff Janes, Tom Lane) + Speed up sorting of uuid, bytea, and + char(n) fields by using abbreviated keys + (Peter Geoghegan) - This change avoids taking an exclusive table lock in some cases - where no truncation is possible. The main benefit comes from - avoiding unnecessary query cancellations on standby servers. + Support for abbreviated keys has also been + added to the non-default operator classes text_pattern_ops, + varchar_pattern_ops, and + bpchar_pattern_ops. Processing of ordered-set + aggregates can also now exploit abbreviated keys. + + + + + + + Speed up CREATE INDEX CONCURRENTLY by treating + TIDs as 64-bit integers during sorting (Peter + Geoghegan) + + + + + + + + + Locking + + + + + + + Reduce contention for the ProcArrayLock (Amit Kapila, + Robert Haas) + + + + + + + Improve performance by moving buffer content locks into the buffer + descriptors (Andres Freund, Simon Riggs) + + + + + + + Replace shared-buffer header spinlocks with atomic operations to + improve scalability (Alexander Korotkov, Andres Freund) + + + + + + + Use atomic operations, rather than a spinlock, to protect an + LWLock's wait queue (Andres Freund) + + + + + + + Partition the shared hash table freelist to reduce contention on + multi-CPU-socket servers (Aleksander Alekseev) @@ -531,111 +606,138 @@ This commit is also listed under libpq and psql This change avoids substantial replication delays that sometimes - occurre while replaying such operations. + occurred while replaying such operations. + + + + + + Optimizer Statistics + + + - Avoid computing GROUP BY columns if they are - functionally dependent on other columns (David Rowley) + Improve ANALYZE's estimates for columns with many nulls + (Tomas Vondra, Alex Shulgin) - If a GROUP BY clause includes all columns of a - non-deferred primary key, as well as other columns of the same - table, those other columns are redundant and can be dropped - from the grouping. This saves computation in many common cases. + Previously ANALYZE tended to underestimate the number + of non-NULL distinct values in a column with many + NULLs, and was also inaccurate in computing the + most-common values. - When appropriate, postpone evaluation of SELECT - output expressions until after an ORDER BY sort - (Konstantin Knizhnik) - - - - This change ensures that volatile or expensive functions in the - output list are executed in the order suggested by ORDER - BY, and that they are not evaluated more times than required - when there is a LIMIT clause. Previously, these - properties held if the ordering was performed by an index scan or - pre-merge-join sort, but not if it was performed by a top-level - sort. + Improve planner's estimate of the number of distinct values in + a query result (Tomas Vondra) - Where feasible, trigger kernel writeback after a configurable - number of writes, to prevent accumulation of dirty data in kernel - disk buffers (Fabien Coelho, Andres Freund) + Use foreign key relationships to infer selectivity for join + predicates (Tomas Vondra, David Rowley) - PostgreSQL writes data to the kernel's disk cache, - from where it will be flushed to physical storage in due time. - Many operating systems are not smart about managing this and allow - large amounts of dirty data to accumulate before deciding to flush - it all at once, leading to long delays for new I/O requests. - This change attempts to alleviate this problem by explicitly - requesting data flushes after a configurable interval. + If a table t has a foreign key restriction, say + (a,b) REFERENCES r (x,y), then a WHERE + condition such as t.a = r.x AND t.b = r.y cannot + select more than one r row per t row. + The planner formerly considered AND conditions + to be independent and would often drastically misestimate + selectivity as a result. Now it compares the WHERE + conditions to applicable foreign key constraints and produces + better estimates. + + + + + + + + <command>VACUUM</> + + + + - On Linux, sync_file_range() is used for this purpose, - and the feature is on by default on Linux because that function has few - downsides. This sync capability is also available on other platforms - that have msync() or posix_fadvise(), - but those interfaces have some undesirable side-effects so the - feature is disabled by default on non-Linux platforms. + Avoid re-vacuuming pages containing only frozen tuples (Masahiko + Sawada, Robert Haas, Andres Freund) - The new configuration parameters , , , and control this behavior. + Formerly, anti-wraparound vacuum had to visit every page of + a table, even pages where there was nothing to do. Now, pages + containing only already-frozen tuples are identified in the table's + visibility map, and can be skipped by vacuum even when doing + transaction wraparound prevention. This should greatly reduce the + cost of maintaining large tables containing mostly-unchanged data. + + + + If necessary, vacuum can be forced to process all-frozen + pages using the new DISABLE_PAGE_SKIPPING option. + Normally, this should never be needed but it might help in + recovering from visibility-map corruption. - Perform checkpoint writes in sorted order (Fabien Coelho, - Andres Freund) + Avoid useless heap-truncation attempts during VACUUM + (Jeff Janes, Tom Lane) - Previously, checkpoints wrote out dirty pages in whatever order - they happen to appear in shared buffers, which usually is nearly - random. That performs poorly, especially on rotating media. - This change causes checkpoint-driven writes to be done in order - by file and block number, and to be balanced across tablespaces. + This change avoids taking an exclusive table lock in some cases + where no truncation is possible. The main benefit comes from + avoiding unnecessary query cancellations on standby servers. + + + + + + General Performance + + + + + Avoid computing GROUP BY columns if they are + functionally dependent on other columns (David Rowley) + + + + If a GROUP BY clause includes all columns of a + non-deferred primary key, as well as other columns of the same + table, those other columns are redundant and can be dropped + from the grouping. This saves computation in many common cases. + + + + + + + When appropriate, postpone evaluation of SELECT + output expressions until after an ORDER BY sort + (Konstantin Knizhnik) + + + + This change ensures that volatile or expensive functions in the + output list are executed in the order suggested by ORDER + BY, and that they are not evaluated more times than required + when there is a LIMIT clause. Previously, these + properties held if the ordering was performed by an index scan or + pre-merge-join sort, but not if it was performed by a top-level + sort. + + + + + @@ -687,25 +828,63 @@ This commit is also listed under libpq and psql - Use foreign key relationships to infer selectivity for join - predicates (Tomas Vondra, David Rowley) + Perform checkpoint writes in sorted order (Fabien Coelho, + Andres Freund) - If a table t has a foreign key restriction, say - (a,b) REFERENCES r (x,y), then a WHERE - condition such as t.a = r.x AND t.b = r.y cannot - select more than one r row per t row. - The planner formerly considered AND conditions - to be independent and would often drastically misestimate - selectivity as a result. Now it compares the WHERE - conditions to applicable foreign key constraints and produces - better estimates. + Previously, checkpoints wrote out dirty pages in whatever order + they happen to appear in shared buffers, which usually is nearly + random. That performs poorly, especially on rotating media. + This change causes checkpoint-driven writes to be done in order + by file and block number, and to be balanced across tablespaces. + + + + + + + Where feasible, trigger kernel writeback after a configurable + number of writes, to prevent accumulation of dirty data in kernel + disk buffers (Fabien Coelho, Andres Freund) + + + + PostgreSQL writes data to the kernel's disk cache, + from where it will be flushed to physical storage in due time. + Many operating systems are not smart about managing this and allow + large amounts of dirty data to accumulate before deciding to flush + it all at once, leading to long delays for new I/O requests. + This change attempts to alleviate this problem by explicitly + requesting data flushes after a configurable interval. + + + + On Linux, sync_file_range() is used for this purpose, + and the feature is on by default on Linux because that function has few + downsides. This sync capability is also available on other platforms + that have msync() or posix_fadvise(), + but those interfaces have some undesirable side-effects so the + feature is disabled by default on non-Linux platforms. + + + + The new configuration parameters , , , and control this behavior. @@ -808,35 +987,6 @@ This commit is also listed under libpq and psql - - Improve ANALYZE's estimates for columns with many nulls - (Tomas Vondra, Alex Shulgin) - - - - Previously ANALYZE tended to underestimate the number - of non-NULL distinct values in a column with many - NULLs, and was also inaccurate in computing the - most-common values. - - - - - - - Improve planner's estimate of the number of distinct values in - a query result (Tomas Vondra) - - - - - @@ -851,69 +1001,6 @@ This commit is also listed under libpq and psql - - Improve sorting performance by using quicksort, not replacement - selection sort, when performing external sort steps (Peter - Geoghegan) - - - - The new approach makes better use of the CPU cache - for typical cache sizes and data volumes. Where necessary, - the behavior can be adjusted via the new configuration parameter - . - - - - - - - Speed up text sorts where the same string occurs multiple times - (Peter Geoghegan) - - - - - - - Speed up sorting of uuid, bytea, and - char(n) fields by using abbreviated keys - (Peter Geoghegan) - - - - Support for abbreviated keys has also been - added to the non-default operator classes text_pattern_ops, - varchar_pattern_ops, and - bpchar_pattern_ops. Processing of ordered-set - aggregates can also now exploit abbreviated keys. - - - - - - - Speed up CREATE INDEX CONCURRENTLY by treating - TIDs as 64-bit integers during sorting (Peter - Geoghegan) - - - - - @@ -924,57 +1011,6 @@ This commit is also listed under libpq and psql - - Reduce contention for the ProcArrayLock (Amit Kapila, - Robert Haas) - - - - - - - Improve performance by moving buffer content locks into the buffer - descriptors (Andres Freund, Simon Riggs) - - - - - - - Replace shared-buffer header spinlocks with atomic operations to - improve scalability (Alexander Korotkov, Andres Freund) - - - - - - - Use atomic operations, rather than a spinlock, to protect an - LWLock's wait queue (Andres Freund) - - - - - - - Partition the shared hash table freelist to reduce contention on - multi-CPU-socket servers (Aleksander Alekseev) - - - - - @@ -1017,6 +1053,21 @@ This commit is also listed under libpq and psql + + Add pg_control_system(), + pg_control_checkpoint(), + pg_control_recovery(), and + pg_control_init() functions to expose fields of + pg_control to SQL (Joe Conway, Michael + Paquier) + + + + + @@ -1082,21 +1133,6 @@ This commit is also listed under libpq and psql - - Add pg_control_system(), - pg_control_checkpoint(), - pg_control_recovery(), and - pg_control_init() functions to expose fields of - pg_control to SQL (Joe Conway, Michael - Paquier) - - - - - @@ -1219,18 +1255,18 @@ This commit is also listed under libpq and psql - Add configure option - This allows the use of systemd service units of - type notify, which greatly simplifies the management - of PostgreSQL under systemd. + This behavior is controlled by the new configuration parameter + . It can + be useful to prevent forgotten transactions from holding locks + or preventing vacuum cleanup for too long. @@ -1247,18 +1283,18 @@ This commit is also listed under libpq and psql - Allow sessions to be terminated automatically if they are in - idle-in-transaction state for too long (Vik Fearing) + Add configure option - This behavior is controlled by the new configuration parameter - . It can - be useful to prevent forgotten transactions from holding locks - or preventing vacuum cleanup for too long. + This allows the use of systemd service units of + type notify, which greatly simplifies the management + of PostgreSQL under systemd. @@ -1857,6 +1893,30 @@ XXX this is pending backpatch, may need to remove + + Improve full-text search to support searching for phrases, that + is, lexemes appearing adjacent to each other in a specific order, + or with a specified distance between them (Teodor Sigaev, Oleg + Bartunov, Dmitry Ivanov) + + + + A phrase-search query can be specified in tsquery + input using the new operators <-> and + <N>. The former means + that the lexemes before and after it must appear adjacent to + each other in that order. The latter means they must be exactly + N lexemes apart. + + + + + @@ -1917,30 +1977,6 @@ XXX this is pending backpatch, may need to remove - - Improve full-text search to support searching for phrases, that - is, lexemes appearing adjacent to each other in a specific order, - or with a specified distance between them (Teodor Sigaev, Oleg - Bartunov, Dmitry Ivanov) - - - - A phrase-search query can be specified in tsquery - input using the new operators <-> and - <N>. The former means - that the lexemes before and after it must appear adjacent to - each other in that order. The latter means they must be exactly - N lexemes apart. - - - - - + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/src/sgml/stylesheet-speedup-xhtml.xsl b/doc/src/sgml/stylesheet-speedup-xhtml.xsl new file mode 100644 index 0000000000..8428dc5870 --- /dev/null +++ b/doc/src/sgml/stylesheet-speedup-xhtml.xsl @@ -0,0 +1,252 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + , + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Error: If you change $chunk.section.depth, then you must update the performance-optimized chunk-all-sections-template. + + + + + + + + + + + + + + + diff --git a/doc/src/sgml/stylesheet.xsl b/doc/src/sgml/stylesheet.xsl index 7967b361dd..631fcc4edf 100644 --- a/doc/src/sgml/stylesheet.xsl +++ b/doc/src/sgml/stylesheet.xsl @@ -6,6 +6,7 @@ + -- cgit v1.2.3 From 5285c5e873d8b622da7007c1628e5afa80f372fb Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Thu, 18 Aug 2016 21:41:10 -0400 Subject: doc: requirepeer is a way to avoid spoofing We already mentioned unix_socket_directories as an option. Reported-by: https://www.postgresql.org/message-id/45016837-6cf3-3136-f959-763d06a28076%402ndquadrant.com Backpatch-through: 9.6 --- doc/src/sgml/runtime.sgml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/runtime.sgml b/doc/src/sgml/runtime.sgml index 60a06590fe..98752c2875 100644 --- a/doc/src/sgml/runtime.sgml +++ b/doc/src/sgml/runtime.sgml @@ -1922,7 +1922,7 @@ pg_dumpall -p 5432 | psql -d postgres -p 5433 - The simplest way to prevent spoofing for local + On way to prevent spoofing of local connections is to use a Unix domain socket directory () that has write permission only for a trusted local user. This prevents a malicious user from creating @@ -1934,6 +1934,13 @@ pg_dumpall -p 5432 | psql -d postgres -p 5433 /tmp cleanup script to prevent removal of the symbolic link. + + Another option for local connections is for clients to use + requirepeer + to specify the required owner of the server process connected to + the socket. + + To prevent spoofing on TCP connections, the best solution is to use SSL certificates and make sure that clients check the server's certificate. -- cgit v1.2.3 From f2e016f8d55ee029c5d6ec853ff6802aaf49fb0a Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Mon, 22 Aug 2016 17:20:44 -0400 Subject: doc: fix typo in recent patch Reported-by: Jeff Janes Backpatch-through: 9.6 --- doc/src/sgml/runtime.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/runtime.sgml b/doc/src/sgml/runtime.sgml index 98752c2875..ef0139c365 100644 --- a/doc/src/sgml/runtime.sgml +++ b/doc/src/sgml/runtime.sgml @@ -1922,7 +1922,7 @@ pg_dumpall -p 5432 | psql -d postgres -p 5433 - On way to prevent spoofing of local + One way to prevent spoofing of local connections is to use a Unix domain socket directory () that has write permission only for a trusted local user. This prevents a malicious user from creating -- cgit v1.2.3 From 86f31695f3b54211226949de519063bbf248e8c4 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Tue, 23 Aug 2016 10:30:52 -0400 Subject: Add txid_current_ifassigned(). Add a variant of txid_current() that returns NULL if no transaction ID is assigned. This version can be used even on a standby server, although it will always return NULL since no transaction IDs can be assigned during recovery. Craig Ringer, per suggestion from Jim Nasby. Reviewed by Petr Jelinek and by me. --- doc/src/sgml/func.sgml | 9 +++++++++ src/backend/utils/adt/txid.c | 21 +++++++++++++++++++++ src/include/catalog/pg_proc.h | 2 ++ src/include/utils/builtins.h | 1 + src/test/regress/expected/txid.out | 16 ++++++++++++++++ src/test/regress/sql/txid.sql | 7 +++++++ 6 files changed, 56 insertions(+) (limited to 'doc/src') diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 169a385a9c..6355300d9d 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -17119,6 +17119,10 @@ SELECT collation for ('foo' COLLATE "de_DE"); txid_current + + txid_current_if_assigned + + txid_current_snapshot @@ -17159,6 +17163,11 @@ SELECT collation for ('foo' COLLATE "de_DE"); bigint get current transaction ID, assigning a new one if the current transaction does not have one + + txid_current_if_assigned() + bigint + same as txid_current() but returns null instead of assigning an xid if none is already assigned + txid_current_snapshot() txid_snapshot diff --git a/src/backend/utils/adt/txid.c b/src/backend/utils/adt/txid.c index c2069a9923..276075e293 100644 --- a/src/backend/utils/adt/txid.c +++ b/src/backend/utils/adt/txid.c @@ -376,6 +376,27 @@ txid_current(PG_FUNCTION_ARGS) PG_RETURN_INT64(val); } +/* + * Same as txid_current() but doesn't assign a new xid if there isn't one + * yet. + */ +Datum +txid_current_if_assigned(PG_FUNCTION_ARGS) +{ + txid val; + TxidEpoch state; + TransactionId topxid = GetTopTransactionIdIfAny(); + + if (topxid == InvalidTransactionId) + PG_RETURN_NULL(); + + load_xid_epoch(&state); + + val = convert_xid(topxid, &state); + + PG_RETURN_INT64(val); +} + /* * txid_current_snapshot() returns txid_snapshot * diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index 6fed7a0d19..050a98c397 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -4904,6 +4904,8 @@ DATA(insert OID = 2942 ( txid_snapshot_send PGNSP PGUID 12 1 0 0 0 f f f f t DESCR("I/O"); DATA(insert OID = 2943 ( txid_current PGNSP PGUID 12 1 0 0 0 f f f f t f s u 0 0 20 "" _null_ _null_ _null_ _null_ _null_ txid_current _null_ _null_ _null_ )); DESCR("get current transaction ID"); +DATA(insert OID = 3348 ( txid_current_if_assigned PGNSP PGUID 12 1 0 0 0 f f f f t f s u 0 0 20 "" _null_ _null_ _null_ _null_ _null_ txid_current_if_assigned _null_ _null_ _null_ )); +DESCR("get current transaction ID"); DATA(insert OID = 2944 ( txid_current_snapshot PGNSP PGUID 12 1 0 0 0 f f f f t f s s 0 0 2970 "" _null_ _null_ _null_ _null_ _null_ txid_current_snapshot _null_ _null_ _null_ )); DESCR("get current snapshot"); DATA(insert OID = 2945 ( txid_snapshot_xmin PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 20 "2970" _null_ _null_ _null_ _null_ _null_ txid_snapshot_xmin _null_ _null_ _null_ )); diff --git a/src/include/utils/builtins.h b/src/include/utils/builtins.h index 40e25c8824..2ae212a9c3 100644 --- a/src/include/utils/builtins.h +++ b/src/include/utils/builtins.h @@ -1221,6 +1221,7 @@ extern Datum txid_snapshot_out(PG_FUNCTION_ARGS); extern Datum txid_snapshot_recv(PG_FUNCTION_ARGS); extern Datum txid_snapshot_send(PG_FUNCTION_ARGS); extern Datum txid_current(PG_FUNCTION_ARGS); +extern Datum txid_current_if_assigned(PG_FUNCTION_ARGS); extern Datum txid_current_snapshot(PG_FUNCTION_ARGS); extern Datum txid_snapshot_xmin(PG_FUNCTION_ARGS); extern Datum txid_snapshot_xmax(PG_FUNCTION_ARGS); diff --git a/src/test/regress/expected/txid.out b/src/test/regress/expected/txid.out index ddd217eb10..802ccb949f 100644 --- a/src/test/regress/expected/txid.out +++ b/src/test/regress/expected/txid.out @@ -238,3 +238,19 @@ SELECT txid_snapshot '1:9223372036854775808:3'; ERROR: invalid input syntax for type txid_snapshot: "1:9223372036854775808:3" LINE 1: SELECT txid_snapshot '1:9223372036854775808:3'; ^ +-- test txid_current_if_assigned +BEGIN; +SELECT txid_current_if_assigned() IS NULL; + ?column? +---------- + t +(1 row) + +SELECT txid_current() \gset +SELECT txid_current_if_assigned() IS NOT DISTINCT FROM BIGINT :'txid_current'; + ?column? +---------- + t +(1 row) + +COMMIT; diff --git a/src/test/regress/sql/txid.sql b/src/test/regress/sql/txid.sql index b6650b922e..4aefd9e64d 100644 --- a/src/test/regress/sql/txid.sql +++ b/src/test/regress/sql/txid.sql @@ -52,3 +52,10 @@ select txid_visible_in_snapshot('1000100010001015', '1000100010001000:1000100010 -- test 64bit overflow SELECT txid_snapshot '1:9223372036854775807:3'; SELECT txid_snapshot '1:9223372036854775808:3'; + +-- test txid_current_if_assigned +BEGIN; +SELECT txid_current_if_assigned() IS NULL; +SELECT txid_current() \gset +SELECT txid_current_if_assigned() IS NOT DISTINCT FROM BIGINT :'txid_current'; +COMMIT; -- cgit v1.2.3 From d2ddee63b43b27d6c6af169342af10db19bd3a1a Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 23 Aug 2016 12:10:25 -0400 Subject: Improve SP-GiST opclass API to better support unlabeled nodes. Previously, the spgSplitTuple action could only create a new upper tuple containing a single labeled node. This made it useless for opclasses that prefer to work with fixed sets of nodes (labeled or otherwise), which meant that restrictive prefixes could not be used with such node definitions. Change the output field set for the choose() method to allow it to specify any valid node set for the new upper tuple, and to specify which of these nodes to place the modified lower tuple in. In addition to its primary use for fixed node sets, this feature could allow existing opclasses that use variable node sets to skip a separate spgAddNode action when splitting a tuple, by setting up the node needed for the incoming value as part of the spgSplitTuple action. However, care would have to be taken to add the extra node only when it would not make the tuple bigger than before. (spgAddNode can enlarge the tuple, spgSplitTuple can't.) This is a prerequisite for an upcoming SP-GiST inet opclass, but is being committed separately to increase the visibility of the API change. In passing, improve the documentation about the traverse-values feature that was added by commit ccd6eb49a. Emre Hasegeli, with cosmetic adjustments and documentation rework by me Discussion: --- doc/src/sgml/spgist.sgml | 115 ++++++++++++++++++-------------- src/backend/access/spgist/spgdoinsert.c | 39 +++++++++-- src/backend/access/spgist/spgtextproc.c | 12 +++- src/include/access/spgist.h | 12 ++-- 4 files changed, 115 insertions(+), 63 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/spgist.sgml b/doc/src/sgml/spgist.sgml index f40c790612..dfa62adb55 100644 --- a/doc/src/sgml/spgist.sgml +++ b/doc/src/sgml/spgist.sgml @@ -114,7 +114,7 @@ box_ops - box + box << &< @@ -183,11 +183,14 @@ Inner tuples are more complex, since they are branching points in the search tree. Each inner tuple contains a set of one or more nodes, which represent groups of similar leaf values. - A node contains a downlink that leads to either another, lower-level inner - tuple, or a short list of leaf tuples that all lie on the same index page. - Each node has a label that describes it; for example, + A node contains a downlink that leads either to another, lower-level inner + tuple, or to a short list of leaf tuples that all lie on the same index page. + Each node normally has a label that describes it; for example, in a radix tree the node label could be the next character of the string - value. Optionally, an inner tuple can have a prefix value + value. (Alternatively, an operator class can omit the node labels, if it + works with a fixed set of nodes for all inner tuples; + see .) + Optionally, an inner tuple can have a prefix value that describes all its members. In a radix tree this could be the common prefix of the represented strings. The prefix value is not necessarily really a prefix, but can be any data needed by the operator class; @@ -202,7 +205,8 @@ tuple, so the SP-GiST core provides the possibility for operator classes to manage level counting while descending the tree. There is also support for incrementally reconstructing the represented - value when that is needed. + value when that is needed, and for passing down additional data (called + traverse values) during a tree descent. @@ -343,10 +347,13 @@ typedef struct spgChooseOut } addNode; struct /* results for spgSplitTuple */ { - /* Info to form new inner tuple with one node */ + /* Info to form new upper-level inner tuple with one child tuple */ bool prefixHasPrefix; /* tuple should have a prefix? */ Datum prefixPrefixDatum; /* if so, its value */ - Datum nodeLabel; /* node's label */ + int prefixNNodes; /* number of nodes */ + Datum *prefixNodeLabels; /* their labels (or NULL for + * no labels) */ + int childNodeN; /* which node gets child tuple */ /* Info to form new lower-level inner tuple with all old nodes */ bool postfixHasPrefix; /* tuple should have a prefix? */ @@ -416,29 +423,33 @@ typedef struct spgChooseOut set resultType to spgSplitTuple. This action moves all the existing nodes into a new lower-level inner tuple, and replaces the existing inner tuple with a tuple - having a single node that links to the new lower-level inner tuple. + having a single downlink pointing to the new lower-level inner tuple. Set prefixHasPrefix to indicate whether the new upper tuple should have a prefix, and if so set prefixPrefixDatum to the prefix value. This new prefix value must be sufficiently less restrictive than the original - to accept the new value to be indexed, and it should be no longer - than the original prefix. - Set nodeLabel to the label to be used for the - node that will point to the new lower-level inner tuple. + to accept the new value to be indexed. + Set prefixNNodes to the number of nodes needed in the + new tuple, and set prefixNodeLabels to a palloc'd array + holding their labels, or to NULL if node labels are not required. + Note that the total size of the new upper tuple must be no more + than the total size of the tuple it is replacing; this constrains + the lengths of the new prefix and new labels. + Set childNodeN to the index (from zero) of the node + that will downlink to the new lower-level inner tuple. Set postfixHasPrefix to indicate whether the new lower-level inner tuple should have a prefix, and if so set postfixPrefixDatum to the prefix value. The - combination of these two prefixes and the additional label must - have the same meaning as the original prefix, because there is - no opportunity to alter the node labels that are moved to the new - lower-level tuple, nor to change any child index entries. + combination of these two prefixes and the downlink node's label + (if any) must have the same meaning as the original prefix, because + there is no opportunity to alter the node labels that are moved to + the new lower-level tuple, nor to change any child index entries. After the node has been split, the choose function will be called again with the replacement inner tuple. - That call will usually result in an spgAddNode result, - since presumably the node label added in the split step will not - match the new value; so after that, there will be a third call - that finally returns spgMatchNode and allows the - insertion to descend to the leaf level. + That call may return an spgAddNode result, if no suitable + node was created by the spgSplitTuple action. Eventually + choose must return spgMatchNode to + allow the insertion to descend to the next level. @@ -492,9 +503,8 @@ typedef struct spgPickSplitOut prefixDatum to the prefix value. Set nNodes to indicate the number of nodes that the new inner tuple will contain, and - set nodeLabels to an array of their label values. - (If the nodes do not require labels, set nodeLabels - to NULL; see for details.) + set nodeLabels to an array of their label values, + or to NULL if node labels are not required. Set mapTuplesToNodes to an array that gives the index (from zero) of the node that each leaf tuple should be assigned to. Set leafTupleDatums to an array of the values to @@ -561,7 +571,7 @@ typedef struct spgInnerConsistentIn Datum reconstructedValue; /* value reconstructed at parent */ void *traversalValue; /* opclass-specific traverse value */ - MemoryContext traversalMemoryContext; + MemoryContext traversalMemoryContext; /* put new traverse values here */ int level; /* current level (counting from zero) */ bool returnData; /* original data must be returned? */ @@ -580,7 +590,6 @@ typedef struct spgInnerConsistentOut int *levelAdds; /* increment level by this much for each */ Datum *reconstructedValues; /* associated reconstructed values */ void **traversalValues; /* opclass-specific traverse values */ - } spgInnerConsistentOut; @@ -599,6 +608,11 @@ typedef struct spgInnerConsistentOut parent tuple; it is (Datum) 0 at the root level or if the inner_consistent function did not provide a value at the parent level. + traversalValue is a pointer to any traverse data + passed down from the previous call of inner_consistent + on the parent index tuple, or NULL at the root level. + traversalMemoryContext is the memory context in which + to store output traverse values (see below). level is the current inner tuple's level, starting at zero for the root level. returnData is true if reconstructed data is @@ -615,9 +629,6 @@ typedef struct spgInnerConsistentOut inner tuple, and nodeLabels is an array of their label values, or NULL if the nodes do not have labels. - traversalValue is a pointer to data that - inner_consistent gets when called on child nodes from an - outer call of inner_consistent on parent nodes. @@ -633,17 +644,19 @@ typedef struct spgInnerConsistentOut reconstructedValues to an array of the values reconstructed for each child node to be visited; otherwise, leave reconstructedValues as NULL. + If it is desired to pass down additional out-of-band information + (traverse values) to lower levels of the tree search, + set traversalValues to an array of the appropriate + traverse values, one for each child node to be visited; otherwise, + leave traversalValues as NULL. Note that the inner_consistent function is responsible for palloc'ing the - nodeNumbers, levelAdds and - reconstructedValues arrays. - Sometimes accumulating some information is needed, while - descending from parent to child node was happened. In this case - traversalValues array keeps pointers to - specific data you need to accumulate for every child node. - Memory for traversalValues should be allocated in - the default context, but each element of it should be allocated in - traversalMemoryContext. + nodeNumbers, levelAdds, + reconstructedValues, and + traversalValues arrays in the current memory context. + However, any output traverse values pointed to by + the traversalValues array should be allocated + in traversalMemoryContext. @@ -670,8 +683,8 @@ typedef struct spgLeafConsistentIn ScanKey scankeys; /* array of operators and comparison values */ int nkeys; /* length of array */ - void *traversalValue; /* opclass-specific traverse value */ Datum reconstructedValue; /* value reconstructed at parent */ + void *traversalValue; /* opclass-specific traverse value */ int level; /* current level (counting from zero) */ bool returnData; /* original data must be returned? */ @@ -700,6 +713,9 @@ typedef struct spgLeafConsistentOut parent tuple; it is (Datum) 0 at the root level or if the inner_consistent function did not provide a value at the parent level. + traversalValue is a pointer to any traverse data + passed down from the previous call of inner_consistent + on the parent index tuple, or NULL at the root level. level is the current leaf tuple's level, starting at zero for the root level. returnData is true if reconstructed data is @@ -797,7 +813,10 @@ typedef struct spgLeafConsistentOut point. In such a case the code typically works with the nodes by number, and there is no need for explicit node labels. To suppress node labels (and thereby save some space), the picksplit - function can return NULL for the nodeLabels array. + function can return NULL for the nodeLabels array, + and likewise the choose function can return NULL for + the prefixNodeLabels array during + a spgSplitTuple action. This will in turn result in nodeLabels being NULL during subsequent calls to choose and inner_consistent. In principle, node labels could be used for some inner tuples and omitted @@ -807,10 +826,7 @@ typedef struct spgLeafConsistentOut When working with an inner tuple having unlabeled nodes, it is an error for choose to return spgAddNode, since the set - of nodes is supposed to be fixed in such cases. Also, there is no - provision for generating an unlabeled node in spgSplitTuple - actions, since it is expected that an spgAddNode action will - be needed as well. + of nodes is supposed to be fixed in such cases. @@ -859,11 +875,10 @@ typedef struct spgLeafConsistentOut The PostgreSQL source distribution includes - several examples of index operator classes for - SP-GiST. The core system currently provides radix - trees over text columns and two types of trees over points: quad-tree and - k-d tree. Look into src/backend/access/spgist/ to see the - code. + several examples of index operator classes for SP-GiST, + as described in . Look + into src/backend/access/spgist/ + and src/backend/utils/adt/ to see the code. diff --git a/src/backend/access/spgist/spgdoinsert.c b/src/backend/access/spgist/spgdoinsert.c index f090ca528b..6fc04b224d 100644 --- a/src/backend/access/spgist/spgdoinsert.c +++ b/src/backend/access/spgist/spgdoinsert.c @@ -1705,17 +1705,40 @@ spgSplitNodeAction(Relation index, SpGistState *state, /* Should not be applied to nulls */ Assert(!SpGistPageStoresNulls(current->page)); + /* Check opclass gave us sane values */ + if (out->result.splitTuple.prefixNNodes <= 0 || + out->result.splitTuple.prefixNNodes > SGITMAXNNODES) + elog(ERROR, "invalid number of prefix nodes: %d", + out->result.splitTuple.prefixNNodes); + if (out->result.splitTuple.childNodeN < 0 || + out->result.splitTuple.childNodeN >= + out->result.splitTuple.prefixNNodes) + elog(ERROR, "invalid child node number: %d", + out->result.splitTuple.childNodeN); + /* - * Construct new prefix tuple, containing a single node with the specified - * label. (We'll update the node's downlink to point to the new postfix - * tuple, below.) + * Construct new prefix tuple with requested number of nodes. We'll fill + * in the childNodeN'th node's downlink below. */ - node = spgFormNodeTuple(state, out->result.splitTuple.nodeLabel, false); + nodes = (SpGistNodeTuple *) palloc(sizeof(SpGistNodeTuple) * + out->result.splitTuple.prefixNNodes); + + for (i = 0; i < out->result.splitTuple.prefixNNodes; i++) + { + Datum label = (Datum) 0; + bool labelisnull; + + labelisnull = (out->result.splitTuple.prefixNodeLabels == NULL); + if (!labelisnull) + label = out->result.splitTuple.prefixNodeLabels[i]; + nodes[i] = spgFormNodeTuple(state, label, labelisnull); + } prefixTuple = spgFormInnerTuple(state, out->result.splitTuple.prefixHasPrefix, out->result.splitTuple.prefixPrefixDatum, - 1, &node); + out->result.splitTuple.prefixNNodes, + nodes); /* it must fit in the space that innerTuple now occupies */ if (prefixTuple->size > innerTuple->size) @@ -1807,10 +1830,12 @@ spgSplitNodeAction(Relation index, SpGistState *state, * the postfix tuple first.) We have to update the local copy of the * prefixTuple too, because that's what will be written to WAL. */ - spgUpdateNodeLink(prefixTuple, 0, postfixBlkno, postfixOffset); + spgUpdateNodeLink(prefixTuple, out->result.splitTuple.childNodeN, + postfixBlkno, postfixOffset); prefixTuple = (SpGistInnerTuple) PageGetItem(current->page, PageGetItemId(current->page, current->offnum)); - spgUpdateNodeLink(prefixTuple, 0, postfixBlkno, postfixOffset); + spgUpdateNodeLink(prefixTuple, out->result.splitTuple.childNodeN, + postfixBlkno, postfixOffset); MarkBufferDirty(current->buffer); diff --git a/src/backend/access/spgist/spgtextproc.c b/src/backend/access/spgist/spgtextproc.c index e0d8f30ef1..852a9b00fa 100644 --- a/src/backend/access/spgist/spgtextproc.c +++ b/src/backend/access/spgist/spgtextproc.c @@ -212,9 +212,14 @@ spg_text_choose(PG_FUNCTION_ARGS) out->result.splitTuple.prefixPrefixDatum = formTextDatum(prefixStr, commonLen); } - out->result.splitTuple.nodeLabel = + out->result.splitTuple.prefixNNodes = 1; + out->result.splitTuple.prefixNodeLabels = + (Datum *) palloc(sizeof(Datum)); + out->result.splitTuple.prefixNodeLabels[0] = Int16GetDatum(*(unsigned char *) (prefixStr + commonLen)); + out->result.splitTuple.childNodeN = 0; + if (prefixSize - commonLen == 1) { out->result.splitTuple.postfixHasPrefix = false; @@ -280,7 +285,10 @@ spg_text_choose(PG_FUNCTION_ARGS) out->resultType = spgSplitTuple; out->result.splitTuple.prefixHasPrefix = in->hasPrefix; out->result.splitTuple.prefixPrefixDatum = in->prefixDatum; - out->result.splitTuple.nodeLabel = Int16GetDatum(-2); + out->result.splitTuple.prefixNNodes = 1; + out->result.splitTuple.prefixNodeLabels = (Datum *) palloc(sizeof(Datum)); + out->result.splitTuple.prefixNodeLabels[0] = Int16GetDatum(-2); + out->result.splitTuple.childNodeN = 0; out->result.splitTuple.postfixHasPrefix = false; } else diff --git a/src/include/access/spgist.h b/src/include/access/spgist.h index f39a2d6938..a953a5a401 100644 --- a/src/include/access/spgist.h +++ b/src/include/access/spgist.h @@ -90,10 +90,13 @@ typedef struct spgChooseOut } addNode; struct /* results for spgSplitTuple */ { - /* Info to form new inner tuple with one node */ + /* Info to form new upper-level inner tuple with one child tuple */ bool prefixHasPrefix; /* tuple should have a prefix? */ Datum prefixPrefixDatum; /* if so, its value */ - Datum nodeLabel; /* node's label */ + int prefixNNodes; /* number of nodes */ + Datum *prefixNodeLabels; /* their labels (or NULL for + * no labels) */ + int childNodeN; /* which node gets child tuple */ /* Info to form new lower-level inner tuple with all old nodes */ bool postfixHasPrefix; /* tuple should have a prefix? */ @@ -134,7 +137,8 @@ typedef struct spgInnerConsistentIn Datum reconstructedValue; /* value reconstructed at parent */ void *traversalValue; /* opclass-specific traverse value */ - MemoryContext traversalMemoryContext; + MemoryContext traversalMemoryContext; /* put new traverse values + * here */ int level; /* current level (counting from zero) */ bool returnData; /* original data must be returned? */ @@ -163,8 +167,8 @@ typedef struct spgLeafConsistentIn ScanKey scankeys; /* array of operators and comparison values */ int nkeys; /* length of array */ - void *traversalValue; /* opclass-specific traverse value */ Datum reconstructedValue; /* value reconstructed at parent */ + void *traversalValue; /* opclass-specific traverse value */ int level; /* current level (counting from zero) */ bool returnData; /* original data must be returned? */ -- cgit v1.2.3 From ff066481b0485b1a4e414de3abcaae0bda02b1e1 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Tue, 23 Aug 2016 12:45:33 -0400 Subject: doc: fix incorrect 'literal' tags Discussion: dcc4113d-1eda-4f60-d1c5-f50eee160bad@gmail.com Author: Alexander Law Backpatch-through: 9.6 --- doc/src/sgml/pgstandby.sgml | 2 +- doc/src/sgml/ref/pg_xlogdump.sgml | 2 +- doc/src/sgml/ref/pgbench.sgml | 2 +- doc/src/sgml/ref/psql-ref.sgml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/pgstandby.sgml b/doc/src/sgml/pgstandby.sgml index fb3f32eaaa..80c6f60062 100644 --- a/doc/src/sgml/pgstandby.sgml +++ b/doc/src/sgml/pgstandby.sgml @@ -363,7 +363,7 @@ recovery_end_command = 'del C:\pgsql.trigger.5442' The copy command on Windows sets the final file size before the file is completely copied, which would ordinarily confuse pg_standby. Therefore - pg_standby waits sleeptime + pg_standby waits sleeptime seconds once it sees the proper file size. GNUWin32's cp sets the file size only after the file copy is complete. diff --git a/doc/src/sgml/ref/pg_xlogdump.sgml b/doc/src/sgml/ref/pg_xlogdump.sgml index 296f1acc24..177caab00d 100644 --- a/doc/src/sgml/ref/pg_xlogdump.sgml +++ b/doc/src/sgml/ref/pg_xlogdump.sgml @@ -153,7 +153,7 @@ PostgreSQL documentation Timeline from which to read log records. The default is to use the - value in startseg, if that is specified; otherwise, the + value in startseg, if that is specified; otherwise, the default is 1. diff --git a/doc/src/sgml/ref/pgbench.sgml b/doc/src/sgml/ref/pgbench.sgml index f58da351f9..285608d508 100644 --- a/doc/src/sgml/ref/pgbench.sgml +++ b/doc/src/sgml/ref/pgbench.sgml @@ -433,7 +433,7 @@ pgbench options dbname sec - Show progress report every sec seconds. The report + Show progress report every sec seconds. The report includes the time since the beginning of the run, the tps since the last report, and the transaction latency average and standard deviation since the last report. Under throttling ( - If this is on, you should create users as username@dbname. - When username is passed by a connecting client, + If this is on, you should create users as username@dbname. + When username is passed by a connecting client, @ and the database name are appended to the user name and that database-specific user name is looked up by the server. Note that when you create users with names containing diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 6355300d9d..5c1c4f69fb 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -13800,7 +13800,7 @@ SELECT xmlagg(x) FROM (SELECT x FROM test ORDER BY y DESC) AS tab; No multiple continuous percentile: returns an array of results matching - the shape of the fractions parameter, with each + the shape of the fractions parameter, with each non-null element replaced by the value corresponding to that percentile @@ -13845,7 +13845,7 @@ SELECT xmlagg(x) FROM (SELECT x FROM test ORDER BY y DESC) AS tab; No multiple discrete percentile: returns an array of results matching the - shape of the fractions parameter, with each non-null + shape of the fractions parameter, with each non-null element replaced by the input value corresponding to that percentile @@ -16850,7 +16850,7 @@ SELECT pg_type_is_visible('myschema.widget'::regtype); pg_options_to_table returns the set of storage option name/value pairs - (option_name/option_value) when passed + (option_name/option_value) when passed pg_class.reloptions or pg_attribute.attoptions. diff --git a/doc/src/sgml/ref/create_sequence.sgml b/doc/src/sgml/ref/create_sequence.sgml index c9591462ee..62ae379226 100644 --- a/doc/src/sgml/ref/create_sequence.sgml +++ b/doc/src/sgml/ref/create_sequence.sgml @@ -349,7 +349,7 @@ END; - The standard's AS <data type> expression is not + The standard's AS data_type expression is not supported. diff --git a/doc/src/sgml/ref/set_role.sgml b/doc/src/sgml/ref/set_role.sgml index aff3792199..a97ceabcff 100644 --- a/doc/src/sgml/ref/set_role.sgml +++ b/doc/src/sgml/ref/set_role.sgml @@ -127,7 +127,7 @@ SELECT SESSION_USER, CURRENT_USER; PostgreSQL - allows identifier syntax ("rolename"), while + allows identifier syntax ("rolename"), while the SQL standard requires the role name to be written as a string literal. SQL does not allow this command during a transaction; PostgreSQL does not make this diff --git a/doc/src/sgml/ref/set_session_auth.sgml b/doc/src/sgml/ref/set_session_auth.sgml index 4ac2128950..96d279aaf9 100644 --- a/doc/src/sgml/ref/set_session_auth.sgml +++ b/doc/src/sgml/ref/set_session_auth.sgml @@ -101,7 +101,7 @@ SELECT SESSION_USER, CURRENT_USER; The SQL standard allows some other expressions to appear in place of the literal user_name, but these options are not important in practice. PostgreSQL - allows identifier syntax ("username"), which SQL + allows identifier syntax ("username"), which SQL does not. SQL does not allow this command during a transaction; PostgreSQL does not make this restriction because there is no reason to. diff --git a/doc/src/sgml/textsearch.sgml b/doc/src/sgml/textsearch.sgml index be5974a4ff..5a70d7db80 100644 --- a/doc/src/sgml/textsearch.sgml +++ b/doc/src/sgml/textsearch.sgml @@ -3622,10 +3622,10 @@ SELECT plainto_tsquery('supernovae stars'); - The optional parameter PATTERN can be the name of + The optional parameter PATTERN can be the name of a text search object, optionally schema-qualified. If - PATTERN is omitted then information about all - visible objects will be displayed. PATTERN can be a + PATTERN is omitted then information about all + visible objects will be displayed. PATTERN can be a regular expression and can provide separate patterns for the schema and object names. The following examples illustrate this: diff --git a/doc/src/sgml/xfunc.sgml b/doc/src/sgml/xfunc.sgml index d8d2e9e490..de6a466efc 100644 --- a/doc/src/sgml/xfunc.sgml +++ b/doc/src/sgml/xfunc.sgml @@ -204,8 +204,8 @@ SELECT clean_emp(); If an argument is of a composite type, then the dot notation, - e.g., argname.fieldname or - $1.fieldname, can be used to access attributes of the + e.g., argname.fieldname or + $1.fieldname, can be used to access attributes of the argument. Again, you might need to qualify the argument's name with the function name to make the form with an argument name unambiguous. @@ -527,7 +527,8 @@ LINE 1: SELECT new_emp().name; Another option is to use functional notation for extracting an attribute. The simple way to explain this is that we can use the - notations attribute(table) and table.attribute + notations attribute(table) + and table.attribute interchangeably. @@ -1305,12 +1306,15 @@ CREATE FUNCTION test(smallint, double precision) RETURNS ... A function that takes a single argument of a composite type should generally not have the same name as any attribute (field) of that type. - Recall that attribute(table) is considered equivalent - to table.attribute. In the case that there is an + Recall that attribute(table) + is considered equivalent + to table.attribute. + In the case that there is an ambiguity between a function on a composite type and an attribute of the composite type, the attribute will always be used. It is possible to override that choice by schema-qualifying the function name - (that is, schema.func(table)) but it's better to + (that is, schema.func(table) + ) but it's better to avoid the problem by not choosing conflicting names. @@ -2818,7 +2822,7 @@ HeapTuple heap_form_tuple(TupleDesc tupdesc, Datum *values, bool *isnull) HeapTuple BuildTupleFromCStrings(AttInMetadata *attinmeta, char **values) to build a HeapTuple given user data - in C string form. values is an array of C strings, + in C string form. values is an array of C strings, one for each attribute of the return row. Each C string should be in the form expected by the input function of the attribute data type. In order to return a null value for one of the attributes, -- cgit v1.2.3 From 26fa446da64716f12ab3a623434c644fcb344b2e Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 26 Aug 2016 16:20:17 -0400 Subject: Add a nonlocalized version of the severity field to client error messages. This has been requested a few times, but the use-case for it was never entirely clear. The reason for adding it now is that transmission of error reports from parallel workers fails when NLS is active, because pq_parse_errornotice() wrongly assumes that the existing severity field is nonlocalized. There are other ways we could have fixed that, but the other options were basically kluges, whereas this way provides something that's at least arguably a useful feature along with the bug fix. Per report from Jakob Egger. Back-patch into 9.6, because otherwise parallel query is essentially unusable in non-English locales. The problem exists in 9.5 as well, but we don't want to risk changing on-the-wire behavior in 9.5 (even though the possibility of new error fields is specifically called out in the protocol document). It may be sufficient to leave the issue unfixed in 9.5, given the very limited usefulness of pq_parse_errornotice in that version. Discussion: --- doc/src/sgml/libpq.sgml | 16 ++++++++++++++++ doc/src/sgml/protocol.sgml | 19 +++++++++++++++++++ src/backend/libpq/pqmq.c | 26 +++++++++++++++++++++----- src/backend/utils/error/elog.c | 33 ++++++++++++++++++++------------- src/include/postgres_ext.h | 1 + src/interfaces/libpq/fe-exec.c | 1 + 6 files changed, 78 insertions(+), 18 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/libpq.sgml b/doc/src/sgml/libpq.sgml index f22e3da047..2f9350b10e 100644 --- a/doc/src/sgml/libpq.sgml +++ b/doc/src/sgml/libpq.sgml @@ -2767,6 +2767,22 @@ char *PQresultErrorField(const PGresult *res, int fieldcode); + + PG_DIAG_SEVERITY_NONLOCALIZED + + + The severity; the field contents are ERROR, + FATAL, or PANIC (in an error message), + or WARNING, NOTICE, DEBUG, + INFO, or LOG (in a notice message). + This is identical to the PG_DIAG_SEVERITY field except + that the contents are never localized. This is present only in + reports generated by PostgreSQL versions 9.6 + and later. + + + + PG_DIAG_SQLSTATE diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index 9c96d8fc44..68b0941029 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -4882,6 +4882,25 @@ message. + + +V + + + + Severity: the field contents are + ERROR, FATAL, or + PANIC (in an error message), or + WARNING, NOTICE, DEBUG, + INFO, or LOG (in a notice message). + This is identical to the S field except + that the contents are never localized. This is present only in + messages generated by PostgreSQL versions 9.6 + and later. + + + + C diff --git a/src/backend/libpq/pqmq.c b/src/backend/libpq/pqmq.c index 921242fbc4..bfe66c6c44 100644 --- a/src/backend/libpq/pqmq.c +++ b/src/backend/libpq/pqmq.c @@ -237,10 +237,26 @@ pq_parse_errornotice(StringInfo msg, ErrorData *edata) switch (code) { case PG_DIAG_SEVERITY: + /* ignore, trusting we'll get a nonlocalized version */ + break; + case PG_DIAG_SEVERITY_NONLOCALIZED: if (strcmp(value, "DEBUG") == 0) - edata->elevel = DEBUG1; /* or some other DEBUG level */ + { + /* + * We can't reconstruct the exact DEBUG level, but + * presumably it was >= client_min_messages, so select + * DEBUG1 to ensure we'll pass it on to the client. + */ + edata->elevel = DEBUG1; + } else if (strcmp(value, "LOG") == 0) - edata->elevel = LOG; /* can't be COMMERROR */ + { + /* + * It can't be LOG_SERVER_ONLY, or the worker wouldn't + * have sent it to us; so LOG is the correct value. + */ + edata->elevel = LOG; + } else if (strcmp(value, "INFO") == 0) edata->elevel = INFO; else if (strcmp(value, "NOTICE") == 0) @@ -254,11 +270,11 @@ pq_parse_errornotice(StringInfo msg, ErrorData *edata) else if (strcmp(value, "PANIC") == 0) edata->elevel = PANIC; else - elog(ERROR, "unknown error severity"); + elog(ERROR, "unrecognized error severity: \"%s\"", value); break; case PG_DIAG_SQLSTATE: if (strlen(value) != 5) - elog(ERROR, "malformed sql state"); + elog(ERROR, "invalid SQLSTATE: \"%s\"", value); edata->sqlerrcode = MAKE_SQLSTATE(value[0], value[1], value[2], value[3], value[4]); break; @@ -308,7 +324,7 @@ pq_parse_errornotice(StringInfo msg, ErrorData *edata) edata->funcname = pstrdup(value); break; default: - elog(ERROR, "unknown error field: %d", (int) code); + elog(ERROR, "unrecognized error field code: %d", (int) code); break; } } diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index 03c4a39761..224ee7801c 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -2753,7 +2753,7 @@ write_csvlog(ErrorData *edata) appendStringInfoChar(&buf, ','); /* Error severity */ - appendStringInfoString(&buf, error_severity(edata->elevel)); + appendStringInfoString(&buf, _(error_severity(edata->elevel))); appendStringInfoChar(&buf, ','); /* SQL state code */ @@ -2870,7 +2870,7 @@ send_message_to_server_log(ErrorData *edata) formatted_log_time[0] = '\0'; log_line_prefix(&buf, edata); - appendStringInfo(&buf, "%s: ", error_severity(edata->elevel)); + appendStringInfo(&buf, "%s: ", _(error_severity(edata->elevel))); if (Log_error_verbosity >= PGERROR_VERBOSE) appendStringInfo(&buf, "%s: ", unpack_sql_state(edata->sqlerrcode)); @@ -3153,12 +3153,16 @@ send_message_to_frontend(ErrorData *edata) if (PG_PROTOCOL_MAJOR(FrontendProtocol) >= 3) { /* New style with separate fields */ + const char *sev; char tbuf[12]; int ssval; int i; + sev = error_severity(edata->elevel); pq_sendbyte(&msgbuf, PG_DIAG_SEVERITY); - err_sendstring(&msgbuf, error_severity(edata->elevel)); + err_sendstring(&msgbuf, _(sev)); + pq_sendbyte(&msgbuf, PG_DIAG_SEVERITY_NONLOCALIZED); + err_sendstring(&msgbuf, sev); /* unpack MAKE_SQLSTATE code */ ssval = edata->sqlerrcode; @@ -3277,7 +3281,7 @@ send_message_to_frontend(ErrorData *edata) initStringInfo(&buf); - appendStringInfo(&buf, "%s: ", error_severity(edata->elevel)); + appendStringInfo(&buf, "%s: ", _(error_severity(edata->elevel))); if (edata->show_funcname && edata->funcname) appendStringInfo(&buf, "%s: ", edata->funcname); @@ -3587,7 +3591,10 @@ get_errno_symbol(int errnum) /* - * error_severity --- get localized string representing elevel + * error_severity --- get string representing elevel + * + * The string is not localized here, but we mark the strings for translation + * so that callers can invoke _() on the result. */ static const char * error_severity(int elevel) @@ -3601,29 +3608,29 @@ error_severity(int elevel) case DEBUG3: case DEBUG4: case DEBUG5: - prefix = _("DEBUG"); + prefix = gettext_noop("DEBUG"); break; case LOG: case LOG_SERVER_ONLY: - prefix = _("LOG"); + prefix = gettext_noop("LOG"); break; case INFO: - prefix = _("INFO"); + prefix = gettext_noop("INFO"); break; case NOTICE: - prefix = _("NOTICE"); + prefix = gettext_noop("NOTICE"); break; case WARNING: - prefix = _("WARNING"); + prefix = gettext_noop("WARNING"); break; case ERROR: - prefix = _("ERROR"); + prefix = gettext_noop("ERROR"); break; case FATAL: - prefix = _("FATAL"); + prefix = gettext_noop("FATAL"); break; case PANIC: - prefix = _("PANIC"); + prefix = gettext_noop("PANIC"); break; default: prefix = "???"; diff --git a/src/include/postgres_ext.h b/src/include/postgres_ext.h index 74c344c704..ae2f087798 100644 --- a/src/include/postgres_ext.h +++ b/src/include/postgres_ext.h @@ -49,6 +49,7 @@ typedef PG_INT64_TYPE pg_int64; * applications. */ #define PG_DIAG_SEVERITY 'S' +#define PG_DIAG_SEVERITY_NONLOCALIZED 'V' #define PG_DIAG_SQLSTATE 'C' #define PG_DIAG_MESSAGE_PRIMARY 'M' #define PG_DIAG_MESSAGE_DETAIL 'D' diff --git a/src/interfaces/libpq/fe-exec.c b/src/interfaces/libpq/fe-exec.c index d1b91c841c..a9ba54628f 100644 --- a/src/interfaces/libpq/fe-exec.c +++ b/src/interfaces/libpq/fe-exec.c @@ -824,6 +824,7 @@ pqInternalNotice(const PGNoticeHooks *hooks, const char *fmt,...) */ pqSaveMessageField(res, PG_DIAG_MESSAGE_PRIMARY, msgBuf); pqSaveMessageField(res, PG_DIAG_SEVERITY, libpq_gettext("NOTICE")); + pqSaveMessageField(res, PG_DIAG_SEVERITY_NONLOCALIZED, "NOTICE"); /* XXX should provide a SQLSTATE too? */ /* -- cgit v1.2.3 From a6f0dc701b2f84839761783e87c49d43cd3e31df Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 28 Aug 2016 12:37:23 -0400 Subject: Update 9.6 release notes through today. --- doc/src/sgml/config.sgml | 24 ++++++--------- doc/src/sgml/release-9.6.sgml | 71 +++++++++++++++++++++++++++++++++++++++---- 2 files changed, 75 insertions(+), 20 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 4db9c81e56..7c483c6ef3 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -5388,9 +5388,10 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; Process Title - These settings control how the process title as seen - by ps is modified. See - for details. + These settings control how process titles of server processes are + modified. Process titles are typically viewed using programs like + ps or, on Windows, Process Explorer. + See for details. @@ -5403,18 +5404,14 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; Sets the cluster name that appears in the process title for all - processes in this cluster. The name can be any string of less than - NAMEDATALEN characters (64 characters in a standard + server processes in this cluster. The name can be any string of less + than NAMEDATALEN characters (64 characters in a standard build). Only printable ASCII characters may be used in the cluster_name value. Other characters will be replaced with question marks (?). No name is shown if this parameter is set to the empty string '' (which is the default). This parameter can only be set at server start. - - The process title is typically viewed using programs like - ps or, on Windows, Process Explorer. - @@ -5427,11 +5424,10 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; Enables updating of the process title every time a new SQL command - is received by the server. The process title is typically viewed - by the ps command, - or in Windows by using the Process Explorer. - This value defaults to off on Windows platforms due to the - platform's significant overhead for updating the process title. + is received by the server. + This setting defaults to on on most platforms, but it + defaults to off on Windows due to that platform's larger + overhead for updating the process title. Only superusers can change this setting. diff --git a/doc/src/sgml/release-9.6.sgml b/doc/src/sgml/release-9.6.sgml index 8d7356e27f..6ec7583485 100644 --- a/doc/src/sgml/release-9.6.sgml +++ b/doc/src/sgml/release-9.6.sgml @@ -7,7 +7,7 @@ Release Date 2016-??-?? - Current as of 2016-08-08 (commit 34927b292) + Current as of 2016-08-27 (commit b9fe6cbc8) @@ -348,6 +348,7 @@ This commit is also listed under libpq and psql 2016-04-27 [59eb55127] Fix EXPLAIN VERBOSE output for parallel aggregate. 2016-06-09 [c9ce4a1c6] Eliminate "parallel degree" terminology. 2016-06-16 [75be66464] Invent min_parallel_relation_size GUC to replace a hard- +2016-08-16 [f85b1a841] Disable parallel query by default. --> Parallel queries (Robert Haas, Amit Kapila, David Rowley, @@ -365,9 +366,11 @@ This commit is also listed under libpq and psql - Use of parallel query execution can be controlled - through the new configuration parameters , + Parallel query execution is not (yet) enabled by default. + To allow it, set the new configuration + parameter to a + value larger than zero. Additional control over use of parallelism + is available through other new configuration parameters , , , and + + + + Disable by default on + Windows (Takayuki Tsunakawa) + + + + The overhead of updating the process title is much larger on Windows + than most other platforms, and it is also less useful to do it since + most Windows users do not have tools that can display process titles. + + + @@ -2431,6 +2450,22 @@ XXX this is pending backpatch, may need to remove + + Add a nonlocalized version of the severity field in error and notice + messages (Tom Lane) + + + + This change allows client code to determine severity of an error or + notice without having to worry about localized variants of the + severity strings. + + + + + @@ -2957,6 +2992,25 @@ This commit is also listed under libpq and PL/pgSQL + + Add macros to make AllocSetContextCreate() calls simpler + and safer (Tom Lane) + + + + Writing out the individual sizing parameters for a memory context + is now deprecated in favor of using one of the new + macros ALLOCSET_DEFAULT_SIZES, + ALLOCSET_SMALL_SIZES, + or ALLOCSET_START_SMALL_SIZES. + Existing code continues to work, however. + + + + + @@ -3038,19 +3092,24 @@ This commit is also listed under libpq and PL/pgSQL Restructure index access method API to hide most of - it at the C level (Alexander Korotkov) + it at the C level (Alexander Korotkov, Andrew Gierth) This change modernizes the index AM API to look more like the designs we have adopted for foreign data wrappers and tablesample handlers. This simplifies the C code - and should make it more feasible to define index access methods in + and makes it much more practical to define index access methods in installable extensions. A consequence is that most of the columns of the pg_am system catalog have disappeared. + New inspection + functions have been added to allow SQL queries to determine + index AM properties that used to be discoverable + from pg_am. -- cgit v1.2.3 From 39d866433e6fb1c385eee8dc67843097b8703add Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 28 Aug 2016 17:40:06 -0400 Subject: Make another editorial pass over the 9.6 release notes. I think they're pretty much release-quality now. --- doc/src/sgml/func.sgml | 28 +-- doc/src/sgml/release-9.6.sgml | 447 +++++++++++++++++++++++------------------- doc/src/sgml/spgist.sgml | 1 + 3 files changed, 264 insertions(+), 212 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 5c1c4f69fb..5148095fb3 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -18580,7 +18580,7 @@ postgres=# SELECT * FROM pg_xlogfile_name_offset(pg_stop_backup()); pg_replication_origin_drop(node_name text) - void + void Delete a previously created replication origin, including any @@ -18612,7 +18612,7 @@ postgres=# SELECT * FROM pg_xlogfile_name_offset(pg_stop_backup()); pg_replication_origin_session_setup(node_name text) - void + void Mark the current session as replaying from the given @@ -18630,7 +18630,7 @@ postgres=# SELECT * FROM pg_xlogfile_name_offset(pg_stop_backup()); pg_replication_origin_session_reset() - void + void Cancel the effects @@ -18679,7 +18679,7 @@ postgres=# SELECT * FROM pg_xlogfile_name_offset(pg_stop_backup()); pg_replication_origin_xact_setup(origin_lsn pg_lsn, origin_timestamp timestamptz) - void + void Mark the current transaction as replaying a transaction that has @@ -18698,7 +18698,7 @@ postgres=# SELECT * FROM pg_xlogfile_name_offset(pg_stop_backup()); pg_replication_origin_xact_reset() - void + void Cancel the effects of @@ -18714,7 +18714,7 @@ postgres=# SELECT * FROM pg_xlogfile_name_offset(pg_stop_backup()); pg_replication_origin_advance(node_name text, pos pg_lsn) - void + void Set replication progress for the given node to the given @@ -19174,7 +19174,7 @@ postgres=# SELECT * FROM pg_xlogfile_name_offset(pg_stop_backup()); - brin_summarize_new_values(index_oid regclass) + brin_summarize_new_values(index regclass) integer summarize page ranges not already summarized @@ -19191,8 +19191,8 @@ postgres=# SELECT * FROM pg_xlogfile_name_offset(pg_stop_backup()); - brin_summarize_new_values receives a BRIN index OID as - argument and inspects the index to find page ranges in the base table + brin_summarize_new_values accepts the OID or name of a + BRIN index and inspects the index to find page ranges in the base table that are not currently summarized by the index; for any such range it creates a new summary index tuple by scanning the table pages. It returns the number of new page range summaries that were inserted @@ -19201,12 +19201,12 @@ postgres=# SELECT * FROM pg_xlogfile_name_offset(pg_stop_backup()); gin_clean_pending_list accepts the OID or name of - a GIN index and cleans up the pending list of the specified GIN index + a GIN index and cleans up the pending list of the specified index by moving entries in it to the main GIN data structure in bulk. - It returns the number of pages cleaned up from the pending list. - Note that if the argument is a GIN index built with fastupdate - option disabled, the cleanup does not happen and the return value is 0 - because the index doesn't have a pending list. + It returns the number of pages removed from the pending list. + Note that if the argument is a GIN index built with + the fastupdate option disabled, no cleanup happens and the + return value is 0, because the index doesn't have a pending list. Please see and for details of the pending list and fastupdate option. diff --git a/doc/src/sgml/release-9.6.sgml b/doc/src/sgml/release-9.6.sgml index 6ec7583485..895d88e768 100644 --- a/doc/src/sgml/release-9.6.sgml +++ b/doc/src/sgml/release-9.6.sgml @@ -23,33 +23,33 @@ - Parallel sequential scans, joins and aggregates + Parallel execution of sequential scans, joins and aggregates - Eliminate repetitive scanning of old data by autovacuum + Autovacuum no longer performs repetitive scanning of old data - Synchronous replication now allows multiple standby servers, for + Synchronous replication now allows multiple standby servers for increased reliability - Allow full-text search for phrases (multiple adjacent words) + Full-text search can now search for phrases (multiple adjacent words) - Support foreign/remote joins, sorts, and UPDATEs in - postgres_fdw + postgres_fdw now supports remote joins, sorts, + UPDATEs, and DELETEs @@ -87,25 +87,6 @@ - - Change the column name in the - information_schema.routines - view from result_cast_character_set_name - to result_cast_char_set_name (Clément - Prévost) - - - - The SQL:2011 standard specifies the longer name, but that appears - to be a mistake, because adjacent column names use the shorter - style, as do other information_schema views. - - - - - @@ -117,7 +98,7 @@ Historically a process has only been shown as waiting if it was - waiting for a heavy weight lock. Now waits for light weight locks + waiting for a heavyweight lock. Now waits for lightweight locks and buffer pins are also shown in pg_stat_activity. Also, the type of lock being waited for is now visible. These changes replace the waiting column with @@ -149,18 +130,18 @@ Make extract() behave - more reasonably with infinite inputs (Vitaly Burovoy) + more reasonably with infinite inputs (Vitaly Burovoy) Historically the extract() function just returned zero given an infinite timestamp, regardless of the given - unit name. Make it return infinity + field name. Make it return infinity or -infinity as appropriate when the requested field is one that is monotonically increasing (e.g, year, epoch), or NULL when it is not (e.g., day, hour). Also, - throw the expected error for bad unit names. + throw the expected error for bad field names. @@ -186,8 +167,8 @@ This commit is also listed under libpq and psql 2016-03-29 [61d66c44f] Fix support of digits in email/hostnames. --> - Fix text search parser to allow leading digits in email - and host tokens (Artur Zakirov) + Fix the default text search parser to allow leading digits + in email and host tokens (Artur Zakirov) @@ -205,18 +186,18 @@ This commit is also listed under libpq and psql 2016-03-16 [9a206d063] Improve script generating unaccent rules --> - Extend contrib/unaccent's standard - unaccent.rules file to handle all diacritics - known to Unicode, and expand ligatures correctly (Thomas Munro, + Extend contrib/unaccent's + standard unaccent.rules file to handle all diacritics + known to Unicode, and to expand ligatures correctly (Thomas Munro, Léonard Benedetti) - The previous version omitted some less-common letters with - diacritic marks. It now also expands ligatures into separate - letters. Installations that use this rules file may wish to - rebuild tsvector columns and indexes that depend on - the result. + The previous version neglected to convert some less-common letters + with diacritic marks. Also, ligatures are now expanded into + separate letters. Installations that use this rules file may wish + to rebuild tsvector columns and indexes that depend on the + result. @@ -258,17 +239,38 @@ This commit is also listed under libpq and psql + + Change a column name in the + information_schema.routines + view from result_cast_character_set_name + to result_cast_char_set_name (Clément + Prévost) + + + + The SQL:2011 standard specifies the longer name, but that appears + to be a mistake, because adjacent column names use the shorter + style, as do other information_schema views. + + + + + - Support multiple and - command-line options (Pavel Stehule, Catalin Iacob) + psql's option no longer implies + + (Pavel Stehule, Catalin Iacob) - To allow this with sane behavior, one backwards incompatibility - had to be introduced: no longer implies - . + Write (or its + abbreviation ) explicitly to obtain the old + behavior. Scripts modified this way will still work with old + versions of psql. @@ -277,7 +279,7 @@ This commit is also listed under libpq and psql 2015-07-02 [5671aaca8] Improve pg_restore's -t switch to match all types of rel --> - Improve pg_restore's switch to + Improve pg_restore's option to match all types of relations, not only plain tables (Craig Ringer) @@ -287,7 +289,7 @@ This commit is also listed under libpq and psql 2016-02-12 [59a884e98] Change delimiter used for display of NextXID --> - Change the display format of NextXID in + Change the display format used for NextXID in pg_controldata and related places (Joe Conway, Bruce Momjian) @@ -383,8 +385,8 @@ This commit is also listed under libpq and psql 2015-09-16 [7aea8e4f2] Determine whether it's safe to attempt a parallel plan f --> - Provide infrastructure for marking the parallel-safe status of - functions (Robert Haas, Amit Kapila) + Provide infrastructure for marking the parallel-safety status of + functions (Robert Haas, Amit Kapila) @@ -432,8 +434,12 @@ This commit is also listed under libpq and psql Add gin_clean_pending_list() function to allow manual invocation of pending-list cleanup for a - GIN index, separately from vacuuming or analyzing the parent table - (Jeff Janes) + GIN index (Jeff Janes) + + + + Formerly, such cleanup happened only as a byproduct of vacuuming or + analyzing the parent table. @@ -711,7 +717,7 @@ This commit is also listed under libpq and psql If necessary, vacuum can be forced to process all-frozen pages using the new DISABLE_PAGE_SKIPPING option. - Normally, this should never be needed but it might help in + Normally this should never be needed, but it might help in recovering from visibility-map corruption. @@ -774,7 +780,7 @@ This commit is also listed under libpq and psql 2016-02-11 [d4c3a156c] Remove GROUP BY columns that are functionally dependent --> - Avoid computing GROUP BY columns if they are + Ignore GROUP BY columns that are functionally dependent on other columns (David Rowley) @@ -788,43 +794,21 @@ This commit is also listed under libpq and psql - - When appropriate, postpone evaluation of SELECT - output expressions until after an ORDER BY sort - (Konstantin Knizhnik) - - - - This change ensures that volatile or expensive functions in the - output list are executed in the order suggested by ORDER - BY, and that they are not evaluated more times than required - when there is a LIMIT clause. Previously, these - properties held if the ordering was performed by an index scan or - pre-merge-join sort, but not if it was performed by a top-level - sort. - - - - - Allow use of an index-only scan on a partial index when the index's WHERE - clause references columns which are not indexed (Tomas Vondra, + clause references columns that are not indexed (Tomas Vondra, Kyotaro Horiguchi) - For example, CREATE INDEX tidx_partial ON t(b) WHERE a - > 0 could not previously be used for an index-only scan by a - query that only referenced a in its WHERE - clause because a is not an indexed value like - b is. + For example, an index defined by CREATE INDEX tidx_partial + ON t(b) WHERE a > 0 can now be used for an index-only scan by + a query that specifies WHERE a > 0 and does not + otherwise use a. Previously this was disallowed + because a is not listed as an index column. @@ -875,11 +859,12 @@ This commit is also listed under libpq and psql On Linux, sync_file_range() is used for this purpose, - and the feature is on by default on Linux because that function has few - downsides. This sync capability is also available on other platforms - that have msync() or posix_fadvise(), - but those interfaces have some undesirable side-effects so the - feature is disabled by default on non-Linux platforms. + and the feature is on by default on Linux because that function has + few downsides. This flushing capability is also available on other + platforms if they have msync() + or posix_fadvise(), but those interfaces have some + undesirable side-effects so the feature is disabled by default on + non-Linux platforms. @@ -902,8 +887,8 @@ This commit is also listed under libpq and psql - For example, SELECT AVG(x), SUM(x) FROM x can use a - single per-row compuation for both aggregates. + For example, SELECT AVG(x), VARIANCE(x) FROM tab can use + a single per-row computation for both aggregates. @@ -913,9 +898,9 @@ This commit is also listed under libpq and psql --> Speed up visibility tests for recently-created tuples by checking - our transaction snapshot, not pg_clog, to decide - if the source transaction should be considered committed (Jeff - Janes, Tom Lane) + the current transaction's snapshot, not pg_clog, to + decide if the source transaction should be considered committed + (Jeff Janes, Tom Lane) @@ -940,10 +925,11 @@ This commit is also listed under libpq and psql Two-phase commit information is now written only to WAL - during PREPARE TRANSACTION, and read from - WAL during COMMIT PREPARED. A separate - state file is created only if the pending transaction does not - get committed or aborted by the time of the next checkpoint. + during PREPARE TRANSACTION, and will be read back from + WAL during COMMIT PREPARED if that happens + soon thereafter. A separate state file is created only if the + pending transaction does not get committed or aborted by the time + of the next checkpoint. @@ -1142,9 +1128,9 @@ This commit is also listed under libpq and psql This function returns an array of the process IDs of any sessions that are blocking the session with the given process ID. Historically users have obtained such information using a self-join - on the pg_locks view. However, it is unreasonably + on the pg_locks view. However, it is unreasonably tedious to do it that way with any modicum of correctness, and - the addition of parallel queries has made the approach entirely + the addition of parallel queries has made the old approach entirely impractical, since locks might be held or awaited by child worker processes rather than the session's main process. @@ -1181,7 +1167,7 @@ This commit is also listed under libpq and psql - The memory usage dump output to the postmaster log during an + The memory usage dump that is output to the postmaster log during an out-of-memory failure now summarizes statistics when there are a large number of memory contexts, rather than possibly generating a very large report. There is also a grand total @@ -1203,7 +1189,8 @@ This commit is also listed under libpq and psql 2016-04-08 [34c33a1f0] Add BSD authentication method. --> - Add an bsd authentication method to allow the use of + Add a bsd authentication + method to allow use of the BSD Authentication service for PostgreSQL client authentication (Marisa Emerson) @@ -1219,9 +1206,10 @@ This commit is also listed under libpq and psql 2016-04-08 [2f1d2b7a7] Set PAM_RHOST item for PAM authentication --> - When using PAM authentication, provide the client - IP address or host name to PAM modules via the - PAM_RHOST item (Grzegorz Sampolski) + When using PAM + authentication, provide the client IP address or host name + to PAM modules via the PAM_RHOST item + (Grzegorz Sampolski) @@ -1230,8 +1218,8 @@ This commit is also listed under libpq and psql 2016-01-07 [5e0b5dcab] Provide more detail in postmaster log for password authe --> - Provide detail in the postmaster log during more password - authentication failures (Tom Lane) + Provide detail in the postmaster log for more types of password + authentication failure (Tom Lane) @@ -1245,8 +1233,8 @@ This commit is also listed under libpq and psql 2015-09-06 [643beffe8] Support RADIUS passwords up to 128 characters --> - Support RADIUS passwords up to 128 characters long - (Marko Tiikkaja) + Support RADIUS passwords + up to 128 characters long (Marko Tiikkaja) @@ -1255,7 +1243,8 @@ This commit is also listed under libpq and psql 2016-04-08 [35e2e357c] Add authentication parameters compat_realm and upn_usena --> - Add new SSPI authentication parameters + Add new SSPI + authentication parameters compat_realm and upn_username to control whether NetBIOS or Kerberos realm names and user names are used during SSPI @@ -1302,30 +1291,13 @@ This commit is also listed under libpq and psql - - Add configure option - - - This allows the use of systemd service units of - type notify, which greatly simplifies the management - of PostgreSQL under systemd. - - - - - - Add log_line_prefix option %n to print - the current time as a Unix epoch, with milliseconds (Tomas Vondra, - Jeff Davis) + Add option %n to + print the current time in Unix epoch form, with milliseconds (Tomas + Vondra, Jeff Davis) @@ -1362,6 +1334,23 @@ This commit is also listed under libpq and psql + + Add configure option + + + This allows the use of systemd service units of + type notify, which greatly simplifies the management + of PostgreSQL under systemd. + + + + + @@ -1407,8 +1396,8 @@ This commit is also listed under libpq and psql but that is unsafe and inefficient. It also prevents a new postmaster from being started until the last old backend has exited. Backends will detect postmaster death when waiting for - client I/O, so the exit will not be instantaneous, but it in most - cases should happen no later than the end of the current query. + client I/O, so the exit will not be instantaneous, but it should + happen no later than the end of the current query. @@ -1541,7 +1530,8 @@ XXX this is pending backpatch, may need to remove --> Add a option to - pg_basebackup (Peter Eisentraut) + pg_basebackup + (Peter Eisentraut) @@ -1612,6 +1602,28 @@ XXX this is pending backpatch, may need to remove + + When appropriate, postpone evaluation of SELECT + output expressions until after an ORDER BY sort + (Konstantin Knizhnik) + + + + This change ensures that volatile or expensive functions in the + output list are executed in the order suggested by ORDER + BY, and that they are not evaluated more times than required + when there is a LIMIT clause. Previously, these + properties held if the ordering was performed by an index scan or + pre-merge-join sort, but not if it was performed by a top-level + sort. + + + + + @@ -1663,8 +1675,8 @@ XXX this is pending backpatch, may need to remove Previously, the foreign join pushdown infrastructure left the question of security entirely up to individual foreign data - wrappers, but it would be too easy for an FDW to - inadvertently open up subtle security hole. So, make it the core + wrappers, but that made it too easy for an FDW to + inadvertently create subtle security holes. So, make it the core code's job to determine which role ID will access each table, and do not attempt join pushdown unless the role is the same for all relevant relations. @@ -1707,7 +1719,7 @@ XXX this is pending backpatch, may need to remove This command allows a database object to be marked as depending - on an extension, so that it will be automatically dropped if + on an extension, so that it will be dropped automatically if the extension is dropped (without needing CASCADE). However, the object is not part of the extension, and thus will be dumped separately by pg_dump. @@ -1777,8 +1789,8 @@ XXX this is pending backpatch, may need to remove --> Add a CASCADE option to CREATE - EXTENSION to automatically create extensions it depends - on (Petr Jelínek) + EXTENSION to automatically create any extensions the + requested one depends on (Petr Jelínek) @@ -1803,7 +1815,7 @@ XXX this is pending backpatch, may need to remove - This is possible because the table has no existing rows. This matches + This is safe because the table has no existing rows. This matches the longstanding behavior of FOREIGN KEY constraints. @@ -1918,10 +1930,10 @@ XXX this is pending backpatch, may need to remove 2016-06-27 [6734a1cac] Change predecence of phrase operator. --> - Improve full-text search to support searching for phrases, that - is, lexemes appearing adjacent to each other in a specific order, - or with a specified distance between them (Teodor Sigaev, Oleg - Bartunov, Dmitry Ivanov) + Improve full-text search to support + searching for phrases, that is, lexemes appearing adjacent to each + other in a specific order, or with a specified distance between + them (Teodor Sigaev, Oleg Bartunov, Dmitry Ivanov) @@ -2001,9 +2013,10 @@ XXX this is pending backpatch, may need to remove 2016-03-17 [f4ceed6ce] Improve support of Hunspell --> - Upgrade the ispell dictionary to handle modern - Hunspell files and support more languages - (Artur Zakirov) + Upgrade + the ispell + dictionary type to handle modern Hunspell files and + support more languages (Artur Zakirov) @@ -2012,7 +2025,9 @@ XXX this is pending backpatch, may need to remove 2015-10-30 [12c9a0400] Implement lookbehind constraints in our regular-expressi --> - Implement look-behind constraints in regular expressions (Tom Lane) + Implement look-behind constraints + in regular expressions + (Tom Lane) @@ -2044,7 +2059,7 @@ XXX this is pending backpatch, may need to remove 2015-11-07 [c5e86ea93] Add "xid <> xid" and "xid <> int4" operators. --> - Add transaction id operators xid <> + Add transaction ID operators xid <> xid and xid <> int4, for consistency with the corresponding equality operators (Michael Paquier) @@ -2090,8 +2105,10 @@ XXX this is pending backpatch, may need to remove 2016-01-05 [abb173392] Add scale(numeric) --> - Add a scale(numeric) function to extract the display - scale of a numeric value (Marko Tiikkaja) + Add a scale(numeric) + function to extract the display scale of a numeric value + (Marko Tiikkaja) @@ -2109,7 +2126,7 @@ XXX this is pending backpatch, may need to remove measures its argument in degrees, whereas sin() measures in radians. These functions go to some lengths to deliver exact results for values where an exact result can be - expected, e.g. sind(30) = 0.5. + expected, for instance sind(30) = 0.5. @@ -2125,9 +2142,9 @@ XXX this is pending backpatch, may need to remove The POSIX standard says that these functions should - return NaN for NaN input, and should throw an error for - out-of-range inputs including infinity. Previously our - behavior varied across platforms. + return NaN for NaN input, and should throw + an error for out-of-range inputs including infinity. + Previously our behavior varied across platforms. @@ -2136,9 +2153,10 @@ XXX this is pending backpatch, may need to remove 2016-03-29 [e511d878f] Allow to_timestamp(float8) to convert float infinity to --> - Make to_timestamp(float8) convert float - infinity to timestamp infinity (Vitaly - Burovoy) + Make to_timestamp(float8) + convert float infinity to + timestamp infinity (Vitaly Burovoy) @@ -2170,11 +2188,12 @@ XXX this is pending backpatch, may need to remove 2015-09-17 [9acb9007d] Fix oversight in tsearch type check --> - Allow ts_stat_sql() and - tsvector_update_trigger() to operate on values that - are of types binary-compatible with the expected argument type, - not just that argument type; for example allow citext - where text is expected (Teodor Sigaev) + Allow ts_stat() + and tsvector_update_trigger() + to operate on values that are of types binary-compatible with the + expected argument type, not only that argument type; for example + allow citext where text is expected (Teodor + Sigaev) @@ -2216,8 +2235,9 @@ XXX this is pending backpatch, may need to remove In to_number(), - interpret V as dividing by 10 to the power of the - number of digits following V (Bruce Momjian) + interpret a V format code as dividing by 10 to the + power of the number of digits following V (Bruce + Momjian) @@ -2231,8 +2251,10 @@ XXX this is pending backpatch, may need to remove 2016-01-05 [ea0d494da] Make the to_reg*() functions accept text not cstring. --> - Make the to_reg*() functions accept type text - not cstring (Petr Korobeinikov) + Make the to_reg*() + functions accept type text not cstring + (Petr Korobeinikov) @@ -2289,7 +2311,7 @@ XXX this is pending backpatch, may need to remove This allows avoiding an error for an unrecognized parameter - name, and instead return a NULL. + name, instead returning a NULL. @@ -2348,7 +2370,7 @@ XXX this is pending backpatch, may need to remove In PL/pgSQL, detect mismatched CONTINUE and EXIT statements while - compiling PL/pgSQL functions, rather than at execution time + compiling a function, rather than at execution time (Jim Nasby) @@ -2453,8 +2475,9 @@ XXX this is pending backpatch, may need to remove 2016-08-26 [e796d0aba] Add a nonlocalized version of the severity field to clie --> - Add a nonlocalized version of the severity field in error and notice - messages (Tom Lane) + Add a nonlocalized version of + the severity field in + error and notice messages (Tom Lane) @@ -2495,6 +2518,8 @@ This commit is also listed under psql and PL/pgSQL + This is done with the new function PQresultVerboseErrorMessage(). This supports psql's new \errverbose feature, and may be useful for other clients as well. @@ -2541,8 +2566,10 @@ This commit is also listed under psql and PL/pgSQL 2015-09-14 [d02426029] Check existency of table/schema for -t/-n option (pg_dum --> - Add a @@ -2594,6 +2621,22 @@ This commit is also listed under psql and PL/pgSQL + + Support multiple and + command-line options (Pavel Stehule, Catalin Iacob) + + + + The specified operations are carried out in the order in which the + options are given, and then psql terminates. + + + + + @@ -2679,7 +2722,7 @@ This commit is also listed under psql and PL/pgSQL 2016-06-15 [9901d8ac2] Use strftime("%c") to format timestamps in psql's \watch --> - Improve the headers output of the \watch command + Improve the headers output by the \watch command (Michael Paquier, Tom Lane) @@ -2790,8 +2833,9 @@ This commit is also listed under libpq and PL/pgSQL This change allows SQL commands in scripts to span multiple lines. Existing custom scripts will need to be modified to add a semicolon - at the end of each line if missing. (Doing so does not break - the script for use with older versions of pgbench.) + at the end of each line that does not have one already. (Doing so + does not break the script for use with older versions + of pgbench.) @@ -2960,8 +3004,9 @@ This commit is also listed under libpq and PL/pgSQL 2015-12-01 [e50cda784] Use pg_rewind when target timeline was switched --> - Improve pg_rewind so that it can work when the - target timeline changes (Alexander Korotkov) + Improve pg_rewind + so that it can work when the target timeline changes (Alexander + Korotkov) @@ -3095,8 +3140,9 @@ This commit is also listed under libpq and PL/pgSQL 2016-08-13 [ed0097e4f] Add SQL-accessible functions for inspecting index AM pro --> - Restructure index access method API to hide most of - it at the C level (Alexander Korotkov, Andrew Gierth) + Restructure index access + method API to hide most of it at + the C level (Alexander Korotkov, Andrew Gierth) @@ -3118,9 +3164,11 @@ This commit is also listed under libpq and PL/pgSQL 2016-04-06 [6c268df12] Add new catalog called pg_init_privs --> - Add pg_init_privs system catalog to hold original - privileges of initdb-created and extension-created - objects (Stephen Frost) + Add pg_init_privs + system catalog to hold original privileges + of initdb-created and extension-created objects + (Stephen Frost) @@ -3315,7 +3363,7 @@ This commit is also listed under libpq and PL/pgSQL This is somewhat like the reconstructed value, but it - could be any arbitrary chunk of data, it need not be of the same + could be any arbitrary chunk of data, not necessarily of the same data type as the indexed column. @@ -3367,9 +3415,10 @@ This commit is also listed under libpq and PL/pgSQL 2016-03-13 [7a8d87483] Rename auto_explain.sample_ratio to sample_rate --> - Add configuration parameter auto_explain.sample_rate - to allow contrib/auto_explain to capture just a - configurable fraction of all queries (Craig Ringer, Julien Rouhaud) + Add configuration parameter auto_explain.sample_rate to + allow contrib/auto_explain + to capture just a configurable fraction of all queries (Craig + Ringer, Julien Rouhaud) @@ -3383,9 +3432,9 @@ This commit is also listed under libpq and PL/pgSQL 2016-04-01 [9ee014fc8] Bloom index contrib module --> - Add contrib/bloom module that implements an index - access method based on Bloom filtering (Teodor Sigaev, Alexander - Korotkov) + Add contrib/bloom module that + implements an index access method based on Bloom filtering (Teodor + Sigaev, Alexander Korotkov) @@ -3401,9 +3450,9 @@ This commit is also listed under libpq and PL/pgSQL 2015-12-28 [81ee726d8] Code and docs review for cube kNN support. --> - In contrib/cube, introduce distance operators for - cubes, and support kNN-style searches in GiST indexes on cube - columns (Stas Kelvich) + In contrib/cube, introduce + distance operators for cubes, and support kNN-style searches in + GiST indexes on cube columns (Stas Kelvich) @@ -3434,8 +3483,9 @@ This commit is also listed under libpq and PL/pgSQL --> Add selectivity estimation functions for - contrib/intarray operators to improve plans for - queries using those operators (Yury Zhuravlev, Alexander Korotkov) + contrib/intarray operators + to improve plans for queries using those operators (Yury Zhuravlev, + Alexander Korotkov) @@ -3470,7 +3520,8 @@ This commit is also listed under libpq and PL/pgSQL --> Add support for word similarity to - contrib/pg_trgm (Alexander Korotkov, Artur Zakirov) + contrib/pg_trgm + (Alexander Korotkov, Artur Zakirov) @@ -3486,9 +3537,8 @@ This commit is also listed under libpq and PL/pgSQL --> Add configuration parameter - pg_trgm.similarity_threshold for contrib/pg_trgm's similarity - threshold (Artur Zakirov) + pg_trgm.similarity_threshold for + contrib/pg_trgm's similarity threshold (Artur Zakirov) @@ -3525,8 +3575,9 @@ This commit is also listed under libpq and PL/pgSQL 2016-06-17 [71d05a2c7] pg_visibility: Add pg_truncate_visibility_map function. --> - Add contrib/pg_visibility module to allow examining - table visibility maps (Robert Haas) + Add contrib/pg_visibility module + to allow examining table visibility maps (Robert Haas) @@ -3545,7 +3596,7 @@ This commit is also listed under libpq and PL/pgSQL - <filename>postgres_fdw</> + <link linkend="postgres-fdw"><filename>postgres_fdw</></> @@ -3597,7 +3648,7 @@ This commit is also listed under libpq and PL/pgSQL - Formerly, this involved sending a SELECT FOR UPDATE + Formerly, remote updates involved sending a SELECT FOR UPDATE command and then updating or deleting the selected rows one-by-one. While that is still necessary if the operation requires any local processing, it can now be done remotely if all elements of the diff --git a/doc/src/sgml/spgist.sgml b/doc/src/sgml/spgist.sgml index d60aa23f33..cd4a8d07c4 100644 --- a/doc/src/sgml/spgist.sgml +++ b/doc/src/sgml/spgist.sgml @@ -674,6 +674,7 @@ typedef struct spgInnerConsistentOut However, any output traverse values pointed to by the traversalValues array should be allocated in traversalMemoryContext. + Each traverse value must be a single palloc'd chunk. -- cgit v1.2.3 From 9b7cd59af1afcfbd786921d5cf73befb5fefa2f7 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 29 Aug 2016 20:16:02 +0300 Subject: Remove support for OpenSSL versions older than 0.9.8. OpenSSL officially only supports 1.0.1 and newer. Some OS distributions still provide patches for 0.9.8, but anything older than that is not interesting anymore. Let's simplify things by removing compatibility code. Andreas Karlsson, with small changes by me. --- contrib/pgcrypto/openssl.c | 152 +------------------------------ doc/src/sgml/installation.sgml | 39 +++----- doc/src/sgml/libpq.sgml | 3 +- doc/src/sgml/pgcrypto.sgml | 18 +--- src/backend/libpq/be-secure-openssl.c | 8 +- src/interfaces/libpq/fe-secure-openssl.c | 4 - src/interfaces/libpq/libpq-int.h | 2 +- 7 files changed, 20 insertions(+), 206 deletions(-) (limited to 'doc/src') diff --git a/contrib/pgcrypto/openssl.c b/contrib/pgcrypto/openssl.c index 976af70591..ffab5d2bb0 100644 --- a/contrib/pgcrypto/openssl.c +++ b/contrib/pgcrypto/openssl.c @@ -37,6 +37,7 @@ #include #include #include +#include #include #include @@ -46,155 +47,6 @@ #define MAX_KEY (512/8) #define MAX_IV (128/8) -/* - * Compatibility with OpenSSL 0.9.6 - * - * It needs AES and newer DES and digest API. - */ -#if OPENSSL_VERSION_NUMBER >= 0x00907000L - -/* - * Nothing needed for OpenSSL 0.9.7+ - */ - -#include -#else /* old OPENSSL */ - -/* - * Emulate OpenSSL AES. - */ - -#include "rijndael.c" - -#define AES_ENCRYPT 1 -#define AES_DECRYPT 0 -#define AES_KEY rijndael_ctx - -static int -AES_set_encrypt_key(const uint8 *key, int kbits, AES_KEY *ctx) -{ - aes_set_key(ctx, key, kbits, 1); - return 0; -} - -static int -AES_set_decrypt_key(const uint8 *key, int kbits, AES_KEY *ctx) -{ - aes_set_key(ctx, key, kbits, 0); - return 0; -} - -static void -AES_ecb_encrypt(const uint8 *src, uint8 *dst, AES_KEY *ctx, int enc) -{ - memcpy(dst, src, 16); - if (enc) - aes_ecb_encrypt(ctx, dst, 16); - else - aes_ecb_decrypt(ctx, dst, 16); -} - -static void -AES_cbc_encrypt(const uint8 *src, uint8 *dst, int len, AES_KEY *ctx, uint8 *iv, int enc) -{ - memcpy(dst, src, len); - if (enc) - { - aes_cbc_encrypt(ctx, iv, dst, len); - memcpy(iv, dst + len - 16, 16); - } - else - { - aes_cbc_decrypt(ctx, iv, dst, len); - memcpy(iv, src + len - 16, 16); - } -} - -/* - * Emulate DES_* API - */ - -#define DES_key_schedule des_key_schedule -#define DES_cblock des_cblock -#define DES_set_key(k, ks) \ - des_set_key((k), *(ks)) -#define DES_ecb_encrypt(i, o, k, e) \ - des_ecb_encrypt((i), (o), *(k), (e)) -#define DES_ncbc_encrypt(i, o, l, k, iv, e) \ - des_ncbc_encrypt((i), (o), (l), *(k), (iv), (e)) -#define DES_ecb3_encrypt(i, o, k1, k2, k3, e) \ - des_ecb3_encrypt((des_cblock *)(i), (des_cblock *)(o), \ - *(k1), *(k2), *(k3), (e)) -#define DES_ede3_cbc_encrypt(i, o, l, k1, k2, k3, iv, e) \ - des_ede3_cbc_encrypt((i), (o), \ - (l), *(k1), *(k2), *(k3), (iv), (e)) - -/* - * Emulate newer digest API. - */ - -static void -EVP_MD_CTX_init(EVP_MD_CTX *ctx) -{ - memset(ctx, 0, sizeof(*ctx)); -} - -static int -EVP_MD_CTX_cleanup(EVP_MD_CTX *ctx) -{ - px_memset(ctx, 0, sizeof(*ctx)); - return 1; -} - -static int -EVP_DigestInit_ex(EVP_MD_CTX *ctx, const EVP_MD *md, void *engine) -{ - EVP_DigestInit(ctx, md); - return 1; -} - -static int -EVP_DigestFinal_ex(EVP_MD_CTX *ctx, unsigned char *res, unsigned int *len) -{ - EVP_DigestFinal(ctx, res, len); - return 1; -} -#endif /* old OpenSSL */ - -/* - * Provide SHA2 for older OpenSSL < 0.9.8 - */ -#if OPENSSL_VERSION_NUMBER < 0x00908000L - -#include "sha2.c" -#include "internal-sha2.c" - -typedef void (*init_f) (PX_MD *md); - -static int -compat_find_digest(const char *name, PX_MD **res) -{ - init_f init = NULL; - - if (pg_strcasecmp(name, "sha224") == 0) - init = init_sha224; - else if (pg_strcasecmp(name, "sha256") == 0) - init = init_sha256; - else if (pg_strcasecmp(name, "sha384") == 0) - init = init_sha384; - else if (pg_strcasecmp(name, "sha512") == 0) - init = init_sha512; - else - return PXE_NO_HASH; - - *res = px_alloc(sizeof(PX_MD)); - init(*res); - return 0; -} -#else -#define compat_find_digest(name, res) (PXE_NO_HASH) -#endif - /* * Hashes */ @@ -275,7 +127,7 @@ px_find_digest(const char *name, PX_MD **res) md = EVP_get_digestbyname(name); if (md == NULL) - return compat_find_digest(name, res); + return PXE_NO_HASH; digest = px_alloc(sizeof(*digest)); digest->algo = md; diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml index a9968756e6..14a6d57aea 100644 --- a/doc/src/sgml/installation.sgml +++ b/doc/src/sgml/installation.sgml @@ -252,10 +252,17 @@ su - postgres - You need Kerberos, OpenSSL, - OpenLDAP, and/or - PAM, if you want to support authentication or - encryption using those services. + You need OpenSSL, if you want to support + encrypted client connections. The minimum required version is + 0.9.8. + + + + + + You need Kerberos, OpenLDAP, + and/or PAM, if you want to support authentication + using those services. @@ -2826,30 +2833,6 @@ MANPATH=/usr/lib/scohelp/%L/man:/usr/dt/man:/usr/man:/usr/share/man:scohelp:/usr - - Problems with OpenSSL - - - When you build PostgreSQL with OpenSSL support you might get - compilation errors in the following files: - - src/backend/libpq/crypt.c - src/backend/libpq/password.c - src/interfaces/libpq/fe-auth.c - src/interfaces/libpq/fe-connect.c - - - This is because of a namespace conflict between the standard - /usr/include/crypt.h header and the header - files provided by OpenSSL. - - - - Upgrading your OpenSSL installation to version 0.9.6a fixes this - problem. Solaris 9 and above has a newer version of OpenSSL. - - - configure Complains About a Failed Test Program diff --git a/doc/src/sgml/libpq.sgml b/doc/src/sgml/libpq.sgml index 2f9350b10e..4e34f00e44 100644 --- a/doc/src/sgml/libpq.sgml +++ b/doc/src/sgml/libpq.sgml @@ -1238,8 +1238,7 @@ postgresql://%2Fvar%2Flib%2Fpostgresql/dbname If set to 1 (default), data sent over SSL connections will be - compressed (this requires OpenSSL version - 0.9.8 or later). + compressed. If set to 0, compression will be disabled (this requires OpenSSL 1.0.0 or later). This parameter is ignored if a connection without SSL is made, diff --git a/doc/src/sgml/pgcrypto.sgml b/doc/src/sgml/pgcrypto.sgml index c4cefde4f7..bf514aacf3 100644 --- a/doc/src/sgml/pgcrypto.sgml +++ b/doc/src/sgml/pgcrypto.sgml @@ -1184,12 +1184,12 @@ gen_random_uuid() returns uuid SHA224/256/384/512 yes - yes (Note 1) + yes Other digest algorithms no - yes (Note 2) + yes (Note 1) Blowfish @@ -1199,7 +1199,7 @@ gen_random_uuid() returns uuid AES yes - yes (Note 3) + yes DES/3DES/CAST5 @@ -1230,12 +1230,6 @@ gen_random_uuid() returns uuid - - - SHA2 algorithms were added to OpenSSL in version 0.9.8. For - older versions, pgcrypto will use built-in code. - - Any digest algorithm OpenSSL supports is automatically picked up. @@ -1243,12 +1237,6 @@ gen_random_uuid() returns uuid explicitly. - - - AES is included in OpenSSL since version 0.9.7. For - older versions, pgcrypto will use built-in code. - - diff --git a/src/backend/libpq/be-secure-openssl.c b/src/backend/libpq/be-secure-openssl.c index f6adb155c6..e5f434ca17 100644 --- a/src/backend/libpq/be-secure-openssl.c +++ b/src/backend/libpq/be-secure-openssl.c @@ -53,10 +53,8 @@ #include #include -#if SSLEAY_VERSION_NUMBER >= 0x0907000L #include -#endif -#if (OPENSSL_VERSION_NUMBER >= 0x0090800fL) && !defined(OPENSSL_NO_ECDH) +#ifndef OPENSSL_NO_ECDH #include #endif @@ -166,9 +164,7 @@ be_tls_init(void) if (!SSL_context) { -#if SSLEAY_VERSION_NUMBER >= 0x0907000L OPENSSL_config(NULL); -#endif SSL_library_init(); SSL_load_error_strings(); @@ -978,7 +974,7 @@ info_cb(const SSL *ssl, int type, int args) static void initialize_ecdh(void) { -#if (OPENSSL_VERSION_NUMBER >= 0x0090800fL) && !defined(OPENSSL_NO_ECDH) +#ifndef OPENSSL_NO_ECDH EC_KEY *ecdh; int nid; diff --git a/src/interfaces/libpq/fe-secure-openssl.c b/src/interfaces/libpq/fe-secure-openssl.c index f6ce1c7a13..d8716128ec 100644 --- a/src/interfaces/libpq/fe-secure-openssl.c +++ b/src/interfaces/libpq/fe-secure-openssl.c @@ -54,9 +54,7 @@ #endif #include -#if (SSLEAY_VERSION_NUMBER >= 0x00907000L) #include -#endif #ifdef USE_SSL_ENGINE #include #endif @@ -848,9 +846,7 @@ pgtls_init(PGconn *conn) { if (pq_init_ssl_lib) { -#if SSLEAY_VERSION_NUMBER >= 0x00907000L OPENSSL_config(NULL); -#endif SSL_library_init(); SSL_load_error_strings(); } diff --git a/src/interfaces/libpq/libpq-int.h b/src/interfaces/libpq/libpq-int.h index 1183323a44..a94ead04ff 100644 --- a/src/interfaces/libpq/libpq-int.h +++ b/src/interfaces/libpq/libpq-int.h @@ -77,7 +77,7 @@ typedef struct #include #include -#if (SSLEAY_VERSION_NUMBER >= 0x00907000L) && !defined(OPENSSL_NO_ENGINE) +#ifndef OPENSSL_NO_ENGINE #define USE_SSL_ENGINE #endif #endif /* USE_OPENSSL */ -- cgit v1.2.3 From 39b691f251167bbb3d49203abfb39d430f68f411 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 2 Sep 2016 17:29:31 -0400 Subject: Don't require dynamic timezone abbreviations to match underlying time zone. Previously, we threw an error if a dynamic timezone abbreviation did not match any abbreviation recorded in the referenced IANA time zone entry. That seemed like a good consistency check at the time, but it turns out that a number of the abbreviations in the IANA database are things that Olson and crew made up out of whole cloth. Their current policy is to remove such names in favor of using simple numeric offsets. Perhaps unsurprisingly, a lot of these made-up abbreviations have varied in meaning over time, which meant that our commit b2cbced9e and later changes made them into dynamic abbreviations. So with newer IANA database versions that don't mention these abbreviations at all, we fail, as reported in bug #14307 from Neil Anderson. It's worse than just a few unused-in-the-wild abbreviations not working, because the pg_timezone_abbrevs view stops working altogether (since its underlying function tries to compute the whole view result in one call). We considered deleting these abbreviations from our abbreviations list, but the problem with that is that we can't stay ahead of possible future IANA changes. Instead, let's leave the abbreviations list alone, and treat any "orphaned" dynamic abbreviation as just meaning the referenced time zone. It will behave a bit differently than it used to, in that you can't any longer override the zone's standard vs. daylight rule by using the "wrong" abbreviation of a pair, but that's better than failing entirely. (Also, this solution can be interpreted as adding a small new feature, which is that any abbreviation a user wants can be defined as referencing a time zone name.) Back-patch to all supported branches, since this problem affects all of them when using tzdata 2016f or newer. Report: <20160902031551.15674.67337@wrigleys.postgresql.org> Discussion: <6189.1472820913@sss.pgh.pa.us> --- doc/src/sgml/catalogs.sgml | 7 +++ doc/src/sgml/datetime.sgml | 41 +++++++++++---- src/backend/utils/adt/datetime.c | 85 +++++++++++++++++++++++-------- src/test/regress/expected/timestamptz.out | 20 ++++++++ src/test/regress/sql/timestamptz.sql | 11 ++++ 5 files changed, 132 insertions(+), 32 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 4e09e06aed..322d8d6dc7 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -9811,6 +9811,13 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx + + While most timezone abbreviations represent fixed offsets from UTC, + there are some that have historically varied in value + (see for more information). + In such cases this view presents their current meaning. + + diff --git a/doc/src/sgml/datetime.sgml b/doc/src/sgml/datetime.sgml index ffd0715128..ef9139f9e3 100644 --- a/doc/src/sgml/datetime.sgml +++ b/doc/src/sgml/datetime.sgml @@ -384,19 +384,38 @@ A zone_abbreviation is just the abbreviation - being defined. The offset is the equivalent - offset in seconds from UTC, positive being east from Greenwich and - negative being west. For example, -18000 would be five hours west - of Greenwich, or North American east coast standard time. D - indicates that the zone name represents local daylight-savings time rather - than standard time. Alternatively, a time_zone_name can - be given, in which case that time zone definition is consulted, and the - abbreviation's meaning in that zone is used. This alternative is - recommended only for abbreviations whose meaning has historically varied, - as looking up the meaning is noticeably more expensive than just using - a fixed integer value. + being defined. An offset is an integer giving + the equivalent offset in seconds from UTC, positive being east from + Greenwich and negative being west. For example, -18000 would be five + hours west of Greenwich, or North American east coast standard time. + D indicates that the zone name represents local + daylight-savings time rather than standard time. + + Alternatively, a time_zone_name can be given, referencing + a zone name defined in the IANA timezone database. The zone's definition + is consulted to see whether the abbreviation is or has been in use in + that zone, and if so, the appropriate meaning is used — that is, + the meaning that was currently in use at the timestamp whose value is + being determined, or the meaning in use immediately before that if it + wasn't current at that time, or the oldest meaning if it was used only + after that time. This behavior is essential for dealing with + abbreviations whose meaning has historically varied. It is also allowed + to define an abbreviation in terms of a zone name in which that + abbreviation does not appear; then using the abbreviation is just + equivalent to writing out the zone name. + + + + + Using a simple integer offset is preferred + when defining an abbreviation whose offset from UTC has never changed, + as such abbreviations are much cheaper to process than those that + require consulting a time zone definition. + + + The @INCLUDE syntax allows inclusion of another file in the .../share/timezonesets/ directory. Inclusion can be nested, diff --git a/src/backend/utils/adt/datetime.c b/src/backend/utils/adt/datetime.c index 965c3b4ff0..45ba7cd906 100644 --- a/src/backend/utils/adt/datetime.c +++ b/src/backend/utils/adt/datetime.c @@ -56,8 +56,9 @@ static void AdjustFractDays(double frac, struct pg_tm * tm, fsec_t *fsec, int scale); static int DetermineTimeZoneOffsetInternal(struct pg_tm * tm, pg_tz *tzp, pg_time_t *tp); -static int DetermineTimeZoneAbbrevOffsetInternal(pg_time_t t, const char *abbr, - pg_tz *tzp, int *isdst); +static bool DetermineTimeZoneAbbrevOffsetInternal(pg_time_t t, + const char *abbr, pg_tz *tzp, + int *offset, int *isdst); static pg_tz *FetchDynamicTimeZone(TimeZoneAbbrevTable *tbl, const datetkn *tp); @@ -1689,19 +1690,40 @@ overflow: * This differs from the behavior of DetermineTimeZoneOffset() in that a * standard-time or daylight-time abbreviation forces use of the corresponding * GMT offset even when the zone was then in DS or standard time respectively. + * (However, that happens only if we can match the given abbreviation to some + * abbreviation that appears in the IANA timezone data. Otherwise, we fall + * back to doing DetermineTimeZoneOffset().) */ int DetermineTimeZoneAbbrevOffset(struct pg_tm * tm, const char *abbr, pg_tz *tzp) { pg_time_t t; + int zone_offset; + int abbr_offset; + int abbr_isdst; /* * Compute the UTC time we want to probe at. (In event of overflow, we'll * probe at the epoch, which is a bit random but probably doesn't matter.) */ - (void) DetermineTimeZoneOffsetInternal(tm, tzp, &t); + zone_offset = DetermineTimeZoneOffsetInternal(tm, tzp, &t); - return DetermineTimeZoneAbbrevOffsetInternal(t, abbr, tzp, &tm->tm_isdst); + /* + * Try to match the abbreviation to something in the zone definition. + */ + if (DetermineTimeZoneAbbrevOffsetInternal(t, abbr, tzp, + &abbr_offset, &abbr_isdst)) + { + /* Success, so use the abbrev-specific answers. */ + tm->tm_isdst = abbr_isdst; + return abbr_offset; + } + + /* + * No match, so use the answers we already got from + * DetermineTimeZoneOffsetInternal. + */ + return zone_offset; } @@ -1715,19 +1737,41 @@ DetermineTimeZoneAbbrevOffsetTS(TimestampTz ts, const char *abbr, pg_tz *tzp, int *isdst) { pg_time_t t = timestamptz_to_time_t(ts); + int zone_offset; + int abbr_offset; + int tz; + struct pg_tm tm; + fsec_t fsec; - return DetermineTimeZoneAbbrevOffsetInternal(t, abbr, tzp, isdst); + /* + * If the abbrev matches anything in the zone data, this is pretty easy. + */ + if (DetermineTimeZoneAbbrevOffsetInternal(t, abbr, tzp, + &abbr_offset, isdst)) + return abbr_offset; + + /* + * Else, break down the timestamp so we can use DetermineTimeZoneOffset. + */ + if (timestamp2tm(ts, &tz, &tm, &fsec, NULL, tzp) != 0) + ereport(ERROR, + (errcode(ERRCODE_DATETIME_VALUE_OUT_OF_RANGE), + errmsg("timestamp out of range"))); + + zone_offset = DetermineTimeZoneOffset(&tm, tzp); + *isdst = tm.tm_isdst; + return zone_offset; } /* DetermineTimeZoneAbbrevOffsetInternal() * * Workhorse for above two functions: work from a pg_time_t probe instant. - * DST status is returned into *isdst. + * On success, return GMT offset and DST status into *offset and *isdst. */ -static int -DetermineTimeZoneAbbrevOffsetInternal(pg_time_t t, const char *abbr, - pg_tz *tzp, int *isdst) +static bool +DetermineTimeZoneAbbrevOffsetInternal(pg_time_t t, const char *abbr, pg_tz *tzp, + int *offset, int *isdst) { char upabbr[TZ_STRLEN_MAX + 1]; unsigned char *p; @@ -1739,18 +1783,17 @@ DetermineTimeZoneAbbrevOffsetInternal(pg_time_t t, const char *abbr, *p = pg_toupper(*p); /* Look up the abbrev's meaning at this time in this zone */ - if (!pg_interpret_timezone_abbrev(upabbr, - &t, - &gmtoff, - isdst, - tzp)) - ereport(ERROR, - (errcode(ERRCODE_CONFIG_FILE_ERROR), - errmsg("time zone abbreviation \"%s\" is not used in time zone \"%s\"", - abbr, pg_get_timezone_name(tzp)))); - - /* Change sign to agree with DetermineTimeZoneOffset() */ - return (int) -gmtoff; + if (pg_interpret_timezone_abbrev(upabbr, + &t, + &gmtoff, + isdst, + tzp)) + { + /* Change sign to agree with DetermineTimeZoneOffset() */ + *offset = (int) -gmtoff; + return true; + } + return false; } diff --git a/src/test/regress/expected/timestamptz.out b/src/test/regress/expected/timestamptz.out index 67f26db204..2bfc13ad72 100644 --- a/src/test/regress/expected/timestamptz.out +++ b/src/test/regress/expected/timestamptz.out @@ -2603,3 +2603,23 @@ SELECT '2007-12-09 07:30:00 UTC'::timestamptz AT TIME ZONE 'VET'; Sun Dec 09 03:00:00 2007 (1 row) +-- +-- Test that the pg_timezone_names and pg_timezone_abbrevs views are +-- more-or-less working. We can't test their contents in any great detail +-- without the outputs changing anytime IANA updates the underlying data, +-- but it seems reasonable to expect at least one entry per major meridian. +-- (At the time of writing, the actual counts are around 38 because of +-- zones using fractional GMT offsets, so this is a pretty loose test.) +-- +select count(distinct utc_offset) >= 24 as ok from pg_timezone_names; + ok +---- + t +(1 row) + +select count(distinct utc_offset) >= 24 as ok from pg_timezone_abbrevs; + ok +---- + t +(1 row) + diff --git a/src/test/regress/sql/timestamptz.sql b/src/test/regress/sql/timestamptz.sql index c023095bb8..ce9d1c2fa1 100644 --- a/src/test/regress/sql/timestamptz.sql +++ b/src/test/regress/sql/timestamptz.sql @@ -468,3 +468,14 @@ SELECT '2007-12-09 07:00:00 UTC'::timestamptz AT TIME ZONE 'VET'; SELECT '2007-12-09 07:00:01 UTC'::timestamptz AT TIME ZONE 'VET'; SELECT '2007-12-09 07:29:59 UTC'::timestamptz AT TIME ZONE 'VET'; SELECT '2007-12-09 07:30:00 UTC'::timestamptz AT TIME ZONE 'VET'; + +-- +-- Test that the pg_timezone_names and pg_timezone_abbrevs views are +-- more-or-less working. We can't test their contents in any great detail +-- without the outputs changing anytime IANA updates the underlying data, +-- but it seems reasonable to expect at least one entry per major meridian. +-- (At the time of writing, the actual counts are around 38 because of +-- zones using fractional GMT offsets, so this is a pretty loose test.) +-- +select count(distinct utc_offset) >= 24 as ok from pg_timezone_names; +select count(distinct utc_offset) >= 24 as ok from pg_timezone_abbrevs; -- cgit v1.2.3 From 0c40ab3a88edf654165e562deee0c303a6ebef5e Mon Sep 17 00:00:00 2001 From: Simon Riggs Date: Sat, 3 Sep 2016 16:19:11 +0100 Subject: Fix wording of logical decoding concepts Be specific about conditions under which we emit >1 copy of message Craig Ringer --- doc/src/sgml/logicaldecoding.sgml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/logicaldecoding.sgml b/doc/src/sgml/logicaldecoding.sgml index c42082002e..484915d042 100644 --- a/doc/src/sgml/logicaldecoding.sgml +++ b/doc/src/sgml/logicaldecoding.sgml @@ -12,7 +12,6 @@ Changes are sent out in streams identified by logical replication slots. - Each stream outputs each change exactly once. @@ -204,8 +203,7 @@ $ pg_recvlogical -d postgres --slot test --drop-slot In the context of logical replication, a slot represents a stream of changes that can be replayed to a client in the order they were made on the origin server. Each slot streams a sequence of changes from a single - database, sending each change exactly once (except when peeking forward - in the stream). + database. @@ -221,6 +219,20 @@ $ pg_recvlogical -d postgres --slot test --drop-slot independently of the connection using them and are crash-safe. + + A logical slot will emit each change just once in normal operation. + The current position of each slot is persisted only at checkpoint, so in + the case of a crash the slot may return to an earlier LSN, which will + then cause recent changes to be resent when the server restarts. + Logical decoding clients are responsible for avoiding ill effects from + handling the same message more than once. Clients may wish to record + the last LSN they saw when decoding and skip over any repeated data or + (when using the replication protocol) request that decoding start from + that LSN rather than letting the server determine the start point. + The Replication Progress Tracking feature is designed for this purpose, + refer to replication origins. + + Multiple independent slots may exist for a single database. Each slot has its own state, allowing different consumers to receive changes from -- cgit v1.2.3 From 35250b6ad7a8ece5cfe54c0316c180df19f36c13 Mon Sep 17 00:00:00 2001 From: Simon Riggs Date: Sat, 3 Sep 2016 17:48:01 +0100 Subject: New recovery target recovery_target_lsn Michael Paquier --- doc/src/sgml/recovery-config.sgml | 24 +++++++-- src/backend/access/transam/recovery.conf.sample | 6 ++- src/backend/access/transam/xlog.c | 70 +++++++++++++++++++++++++ src/include/access/xlog.h | 1 + src/test/recovery/t/003_recovery_targets.pl | 28 ++++++++-- 5 files changed, 120 insertions(+), 9 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/recovery-config.sgml b/doc/src/sgml/recovery-config.sgml index 26af221745..de3fb10f5b 100644 --- a/doc/src/sgml/recovery-config.sgml +++ b/doc/src/sgml/recovery-config.sgml @@ -157,9 +157,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows By default, recovery will recover to the end of the WAL log. The following parameters can be used to specify an earlier stopping point. At most one of recovery_target, - recovery_target_name, recovery_target_time, or - recovery_target_xid can be used; if more than one of these - is specified in the configuration file, the last entry will be used. + recovery_target_lsn, recovery_target_name, + recovery_target_time, or recovery_target_xid + can be used; if more than one of these is specified in the configuration + file, the last entry will be used. @@ -232,6 +233,23 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows + + + recovery_target_lsn (pg_lsn) + + recovery_target_lsn recovery parameter + + + + + This parameter specifies the LSN of the transaction log location up + to which recovery will proceed. The precise stopping point is also + influenced by . This + parameter is parsed using the system data type + pg_lsn. + + + diff --git a/src/backend/access/transam/recovery.conf.sample b/src/backend/access/transam/recovery.conf.sample index b777400d03..7a16751541 100644 --- a/src/backend/access/transam/recovery.conf.sample +++ b/src/backend/access/transam/recovery.conf.sample @@ -67,8 +67,8 @@ # must set a recovery target. # # You may set a recovery target either by transactionId, by name, -# or by timestamp. Recovery may either include or exclude the -# transaction(s) with the recovery target value (ie, stop either +# by timestamp or by WAL position (LSN). Recovery may either include or +# exclude the transaction(s) with the recovery target value (ie, stop either # just after or just before the given target, respectively). # # @@ -78,6 +78,8 @@ # #recovery_target_xid = '' # +#recovery_target_lsn = '' # e.g. '0/70006B8' +# #recovery_target_inclusive = true # # diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 0b991bb91d..2189c22c64 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -67,6 +67,7 @@ #include "utils/builtins.h" #include "utils/guc.h" #include "utils/memutils.h" +#include "utils/pg_lsn.h" #include "utils/ps_status.h" #include "utils/relmapper.h" #include "utils/snapmgr.h" @@ -254,6 +255,7 @@ static RecoveryTargetAction recoveryTargetAction = RECOVERY_TARGET_ACTION_PAUSE; static TransactionId recoveryTargetXid; static TimestampTz recoveryTargetTime; static char *recoveryTargetName; +static XLogRecPtr recoveryTargetLSN; static int recovery_min_apply_delay = 0; static TimestampTz recoveryDelayUntilTime; @@ -275,6 +277,7 @@ static bool fast_promote = false; */ static TransactionId recoveryStopXid; static TimestampTz recoveryStopTime; +static XLogRecPtr recoveryStopLSN; static char recoveryStopName[MAXFNAMELEN]; static bool recoveryStopAfter; @@ -5078,6 +5081,23 @@ readRecoveryCommandFile(void) (errmsg_internal("recovery_target_name = '%s'", recoveryTargetName))); } + else if (strcmp(item->name, "recovery_target_lsn") == 0) + { + recoveryTarget = RECOVERY_TARGET_LSN; + + /* + * Convert the LSN string given by the user to XLogRecPtr form. + */ + recoveryTargetLSN = + DatumGetLSN(DirectFunctionCall3(pg_lsn_in, + CStringGetDatum(item->value), + ObjectIdGetDatum(InvalidOid), + Int32GetDatum(-1))); + ereport(DEBUG2, + (errmsg_internal("recovery_target_lsn = '%X/%X'", + (uint32) (recoveryTargetLSN >> 32), + (uint32) recoveryTargetLSN))); + } else if (strcmp(item->name, "recovery_target") == 0) { if (strcmp(item->value, "immediate") == 0) @@ -5400,8 +5420,26 @@ recoveryStopsBefore(XLogReaderState *record) recoveryStopAfter = false; recoveryStopXid = InvalidTransactionId; + recoveryStopLSN = InvalidXLogRecPtr; + recoveryStopTime = 0; + recoveryStopName[0] = '\0'; + return true; + } + + /* Check if target LSN has been reached */ + if (recoveryTarget == RECOVERY_TARGET_LSN && + !recoveryTargetInclusive && + record->ReadRecPtr >= recoveryTargetLSN) + { + recoveryStopAfter = false; + recoveryStopXid = InvalidTransactionId; + recoveryStopLSN = record->ReadRecPtr; recoveryStopTime = 0; recoveryStopName[0] = '\0'; + ereport(LOG, + (errmsg("recovery stopping before WAL position (LSN) \"%X/%X\"", + (uint32) (recoveryStopLSN >> 32), + (uint32) recoveryStopLSN))); return true; } @@ -5479,6 +5517,7 @@ recoveryStopsBefore(XLogReaderState *record) recoveryStopAfter = false; recoveryStopXid = recordXid; recoveryStopTime = recordXtime; + recoveryStopLSN = InvalidXLogRecPtr; recoveryStopName[0] = '\0'; if (isCommit) @@ -5532,6 +5571,7 @@ recoveryStopsAfter(XLogReaderState *record) { recoveryStopAfter = true; recoveryStopXid = InvalidTransactionId; + recoveryStopLSN = InvalidXLogRecPtr; (void) getRecordTimestamp(record, &recoveryStopTime); strlcpy(recoveryStopName, recordRestorePointData->rp_name, MAXFNAMELEN); @@ -5543,6 +5583,23 @@ recoveryStopsAfter(XLogReaderState *record) } } + /* Check if the target LSN has been reached */ + if (recoveryTarget == RECOVERY_TARGET_LSN && + recoveryTargetInclusive && + record->ReadRecPtr >= recoveryTargetLSN) + { + recoveryStopAfter = true; + recoveryStopXid = InvalidTransactionId; + recoveryStopLSN = record->ReadRecPtr; + recoveryStopTime = 0; + recoveryStopName[0] = '\0'; + ereport(LOG, + (errmsg("recovery stopping after WAL position (LSN) \"%X/%X\"", + (uint32) (recoveryStopLSN >> 32), + (uint32) recoveryStopLSN))); + return true; + } + if (rmid != RM_XACT_ID) return false; @@ -5598,6 +5655,7 @@ recoveryStopsAfter(XLogReaderState *record) recoveryStopAfter = true; recoveryStopXid = recordXid; recoveryStopTime = recordXtime; + recoveryStopLSN = InvalidXLogRecPtr; recoveryStopName[0] = '\0'; if (xact_info == XLOG_XACT_COMMIT || @@ -5629,6 +5687,7 @@ recoveryStopsAfter(XLogReaderState *record) recoveryStopAfter = true; recoveryStopXid = InvalidTransactionId; recoveryStopTime = 0; + recoveryStopLSN = InvalidXLogRecPtr; recoveryStopName[0] = '\0'; return true; } @@ -6055,6 +6114,11 @@ StartupXLOG(void) ereport(LOG, (errmsg("starting point-in-time recovery to \"%s\"", recoveryTargetName))); + else if (recoveryTarget == RECOVERY_TARGET_LSN) + ereport(LOG, + (errmsg("starting point-in-time recovery to WAL position (LSN) \"%X/%X\"", + (uint32) (recoveryTargetLSN >> 32), + (uint32) recoveryTargetLSN))); else if (recoveryTarget == RECOVERY_TARGET_IMMEDIATE) ereport(LOG, (errmsg("starting point-in-time recovery to earliest consistent point"))); @@ -7124,6 +7188,12 @@ StartupXLOG(void) "%s %s\n", recoveryStopAfter ? "after" : "before", timestamptz_to_str(recoveryStopTime)); + else if (recoveryTarget == RECOVERY_TARGET_LSN) + snprintf(reason, sizeof(reason), + "%s LSN %X/%X\n", + recoveryStopAfter ? "after" : "before", + (uint32 ) (recoveryStopLSN >> 32), + (uint32) recoveryStopLSN); else if (recoveryTarget == RECOVERY_TARGET_NAME) snprintf(reason, sizeof(reason), "at restore point \"%s\"", diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index 14b7f7f459..c9f332c908 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -83,6 +83,7 @@ typedef enum RECOVERY_TARGET_XID, RECOVERY_TARGET_TIME, RECOVERY_TARGET_NAME, + RECOVERY_TARGET_LSN, RECOVERY_TARGET_IMMEDIATE } RecoveryTargetType; diff --git a/src/test/recovery/t/003_recovery_targets.pl b/src/test/recovery/t/003_recovery_targets.pl index d1f6d78388..a82545bf6f 100644 --- a/src/test/recovery/t/003_recovery_targets.pl +++ b/src/test/recovery/t/003_recovery_targets.pl @@ -3,7 +3,7 @@ use strict; use warnings; use PostgresNode; use TestLib; -use Test::More tests => 7; +use Test::More tests => 9; # Create and test a standby from given backup, with a certain # recovery target. @@ -86,6 +86,16 @@ my $lsn4 = $node_master->safe_psql('postgres', "SELECT pg_create_restore_point('$recovery_name');"); +# And now for a recovery target LSN +$node_master->safe_psql('postgres', + "INSERT INTO tab_int VALUES (generate_series(4001,5000))"); +my $recovery_lsn = $node_master->safe_psql('postgres', "SELECT pg_current_xlog_location()"); +my $lsn5 = + $node_master->safe_psql('postgres', "SELECT pg_current_xlog_location();"); + +$node_master->safe_psql('postgres', + "INSERT INTO tab_int VALUES (generate_series(5001,6000))"); + # Force archiving of WAL file $node_master->safe_psql('postgres', "SELECT pg_switch_xlog()"); @@ -102,6 +112,9 @@ test_recovery_standby('time', 'standby_3', $node_master, \@recovery_params, @recovery_params = ("recovery_target_name = '$recovery_name'"); test_recovery_standby('name', 'standby_4', $node_master, \@recovery_params, "4000", $lsn4); +@recovery_params = ("recovery_target_lsn = '$recovery_lsn'"); +test_recovery_standby('LSN', 'standby_5', $node_master, \@recovery_params, + "5000", $lsn5); # Multiple targets # Last entry has priority (note that an array respects the order of items @@ -111,16 +124,23 @@ test_recovery_standby('name', 'standby_4', $node_master, \@recovery_params, "recovery_target_xid = '$recovery_txid'", "recovery_target_time = '$recovery_time'"); test_recovery_standby('name + XID + time', - 'standby_5', $node_master, \@recovery_params, "3000", $lsn3); + 'standby_6', $node_master, \@recovery_params, "3000", $lsn3); @recovery_params = ( "recovery_target_time = '$recovery_time'", "recovery_target_name = '$recovery_name'", "recovery_target_xid = '$recovery_txid'"); test_recovery_standby('time + name + XID', - 'standby_6', $node_master, \@recovery_params, "2000", $lsn2); + 'standby_7', $node_master, \@recovery_params, "2000", $lsn2); @recovery_params = ( "recovery_target_xid = '$recovery_txid'", "recovery_target_time = '$recovery_time'", "recovery_target_name = '$recovery_name'"); test_recovery_standby('XID + time + name', - 'standby_7', $node_master, \@recovery_params, "4000", $lsn4); + 'standby_8', $node_master, \@recovery_params, "4000", $lsn4); +@recovery_params = ( + "recovery_target_xid = '$recovery_txid'", + "recovery_target_time = '$recovery_time'", + "recovery_target_name = '$recovery_name'", + "recovery_target_lsn = '$recovery_lsn'",); +test_recovery_standby('XID + time + name + LSN', + 'standby_9', $node_master, \@recovery_params, "5000", $lsn5); -- cgit v1.2.3 From 6591f4226c81104f7746da6a5c00519919c560ae Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 3 Sep 2016 15:29:03 -0400 Subject: Improve readability of the output of psql's \timing command. In addition to the existing decimal-milliseconds output value, display the same value in mm:ss.fff format if it exceeds one second. Tack on hours and even days fields if the interval is large enough. This avoids needing mental arithmetic to convert the values into customary time units. Corey Huinker, reviewed by Gerdan Santos; bikeshedding by many Discussion: --- doc/src/sgml/ref/psql-ref.sgml | 7 ++++-- src/bin/psql/common.c | 56 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 4 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml index 8a66ce7983..4806e77be7 100644 --- a/doc/src/sgml/ref/psql-ref.sgml +++ b/doc/src/sgml/ref/psql-ref.sgml @@ -2789,8 +2789,11 @@ testdb=> \setenv LESS -imx4F \timing [ on | off ] - Without parameter, toggles a display of how long each SQL statement - takes, in milliseconds. With parameter, sets same. + With a parameter, turns displaying of how long each SQL statement + takes on or off. Without a parameter, toggles the display between + on and off. The display is in milliseconds; intervals longer than + 1 second are also shown in minutes:seconds format, with hours and + days fields added if needed. diff --git a/src/bin/psql/common.c b/src/bin/psql/common.c index 7399950284..a7789dfa53 100644 --- a/src/bin/psql/common.c +++ b/src/bin/psql/common.c @@ -10,6 +10,7 @@ #include #include +#include #include #ifndef WIN32 #include /* for write() */ @@ -531,6 +532,57 @@ ClearOrSaveResult(PGresult *result) } +/* + * Print microtiming output. Always print raw milliseconds; if the interval + * is >= 1 second, also break it down into days/hours/minutes/seconds. + */ +static void +PrintTiming(double elapsed_msec) +{ + double seconds; + double minutes; + double hours; + double days; + + if (elapsed_msec < 1000.0) + { + /* This is the traditional (pre-v10) output format */ + printf(_("Time: %.3f ms\n"), elapsed_msec); + return; + } + + /* + * Note: we could print just seconds, in a format like %06.3f, when the + * total is less than 1min. But that's hard to interpret unless we tack + * on "s" or otherwise annotate it. Forcing the display to include + * minutes seems like a better solution. + */ + seconds = elapsed_msec / 1000.0; + minutes = floor(seconds / 60.0); + seconds -= 60.0 * minutes; + if (minutes < 60.0) + { + printf(_("Time: %.3f ms (%02d:%06.3f)\n"), + elapsed_msec, (int) minutes, seconds); + return; + } + + hours = floor(minutes / 60.0); + minutes -= 60.0 * hours; + if (hours < 24.0) + { + printf(_("Time: %.3f ms (%02d:%02d:%06.3f)\n"), + elapsed_msec, (int) hours, (int) minutes, seconds); + return; + } + + days = floor(hours / 24.0); + hours -= 24.0 * days; + printf(_("Time: %.3f ms (%.0f d %02d:%02d:%06.3f)\n"), + elapsed_msec, days, (int) hours, (int) minutes, seconds); +} + + /* * PSQLexec * @@ -679,7 +731,7 @@ PSQLexecWatch(const char *query, const printQueryOpt *opt) /* Possible microtiming output */ if (pset.timing) - printf(_("Time: %.3f ms\n"), elapsed_msec); + PrintTiming(elapsed_msec); return 1; } @@ -1332,7 +1384,7 @@ SendQuery(const char *query) /* Possible microtiming output */ if (pset.timing) - printf(_("Time: %.3f ms\n"), elapsed_msec); + PrintTiming(elapsed_msec); /* check for events that may occur during query execution */ -- cgit v1.2.3 From 5a072244919a92b2c757b2e3985191f02d674627 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 4 Sep 2016 13:19:54 -0400 Subject: Update release notes to mention need for ALTER EXTENSION UPDATE. Maybe we ought to make pg_upgrade do this for you, but it won't happen in 9.6, so call out the need for it as a migration consideration. --- doc/src/sgml/release-9.6.sgml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 'doc/src') diff --git a/doc/src/sgml/release-9.6.sgml b/doc/src/sgml/release-9.6.sgml index 895d88e768..578c3d1fdb 100644 --- a/doc/src/sgml/release-9.6.sgml +++ b/doc/src/sgml/release-9.6.sgml @@ -303,6 +303,26 @@ This commit is also listed under libpq and psql + + + + Update extension functions to be marked parallel-safe where + appropriate (Andreas Karlsson) + + + + Many of the standard extensions have been updated to allow their + functions to be executed within parallel query worker processes. + These changes will not take effect in + databases pg_upgrade'd from prior versions unless + you apply ALTER EXTENSION UPDATE to each such extension + (in each database of a cluster). + + + -- cgit v1.2.3 From ec03f4121cec6cf885bf40d9dfb53b8368251e99 Mon Sep 17 00:00:00 2001 From: Simon Riggs Date: Mon, 5 Sep 2016 09:47:49 +0100 Subject: Document LSN acronym in WAL Internals We previously didn't mention what an LSN actually was. Simon Riggs and Michael Paquier --- doc/src/sgml/acronyms.sgml | 10 ++++++++++ doc/src/sgml/wal.sgml | 16 ++++++++++++++++ 2 files changed, 26 insertions(+) (limited to 'doc/src') diff --git a/doc/src/sgml/acronyms.sgml b/doc/src/sgml/acronyms.sgml index 38f111ef9d..bf2273fa8a 100644 --- a/doc/src/sgml/acronyms.sgml +++ b/doc/src/sgml/acronyms.sgml @@ -380,6 +380,16 @@ + + LSN + + + Log Sequence Number, see pg_lsn + and WAL Internals. + + + + MSVC diff --git a/doc/src/sgml/wal.sgml b/doc/src/sgml/wal.sgml index 503ea8a2a7..9ae6547721 100644 --- a/doc/src/sgml/wal.sgml +++ b/doc/src/sgml/wal.sgml @@ -724,6 +724,10 @@ WAL Internals + + LSN + + WAL is automatically enabled; no action is required from the administrator except ensuring that the @@ -732,6 +736,18 @@ linkend="wal-configuration">). + + WAL records are appended to the WAL + logs as each new record is written. The insert position is described by + a Log Sequence Number (LSN) that is a byte offset into + the logs, increasing monotonically with each new record. + LSN values are returned as the datatype + pg_lsn. Values can be + compared to calculate the volume of WAL data that + separates them, so they are used to measure the progress of replication + and recovery. + + WAL logs are stored in the directory pg_xlog under the data directory, as a set of -- cgit v1.2.3 From 15bc038f9bcd1a9af3f625caffafc7c20322202d Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 5 Sep 2016 12:59:55 -0400 Subject: Relax transactional restrictions on ALTER TYPE ... ADD VALUE. To prevent possibly breaking indexes on enum columns, we must keep uncommitted enum values from getting stored in tables, unless we can be sure that any such column is new in the current transaction. Formerly, we enforced this by disallowing ALTER TYPE ... ADD VALUE from being executed at all in a transaction block, unless the target enum type had been created in the current transaction. This patch removes that restriction, and instead insists that an uncommitted enum value can't be referenced unless it belongs to an enum type created in the same transaction as the value. Per discussion, this should be a bit less onerous. It does require each function that could possibly return a new enum value to SQL operations to check this restriction, but there aren't so many of those that this seems unmaintainable. Andrew Dunstan and Tom Lane Discussion: <4075.1459088427@sss.pgh.pa.us> --- doc/src/sgml/ref/alter_type.sgml | 6 ++- src/backend/commands/typecmds.c | 21 +------- src/backend/tcop/utility.c | 2 +- src/backend/utils/adt/enum.c | 104 +++++++++++++++++++++++++++++++++++++ src/backend/utils/errcodes.txt | 1 + src/include/commands/typecmds.h | 2 +- src/test/regress/expected/enum.out | 65 +++++++++++++++++++---- src/test/regress/sql/enum.sql | 30 ++++++++--- 8 files changed, 191 insertions(+), 40 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/alter_type.sgml b/doc/src/sgml/ref/alter_type.sgml index 9789881a5c..aec73f6285 100644 --- a/doc/src/sgml/ref/alter_type.sgml +++ b/doc/src/sgml/ref/alter_type.sgml @@ -266,8 +266,10 @@ ALTER TYPE name ADD VALUE [ IF NOT Notes - ALTER TYPE ... ADD VALUE (the form that adds a new value to an - enum type) cannot be executed inside a transaction block. + If ALTER TYPE ... ADD VALUE (the form that adds a new value to + an enum type) is executed inside a transaction block, the new value cannot + be used until after the transaction has been committed, except in the case + that the enum type itself was created earlier in the same transaction. diff --git a/src/backend/commands/typecmds.c b/src/backend/commands/typecmds.c index ce04211067..8e7be78f65 100644 --- a/src/backend/commands/typecmds.c +++ b/src/backend/commands/typecmds.c @@ -1221,7 +1221,7 @@ DefineEnum(CreateEnumStmt *stmt) * Adds a new label to an existing enum. */ ObjectAddress -AlterEnum(AlterEnumStmt *stmt, bool isTopLevel) +AlterEnum(AlterEnumStmt *stmt) { Oid enum_type_oid; TypeName *typename; @@ -1236,25 +1236,6 @@ AlterEnum(AlterEnumStmt *stmt, bool isTopLevel) if (!HeapTupleIsValid(tup)) elog(ERROR, "cache lookup failed for type %u", enum_type_oid); - /* - * Ordinarily we disallow adding values within transaction blocks, because - * we can't cope with enum OID values getting into indexes and then having - * their defining pg_enum entries go away. However, it's okay if the enum - * type was created in the current transaction, since then there can be no - * such indexes that wouldn't themselves go away on rollback. (We support - * this case because pg_dump --binary-upgrade needs it.) We test this by - * seeing if the pg_type row has xmin == current XID and is not - * HEAP_UPDATED. If it is HEAP_UPDATED, we can't be sure whether the type - * was created or only modified in this xact. So we are disallowing some - * cases that could theoretically be safe; but fortunately pg_dump only - * needs the simplest case. - */ - if (HeapTupleHeaderGetXmin(tup->t_data) == GetCurrentTransactionId() && - !(tup->t_data->t_infomask & HEAP_UPDATED)) - /* safe to do inside transaction block */ ; - else - PreventTransactionChain(isTopLevel, "ALTER TYPE ... ADD"); - /* Check it's an enum and check user has permission to ALTER the enum */ checkEnumOwner(tup); diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index ac50c2a03d..ac64135d5d 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -1359,7 +1359,7 @@ ProcessUtilitySlow(Node *parsetree, break; case T_AlterEnumStmt: /* ALTER TYPE (enum) */ - address = AlterEnum((AlterEnumStmt *) parsetree, isTopLevel); + address = AlterEnum((AlterEnumStmt *) parsetree); break; case T_ViewStmt: /* CREATE VIEW */ diff --git a/src/backend/utils/adt/enum.c b/src/backend/utils/adt/enum.c index 135a54428a..47d5355027 100644 --- a/src/backend/utils/adt/enum.c +++ b/src/backend/utils/adt/enum.c @@ -19,6 +19,7 @@ #include "catalog/indexing.h" #include "catalog/pg_enum.h" #include "libpq/pqformat.h" +#include "storage/procarray.h" #include "utils/array.h" #include "utils/builtins.h" #include "utils/fmgroids.h" @@ -31,6 +32,93 @@ static Oid enum_endpoint(Oid enumtypoid, ScanDirection direction); static ArrayType *enum_range_internal(Oid enumtypoid, Oid lower, Oid upper); +/* + * Disallow use of an uncommitted pg_enum tuple. + * + * We need to make sure that uncommitted enum values don't get into indexes. + * If they did, and if we then rolled back the pg_enum addition, we'd have + * broken the index because value comparisons will not work reliably without + * an underlying pg_enum entry. (Note that removal of the heap entry + * containing an enum value is not sufficient to ensure that it doesn't appear + * in upper levels of indexes.) To do this we prevent an uncommitted row from + * being used for any SQL-level purpose. This is stronger than necessary, + * since the value might not be getting inserted into a table or there might + * be no index on its column, but it's easy to enforce centrally. + * + * However, it's okay to allow use of uncommitted values belonging to enum + * types that were themselves created in the same transaction, because then + * any such index would also be new and would go away altogether on rollback. + * (This case is required by pg_upgrade.) + * + * This function needs to be called (directly or indirectly) in any of the + * functions below that could return an enum value to SQL operations. + */ +static void +check_safe_enum_use(HeapTuple enumval_tup) +{ + TransactionId xmin; + Form_pg_enum en; + HeapTuple enumtyp_tup; + + /* + * If the row is hinted as committed, it's surely safe. This provides a + * fast path for all normal use-cases. + */ + if (HeapTupleHeaderXminCommitted(enumval_tup->t_data)) + return; + + /* + * Usually, a row would get hinted as committed when it's read or loaded + * into syscache; but just in case not, let's check the xmin directly. + */ + xmin = HeapTupleHeaderGetXmin(enumval_tup->t_data); + if (!TransactionIdIsInProgress(xmin) && + TransactionIdDidCommit(xmin)) + return; + + /* It is a new enum value, so check to see if the whole enum is new */ + en = (Form_pg_enum) GETSTRUCT(enumval_tup); + enumtyp_tup = SearchSysCache1(TYPEOID, ObjectIdGetDatum(en->enumtypid)); + if (!HeapTupleIsValid(enumtyp_tup)) + elog(ERROR, "cache lookup failed for type %u", en->enumtypid); + + /* + * We insist that the type have been created in the same (sub)transaction + * as the enum value. It would be safe to allow the type's originating + * xact to be a subcommitted child of the enum value's xact, but not vice + * versa (since we might now be in a subxact of the type's originating + * xact, which could roll back along with the enum value's subxact). The + * former case seems a sufficiently weird usage pattern as to not be worth + * spending code for, so we're left with a simple equality check. + * + * We also insist that the type's pg_type row not be HEAP_UPDATED. If it + * is, we can't tell whether the row was created or only modified in the + * apparent originating xact, so it might be older than that xact. (We do + * not worry whether the enum value is HEAP_UPDATED; if it is, we might + * think it's too new and throw an unnecessary error, but we won't allow + * an unsafe case.) + */ + if (xmin == HeapTupleHeaderGetXmin(enumtyp_tup->t_data) && + !(enumtyp_tup->t_data->t_infomask & HEAP_UPDATED)) + { + /* same (sub)transaction, so safe */ + ReleaseSysCache(enumtyp_tup); + return; + } + + /* + * There might well be other tests we could do here to narrow down the + * unsafe conditions, but for now just raise an exception. + */ + ereport(ERROR, + (errcode(ERRCODE_UNSAFE_NEW_ENUM_VALUE_USAGE), + errmsg("unsafe use of new value \"%s\" of enum type %s", + NameStr(en->enumlabel), + format_type_be(en->enumtypid)), + errhint("New enum values must be committed before they can be used."))); +} + + /* Basic I/O support */ Datum @@ -59,6 +147,9 @@ enum_in(PG_FUNCTION_ARGS) format_type_be(enumtypoid), name))); + /* check it's safe to use in SQL */ + check_safe_enum_use(tup); + /* * This comes from pg_enum.oid and stores system oids in user tables. This * oid must be preserved by binary upgrades. @@ -124,6 +215,9 @@ enum_recv(PG_FUNCTION_ARGS) format_type_be(enumtypoid), name))); + /* check it's safe to use in SQL */ + check_safe_enum_use(tup); + enumoid = HeapTupleGetOid(tup); ReleaseSysCache(tup); @@ -327,9 +421,16 @@ enum_endpoint(Oid enumtypoid, ScanDirection direction) enum_tuple = systable_getnext_ordered(enum_scan, direction); if (HeapTupleIsValid(enum_tuple)) + { + /* check it's safe to use in SQL */ + check_safe_enum_use(enum_tuple); minmax = HeapTupleGetOid(enum_tuple); + } else + { + /* should only happen with an empty enum */ minmax = InvalidOid; + } systable_endscan_ordered(enum_scan); index_close(enum_idx, AccessShareLock); @@ -490,6 +591,9 @@ enum_range_internal(Oid enumtypoid, Oid lower, Oid upper) if (left_found) { + /* check it's safe to use in SQL */ + check_safe_enum_use(enum_tuple); + if (cnt >= max) { max *= 2; diff --git a/src/backend/utils/errcodes.txt b/src/backend/utils/errcodes.txt index be924d58bd..e7bdb925ac 100644 --- a/src/backend/utils/errcodes.txt +++ b/src/backend/utils/errcodes.txt @@ -398,6 +398,7 @@ Section: Class 55 - Object Not In Prerequisite State 55006 E ERRCODE_OBJECT_IN_USE object_in_use 55P02 E ERRCODE_CANT_CHANGE_RUNTIME_PARAM cant_change_runtime_param 55P03 E ERRCODE_LOCK_NOT_AVAILABLE lock_not_available +55P04 E ERRCODE_UNSAFE_NEW_ENUM_VALUE_USAGE unsafe_new_enum_value_usage Section: Class 57 - Operator Intervention diff --git a/src/include/commands/typecmds.h b/src/include/commands/typecmds.h index e4c86f1b1d..847b770f00 100644 --- a/src/include/commands/typecmds.h +++ b/src/include/commands/typecmds.h @@ -26,7 +26,7 @@ extern void RemoveTypeById(Oid typeOid); extern ObjectAddress DefineDomain(CreateDomainStmt *stmt); extern ObjectAddress DefineEnum(CreateEnumStmt *stmt); extern ObjectAddress DefineRange(CreateRangeStmt *stmt); -extern ObjectAddress AlterEnum(AlterEnumStmt *stmt, bool isTopLevel); +extern ObjectAddress AlterEnum(AlterEnumStmt *stmt); extern ObjectAddress DefineCompositeType(RangeVar *typevar, List *coldeflist); extern Oid AssignTypeArrayOid(void); diff --git a/src/test/regress/expected/enum.out b/src/test/regress/expected/enum.out index 1a61a5b0df..d4a45a306b 100644 --- a/src/test/regress/expected/enum.out +++ b/src/test/regress/expected/enum.out @@ -560,25 +560,72 @@ DROP TYPE bogus; -- check transactional behaviour of ALTER TYPE ... ADD VALUE -- CREATE TYPE bogus AS ENUM('good'); --- check that we can't add new values to existing enums in a transaction +-- check that we can add new values to existing enums in a transaction +-- but we can't use them BEGIN; -ALTER TYPE bogus ADD VALUE 'bad'; -ERROR: ALTER TYPE ... ADD cannot run inside a transaction block +ALTER TYPE bogus ADD VALUE 'new'; +SAVEPOINT x; +SELECT 'new'::bogus; -- unsafe +ERROR: unsafe use of new value "new" of enum type bogus +LINE 1: SELECT 'new'::bogus; + ^ +HINT: New enum values must be committed before they can be used. +ROLLBACK TO x; +SELECT enum_first(null::bogus); -- safe + enum_first +------------ + good +(1 row) + +SELECT enum_last(null::bogus); -- unsafe +ERROR: unsafe use of new value "new" of enum type bogus +HINT: New enum values must be committed before they can be used. +ROLLBACK TO x; +SELECT enum_range(null::bogus); -- unsafe +ERROR: unsafe use of new value "new" of enum type bogus +HINT: New enum values must be committed before they can be used. +ROLLBACK TO x; COMMIT; +SELECT 'new'::bogus; -- now safe + bogus +------- + new +(1 row) + +SELECT enumlabel, enumsortorder +FROM pg_enum +WHERE enumtypid = 'bogus'::regtype +ORDER BY 2; + enumlabel | enumsortorder +-----------+--------------- + good | 1 + new | 2 +(2 rows) + -- check that we recognize the case where the enum already existed but was --- modified in the current txn +-- modified in the current txn; this should not be considered safe BEGIN; ALTER TYPE bogus RENAME TO bogon; ALTER TYPE bogon ADD VALUE 'bad'; -ERROR: ALTER TYPE ... ADD cannot run inside a transaction block +SELECT 'bad'::bogon; +ERROR: unsafe use of new value "bad" of enum type bogon +LINE 1: SELECT 'bad'::bogon; + ^ +HINT: New enum values must be committed before they can be used. ROLLBACK; DROP TYPE bogus; --- check that we *can* add new values to existing enums in a transaction, --- if the type is new as well +-- check that we can add new values to existing enums in a transaction +-- and use them, if the type is new as well BEGIN; -CREATE TYPE bogus AS ENUM(); -ALTER TYPE bogus ADD VALUE 'good'; +CREATE TYPE bogus AS ENUM('good'); +ALTER TYPE bogus ADD VALUE 'bad'; ALTER TYPE bogus ADD VALUE 'ugly'; +SELECT enum_range(null::bogus); + enum_range +----------------- + {good,bad,ugly} +(1 row) + ROLLBACK; -- -- Cleanup diff --git a/src/test/regress/sql/enum.sql b/src/test/regress/sql/enum.sql index 88a835e8aa..d25e8dedb6 100644 --- a/src/test/regress/sql/enum.sql +++ b/src/test/regress/sql/enum.sql @@ -262,26 +262,42 @@ DROP TYPE bogus; -- CREATE TYPE bogus AS ENUM('good'); --- check that we can't add new values to existing enums in a transaction +-- check that we can add new values to existing enums in a transaction +-- but we can't use them BEGIN; -ALTER TYPE bogus ADD VALUE 'bad'; +ALTER TYPE bogus ADD VALUE 'new'; +SAVEPOINT x; +SELECT 'new'::bogus; -- unsafe +ROLLBACK TO x; +SELECT enum_first(null::bogus); -- safe +SELECT enum_last(null::bogus); -- unsafe +ROLLBACK TO x; +SELECT enum_range(null::bogus); -- unsafe +ROLLBACK TO x; COMMIT; +SELECT 'new'::bogus; -- now safe +SELECT enumlabel, enumsortorder +FROM pg_enum +WHERE enumtypid = 'bogus'::regtype +ORDER BY 2; -- check that we recognize the case where the enum already existed but was --- modified in the current txn +-- modified in the current txn; this should not be considered safe BEGIN; ALTER TYPE bogus RENAME TO bogon; ALTER TYPE bogon ADD VALUE 'bad'; +SELECT 'bad'::bogon; ROLLBACK; DROP TYPE bogus; --- check that we *can* add new values to existing enums in a transaction, --- if the type is new as well +-- check that we can add new values to existing enums in a transaction +-- and use them, if the type is new as well BEGIN; -CREATE TYPE bogus AS ENUM(); -ALTER TYPE bogus ADD VALUE 'good'; +CREATE TYPE bogus AS ENUM('good'); +ALTER TYPE bogus ADD VALUE 'bad'; ALTER TYPE bogus ADD VALUE 'ugly'; +SELECT enum_range(null::bogus); ROLLBACK; -- -- cgit v1.2.3 From 975768f8eae2581b89ceafe8b16a77ff375207fe Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 6 Sep 2016 17:50:53 -0400 Subject: Doc: small improvements for documentation about VACUUM freezing. Mostly, explain how row xmin's used to be replaced by FrozenTransactionId and no longer are. Do a little copy-editing on the side. Per discussion with Egor Rogov. Back-patch to 9.4 where the behavioral change occurred. Discussion: <575D7955.6060209@postgrespro.ru> --- doc/src/sgml/maintenance.sgml | 48 +++++++++++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 13 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/maintenance.sgml b/doc/src/sgml/maintenance.sgml index 2713883019..f87f3e00de 100644 --- a/doc/src/sgml/maintenance.sgml +++ b/doc/src/sgml/maintenance.sgml @@ -389,7 +389,8 @@ - PostgreSQL's MVCC transaction semantics + PostgreSQL's + MVCC transaction semantics depend on being able to compare transaction ID (XID) numbers: a row version with an insertion XID greater than the current transaction's XID is in the future and should not be visible @@ -407,13 +408,10 @@ The reason that periodic vacuuming solves the problem is that VACUUM will mark rows as frozen, indicating that - they were inserted by a transaction which committed sufficiently far in - the past that the effects of the inserting transaction is certain to be - visible, from an MVCC perspective, to all current and future transactions. - PostgreSQL reserves a special XID, - FrozenTransactionId, which does not follow the normal XID - comparison rules and is always considered older - than every normal XID. Normal XIDs are + they were inserted by a transaction that committed sufficiently far in + the past that the effects of the inserting transaction are certain to be + visible to all current and future transactions. + Normal XIDs are compared using modulo-232 arithmetic. This means that for every normal XID, there are two billion XIDs that are older and two billion that are newer; another @@ -423,16 +421,40 @@ the next two billion transactions, no matter which normal XID we are talking about. If the row version still exists after more than two billion transactions, it will suddenly appear to be in the future. To - prevent this, frozen row versions are treated as if the inserting XID were + prevent this, PostgreSQL reserves a special XID, + FrozenTransactionId, which does not follow the normal XID + comparison rules and is always considered older + than every normal XID. + Frozen row versions are treated as if the inserting XID were FrozenTransactionId, so that they will appear to be in the past to all normal transactions regardless of wraparound issues, and so such row versions will be valid until deleted, no matter how long that is. + + + In PostgreSQL versions before 9.4, freezing was + implemented by actually replacing a row's insertion XID + with FrozenTransactionId, which was visible in the + row's xmin system column. Newer versions just set a flag + bit, preserving the row's original xmin for possible + forensic use. However, rows with xmin equal + to FrozenTransactionId (2) may still be found + in databases pg_upgrade'd from pre-9.4 versions. + + + Also, system catalogs may contain rows with xmin equal + to BootstrapTransactionId (1), indicating that they were + inserted during the first phase of initdb. + Like FrozenTransactionId, this special XID is treated as + older than every normal XID. + + + - controls how old an XID value has to be before its row version will be + controls how old an XID value has to be before rows bearing that XID will be frozen. Increasing this setting may avoid unnecessary work if the rows that would otherwise be frozen will soon be modified again, but decreasing this setting increases @@ -442,10 +464,10 @@ VACUUM uses the visibility map - to determine which pages of a relation must be scanned. Normally, it - will skips pages that don't have any dead row versions even if those pages + to determine which pages of a table must be scanned. Normally, it + will skip pages that don't have any dead row versions even if those pages might still have row versions with old XID values. Therefore, normal - scans won't succeed in freezing every row version in the table. + VACUUMs won't always freeze every old row version in the table. Periodically, VACUUM will perform an aggressive vacuum, skipping only those pages which contain neither dead rows nor any unfrozen XID or MXID values. -- cgit v1.2.3 From bd180b607927c7757af17cd6fce0e545e5c48584 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 7 Sep 2016 13:36:08 -0400 Subject: Doc: minor documentation improvements about extensions. Document the formerly-undocumented behavior that schema and comment control-file entries for an extension are honored only during initial installation, whereas other properties are also honored during updates. While at it, do some copy-editing on the recently-added docs for CREATE EXTENSION ... CASCADE, use links for some formerly vague cross references, and make a couple other minor improvements. Back-patch to 9.6 where CASCADE was added. The other parts of this could go further back, but they're probably not important enough to bother. --- doc/src/sgml/extend.sgml | 32 ++++++++++++-------- doc/src/sgml/ref/create_extension.sgml | 54 ++++++++++++++-------------------- 2 files changed, 42 insertions(+), 44 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/extend.sgml b/doc/src/sgml/extend.sgml index f050ff1f66..df88380a23 100644 --- a/doc/src/sgml/extend.sgml +++ b/doc/src/sgml/extend.sgml @@ -335,11 +335,13 @@ by pg_dump. Such a change is usually only sensible if you concurrently make the same change in the extension's script file. (But there are special provisions for tables containing configuration - data; see below.) + data; see .) + In production situations, it's generally better to create an extension + update script to perform changes to extension member objects. - The extension script may set privileges on objects which are part of the + The extension script may set privileges on objects that are part of the extension via GRANT and REVOKE statements. The final set of privileges for each object (if any are set) will be stored in the @@ -453,9 +455,11 @@ comment (string) - A comment (any string) about the extension. Alternatively, - the comment can be set by means of the - command in the script file. + A comment (any string) about the extension. The comment is applied + when initially creating an extension, but not during extension updates + (since that might override user-added comments). Alternatively, + the extension's comment can be set by writing + a command in the script file. @@ -518,7 +522,7 @@ its contained objects into a different schema after initial creation of the extension. The default is false, i.e. the extension is not relocatable. - See below for more information. + See for more information. @@ -529,7 +533,10 @@ This parameter can only be set for non-relocatable extensions. It forces the extension to be loaded into exactly the named schema - and not any other. See below for more information. + and not any other. + The schema parameter is consulted only when + initially creating an extension, not during extension updates. + See for more information. @@ -562,7 +569,8 @@ comments) by the extension mechanism. This provision is commonly used to throw an error if the script file is fed to psql rather than being loaded via CREATE EXTENSION (see example - script below). Without that, users might accidentally load the + script in ). + Without that, users might accidentally load the extension's contents as loose objects rather than as an extension, a state of affairs that's a bit tedious to recover from. @@ -580,7 +588,7 @@ - + Extension Relocatability @@ -678,7 +686,7 @@ SET LOCAL search_path TO @extschema@; - + Extension Configuration Tables @@ -762,7 +770,7 @@ SELECT pg_catalog.pg_extension_config_dump('my_config', 'WHERE NOT standard_entr out but the dump will not be able to be restored directly and user intervention will be required. - + Sequences associated with serial or bigserial columns need to be directly marked to dump their state. Marking their parent @@ -877,7 +885,7 @@ SELECT * FROM pg_extension_update_paths('extension_name'); - + Extension Example diff --git a/doc/src/sgml/ref/create_extension.sgml b/doc/src/sgml/ref/create_extension.sgml index 007d8c9330..14e910115a 100644 --- a/doc/src/sgml/ref/create_extension.sgml +++ b/doc/src/sgml/ref/create_extension.sgml @@ -95,35 +95,21 @@ CREATE EXTENSION [ IF NOT EXISTS ] extension_name If not specified, and the extension's control file does not specify a schema either, the current default object creation schema is used. + - If the extension specifies schema in its control file, - the schema cannot be overridden with SCHEMA clause. - The SCHEMA clause in this case works as follows: - - - - If schema_name matches - the schema in control file, it will be used normally as there is no - conflict. - - - - - If the CASCADE clause is given, the - schema_name will only - be used for the missing required extensions which do not specify - schema in their control files. - - - - - If schema_name is not - the same as the one in extension's control file and the - CASCADE clause is not given, error will be thrown. - - - + If the extension specifies a schema parameter in its + control file, then that schema cannot be overridden with + a SCHEMA clause. Normally, an error will be raised if + a SCHEMA clause is given and it conflicts with the + extension's schema parameter. However, if + the CASCADE clause is also given, + then schema_name is + ignored when it conflicts. The + given schema_name will be + used for installation of any needed extensions that do not + specify schema in their control files. + Remember that the extension itself is not considered to be within any schema: extensions have unqualified names that must be unique @@ -147,7 +133,8 @@ CREATE EXTENSION [ IF NOT EXISTS ] extension_name old_version - FROM old_version + + FROM old_version must be specified when, and only when, you are attempting to install an extension that replaces an old style module that is just a collection of objects not packaged into an extension. This option @@ -174,10 +161,13 @@ CREATE EXTENSION [ IF NOT EXISTS ] extension_name CASCADE - Try to install extension including the required dependencies - recursively. The SCHEMA option will be propagated - to the required extensions. Other options are not recursively - applied when using this clause. + Automatically install any extensions that this extension depends on + that are not already installed. Their dependencies are likewise + automatically installed, recursively. The SCHEMA clause, + if given, applies to all extensions that get installed this way. + Other options of the statement are not applied to + automatically-installed extensions; in particular, their default + versions are always selected. -- cgit v1.2.3 From 0ab9c56d0fe3acc9d4717a9cbac6ef3369275b90 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 7 Sep 2016 16:11:56 -0400 Subject: Support renaming an existing value of an enum type. Not much to be said about this patch: it does what it says on the tin. In passing, rename AlterEnumStmt.skipIfExists to skipIfNewValExists to clarify what it actually does. In the discussion of this patch we considered supporting other similar options, such as IF EXISTS on the type as a whole or IF NOT EXISTS on the target name. This patch doesn't actually add any such feature, but it might happen later. Dagfinn Ilmari Mannsåker, reviewed by Emre Hasegeli Discussion: --- doc/src/sgml/ref/alter_type.sgml | 68 ++++++++++++++++++++++-------- src/backend/catalog/pg_enum.c | 85 ++++++++++++++++++++++++++++++++++++++ src/backend/commands/typecmds.c | 20 ++++++--- src/backend/nodes/copyfuncs.c | 3 +- src/backend/nodes/equalfuncs.c | 3 +- src/backend/parser/gram.y | 20 +++++++-- src/include/catalog/pg_enum.h | 2 + src/include/nodes/parsenodes.h | 3 +- src/test/regress/expected/enum.out | 22 ++++++++++ src/test/regress/sql/enum.sql | 11 +++++ 10 files changed, 208 insertions(+), 29 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/alter_type.sgml b/doc/src/sgml/ref/alter_type.sgml index aec73f6285..fdb4f3367d 100644 --- a/doc/src/sgml/ref/alter_type.sgml +++ b/doc/src/sgml/ref/alter_type.sgml @@ -28,7 +28,8 @@ ALTER TYPE name OWNER TO { name RENAME ATTRIBUTE attribute_name TO new_attribute_name [ CASCADE | RESTRICT ] ALTER TYPE name RENAME TO new_name ALTER TYPE name SET SCHEMA new_schema -ALTER TYPE name ADD VALUE [ IF NOT EXISTS ] new_enum_value [ { BEFORE | AFTER } existing_enum_value ] +ALTER TYPE name ADD VALUE [ IF NOT EXISTS ] new_enum_value [ { BEFORE | AFTER } neighbor_enum_value ] +ALTER TYPE name RENAME VALUE existing_enum_value TO new_enum_value where action is one of: @@ -124,21 +125,13 @@ ALTER TYPE name ADD VALUE [ IF NOT - CASCADE + RENAME VALUE - Automatically propagate the operation to typed tables of the - type being altered, and their descendants. - - - - - - RESTRICT - - - Refuse the operation if the type being altered is the type of a - typed table. This is the default. + This form renames a value of an enum type. + The value's place in the enum's ordering is not affected. + An error will occur if the specified value is not present or the new + name is already present. @@ -241,14 +234,15 @@ ALTER TYPE name ADD VALUE [ IF NOT new_enum_value - The new value to be added to an enum type's list of values. + The new value to be added to an enum type's list of values, + or the new name to be given to an existing value. Like all enum literals, it needs to be quoted. - existing_enum_value + neighbor_enum_value The existing enum value that the new value should be added immediately @@ -258,6 +252,36 @@ ALTER TYPE name ADD VALUE [ IF NOT + + existing_enum_value + + + The existing enum value that should be renamed. + Like all enum literals, it needs to be quoted. + + + + + + CASCADE + + + Automatically propagate the operation to typed tables of the + type being altered, and their descendants. + + + + + + RESTRICT + + + Refuse the operation if the type being altered is the type of a + typed table. This is the default. + + + + @@ -270,6 +294,8 @@ ALTER TYPE name ADD VALUE [ IF NOT an enum type) is executed inside a transaction block, the new value cannot be used until after the transaction has been committed, except in the case that the enum type itself was created earlier in the same transaction. + Likewise, when a pre-existing enum value is renamed, the transaction must + be committed before the renamed value can be used. @@ -323,7 +349,15 @@ ALTER TYPE compfoo ADD ATTRIBUTE f3 int; To add a new value to an enum type in a particular sort position: ALTER TYPE colors ADD VALUE 'orange' AFTER 'red'; - + + + + + To rename an enum value: + +ALTER TYPE colors RENAME VALUE 'purple' TO 'mauve'; + + diff --git a/src/backend/catalog/pg_enum.c b/src/backend/catalog/pg_enum.c index c66f9632c2..1f0ffcfa15 100644 --- a/src/backend/catalog/pg_enum.c +++ b/src/backend/catalog/pg_enum.c @@ -465,6 +465,91 @@ restart: } +/* + * RenameEnumLabel + * Rename a label in an enum set. + */ +void +RenameEnumLabel(Oid enumTypeOid, + const char *oldVal, + const char *newVal) +{ + Relation pg_enum; + HeapTuple enum_tup; + Form_pg_enum en; + CatCList *list; + int nelems; + HeapTuple old_tup; + bool found_new; + int i; + + /* check length of new label is ok */ + if (strlen(newVal) > (NAMEDATALEN - 1)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_NAME), + errmsg("invalid enum label \"%s\"", newVal), + errdetail("Labels must be %d characters or less.", + NAMEDATALEN - 1))); + + /* + * Acquire a lock on the enum type, which we won't release until commit. + * This ensures that two backends aren't concurrently modifying the same + * enum type. Since we are not changing the type's sort order, this is + * probably not really necessary, but there seems no reason not to take + * the lock to be sure. + */ + LockDatabaseObject(TypeRelationId, enumTypeOid, 0, ExclusiveLock); + + pg_enum = heap_open(EnumRelationId, RowExclusiveLock); + + /* Get the list of existing members of the enum */ + list = SearchSysCacheList1(ENUMTYPOIDNAME, + ObjectIdGetDatum(enumTypeOid)); + nelems = list->n_members; + + /* + * Locate the element to rename and check if the new label is already in + * use. (The unique index on pg_enum would catch that anyway, but we + * prefer a friendlier error message.) + */ + old_tup = NULL; + found_new = false; + for (i = 0; i < nelems; i++) + { + enum_tup = &(list->members[i]->tuple); + en = (Form_pg_enum) GETSTRUCT(enum_tup); + if (strcmp(NameStr(en->enumlabel), oldVal) == 0) + old_tup = enum_tup; + if (strcmp(NameStr(en->enumlabel), newVal) == 0) + found_new = true; + } + if (!old_tup) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("\"%s\" is not an existing enum label", + oldVal))); + if (found_new) + ereport(ERROR, + (errcode(ERRCODE_DUPLICATE_OBJECT), + errmsg("enum label \"%s\" already exists", + newVal))); + + /* OK, make a writable copy of old tuple */ + enum_tup = heap_copytuple(old_tup); + en = (Form_pg_enum) GETSTRUCT(enum_tup); + + ReleaseCatCacheList(list); + + /* Update the pg_enum entry */ + namestrcpy(&en->enumlabel, newVal); + simple_heap_update(pg_enum, &enum_tup->t_self, enum_tup); + CatalogUpdateIndexes(pg_enum, enum_tup); + heap_freetuple(enum_tup); + + heap_close(pg_enum, RowExclusiveLock); +} + + /* * RenumberEnumType * Renumber existing enum elements to have sort positions 1..n. diff --git a/src/backend/commands/typecmds.c b/src/backend/commands/typecmds.c index 6cc7106467..41fd2dae7f 100644 --- a/src/backend/commands/typecmds.c +++ b/src/backend/commands/typecmds.c @@ -1241,17 +1241,25 @@ AlterEnum(AlterEnumStmt *stmt) /* Check it's an enum and check user has permission to ALTER the enum */ checkEnumOwner(tup); - /* Add the new label */ - AddEnumLabel(enum_type_oid, stmt->newVal, - stmt->newValNeighbor, stmt->newValIsAfter, - stmt->skipIfExists); + ReleaseSysCache(tup); + + if (stmt->oldVal) + { + /* Rename an existing label */ + RenameEnumLabel(enum_type_oid, stmt->oldVal, stmt->newVal); + } + else + { + /* Add a new label */ + AddEnumLabel(enum_type_oid, stmt->newVal, + stmt->newValNeighbor, stmt->newValIsAfter, + stmt->skipIfNewValExists); + } InvokeObjectPostAlterHook(TypeRelationId, enum_type_oid, 0); ObjectAddressSet(address, TypeRelationId, enum_type_oid); - ReleaseSysCache(tup); - return address; } diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index be2207e318..4f39dad66b 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -3375,10 +3375,11 @@ _copyAlterEnumStmt(const AlterEnumStmt *from) AlterEnumStmt *newnode = makeNode(AlterEnumStmt); COPY_NODE_FIELD(typeName); + COPY_STRING_FIELD(oldVal); COPY_STRING_FIELD(newVal); COPY_STRING_FIELD(newValNeighbor); COPY_SCALAR_FIELD(newValIsAfter); - COPY_SCALAR_FIELD(skipIfExists); + COPY_SCALAR_FIELD(skipIfNewValExists); return newnode; } diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index c4ec4077a6..4800165a91 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -1465,10 +1465,11 @@ static bool _equalAlterEnumStmt(const AlterEnumStmt *a, const AlterEnumStmt *b) { COMPARE_NODE_FIELD(typeName); + COMPARE_STRING_FIELD(oldVal); COMPARE_STRING_FIELD(newVal); COMPARE_STRING_FIELD(newValNeighbor); COMPARE_SCALAR_FIELD(newValIsAfter); - COMPARE_SCALAR_FIELD(skipIfExists); + COMPARE_SCALAR_FIELD(skipIfNewValExists); return true; } diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index b69a77a588..1526c73a1c 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -5257,30 +5257,44 @@ AlterEnumStmt: { AlterEnumStmt *n = makeNode(AlterEnumStmt); n->typeName = $3; + n->oldVal = NULL; n->newVal = $7; n->newValNeighbor = NULL; n->newValIsAfter = true; - n->skipIfExists = $6; + n->skipIfNewValExists = $6; $$ = (Node *) n; } | ALTER TYPE_P any_name ADD_P VALUE_P opt_if_not_exists Sconst BEFORE Sconst { AlterEnumStmt *n = makeNode(AlterEnumStmt); n->typeName = $3; + n->oldVal = NULL; n->newVal = $7; n->newValNeighbor = $9; n->newValIsAfter = false; - n->skipIfExists = $6; + n->skipIfNewValExists = $6; $$ = (Node *) n; } | ALTER TYPE_P any_name ADD_P VALUE_P opt_if_not_exists Sconst AFTER Sconst { AlterEnumStmt *n = makeNode(AlterEnumStmt); n->typeName = $3; + n->oldVal = NULL; n->newVal = $7; n->newValNeighbor = $9; n->newValIsAfter = true; - n->skipIfExists = $6; + n->skipIfNewValExists = $6; + $$ = (Node *) n; + } + | ALTER TYPE_P any_name RENAME VALUE_P Sconst TO Sconst + { + AlterEnumStmt *n = makeNode(AlterEnumStmt); + n->typeName = $3; + n->oldVal = $6; + n->newVal = $8; + n->newValNeighbor = NULL; + n->newValIsAfter = false; + n->skipIfNewValExists = false; $$ = (Node *) n; } ; diff --git a/src/include/catalog/pg_enum.h b/src/include/catalog/pg_enum.h index dd32443b91..901d3adbb9 100644 --- a/src/include/catalog/pg_enum.h +++ b/src/include/catalog/pg_enum.h @@ -67,5 +67,7 @@ extern void EnumValuesDelete(Oid enumTypeOid); extern void AddEnumLabel(Oid enumTypeOid, const char *newVal, const char *neighbor, bool newValIsAfter, bool skipIfExists); +extern void RenameEnumLabel(Oid enumTypeOid, + const char *oldVal, const char *newVal); #endif /* PG_ENUM_H */ diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 3716c2eef9..8d3dcf4d4c 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -2708,10 +2708,11 @@ typedef struct AlterEnumStmt { NodeTag type; List *typeName; /* qualified name (list of Value strings) */ + char *oldVal; /* old enum value's name, if renaming */ char *newVal; /* new enum value's name */ char *newValNeighbor; /* neighboring enum value, if specified */ bool newValIsAfter; /* place new enum value after neighbor? */ - bool skipIfExists; /* no error if label already exists */ + bool skipIfNewValExists; /* no error if new already exists? */ } AlterEnumStmt; /* ---------------------- diff --git a/src/test/regress/expected/enum.out b/src/test/regress/expected/enum.out index d4a45a306b..514d1d01a1 100644 --- a/src/test/regress/expected/enum.out +++ b/src/test/regress/expected/enum.out @@ -556,6 +556,28 @@ CREATE TABLE enumtest_bogus_child(parent bogus REFERENCES enumtest_parent); ERROR: foreign key constraint "enumtest_bogus_child_parent_fkey" cannot be implemented DETAIL: Key columns "parent" and "id" are of incompatible types: bogus and rainbow. DROP TYPE bogus; +-- check renaming a value +ALTER TYPE rainbow RENAME VALUE 'red' TO 'crimson'; +SELECT enumlabel, enumsortorder +FROM pg_enum +WHERE enumtypid = 'rainbow'::regtype +ORDER BY 2; + enumlabel | enumsortorder +-----------+--------------- + crimson | 1 + orange | 2 + yellow | 3 + green | 4 + blue | 5 + purple | 6 +(6 rows) + +-- check that renaming a non-existent value fails +ALTER TYPE rainbow RENAME VALUE 'red' TO 'crimson'; +ERROR: "red" is not an existing enum label +-- check that renaming to an existent value fails +ALTER TYPE rainbow RENAME VALUE 'blue' TO 'green'; +ERROR: enum label "green" already exists -- -- check transactional behaviour of ALTER TYPE ... ADD VALUE -- diff --git a/src/test/regress/sql/enum.sql b/src/test/regress/sql/enum.sql index d25e8dedb6..d7e87143a0 100644 --- a/src/test/regress/sql/enum.sql +++ b/src/test/regress/sql/enum.sql @@ -257,6 +257,17 @@ CREATE TYPE bogus AS ENUM('good', 'bad', 'ugly'); CREATE TABLE enumtest_bogus_child(parent bogus REFERENCES enumtest_parent); DROP TYPE bogus; +-- check renaming a value +ALTER TYPE rainbow RENAME VALUE 'red' TO 'crimson'; +SELECT enumlabel, enumsortorder +FROM pg_enum +WHERE enumtypid = 'rainbow'::regtype +ORDER BY 2; +-- check that renaming a non-existent value fails +ALTER TYPE rainbow RENAME VALUE 'red' TO 'crimson'; +-- check that renaming to an existent value fails +ALTER TYPE rainbow RENAME VALUE 'blue' TO 'green'; + -- -- check transactional behaviour of ALTER TYPE ... ADD VALUE -- -- cgit v1.2.3 From c9cf432ef32a9d29323b9b079178c1a6be126ff8 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Wed, 7 Sep 2016 20:51:28 -0400 Subject: 9.6 release notes: correct summary item about freeze Previously it less precisely talked about autovacuum. Backpatch-through: 9.6 --- doc/src/sgml/release-9.6.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/release-9.6.sgml b/doc/src/sgml/release-9.6.sgml index 578c3d1fdb..ddd280c85a 100644 --- a/doc/src/sgml/release-9.6.sgml +++ b/doc/src/sgml/release-9.6.sgml @@ -29,7 +29,7 @@ - Autovacuum no longer performs repetitive scanning of old data + Avoid scanning pages unnecessarily during vacuum freeze operations -- cgit v1.2.3 From d299eb41dfc7b73dec80f22554b952f01c9d54a4 Mon Sep 17 00:00:00 2001 From: Noah Misch Date: Thu, 8 Sep 2016 01:42:09 -0400 Subject: MSVC: Pass any user-set MSBFLAGS to MSBuild and VCBUILD. This is particularly useful to pass /m, to perform a parallel build. Christian Ullrich, reviewed by Michael Paquier. --- doc/src/sgml/install-windows.sgml | 8 ++++++++ src/tools/msvc/build.pl | 7 ++++--- src/tools/msvc/clean.bat | 2 +- src/tools/msvc/vcregress.pl | 3 ++- 4 files changed, 15 insertions(+), 5 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/install-windows.sgml b/doc/src/sgml/install-windows.sgml index 8cd189c8e1..50116f315d 100644 --- a/doc/src/sgml/install-windows.sgml +++ b/doc/src/sgml/install-windows.sgml @@ -145,6 +145,14 @@ $ENV{PATH}=$ENV{PATH} . ';c:\some\where\bison\bin'; + + To pass additional command line arguments to the Visual Studio build + command (msbuild or vcbuild): + +$ENV{MSBFLAGS}="/m"; + + + Requirements diff --git a/src/tools/msvc/build.pl b/src/tools/msvc/build.pl index 007e3c73b2..52739774c7 100644 --- a/src/tools/msvc/build.pl +++ b/src/tools/msvc/build.pl @@ -38,6 +38,7 @@ my $vcver = Mkvcbuild::mkvcbuild($config); # check what sort of build we are doing my $bconf = $ENV{CONFIG} || "Release"; +my $msbflags = $ENV{MSBFLAGS} || ""; my $buildwhat = $ARGV[1] || ""; if (uc($ARGV[0]) eq 'DEBUG') { @@ -53,16 +54,16 @@ elsif (uc($ARGV[0]) ne "RELEASE") if ($buildwhat and $vcver >= 10.00) { system( - "msbuild $buildwhat.vcxproj /verbosity:normal /p:Configuration=$bconf" + "msbuild $buildwhat.vcxproj $msbflags /verbosity:normal /p:Configuration=$bconf" ); } elsif ($buildwhat) { - system("vcbuild $buildwhat.vcproj $bconf"); + system("vcbuild $msbflags $buildwhat.vcproj $bconf"); } else { - system("msbuild pgsql.sln /verbosity:normal /p:Configuration=$bconf"); + system("msbuild pgsql.sln $msbflags /verbosity:normal /p:Configuration=$bconf"); } # report status diff --git a/src/tools/msvc/clean.bat b/src/tools/msvc/clean.bat index 469b8a24b2..e21e37f6f7 100755 --- a/src/tools/msvc/clean.bat +++ b/src/tools/msvc/clean.bat @@ -107,6 +107,6 @@ REM for /r %%f in (*.sql) do if exist %%f.in del %%f cd %D% REM Clean up ecpg regression test files -msbuild /NoLogo ecpg_regression.proj /t:clean /v:q +msbuild %MSBFLAGS% /NoLogo ecpg_regression.proj /t:clean /v:q goto :eof diff --git a/src/tools/msvc/vcregress.pl b/src/tools/msvc/vcregress.pl index b4f946474f..bcf22677ac 100644 --- a/src/tools/msvc/vcregress.pl +++ b/src/tools/msvc/vcregress.pl @@ -138,8 +138,9 @@ sub check sub ecpgcheck { + my $msbflags = $ENV{MSBFLAGS} || ""; chdir $startdir; - system("msbuild ecpg_regression.proj /p:config=$Config"); + system("msbuild ecpg_regression.proj $msbflags /p:config=$Config"); my $status = $? >> 8; exit $status if $status; InstallTemp(); -- cgit v1.2.3 From f66472428a51fc484bc5ca81791924d06a6f096d Mon Sep 17 00:00:00 2001 From: Simon Riggs Date: Fri, 9 Sep 2016 11:19:21 +0100 Subject: Correct TABLESAMPLE docs Revert to original use of word “sample”, though with clarification, per Tom Lane. Discussion: 29052.1471015383@sss.pgh.pa.us --- doc/src/sgml/ref/select.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/select.sgml b/doc/src/sgml/ref/select.sgml index e0098eb8d3..211e4c320c 100644 --- a/doc/src/sgml/ref/select.sgml +++ b/doc/src/sgml/ref/select.sgml @@ -391,7 +391,7 @@ TABLE [ ONLY ] table_name [ * ] not been changed meanwhile. But different seed values will usually produce different samples. If REPEATABLE is not given then a new random - seed is selected for each query. + sample is selected for each query, based upon a system-generated seed. Note that some add-on sampling methods do not accept REPEATABLE, and will always produce new samples on each use. -- cgit v1.2.3 From 40b449ae84dcf71177d7749a7b0c582b64dc15f0 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 11 Sep 2016 14:15:07 -0400 Subject: Allow CREATE EXTENSION to follow extension update paths. Previously, to update an extension you had to produce both a version-update script and a new base installation script. It's become more and more obvious that that's tedious, duplicative, and error-prone. This patch attempts to improve matters by allowing the new base installation script to be omitted. CREATE EXTENSION will install a requested version if it can find a base script and a chain of update scripts that will get there. As in the existing update logic, shorter chains are preferred if there's more than one possibility, with an arbitrary tie-break rule for chains of equal length. Also adjust the pg_available_extension_versions view to show such versions as installable. While at it, refactor the code so that CASCADE processing works for extensions requested during ApplyExtensionUpdates(). Without this, addition of a new requirement in an updated extension would require creating a new base script, even if there was no other reason to do that. (It would be easy at this point to add a CASCADE option to ALTER EXTENSION UPDATE, to allow the same thing to happen during a manually-commanded version update, but I have not done that here.) Tom Lane, reviewed by Andres Freund Discussion: <20160905005919.jz2m2yh3und2dsuy@alap3.anarazel.de> --- doc/src/sgml/extend.sgml | 41 +++ src/backend/commands/extension.c | 608 +++++++++++++++++++++++++-------------- 2 files changed, 439 insertions(+), 210 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/extend.sgml b/doc/src/sgml/extend.sgml index df88380a23..e19c657d8f 100644 --- a/doc/src/sgml/extend.sgml +++ b/doc/src/sgml/extend.sgml @@ -885,6 +885,47 @@ SELECT * FROM pg_extension_update_paths('extension_name'); + + Installing Extensions using Update Scripts + + + An extension that has been around for awhile will probably exist in + several versions, for which the author will need to write update scripts. + For example, if you have released a foo extension in + versions 1.0, 1.1, and 1.2, there + should be update scripts foo--1.0--1.1.sql + and foo--1.1--1.2.sql. + Before PostgreSQL 10, it was necessary to also create + new script files foo--1.1.sql and foo--1.2.sql + that directly build the newer extension versions, or else the newer + versions could not be installed directly, only by + installing 1.0 and then updating. That was tedious and + duplicative, but now it's unnecessary, because CREATE + EXTENSION can follow update chains automatically. + For example, if only the script + files foo--1.0.sql, foo--1.0--1.1.sql, + and foo--1.1--1.2.sql are available then a request to + install version 1.2 is honored by running those three + scripts in sequence. The processing is the same as if you'd first + installed 1.0 and then updated to 1.2. + (As with ALTER EXTENSION UPDATE, if multiple pathways are + available then the shortest is preferred.) Arranging an extension's + script files in this style can reduce the amount of maintenance effort + needed to produce small updates. + + + + If you use secondary (version-specific) control files with an extension + maintained in this style, keep in mind that each version needs a control + file even if it has no stand-alone installation script, as that control + file will determine how the implicit update to that version is performed. + For example, if foo--1.0.control specifies requires + = 'bar' but foo's other control files do not, the + extension's dependency on bar will be dropped when updating + from 1.0 to another version. + + + Extension Example diff --git a/src/backend/commands/extension.c b/src/backend/commands/extension.c index df49a78e2f..f6c2c8af91 100644 --- a/src/backend/commands/extension.c +++ b/src/backend/commands/extension.c @@ -100,14 +100,25 @@ typedef struct ExtensionVersionInfo static List *find_update_path(List *evi_list, ExtensionVersionInfo *evi_start, ExtensionVersionInfo *evi_target, + bool reject_indirect, bool reinitialize); +static Oid get_required_extension(char *reqExtensionName, + char *extensionName, + char *origSchemaName, + bool cascade, + List *parents, + bool is_create); static void get_available_versions_for_extension(ExtensionControlFile *pcontrol, Tuplestorestate *tupstore, TupleDesc tupdesc); +static Datum convert_requires_to_datum(List *requires); static void ApplyExtensionUpdates(Oid extensionOid, ExtensionControlFile *pcontrol, const char *initialVersion, - List *updateVersions); + List *updateVersions, + char *origSchemaName, + bool cascade, + bool is_create); static char *read_whole_file(const char *filename, int *length); @@ -1071,7 +1082,7 @@ identify_update_path(ExtensionControlFile *control, evi_target = get_ext_ver_info(newVersion, &evi_list); /* Find shortest path */ - result = find_update_path(evi_list, evi_start, evi_target, false); + result = find_update_path(evi_list, evi_start, evi_target, false, false); if (result == NIL) ereport(ERROR, @@ -1086,9 +1097,13 @@ identify_update_path(ExtensionControlFile *control, * Apply Dijkstra's algorithm to find the shortest path from evi_start to * evi_target. * + * If reject_indirect is true, ignore paths that go through installable + * versions. This saves work when the caller will consider starting from + * all installable versions anyway. + * * If reinitialize is false, assume the ExtensionVersionInfo list has not * been used for this before, and the initialization done by get_ext_ver_info - * is still good. + * is still good. Otherwise, reinitialize all transient fields used here. * * Result is a List of names of versions to transition through (the initial * version is *not* included). Returns NIL if no such path. @@ -1097,6 +1112,7 @@ static List * find_update_path(List *evi_list, ExtensionVersionInfo *evi_start, ExtensionVersionInfo *evi_target, + bool reject_indirect, bool reinitialize) { List *result; @@ -1105,6 +1121,8 @@ find_update_path(List *evi_list, /* Caller error if start == target */ Assert(evi_start != evi_target); + /* Caller error if reject_indirect and target is installable */ + Assert(!(reject_indirect && evi_target->installable)); if (reinitialize) { @@ -1131,6 +1149,9 @@ find_update_path(List *evi_list, ExtensionVersionInfo *evi2 = (ExtensionVersionInfo *) lfirst(lc); int newdist; + /* if reject_indirect, treat installable versions as unreachable */ + if (reject_indirect && evi2->installable) + continue; newdist = evi->distance + 1; if (newdist < evi2->distance) { @@ -1166,6 +1187,67 @@ find_update_path(List *evi_list, return result; } +/* + * Given a target version that is not directly installable, find the + * best installation sequence starting from a directly-installable version. + * + * evi_list: previously-collected version update graph + * evi_target: member of that list that we want to reach + * + * Returns the best starting-point version, or NULL if there is none. + * On success, *best_path is set to the path from the start point. + * + * If there's more than one possible start point, prefer shorter update paths, + * and break any ties arbitrarily on the basis of strcmp'ing the starting + * versions' names. + */ +static ExtensionVersionInfo * +find_install_path(List *evi_list, ExtensionVersionInfo *evi_target, + List **best_path) +{ + ExtensionVersionInfo *evi_start = NULL; + ListCell *lc; + + *best_path = NIL; + + /* + * We don't expect to be called for an installable target, but if we are, + * the answer is easy: just start from there, with an empty update path. + */ + if (evi_target->installable) + return evi_target; + + /* Consider all installable versions as start points */ + foreach(lc, evi_list) + { + ExtensionVersionInfo *evi1 = (ExtensionVersionInfo *) lfirst(lc); + List *path; + + if (!evi1->installable) + continue; + + /* + * Find shortest path from evi1 to evi_target; but no need to consider + * paths going through other installable versions. + */ + path = find_update_path(evi_list, evi1, evi_target, true, true); + if (path == NIL) + continue; + + /* Remember best path */ + if (evi_start == NULL || + list_length(path) < list_length(*best_path) || + (list_length(path) == list_length(*best_path) && + strcmp(evi_start->name, evi1->name) < 0)) + { + evi_start = evi1; + *best_path = path; + } + } + + return evi_start; +} + /* * CREATE EXTENSION worker * @@ -1175,17 +1257,16 @@ find_update_path(List *evi_list, * installed, allowing us to error out if we recurse to one of those. */ static ObjectAddress -CreateExtensionInternal(ParseState *pstate, CreateExtensionStmt *stmt, List *parents) +CreateExtensionInternal(char *extensionName, + char *schemaName, + char *versionName, + char *oldVersionName, + bool cascade, + List *parents, + bool is_create) { - DefElem *d_schema = NULL; - DefElem *d_new_version = NULL; - DefElem *d_old_version = NULL; - DefElem *d_cascade = NULL; - char *schemaName = NULL; + char *origSchemaName = schemaName; Oid schemaOid = InvalidOid; - char *versionName; - char *oldVersionName; - bool cascade = false; Oid extowner = GetUserId(); ExtensionControlFile *pcontrol; ExtensionControlFile *control; @@ -1193,87 +1274,43 @@ CreateExtensionInternal(ParseState *pstate, CreateExtensionStmt *stmt, List *par List *requiredExtensions; List *requiredSchemas; Oid extensionOid; - ListCell *lc; ObjectAddress address; + ListCell *lc; /* * Read the primary control file. Note we assume that it does not contain * any non-ASCII data, so there is no need to worry about encoding at this * point. */ - pcontrol = read_extension_control_file(stmt->extname); - - /* - * Read the statement option list - */ - foreach(lc, stmt->options) - { - DefElem *defel = (DefElem *) lfirst(lc); - - if (strcmp(defel->defname, "schema") == 0) - { - if (d_schema) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"), - parser_errposition(pstate, defel->location))); - d_schema = defel; - } - else if (strcmp(defel->defname, "new_version") == 0) - { - if (d_new_version) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"), - parser_errposition(pstate, defel->location))); - d_new_version = defel; - } - else if (strcmp(defel->defname, "old_version") == 0) - { - if (d_old_version) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"), - parser_errposition(pstate, defel->location))); - d_old_version = defel; - } - else if (strcmp(defel->defname, "cascade") == 0) - { - if (d_cascade) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"), - parser_errposition(pstate, defel->location))); - d_cascade = defel; - cascade = defGetBoolean(d_cascade); - } - else - elog(ERROR, "unrecognized option: %s", defel->defname); - } + pcontrol = read_extension_control_file(extensionName); /* * Determine the version to install */ - if (d_new_version && d_new_version->arg) - versionName = strVal(d_new_version->arg); - else if (pcontrol->default_version) - versionName = pcontrol->default_version; - else + if (versionName == NULL) { - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("version to install must be specified"))); - versionName = NULL; /* keep compiler quiet */ + if (pcontrol->default_version) + versionName = pcontrol->default_version; + else + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("version to install must be specified"))); } check_valid_version_name(versionName); /* - * Determine the (unpackaged) version to update from, if any, and then - * figure out what sequence of update scripts we need to apply. + * Figure out which script(s) we need to run to install the desired + * version of the extension. If we do not have a script that directly + * does what is needed, we try to find a sequence of update scripts that + * will get us there. */ - if (d_old_version && d_old_version->arg) + if (oldVersionName) { - oldVersionName = strVal(d_old_version->arg); + /* + * "FROM old_version" was specified, indicating that we're trying to + * update from some unpackaged version of the extension. Locate a + * series of update scripts that will do it. + */ check_valid_version_name(oldVersionName); if (strcmp(oldVersionName, versionName) == 0) @@ -1308,8 +1345,48 @@ CreateExtensionInternal(ParseState *pstate, CreateExtensionStmt *stmt, List *par } else { + /* + * No FROM, so we're installing from scratch. If there is an install + * script for the desired version, we only need to run that one. + */ + char *filename; + struct stat fst; + oldVersionName = NULL; - updateVersions = NIL; + + filename = get_extension_script_filename(pcontrol, NULL, versionName); + if (stat(filename, &fst) == 0) + { + /* Easy, no extra scripts */ + updateVersions = NIL; + } + else + { + /* Look for best way to install this version */ + List *evi_list; + ExtensionVersionInfo *evi_start; + ExtensionVersionInfo *evi_target; + + /* Extract the version update graph from the script directory */ + evi_list = get_ext_ver_list(pcontrol); + + /* Identify the target version */ + evi_target = get_ext_ver_info(versionName, &evi_list); + + /* Identify best path to reach target */ + evi_start = find_install_path(evi_list, evi_target, + &updateVersions); + + /* Fail if no path ... */ + if (evi_start == NULL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("extension \"%s\" has no installation script nor update path for version \"%s\"", + pcontrol->name, versionName))); + + /* Otherwise, install best starting point and then upgrade */ + versionName = evi_start->name; + } } /* @@ -1320,13 +1397,8 @@ CreateExtensionInternal(ParseState *pstate, CreateExtensionStmt *stmt, List *par /* * Determine the target schema to install the extension into */ - if (d_schema && d_schema->arg) + if (schemaName) { - /* - * User given schema, CREATE EXTENSION ... WITH SCHEMA ... - */ - schemaName = strVal(d_schema->arg); - /* If the user is giving us the schema name, it must exist already. */ schemaOid = get_namespace_oid(schemaName, false); } @@ -1374,7 +1446,7 @@ CreateExtensionInternal(ParseState *pstate, CreateExtensionStmt *stmt, List *par else if (!OidIsValid(schemaOid)) { /* - * Neither user nor author of the extension specified schema, use the + * Neither user nor author of the extension specified schema; use the * current default creation namespace, which is the first explicit * entry in the search_path. */ @@ -1415,66 +1487,12 @@ CreateExtensionInternal(ParseState *pstate, CreateExtensionStmt *stmt, List *par Oid reqext; Oid reqschema; - reqext = get_extension_oid(curreq, true); - if (!OidIsValid(reqext)) - { - if (cascade) - { - /* Must install it. */ - CreateExtensionStmt *ces; - ListCell *lc2; - ObjectAddress addr; - List *cascade_parents; - - /* Check extension name validity before trying to cascade. */ - check_valid_extension_name(curreq); - - /* Check for cyclic dependency between extensions. */ - foreach(lc2, parents) - { - char *pname = (char *) lfirst(lc2); - - if (strcmp(pname, curreq) == 0) - ereport(ERROR, - (errcode(ERRCODE_INVALID_RECURSION), - errmsg("cyclic dependency detected between extensions \"%s\" and \"%s\"", - curreq, stmt->extname))); - } - - ereport(NOTICE, - (errmsg("installing required extension \"%s\"", - curreq))); - - /* Build a CREATE EXTENSION statement to pass down. */ - ces = makeNode(CreateExtensionStmt); - ces->extname = curreq; - ces->if_not_exists = false; - - /* Propagate the CASCADE option. */ - ces->options = list_make1(d_cascade); - - /* Propagate the SCHEMA option if given. */ - if (d_schema && d_schema->arg) - ces->options = lappend(ces->options, d_schema); - - /* Add current extension to list of parents to pass down. */ - cascade_parents = - lappend(list_copy(parents), stmt->extname); - - /* Create the required extension. */ - addr = CreateExtensionInternal(pstate, ces, cascade_parents); - - /* Get its newly-assigned OID. */ - reqext = addr.objectId; - } - else - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_OBJECT), - errmsg("required extension \"%s\" is not installed", - curreq), - errhint("Use CREATE EXTENSION ... CASCADE to install required extensions too."))); - } - + reqext = get_required_extension(curreq, + extensionName, + origSchemaName, + cascade, + parents, + is_create); reqschema = get_extension_schema(reqext); requiredExtensions = lappend_oid(requiredExtensions, reqext); requiredSchemas = lappend_oid(requiredSchemas, reqschema); @@ -1510,17 +1528,100 @@ CreateExtensionInternal(ParseState *pstate, CreateExtensionStmt *stmt, List *par * though a series of ALTER EXTENSION UPDATE commands were given */ ApplyExtensionUpdates(extensionOid, pcontrol, - versionName, updateVersions); + versionName, updateVersions, + origSchemaName, cascade, is_create); return address; } +/* + * Get the OID of an extension listed in "requires", possibly creating it. + */ +static Oid +get_required_extension(char *reqExtensionName, + char *extensionName, + char *origSchemaName, + bool cascade, + List *parents, + bool is_create) +{ + Oid reqExtensionOid; + + reqExtensionOid = get_extension_oid(reqExtensionName, true); + if (!OidIsValid(reqExtensionOid)) + { + if (cascade) + { + /* Must install it. */ + ObjectAddress addr; + List *cascade_parents; + ListCell *lc; + + /* Check extension name validity before trying to cascade. */ + check_valid_extension_name(reqExtensionName); + + /* Check for cyclic dependency between extensions. */ + foreach(lc, parents) + { + char *pname = (char *) lfirst(lc); + + if (strcmp(pname, reqExtensionName) == 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_RECURSION), + errmsg("cyclic dependency detected between extensions \"%s\" and \"%s\"", + reqExtensionName, extensionName))); + } + + ereport(NOTICE, + (errmsg("installing required extension \"%s\"", + reqExtensionName))); + + /* Add current extension to list of parents to pass down. */ + cascade_parents = lappend(list_copy(parents), extensionName); + + /* + * Create the required extension. We propagate the SCHEMA option + * if any, and CASCADE, but no other options. + */ + addr = CreateExtensionInternal(reqExtensionName, + origSchemaName, + NULL, + NULL, + cascade, + cascade_parents, + is_create); + + /* Get its newly-assigned OID. */ + reqExtensionOid = addr.objectId; + } + else + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("required extension \"%s\" is not installed", + reqExtensionName), + is_create ? + errhint("Use CREATE EXTENSION ... CASCADE to install required extensions too.") : 0)); + } + + return reqExtensionOid; +} + /* * CREATE EXTENSION */ ObjectAddress CreateExtension(ParseState *pstate, CreateExtensionStmt *stmt) { + DefElem *d_schema = NULL; + DefElem *d_new_version = NULL; + DefElem *d_old_version = NULL; + DefElem *d_cascade = NULL; + char *schemaName = NULL; + char *versionName = NULL; + char *oldVersionName = NULL; + bool cascade = false; + ListCell *lc; + /* Check extension name validity before any filesystem access */ check_valid_extension_name(stmt->extname); @@ -1556,8 +1657,63 @@ CreateExtension(ParseState *pstate, CreateExtensionStmt *stmt) (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("nested CREATE EXTENSION is not supported"))); - /* Finally create the extension. */ - return CreateExtensionInternal(pstate, stmt, NIL); + /* Deconstruct the statement option list */ + foreach(lc, stmt->options) + { + DefElem *defel = (DefElem *) lfirst(lc); + + if (strcmp(defel->defname, "schema") == 0) + { + if (d_schema) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); + d_schema = defel; + schemaName = defGetString(d_schema); + } + else if (strcmp(defel->defname, "new_version") == 0) + { + if (d_new_version) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); + d_new_version = defel; + versionName = defGetString(d_new_version); + } + else if (strcmp(defel->defname, "old_version") == 0) + { + if (d_old_version) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); + d_old_version = defel; + oldVersionName = defGetString(d_old_version); + } + else if (strcmp(defel->defname, "cascade") == 0) + { + if (d_cascade) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); + d_cascade = defel; + cascade = defGetBoolean(d_cascade); + } + else + elog(ERROR, "unrecognized option: %s", defel->defname); + } + + /* Call CreateExtensionInternal to do the real work. */ + return CreateExtensionInternal(stmt->extname, + schemaName, + versionName, + oldVersionName, + cascade, + NIL, + true); } /* @@ -1914,43 +2070,28 @@ get_available_versions_for_extension(ExtensionControlFile *pcontrol, Tuplestorestate *tupstore, TupleDesc tupdesc) { - int extnamelen = strlen(pcontrol->name); - char *location; - DIR *dir; - struct dirent *de; + List *evi_list; + ListCell *lc; - location = get_extension_script_directory(pcontrol); - dir = AllocateDir(location); - /* Note this will fail if script directory doesn't exist */ - while ((de = ReadDir(dir, location)) != NULL) + /* Extract the version update graph from the script directory */ + evi_list = get_ext_ver_list(pcontrol); + + /* For each installable version ... */ + foreach(lc, evi_list) { + ExtensionVersionInfo *evi = (ExtensionVersionInfo *) lfirst(lc); ExtensionControlFile *control; - char *vername; Datum values[7]; bool nulls[7]; + ListCell *lc2; - /* must be a .sql file ... */ - if (!is_extension_script_filename(de->d_name)) - continue; - - /* ... matching extension name followed by separator */ - if (strncmp(de->d_name, pcontrol->name, extnamelen) != 0 || - de->d_name[extnamelen] != '-' || - de->d_name[extnamelen + 1] != '-') - continue; - - /* extract version name from 'extname--something.sql' filename */ - vername = pstrdup(de->d_name + extnamelen + 2); - *strrchr(vername, '.') = '\0'; - - /* ignore it if it's an update script */ - if (strstr(vername, "--")) + if (!evi->installable) continue; /* * Fetch parameters for specific version (pcontrol is not changed) */ - control = read_extension_aux_control_file(pcontrol, vername); + control = read_extension_aux_control_file(pcontrol, evi->name); memset(values, 0, sizeof(values)); memset(nulls, 0, sizeof(nulls)); @@ -1959,7 +2100,7 @@ get_available_versions_for_extension(ExtensionControlFile *pcontrol, values[0] = DirectFunctionCall1(namein, CStringGetDatum(control->name)); /* version */ - values[1] = CStringGetTextDatum(vername); + values[1] = CStringGetTextDatum(evi->name); /* superuser */ values[2] = BoolGetDatum(control->superuser); /* relocatable */ @@ -1974,27 +2115,7 @@ get_available_versions_for_extension(ExtensionControlFile *pcontrol, if (control->requires == NIL) nulls[5] = true; else - { - Datum *datums; - int ndatums; - ArrayType *a; - ListCell *lc; - - ndatums = list_length(control->requires); - datums = (Datum *) palloc(ndatums * sizeof(Datum)); - ndatums = 0; - foreach(lc, control->requires) - { - char *curreq = (char *) lfirst(lc); - - datums[ndatums++] = - DirectFunctionCall1(namein, CStringGetDatum(curreq)); - } - a = construct_array(datums, ndatums, - NAMEOID, - NAMEDATALEN, false, 'c'); - values[5] = PointerGetDatum(a); - } + values[5] = convert_requires_to_datum(control->requires); /* comment */ if (control->comment == NULL) nulls[6] = true; @@ -2002,9 +2123,75 @@ get_available_versions_for_extension(ExtensionControlFile *pcontrol, values[6] = CStringGetTextDatum(control->comment); tuplestore_putvalues(tupstore, tupdesc, values, nulls); + + /* + * Find all non-directly-installable versions that would be installed + * starting from this version, and report them, inheriting the + * parameters that aren't changed in updates from this version. + */ + foreach(lc2, evi_list) + { + ExtensionVersionInfo *evi2 = (ExtensionVersionInfo *) lfirst(lc2); + List *best_path; + + if (evi2->installable) + continue; + if (find_install_path(evi_list, evi2, &best_path) == evi) + { + /* + * Fetch parameters for this version (pcontrol is not changed) + */ + control = read_extension_aux_control_file(pcontrol, evi2->name); + + /* name stays the same */ + /* version */ + values[1] = CStringGetTextDatum(evi2->name); + /* superuser */ + values[2] = BoolGetDatum(control->superuser); + /* relocatable */ + values[3] = BoolGetDatum(control->relocatable); + /* schema stays the same */ + /* requires */ + if (control->requires == NIL) + nulls[5] = true; + else + { + values[5] = convert_requires_to_datum(control->requires); + nulls[5] = false; + } + /* comment stays the same */ + + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + } + } } +} - FreeDir(dir); +/* + * Convert a list of extension names to a name[] Datum + */ +static Datum +convert_requires_to_datum(List *requires) +{ + Datum *datums; + int ndatums; + ArrayType *a; + ListCell *lc; + + ndatums = list_length(requires); + datums = (Datum *) palloc(ndatums * sizeof(Datum)); + ndatums = 0; + foreach(lc, requires) + { + char *curreq = (char *) lfirst(lc); + + datums[ndatums++] = + DirectFunctionCall1(namein, CStringGetDatum(curreq)); + } + a = construct_array(datums, ndatums, + NAMEOID, + NAMEDATALEN, false, 'c'); + return PointerGetDatum(a); } /* @@ -2076,7 +2263,7 @@ pg_extension_update_paths(PG_FUNCTION_ARGS) continue; /* Find shortest path from evi1 to evi2 */ - path = find_update_path(evi_list, evi1, evi2, true); + path = find_update_path(evi_list, evi1, evi2, false, true); /* Emit result row */ memset(values, 0, sizeof(values)); @@ -2808,7 +2995,8 @@ ExecAlterExtensionStmt(ParseState *pstate, AlterExtensionStmt *stmt) * time */ ApplyExtensionUpdates(extensionOid, control, - oldVersionName, updateVersions); + oldVersionName, updateVersions, + NULL, false, false); ObjectAddressSet(address, ExtensionRelationId, extensionOid); @@ -2827,7 +3015,10 @@ static void ApplyExtensionUpdates(Oid extensionOid, ExtensionControlFile *pcontrol, const char *initialVersion, - List *updateVersions) + List *updateVersions, + char *origSchemaName, + bool cascade, + bool is_create) { const char *oldVersionName = initialVersion; ListCell *lcv; @@ -2906,8 +3097,9 @@ ApplyExtensionUpdates(Oid extensionOid, heap_close(extRel, RowExclusiveLock); /* - * Look up the prerequisite extensions for this version, and build - * lists of their OIDs and the OIDs of their target schemas. + * Look up the prerequisite extensions for this version, install them + * if necessary, and build lists of their OIDs and the OIDs of their + * target schemas. */ requiredExtensions = NIL; requiredSchemas = NIL; @@ -2917,16 +3109,12 @@ ApplyExtensionUpdates(Oid extensionOid, Oid reqext; Oid reqschema; - /* - * We intentionally don't use get_extension_oid's default error - * message here, because it would be confusing in this context. - */ - reqext = get_extension_oid(curreq, true); - if (!OidIsValid(reqext)) - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_OBJECT), - errmsg("required extension \"%s\" is not installed", - curreq))); + reqext = get_required_extension(curreq, + control->name, + origSchemaName, + cascade, + NIL, + is_create); reqschema = get_extension_schema(reqext); requiredExtensions = lappend_oid(requiredExtensions, reqext); requiredSchemas = lappend_oid(requiredSchemas, reqschema); -- cgit v1.2.3 From c3c0d7bd701dae4737c974a59ffa9b366110f9c1 Mon Sep 17 00:00:00 2001 From: Simon Riggs Date: Sun, 11 Sep 2016 23:26:18 +0100 Subject: Raise max setting of checkpoint_timeout to 1d Previously checkpoint_timeout was capped at 3600s New max setting is 86400s = 24h = 1d Discussion: 32558.1454471895@sss.pgh.pa.us --- doc/src/sgml/config.sgml | 2 +- src/backend/utils/misc/guc.c | 2 +- src/backend/utils/misc/postgresql.conf.sample | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7c483c6ef3..cd66abc8ba 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -2614,7 +2614,7 @@ include_dir 'conf.d' Maximum time between automatic WAL checkpoints, in seconds. - The valid range is between 30 seconds and one hour. + The valid range is between 30 seconds and one day. The default is five minutes (5min). Increasing this parameter can increase the amount of time needed for crash recovery. diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index c5178f7cad..c72bd6190a 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -2250,7 +2250,7 @@ static struct config_int ConfigureNamesInt[] = GUC_UNIT_S }, &CheckPointTimeout, - 300, 30, 3600, + 300, 30, 86400, NULL, NULL, NULL }, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 6d0666c44f..b1c3aea9ee 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -203,7 +203,7 @@ # - Checkpoints - -#checkpoint_timeout = 5min # range 30s-1h +#checkpoint_timeout = 5min # range 30s-1d #max_wal_size = 1GB #min_wal_size = 80MB #checkpoint_completion_target = 0.5 # checkpoint target duration, 0.0 - 1.0 -- cgit v1.2.3 From 9083353b15c3cf8e7bbac104a81ad42281178cdf Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Mon, 12 Sep 2016 12:00:00 -0400 Subject: pg_basebackup: Clean created directories on failure Like initdb, clean up created data and xlog directories, unless the new -n/--noclean option is specified. Tablespace directories are not cleaned up, but a message is written about that. Reviewed-by: Masahiko Sawada --- doc/src/sgml/ref/pg_basebackup.sgml | 18 +++++ src/bin/pg_basebackup/pg_basebackup.c | 98 ++++++++++++++++++++++++++-- src/bin/pg_basebackup/t/010_pg_basebackup.pl | 10 ++- 3 files changed, 119 insertions(+), 7 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml index 03615da480..9f1eae12d8 100644 --- a/doc/src/sgml/ref/pg_basebackup.sgml +++ b/doc/src/sgml/ref/pg_basebackup.sgml @@ -398,6 +398,24 @@ PostgreSQL documentation + + + + + + By default, when pg_basebackup aborts with an + error, it removes any directories it might have created before + discovering that it cannot finish the job (for example, data directory + and transaction log directory). This option inhibits tidying-up and is + thus useful for debugging. + + + + Note that tablespace directories are not cleaned up either way. + + + + diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index 351a42068f..42f3b273a6 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -58,6 +58,7 @@ static TablespaceList tablespace_dirs = {NULL, NULL}; static char *xlog_dir = ""; static char format = 'p'; /* p(lain)/t(ar) */ static char *label = "pg_basebackup base backup"; +static bool noclean = false; static bool showprogress = false; static int verbose = 0; static int compresslevel = 0; @@ -69,6 +70,13 @@ static int standby_message_timeout = 10 * 1000; /* 10 sec = default */ static pg_time_t last_progress_report = 0; static int32 maxrate = 0; /* no limit by default */ +static bool success = false; +static bool made_new_pgdata = false; +static bool found_existing_pgdata = false; +static bool made_new_xlogdir = false; +static bool found_existing_xlogdir = false; +static bool made_tablespace_dirs = false; +static bool found_tablespace_dirs = false; /* Progress counters */ static uint64 totalsize; @@ -82,6 +90,7 @@ static int bgpipe[2] = {-1, -1}; /* Handle to child process */ static pid_t bgchild = -1; +static bool in_log_streamer = false; /* End position for xlog streaming, empty string if unknown yet */ static XLogRecPtr xlogendptr; @@ -98,7 +107,7 @@ static PQExpBuffer recoveryconfcontents = NULL; /* Function headers */ static void usage(void); static void disconnect_and_exit(int code); -static void verify_dir_is_empty_or_create(char *dirname); +static void verify_dir_is_empty_or_create(char *dirname, bool *created, bool *found); static void progress_report(int tablespacenum, const char *filename, bool force); static void ReceiveTarFile(PGconn *conn, PGresult *res, int rownum); @@ -114,6 +123,69 @@ static const char *get_tablespace_mapping(const char *dir); static void tablespace_list_append(const char *arg); +static void +cleanup_directories_atexit(void) +{ + if (success || in_log_streamer) + return; + + if (!noclean) + { + if (made_new_pgdata) + { + fprintf(stderr, _("%s: removing data directory \"%s\"\n"), + progname, basedir); + if (!rmtree(basedir, true)) + fprintf(stderr, _("%s: failed to remove data directory\n"), + progname); + } + else if (found_existing_pgdata) + { + fprintf(stderr, + _("%s: removing contents of data directory \"%s\"\n"), + progname, basedir); + if (!rmtree(basedir, false)) + fprintf(stderr, _("%s: failed to remove contents of data directory\n"), + progname); + } + + if (made_new_xlogdir) + { + fprintf(stderr, _("%s: removing transaction log directory \"%s\"\n"), + progname, xlog_dir); + if (!rmtree(xlog_dir, true)) + fprintf(stderr, _("%s: failed to remove transaction log directory\n"), + progname); + } + else if (found_existing_xlogdir) + { + fprintf(stderr, + _("%s: removing contents of transaction log directory \"%s\"\n"), + progname, xlog_dir); + if (!rmtree(xlog_dir, false)) + fprintf(stderr, _("%s: failed to remove contents of transaction log directory\n"), + progname); + } + } + else + { + if (made_new_pgdata || found_existing_pgdata) + fprintf(stderr, + _("%s: data directory \"%s\" not removed at user's request\n"), + progname, basedir); + + if (made_new_xlogdir || found_existing_xlogdir) + fprintf(stderr, + _("%s: transaction log directory \"%s\" not removed at user's request\n"), + progname, xlog_dir); + } + + if (made_tablespace_dirs || found_tablespace_dirs) + fprintf(stderr, + _("%s: changes to tablespace directories will not be undone"), + progname); +} + static void disconnect_and_exit(int code) { @@ -253,6 +325,7 @@ usage(void) printf(_(" -c, --checkpoint=fast|spread\n" " set fast or spread checkpointing\n")); printf(_(" -l, --label=LABEL set backup label\n")); + printf(_(" -n, --noclean do not clean up after errors\n")); printf(_(" -P, --progress show progress information\n")); printf(_(" -v, --verbose output verbose messages\n")); printf(_(" -V, --version output version information, then exit\n")); @@ -375,6 +448,8 @@ LogStreamerMain(logstreamer_param *param) { StreamCtl stream; + in_log_streamer = true; + MemSet(&stream, 0, sizeof(stream)); stream.startpos = param->startptr; stream.timeline = param->timeline; @@ -501,7 +576,7 @@ StartLogStreamer(char *startpos, uint32 timeline, char *sysidentifier) * be give and the process ended. */ static void -verify_dir_is_empty_or_create(char *dirname) +verify_dir_is_empty_or_create(char *dirname, bool *created, bool *found) { switch (pg_check_dir(dirname)) { @@ -517,12 +592,16 @@ verify_dir_is_empty_or_create(char *dirname) progname, dirname, strerror(errno)); disconnect_and_exit(1); } + if (created) + *created = true; return; case 1: /* * Exists, empty */ + if (found) + *found = true; return; case 2: case 3: @@ -1683,7 +1762,7 @@ BaseBackup(void) { char *path = (char *) get_tablespace_mapping(PQgetvalue(res, i, 1)); - verify_dir_is_empty_or_create(path); + verify_dir_is_empty_or_create(path, &made_tablespace_dirs, &found_tablespace_dirs); } } @@ -1892,6 +1971,7 @@ main(int argc, char **argv) {"gzip", no_argument, NULL, 'z'}, {"compress", required_argument, NULL, 'Z'}, {"label", required_argument, NULL, 'l'}, + {"noclean", no_argument, NULL, 'n'}, {"dbname", required_argument, NULL, 'd'}, {"host", required_argument, NULL, 'h'}, {"port", required_argument, NULL, 'p'}, @@ -1926,7 +2006,9 @@ main(int argc, char **argv) } } - while ((c = getopt_long(argc, argv, "D:F:r:RT:xX:l:zZ:d:c:h:p:U:s:S:wWvP", + atexit(cleanup_directories_atexit); + + while ((c = getopt_long(argc, argv, "D:F:r:RT:xX:l:nzZ:d:c:h:p:U:s:S:wWvP", long_options, &option_index)) != -1) { switch (c) @@ -2001,6 +2083,9 @@ main(int argc, char **argv) case 'l': label = pg_strdup(optarg); break; + case 'n': + noclean = true; + break; case 'z': #ifdef HAVE_LIBZ compresslevel = Z_DEFAULT_COMPRESSION; @@ -2170,14 +2255,14 @@ main(int argc, char **argv) * unless we are writing to stdout. */ if (format == 'p' || strcmp(basedir, "-") != 0) - verify_dir_is_empty_or_create(basedir); + verify_dir_is_empty_or_create(basedir, &made_new_pgdata, &found_existing_pgdata); /* Create transaction log symlink, if required */ if (strcmp(xlog_dir, "") != 0) { char *linkloc; - verify_dir_is_empty_or_create(xlog_dir); + verify_dir_is_empty_or_create(xlog_dir, &made_new_xlogdir, &found_existing_xlogdir); /* form name of the place where the symlink must go */ linkloc = psprintf("%s/pg_xlog", basedir); @@ -2198,5 +2283,6 @@ main(int argc, char **argv) BaseBackup(); + success = true; return 0; } diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/010_pg_basebackup.pl index 6c33936d25..fd9857d67b 100644 --- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl +++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl @@ -4,7 +4,7 @@ use Cwd; use Config; use PostgresNode; use TestLib; -use Test::More tests => 51; +use Test::More tests => 54; program_help_ok('pg_basebackup'); program_version_ok('pg_basebackup'); @@ -40,6 +40,14 @@ $node->command_fails( [ 'pg_basebackup', '-D', "$tempdir/backup" ], 'pg_basebackup fails because of WAL configuration'); +ok(! -d "$tempdir/backup", 'backup directory was cleaned up'); + +$node->command_fails( + [ 'pg_basebackup', '-D', "$tempdir/backup", '-n' ], + 'failing run with noclean option'); + +ok(-d "$tempdir/backup", 'backup directory was created and left behind'); + open CONF, ">>$pgdata/postgresql.conf"; print CONF "max_replication_slots = 10\n"; print CONF "max_wal_senders = 10\n"; -- cgit v1.2.3 From 42fd984c0b7b53d1bc961c9ed6bb84fe28eae52b Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 12 Sep 2016 19:19:24 -0400 Subject: Docs: assorted minor cleanups. Standardize on "user_name" for a field name in related examples in ddl.sgml; before we had variously "user_name", "username", and "user". The last is flat wrong because it conflicts with a reserved word. Be consistent about entry capitalization in a table in func.sgml. Fix a typo in pgtrgm.sgml. Back-patch to 9.6 and 9.5 as relevant. Alexander Law --- doc/src/sgml/ddl.sgml | 44 ++++++++++++++++++++++---------------------- doc/src/sgml/func.sgml | 2 +- doc/src/sgml/pgtrgm.sgml | 2 +- 3 files changed, 24 insertions(+), 24 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ddl.sgml b/doc/src/sgml/ddl.sgml index a393813b38..f43352c2a9 100644 --- a/doc/src/sgml/ddl.sgml +++ b/doc/src/sgml/ddl.sgml @@ -1629,7 +1629,7 @@ CREATE POLICY account_managers ON accounts TO managers CREATE POLICY user_policy ON users - USING (user = current_user); + USING (user_name = current_user); @@ -1642,7 +1642,7 @@ CREATE POLICY user_policy ON users CREATE POLICY user_policy ON users USING (true) - WITH CHECK (user = current_user); + WITH CHECK (user_name = current_user); @@ -1662,7 +1662,7 @@ CREATE POLICY user_policy ON users -- Simple passwd-file based example CREATE TABLE passwd ( - username text UNIQUE NOT NULL, + user_name text UNIQUE NOT NULL, pwhash text, uid int PRIMARY KEY, gid int NOT NULL, @@ -1696,9 +1696,9 @@ CREATE POLICY all_view ON passwd FOR SELECT USING (true); -- Normal users can update their own records, but -- limit which shells a normal user is allowed to set CREATE POLICY user_mod ON passwd FOR UPDATE - USING (current_user = username) + USING (current_user = user_name) WITH CHECK ( - current_user = username AND + current_user = user_name AND shell IN ('/bin/bash','/bin/sh','/bin/dash','/bin/zsh','/bin/tcsh') ); @@ -1706,7 +1706,7 @@ CREATE POLICY user_mod ON passwd FOR UPDATE GRANT SELECT, INSERT, UPDATE, DELETE ON passwd TO admin; -- Users only get select access on public columns GRANT SELECT - (username, uid, gid, real_name, home_phone, extra_info, home_dir, shell) + (user_name, uid, gid, real_name, home_phone, extra_info, home_dir, shell) ON passwd TO public; -- Allow users to update certain columns GRANT UPDATE @@ -1725,11 +1725,11 @@ GRANT UPDATE postgres=> set role admin; SET postgres=> table passwd; - username | pwhash | uid | gid | real_name | home_phone | extra_info | home_dir | shell -----------+--------+-----+-----+-----------+--------------+------------+-------------+----------- - admin | xxx | 0 | 0 | Admin | 111-222-3333 | | /root | /bin/dash - bob | xxx | 1 | 1 | Bob | 123-456-7890 | | /home/bob | /bin/zsh - alice | xxx | 2 | 1 | Alice | 098-765-4321 | | /home/alice | /bin/zsh + user_name | pwhash | uid | gid | real_name | home_phone | extra_info | home_dir | shell +-----------+--------+-----+-----+-----------+--------------+------------+-------------+----------- + admin | xxx | 0 | 0 | Admin | 111-222-3333 | | /root | /bin/dash + bob | xxx | 1 | 1 | Bob | 123-456-7890 | | /home/bob | /bin/zsh + alice | xxx | 2 | 1 | Alice | 098-765-4321 | | /home/alice | /bin/zsh (3 rows) -- Test what Alice is able to do @@ -1737,26 +1737,26 @@ postgres=> set role alice; SET postgres=> table passwd; ERROR: permission denied for relation passwd -postgres=> select username,real_name,home_phone,extra_info,home_dir,shell from passwd; - username | real_name | home_phone | extra_info | home_dir | shell -----------+-----------+--------------+------------+-------------+----------- - admin | Admin | 111-222-3333 | | /root | /bin/dash - bob | Bob | 123-456-7890 | | /home/bob | /bin/zsh - alice | Alice | 098-765-4321 | | /home/alice | /bin/zsh +postgres=> select user_name,real_name,home_phone,extra_info,home_dir,shell from passwd; + user_name | real_name | home_phone | extra_info | home_dir | shell +-----------+-----------+--------------+------------+-------------+----------- + admin | Admin | 111-222-3333 | | /root | /bin/dash + bob | Bob | 123-456-7890 | | /home/bob | /bin/zsh + alice | Alice | 098-765-4321 | | /home/alice | /bin/zsh (3 rows) -postgres=> update passwd set username = 'joe'; +postgres=> update passwd set user_name = 'joe'; ERROR: permission denied for relation passwd -- Alice is allowed to change her own real_name, but no others postgres=> update passwd set real_name = 'Alice Doe'; UPDATE 1 -postgres=> update passwd set real_name = 'John Doe' where username = 'admin'; +postgres=> update passwd set real_name = 'John Doe' where user_name = 'admin'; UPDATE 0 postgres=> update passwd set shell = '/bin/xx'; ERROR: new row violates WITH CHECK OPTION for "passwd" postgres=> delete from passwd; ERROR: permission denied for relation passwd -postgres=> insert into passwd (username) values ('xxx'); +postgres=> insert into passwd (user_name) values ('xxx'); ERROR: permission denied for relation passwd -- Alice can change her own password; RLS silently prevents updating other rows postgres=> update passwd set pwhash = 'abc'; @@ -2055,7 +2055,7 @@ DROP SCHEMA myschema CASCADE; (since this is one of the ways to restrict the activities of your users to well-defined namespaces). The syntax for that is: -CREATE SCHEMA schemaname AUTHORIZATION username; +CREATE SCHEMA schema_name AUTHORIZATION user_name; You can even omit the schema name, in which case the schema name will be the same as the user name. See username.tablename. + user_name.table_name. This is how PostgreSQL will effectively behave if you create a per-user schema for every user. diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 5148095fb3..47fcb30da0 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -9523,7 +9523,7 @@ CREATE TYPE rainbow AS ENUM ('red', 'orange', 'yellow', 'green', 'blue', 'purple ts_filter(vector tsvector, weights "char"[]) tsvector - Select only elements with given weights from vector + select only elements with given weights from vector ts_filter('fat:2,4 cat:3b rat:5A'::tsvector, '{a,b}') 'cat':3B 'rat':5A diff --git a/doc/src/sgml/pgtrgm.sgml b/doc/src/sgml/pgtrgm.sgml index d362b03cf3..775a7b8be7 100644 --- a/doc/src/sgml/pgtrgm.sgml +++ b/doc/src/sgml/pgtrgm.sgml @@ -104,7 +104,7 @@ the second string a most similar word not a most similar substring. The range of the result is zero (indicating that the two strings are completely dissimilar) to one (indicating that the first string is - identical to one of the word of the second string). + identical to one of the words of the second string). -- cgit v1.2.3 From 5225c66336a1e4b46925e9f169086fc70f49736f Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Thu, 15 Sep 2016 17:24:54 -0400 Subject: Clarify policy on marking inherited constraints as valid. Amit Langote and Robert Haas --- doc/src/sgml/ref/alter_table.sgml | 14 +++++++++----- src/backend/commands/tablecmds.c | 3 ++- 2 files changed, 11 insertions(+), 6 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/alter_table.sgml b/doc/src/sgml/ref/alter_table.sgml index 6f51cbc896..e48ccf21e4 100644 --- a/doc/src/sgml/ref/alter_table.sgml +++ b/doc/src/sgml/ref/alter_table.sgml @@ -1028,11 +1028,15 @@ ALTER TABLE ALL IN TABLESPACE name If a table has any descendant tables, it is not permitted to add, - rename, or change the type of a column, or rename an inherited constraint - in the parent table without doing - the same to the descendants. That is, ALTER TABLE ONLY - will be rejected. This ensures that the descendants always have - columns matching the parent. + rename, or change the type of a column in the parent table without doing + same to the descendants. This ensures that the descendants always have + columns matching the parent. Similarly, a constraint cannot be renamed + in the parent without also renaming it in all descendents, so that + constraints also match between the parent and its descendents. + Also, because selecting from the parent also selects from its descendents, + a constraint on the parent cannot be marked valid unless it is also marked + valid for those descendents. In all of these cases, ALTER TABLE + ONLY will be rejected. diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 86e98148c1..d31276284c 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -6908,7 +6908,8 @@ ATExecValidateConstraint(Relation rel, char *constrName, bool recurse, /* * If we are told not to recurse, there had better not be any - * child tables; else the addition would put them out of step. + * child tables, because we can't mark the constraint on the + * parent valid unless it is valid for all child tables. */ if (!recurse) ereport(ERROR, -- cgit v1.2.3 From 2c8f0d6e53e5dbcf28ee127303b81a6e12942665 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Mon, 19 Sep 2016 13:38:21 -0400 Subject: Update recovery_min_apply_delay docs for remote_apply mode. Bernd Helmle, reviewed by Thomas Munro, tweaked by me. --- doc/src/sgml/recovery-config.sgml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/recovery-config.sgml b/doc/src/sgml/recovery-config.sgml index de3fb10f5b..8c24ae2174 100644 --- a/doc/src/sgml/recovery-config.sgml +++ b/doc/src/sgml/recovery-config.sgml @@ -487,10 +487,17 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows This parameter is intended for use with streaming replication deployments; however, if the parameter is specified it will be honored in all cases. - Synchronous replication is not affected by this setting because there is - not yet any setting to request synchronous apply of transaction commits. + hot_standby_feedback will be delayed by use of this feature which could lead to bloat on the master; use both together with care. + + + + Synchronous replication is affected by this setting when synchronous_commit + is set to remote_apply; every COMMIT + will need to wait to be applied. + + -- cgit v1.2.3 From 6cc54f38a9fe1f4103c45a9858804d1d5d4de0fd Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 19 Sep 2016 21:56:16 +0300 Subject: Remove obsolete warning from docs. Python 2.4 and Fedora 4 are both obsolete at this point, especially unpatched debug builds. Discussion: <85e377b2-d459-396e-59b1-115548bbc059@iki.fi> --- doc/src/sgml/plpython.sgml | 13 ------------- 1 file changed, 13 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/plpython.sgml b/doc/src/sgml/plpython.sgml index 905e757ab6..bb69c752b8 100644 --- a/doc/src/sgml/plpython.sgml +++ b/doc/src/sgml/plpython.sgml @@ -696,19 +696,6 @@ AS $$ $$ LANGUAGE plpythonu; - - - Due to Python - bug #1483133, - some debug versions of Python 2.4 - (configured and compiled with option --with-pydebug) - are known to crash the PostgreSQL server - when using an iterator to return a set result. - Unpatched versions of Fedora 4 contain this bug. - It does not happen in production versions of Python or on patched - versions of Fedora 4. - - -- cgit v1.2.3 From 4f6494cfd26c1dfe708c4598c11eea5fce168fd1 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 20 Sep 2016 12:00:00 -0400 Subject: doc: Correct ALTER USER MAPPING example The existing example threw an error. From: gabrielle --- doc/src/sgml/ref/alter_user_mapping.sgml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/alter_user_mapping.sgml b/doc/src/sgml/ref/alter_user_mapping.sgml index 3a908130d8..3be54afee5 100644 --- a/doc/src/sgml/ref/alter_user_mapping.sgml +++ b/doc/src/sgml/ref/alter_user_mapping.sgml @@ -89,9 +89,9 @@ ALTER USER MAPPING FOR { user_name Examples - Change the password for user mapping bob, server foo: + Change the password for user mapping bob, server foo: -ALTER USER MAPPING FOR bob SERVER foo OPTIONS (user 'bob', password 'public'); +ALTER USER MAPPING FOR bob SERVER foo OPTIONS (SET password 'public'); -- cgit v1.2.3 From 16d1adb35cf887325b7c5dbf473632d557065171 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 20 Sep 2016 12:00:00 -0400 Subject: doc: Fix documentation to match actual make output based on patch from Takeshi Ideriha --- doc/src/sgml/installation.sgml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml index 14a6d57aea..f6de18ed2d 100644 --- a/doc/src/sgml/installation.sgml +++ b/doc/src/sgml/installation.sgml @@ -1510,7 +1510,7 @@ su - postgres will take a few minutes depending on your hardware. The last line displayed should be: -All of PostgreSQL is successfully made. Ready to install. +All of PostgreSQL successfully made. Ready to install. @@ -1523,7 +1523,7 @@ All of PostgreSQL is successfully made. Ready to install. The last line displayed should be: -PostgreSQL, contrib and HTML documentation successfully made. Ready to install. +PostgreSQL, contrib, and documentation successfully made. Ready to install. -- cgit v1.2.3 From 46b55e7f853dc0ef60ae3b1042b883fa4ffac95f Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 20 Sep 2016 12:00:00 -0400 Subject: pg_restore: Add -N option to exclude schemas This is similar to the -N option in pg_dump, except that it doesn't take a pattern, just like the existing -n option in pg_restore. From: Michael Banck --- doc/src/sgml/ref/pg_restore.sgml | 18 +++++++++++++++++- src/bin/pg_dump/pg_backup.h | 1 + src/bin/pg_dump/pg_backup_archiver.c | 5 +++++ src/bin/pg_dump/pg_restore.c | 8 +++++++- 4 files changed, 30 insertions(+), 2 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml index c9069193af..bd5b405314 100644 --- a/doc/src/sgml/ref/pg_restore.sgml +++ b/doc/src/sgml/ref/pg_restore.sgml @@ -302,7 +302,7 @@ - + @@ -314,6 +314,22 @@ + + + + + + Do not restore objects that are in the named schema. Multiple schemas + to be excluded may be specified with multiple + + + When both + + + diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 4afa92f5f6..0a28124cf6 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -99,6 +99,7 @@ typedef struct _restoreOptions SimpleStringList indexNames; SimpleStringList functionNames; SimpleStringList schemaNames; + SimpleStringList schemaExcludeNames; SimpleStringList triggerNames; SimpleStringList tableNames; diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 05bdbdbf02..a69b06f6d7 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -2751,6 +2751,11 @@ _tocEntryRequired(TocEntry *te, teSection curSection, RestoreOptions *ropt) return 0; } + if (ropt->schemaExcludeNames.head != NULL + && te->namespace + && simple_string_list_member(&ropt->schemaExcludeNames, te->namespace)) + return 0; + if (ropt->selTypes) { if (strcmp(te->desc, "TABLE") == 0 || diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c index fb08e6bb8e..b21fd263b0 100644 --- a/src/bin/pg_dump/pg_restore.c +++ b/src/bin/pg_dump/pg_restore.c @@ -85,6 +85,7 @@ main(int argc, char **argv) {"data-only", 0, NULL, 'a'}, {"dbname", 1, NULL, 'd'}, {"exit-on-error", 0, NULL, 'e'}, + {"exclude-schema", 1, NULL, 'N'}, {"file", 1, NULL, 'f'}, {"format", 1, NULL, 'F'}, {"function", 1, NULL, 'P'}, @@ -148,7 +149,7 @@ main(int argc, char **argv) } } - while ((c = getopt_long(argc, argv, "acCd:ef:F:h:I:j:lL:n:Op:P:RsS:t:T:U:vwWx1", + while ((c = getopt_long(argc, argv, "acCd:ef:F:h:I:j:lL:n:N:Op:P:RsS:t:T:U:vwWx1", cmdopts, NULL)) != -1) { switch (c) @@ -196,6 +197,10 @@ main(int argc, char **argv) simple_string_list_append(&opts->schemaNames, optarg); break; + case 'N': /* Do not dump data for this schema */ + simple_string_list_append(&opts->schemaExcludeNames, optarg); + break; + case 'O': opts->noOwner = 1; break; @@ -456,6 +461,7 @@ usage(const char *progname) printf(_(" -L, --use-list=FILENAME use table of contents from this file for\n" " selecting/ordering output\n")); printf(_(" -n, --schema=NAME restore only objects in this schema\n")); + printf(_(" -N, --exclude-schema=NAME do not restore objects in this schema\n")); printf(_(" -O, --no-owner skip restoration of object ownership\n")); printf(_(" -P, --function=NAME(args) restore named function\n")); printf(_(" -s, --schema-only restore only the schema, no data\n")); -- cgit v1.2.3 From 60270e5e00850ee8cc34296e38d0000415c8b152 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Wed, 21 Sep 2016 08:37:02 -0400 Subject: Add more parallel query documentation. Previously, the individual settings were documented, but there was no overall discussion of the capabilities and limitations of the feature. Add that. Patch by me, reviewed by Peter Eisentraut and Álvaro Herrera. --- doc/src/sgml/config.sgml | 5 + doc/src/sgml/filelist.sgml | 1 + doc/src/sgml/parallel.sgml | 472 +++++++++++++++++++++++++++++++++++++++++++++ doc/src/sgml/postgres.sgml | 1 + 4 files changed, 479 insertions(+) create mode 100644 doc/src/sgml/parallel.sgml (limited to 'doc/src') diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index cd66abc8ba..a848a7edd1 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -2027,6 +2027,11 @@ include_dir 'conf.d' as much CPU time, memory, I/O bandwidth, and so forth as a query which uses no workers at all. + + + For more information on parallel query, see + . + diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml index 43837114ba..69649a7da4 100644 --- a/doc/src/sgml/filelist.sgml +++ b/doc/src/sgml/filelist.sgml @@ -24,6 +24,7 @@ + diff --git a/doc/src/sgml/parallel.sgml b/doc/src/sgml/parallel.sgml new file mode 100644 index 0000000000..c80d42dbef --- /dev/null +++ b/doc/src/sgml/parallel.sgml @@ -0,0 +1,472 @@ + + + + Parallel Query + + + parallel query + + + + PostgreSQL can devise query plans which can leverage + multiple CPUs in order to answer queries faster. This feature is known + as parallel query. Many queries cannot benefit from parallel query, either + due to limitations of the current implementation or because there is no + imaginable query plan which is any faster than the serial query plan. + However, for queries that can benefit, the speedup from parallel query + is often very significant. Many queries can run more than twice as fast + when using parallel query, and some queries can run four times faster or + even more. Queries that touch a large amount of data but return only a + few rows to the user will typically benefit most. This chapter explains + some details of how parallel query works and in which situations it can be + used so that users who wish to make use of it can understand what to expect. + + + + How Parallel Query Works + + + When the optimizer determines that parallel query is the fastest execution + strategy for a particular query, it will create a query plan which includes + a Gather node. Here is a simple example: + + +EXPLAIN SELECT * FROM pgbench_accounts WHERE filler LIKE '%x%'; + QUERY PLAN +------------------------------------------------------------------------------------- + Gather (cost=1000.00..217018.43 rows=1 width=97) + Workers Planned: 2 + -> Parallel Seq Scan on pgbench_accounts (cost=0.00..216018.33 rows=1 width=97) + Filter: (filler ~~ '%x%'::text) +(4 rows) + + + + + In all cases, the Gather node will have exactly one + child plan, which is the portion of the plan that will be executed in + parallel. If the Gather node is at the very top of the plan + tree, then the entire query will execute in parallel. If it is somewhere + else in the plan tree, then only that portion of the query will run in + parallel. In the example above, the query accesses only one table, so + there is only one plan node other than the Gather node itself; + since that plan node is a child of the Gather node, it will + run in parallel. + + + + Using EXPLAIN, you can see the number of + workers chosen by the planner. When the Gather node is reached + during query execution, the process which is implementing the user's + session will request a number of background + worker processes equal to the number + of workers chosen by the planner. The total number of background + workers that can exist at any one time is limited by + , so it is possible for a + parallel query to run with fewer workers than planned, or even with + no workers at all. The optimal plan may depend on the number of workers + that are available, so this can result in poor query performance. If this + occurrence is frequent, considering increasing + max_worker_processes so that more workers can be run + simultaneously or alternatively reducing + so that the planner + requests fewer workers. + + + + Every background worker process which is successfully started for a given + parallel query will execute the portion of the plan which is a descendent + of the Gather node. The leader will also execute that portion + of the plan, but it has an additional responsibility: it must also read + all of the tuples generated by the workers. When the parallel portion of + the plan generates only a small number of tuples, the leader will often + behave very much like an additional worker, speeding up query execution. + Conversely, when the parallel portion of the plan generates a large number + of tuples, the leader may be almost entirely occupied with reading the + tuples generated by the workers and performing any further processing + steps which are required by plan nodes above the level of the + Gather node. In such cases, the leader will do very + little of the work of executing the parallel portion of the plan. + + + + + When Can Parallel Query Be Used? + + + There are several settings which can cause the query planner not to + generate a parallel query plan under any circumstances. In order for + any parallel query plans whatsoever to be generated, the following + settings must be configured as indicated. + + + + + + must be set to a + value which is greater than zero. This is a special case of the more + general principle that no more workers should be used than the number + configured via max_parallel_workers_per_gather. + + + + + + must be set to a + value other than none. Parallel query requires dynamic + shared memory in order to pass data between cooperating processes. + + + + + + In addition, the system must not be running in single-user mode. Since + the entire database system is running in single process in this situation, + no background workers will be available. + + + + Even when it is in general possible for parallel query plans to be + generated, the planner will not generate them for a given query + if any of the following are true: + + + + + + The query writes any data or locks any database rows. If a query + contains a data-modifying operation either at the top level or within + a CTE, no parallel plans for that query will be generated. This is a + limitation of the current implementation which could be lifted in a + future release. + + + + + + The query might be suspended during execution. In any situation in + which the system thinks that partial or incremental execution might + occur, no parallel plan is generated. For example, a cursor created + using DECLARE CURSOR will never use + a parallel plan. Similarly, a PL/pgsql loop of the form + FOR x IN query LOOP .. END LOOP will never use a + parallel plan, because the parallel query system is unable to verify + that the code in the loop is safe to execute while parallel query is + active. + + + + + + The query uses any function marked PARALLEL UNSAFE. + Most system-defined functions are PARALLEL SAFE, + but user-defined functions are marked PARALLEL + UNSAFE by default. See the discussion of + . + + + + + + The query is running inside of another query that is already parallel. + For example, if a function called by a parallel query issues an SQL + query itself, that query will never use a parallel plan. This is a + limitation of the current implementation, but it may not be desirable + to remove this limitation, since it could result in a single query + using a very large number of processes. + + + + + + The transaction isolation level is serializable. This is + a limitation of the current implementation. + + + + + + Even when parallel query plan is generated for a particular query, there + are several circumstances under which it will be impossible to execute + that plan in parallel at execution time. If this occurs, the leader + will execute the portion of the plan between below the Gather + node entirely by itself, almost as if the Gather node were + not present. This will happen if any of the following conditions are met: + + + + + + No background workers can be obtained because of the limitation that + the total number of background workers cannot exceed + . + + + + + + The client sends an Execute message with a non-zero fetch count. + See the discussion of the + extended query protocol. + Since libpq currently provides no way to + send such a message, this can only occur when using a client that + does not rely on libpq. If this is a frequent + occurrence, it may be a good idea to set + in sessions + where it is likely, so as to avoid generating query plans that may + be suboptimal when run serially. + + + + + + The transaction isolation level is serializable. This situation + does not normally arise, because parallel query plans are not + generated when the transaction isolation level is serializable. + However, it can happen if the transaction isolation level is changed to + serializable after the plan is generated and before it is executed. + + + + + + + Parallel Plans + + + Because each worker executes the parallel portion of the plan to + completion, it is not possible to simply take an ordinary query plan + and run it using multiple workers. Each worker would produce a full + copy of the output result set, so the query would not run any faster + than normal but would produce incorrect results. Instead, the parallel + portion of the plan must be what is known internally to the query + optimizer as a partial plan; that is, it must constructed + so that each process will which executes the plan will generate only a + subset of the output rows in such a way that each required output row + is guaranteed to be generated by exactly one of the cooperating processes. + + + + Parallel Scans + + + Currently, the only type of scan which has been modified to work with + parallel query is a sequential scan. Therefore, the driving table in + a parallel plan will always be scanned using a + Parallel Seq Scan. The relation's blocks will be divided + among the cooperating processes. Blocks are handed out one at a + time, so that access to the relation remains sequential. Each process + will visit every tuple on the page assigned to it before requesting a new + page. + + + + + Parallel Joins + + + The driving table may be joined to one or more other tables using nested + loops or hash joins. The outer side of the join may be any kind of + non-parallel plan that is otherwise supported by the planner provided that + it is safe to run within a parallel worker. For example, it may be an + index scan which looks up a value based on a column taken from the inner + table. Each worker will execute the outer side of the plan in full, which + is why merge joins are not supported here. The outer side of a merge join + will often involve sorting the entire inner table; even if it involves an + index, it is unlikely to be productive to have multiple processes each + conduct a full index scan of the inner table. + + + + + Parallel Aggregation + + It is not possible to perform the aggregation portion of a query entirely + in parallel. For example, if a query involves selecting + COUNT(*), each worker could compute a total, but those totals + would need to combined in order to produce a final answer. If the query + involved a GROUP BY clause, a separate total would need to + be computed for each group. Even though aggregation can't be done entirely + in parallel, queries involving aggregation are often excellent candidates + for parallel query, because they typically read many rows but return only + a few rows to the client. Queries that return many rows to the client + are often limited by the speed at which the client can read the data, + in which case parallel query cannot help very much. + + + + PostgreSQL supports parallel aggregation by aggregating + twice. First, each process participating in the parallel portion of the + query performs an aggregation step, producing a partial result for each + group of which that process is aware. This is reflected in the plan as + a PartialAggregate node. Second, the partial results are + transferred to the leader via the Gather node. Finally, the + leader re-aggregates the results across all workers in order to produce + the final result. This is reflected in the plan as a + FinalizeAggregate node. + + + + Parallel aggregation is not supported in all situations. Each aggregate + must be safe for parallelism and must + have a combine function. If the aggregate has a transition state of type + internal, it must have serialization and deserialization + functions. See for more details. + Parallel aggregation is not supported for ordered set aggregates or when + the query involves GROUPING SETS. It can only be used when + all joins involved in the query are also part of the parallel portion + of the plan. + + + + + + Parallel Plan Tips + + + If a query that is expected to do so does not produce a parallel plan, + you can try reducing or + . Of course, this plan may turn + out to be slower than the serial plan which the planner preferred, but + this will not always be the case. If you don't get a parallel + plan even with very small values of these settings (e.g. after setting + them both to zero), there may be some reason why the query planner is + unable to generate a parallel plan for your query. See + and + for information on why this may be + the case. + + + + When executing a parallel plan, you can use EXPLAIN (ANALYZE, + VERBOSE) will display per-worker statistics for each plan node. + This may be useful in determining whether the work is being evenly + distributed between all plan nodes and more generally in understanding the + performance characteristics of the plan. + + + + + + + Parallel Safety + + + The planner classifies operations involved in a query as either + parallel safe, parallel restricted, + or parallel unsafe. A parallel safe operation is one which + does not conflict with the use of parallel query. A parallel restricted + operation is one which cannot be performed in a parallel worker, but which + can be performed in the leader while parallel query is in use. Therefore, + parallel restricted operations can never occur below a Gather + node, but can occur elsewhere in a plan which contains a + Gather node. A parallel unsafe operation is one which cannot + be performed while parallel query is in use, not even in the leader. + When a query contains anything which is parallel unsafe, parallel query + is completely disabled for that query. + + + + The following operations are always parallel restricted. + + + + + + Scans of common table expressions (CTEs). + + + + + + Scans of temporary tables. + + + + + + Scans of foreign tables, unless the foreign data wrapper has + an IsForeignScanParallelSafe API which indicates otherwise. + + + + + + Access to an InitPlan or SubPlan. + + + + + + Parallel Labeling for Functions and Aggregates + + + The planner cannot automatically determine whether a user-defined + function or aggregate is parallel safe, parallel restricted, or parallel + unsafe, because this would require predicting every operation which the + function could possibly perform. In general, this is equivalent to the + Halting Problem and therefore impossible. Even for simple functions + where it conceivably be done, we do not try, since this would be expensive + and error-prone. Instead, all user-defined functions are assumed to + be parallel unsafe unless otherwise marked. When using + or + , markings can be set by specifying + PARALLEL SAFE, PARALLEL RESTRICTED, or + PARALLEL UNSAFE as appropriate. When using + , the + PARALLEL option can be specified with SAFE, + RESTRICTED, or UNSAFE as the corresponding value. + + + + Functions and aggregates must be marked PARALLEL UNSAFE if + they write to the database, access sequences, change the transaction state + even temporarily (e.g. a PL/pgsql function which establishes an + EXCEPTION block to catch errors), or make persistent changes to + settings. Similarly, functions must be marked PARALLEL + RESTRICTED if they access temporary tables, client connection state, + cursors, prepared statements, or miscellaneous backend-local state which + the system cannot synchronize across workers. For example, + setseed and random are parallel restricted for + this last reason. + + + + In general, if a function is labeled as being safe when it is restricted or + unsafe, or if it is labeled as being restricted when it is in fact unsafe, + it may throw errors or produce wrong answers when used in a parallel query. + C-language functions could in theory exhibit totally undefined behavior if + mislabeled, since there is no way for the system to protect itself against + arbitrary C code, but in most likely cases the result will be no worse than + for any other function. If in doubt, it is probably best to label functions + as UNSAFE. + + + + If a function executed within a parallel worker acquires locks which are + not held by the leader, for example by querying a table not referenced in + the query, those locks will be released at worker exit, not end of + transaction. If you write a function which does this, and this behavior + difference is important to you, mark such functions as + PARALLEL RESTRICTED + to ensure that they execute only in the leader. + + + + Note that the query planner does not consider deferring the evaluation of + parallel-restricted functions or aggregates involved in the query in + order to obtain a superior plan. So, for example, if a WHERE + clause applied to a particular table is parallel restricted, the query + planner will not consider placing the scan of that table below a + Gather node. In some cases, it would be + possible (and perhaps even efficient) to include the scan of that table in + the parallel portion of the query and defer the evaluation of the + WHERE clause so that it happens above the Gather + node. However, the planner does not do this. + + + + + + + diff --git a/doc/src/sgml/postgres.sgml b/doc/src/sgml/postgres.sgml index 0346d367e5..9143917c49 100644 --- a/doc/src/sgml/postgres.sgml +++ b/doc/src/sgml/postgres.sgml @@ -106,6 +106,7 @@ &textsearch; &mvcc; &perform; + ∥ -- cgit v1.2.3 From e7010ce4794a4c12a6a8bfb0ca1de49b61046847 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 5 Aug 2016 21:35:19 -0400 Subject: pg_ctl: Add wait option to promote action When waiting is selected for the promote action, look into pg_control until the state changes, then use the PQping-based waiting until the server is reachable. Reviewed-by: Michael Paquier --- doc/src/sgml/ref/pg_ctl-ref.sgml | 29 ++++++++++++++++++++------ src/bin/pg_ctl/pg_ctl.c | 45 ++++++++++++++++++++++++++++------------ src/bin/pg_ctl/t/003_promote.pl | 18 +++++++++++++++- 3 files changed, 72 insertions(+), 20 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/pg_ctl-ref.sgml b/doc/src/sgml/ref/pg_ctl-ref.sgml index 6ceb7816dc..a00c355f4a 100644 --- a/doc/src/sgml/ref/pg_ctl-ref.sgml +++ b/doc/src/sgml/ref/pg_ctl-ref.sgml @@ -91,6 +91,8 @@ PostgreSQL documentation pg_ctl + + seconds datadir @@ -361,8 +363,8 @@ PostgreSQL documentation - The maximum number of seconds to wait when waiting for startup or - shutdown to complete. Defaults to the value of the + The maximum number of seconds to wait when waiting for an operation + to complete (see option ). Defaults to the value of the PGCTLTIMEOUT environment variable or, if not set, to 60 seconds. @@ -383,8 +385,23 @@ PostgreSQL documentation - Wait for the startup or shutdown to complete. - Waiting is the default option for shutdowns, but not startups. + Wait for an operation to complete. This is supported for the + modes start, stop, + restart, promote, + and register. + + + + Waiting is the default option for shutdowns, but not startups, + restarts, or promotions. This is mainly for historical reasons; the + waiting option is almost always preferable. If waiting is not + selected, the requested action is triggered, but there is no feedback + about its success. In that case, the server log file or an external + monitoring system would have to be used to check the progress and + success of the operation. + + + When waiting for startup, pg_ctl repeatedly attempts to connect to the server. When waiting for shutdown, pg_ctl waits for @@ -400,8 +417,8 @@ PostgreSQL documentation - Do not wait for startup or shutdown to complete. This is the - default for start and restart modes. + Do not wait for an operation to complete. This is the opposite of the + option . diff --git a/src/bin/pg_ctl/pg_ctl.c b/src/bin/pg_ctl/pg_ctl.c index eb8a67a903..2f0976a9cc 100644 --- a/src/bin/pg_ctl/pg_ctl.c +++ b/src/bin/pg_ctl/pg_ctl.c @@ -1228,7 +1228,34 @@ do_promote(void) exit(1); } - print_msg(_("server promoting\n")); + if (do_wait) + { + DBState state = DB_STARTUP; + + print_msg(_("waiting for server to promote...")); + while (wait_seconds > 0) + { + state = get_control_dbstate(); + if (state == DB_IN_PRODUCTION) + break; + + print_msg("."); + pg_usleep(1000000); /* 1 sec */ + wait_seconds--; + } + if (state == DB_IN_PRODUCTION) + { + print_msg(_(" done\n")); + print_msg(_("server promoted\n")); + } + else + { + print_msg(_(" stopped waiting\n")); + print_msg(_("server is still promoting\n")); + } + } + else + print_msg(_("server promoting\n")); } @@ -2405,18 +2432,10 @@ main(int argc, char **argv) if (!wait_set) { - switch (ctl_command) - { - case RESTART_COMMAND: - case START_COMMAND: - do_wait = false; - break; - case STOP_COMMAND: - do_wait = true; - break; - default: - break; - } + if (ctl_command == STOP_COMMAND) + do_wait = true; + else + do_wait = false; } if (ctl_command == RELOAD_COMMAND) diff --git a/src/bin/pg_ctl/t/003_promote.pl b/src/bin/pg_ctl/t/003_promote.pl index 1461234f2a..0b6090b6eb 100644 --- a/src/bin/pg_ctl/t/003_promote.pl +++ b/src/bin/pg_ctl/t/003_promote.pl @@ -3,7 +3,7 @@ use warnings; use PostgresNode; use TestLib; -use Test::More tests => 9; +use Test::More tests => 12; my $tempdir = TestLib::tempdir; @@ -37,3 +37,19 @@ command_ok([ 'pg_ctl', '-D', $node_standby->data_dir, 'promote' ], ok($node_standby->poll_query_until('postgres', 'SELECT NOT pg_is_in_recovery()'), 'promoted standby is not in recovery'); + +# same again with wait option +$node_standby = get_new_node('standby2'); +$node_standby->init_from_backup($node_primary, 'my_backup', has_streaming => 1); +$node_standby->start; + +is($node_standby->safe_psql('postgres', 'SELECT pg_is_in_recovery()'), + 't', 'standby is in recovery'); + +command_ok([ 'pg_ctl', '-D', $node_standby->data_dir, '-w', 'promote' ], + 'pg_ctl -w promote of standby runs'); + +# no wait here + +is($node_standby->safe_psql('postgres', 'SELECT pg_is_in_recovery()'), + 'f', 'promoted standby is not in recovery'); -- cgit v1.2.3 From 5a7bae0699657315b487896810a30bd7425f6a08 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 23 Sep 2016 14:22:07 -0400 Subject: Doc: fix examples of # operators so they actually work. These worked as-is until around 7.0, but fail in newer versions because there are more operators named "#". Besides it's a bit inconsistent that only two of the examples on this page lack type names on their constants. Report: <20160923081530.1517.75670@wrigleys.postgresql.org> --- doc/src/sgml/func.sgml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 47fcb30da0..3cc69bbffd 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -8365,12 +8365,12 @@ CREATE TYPE rainbow AS ENUM ('red', 'orange', 'yellow', 'green', 'blue', 'purple # Point or box of intersection - '((1,-1),(-1,1))' # '((1,1),(-1,-1))' + box '((1,-1),(-1,1))' # box '((1,1),(-2,-2))' # Number of points in path or polygon - # '((1,0),(0,1),(-1,0))' + # path '((1,0),(0,1),(-1,0))' @-@ -- cgit v1.2.3 From 98c2d3332b30006ff71add99bc9d619c9457e71f Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 24 Sep 2016 16:25:35 -0400 Subject: Do a final round of updates on the 9.6 release notes. Set release date, document a few recent commits, do one last pass of copy-editing. --- doc/src/sgml/catalogs.sgml | 9 +++-- doc/src/sgml/release-9.6.sgml | 83 +++++++++++++++++++++++++++++++++---------- 2 files changed, 71 insertions(+), 21 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 322d8d6dc7..29738b07cb 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -7596,6 +7596,11 @@ application. + + By default, the pg_config view can be read + only by superusers. + + <structname>pg_config</> Columns @@ -7771,8 +7776,8 @@ - The pg_file_settings view can be read only by - superusers. + By default, the pg_file_settings view can be read + only by superusers.
diff --git a/doc/src/sgml/release-9.6.sgml b/doc/src/sgml/release-9.6.sgml index ddd280c85a..5c40910c72 100644 --- a/doc/src/sgml/release-9.6.sgml +++ b/doc/src/sgml/release-9.6.sgml @@ -6,8 +6,7 @@ Release Date - 2016-??-?? - Current as of 2016-08-27 (commit b9fe6cbc8) + 2016-09-29 @@ -56,7 +55,7 @@ Substantial performance improvements, especially in the area of - scalability on multi-CPU-socket servers + scalability on multi-CPU-socket servers @@ -269,7 +268,7 @@ This commit is also listed under libpq and psql Write (or its abbreviation ) explicitly to obtain the old - behavior. Scripts modified this way will still work with old + behavior. Scripts so modified will still work with old versions of psql. @@ -371,6 +370,7 @@ and many others in the same vein 2016-06-09 [c9ce4a1c6] Eliminate "parallel degree" terminology. 2016-06-16 [75be66464] Invent min_parallel_relation_size GUC to replace a hard- 2016-08-16 [f85b1a841] Disable parallel query by default. +2016-09-15 [72ce78162] Make min_parallel_relation_size's default value platform --> Parallel queries (Robert Haas, Amit Kapila, David Rowley, @@ -504,6 +504,7 @@ and many others in the same vein Improve sorting performance by using quicksort, not replacement @@ -693,7 +694,7 @@ and many others in the same vein (a,b) REFERENCES r (x,y), then a WHERE condition such as t.a = r.x AND t.b = r.y cannot select more than one r row per t row. - The planner formerly considered AND conditions + The planner formerly considered these AND conditions to be independent and would often drastically misestimate selectivity as a result. Now it compares the WHERE conditions to applicable foreign key constraints and produces @@ -731,7 +732,7 @@ and many others in the same vein containing only already-frozen tuples are identified in the table's visibility map, and can be skipped by vacuum even when doing transaction wraparound prevention. This should greatly reduce the - cost of maintaining large tables containing mostly-unchanged data. + cost of maintaining large tables containing mostly-unchanging data. @@ -872,7 +873,8 @@ and many others in the same vein from where it will be flushed to physical storage in due time. Many operating systems are not smart about managing this and allow large amounts of dirty data to accumulate before deciding to flush - it all at once, leading to long delays for new I/O requests. + it all at once, causing long delays for new I/O requests until the + flushing finishes. This change attempts to alleviate this problem by explicitly requesting data flushes after a configurable interval. @@ -1209,7 +1211,7 @@ and many others in the same vein 2016-04-08 [34c33a1f0] Add BSD authentication method. --> - Add a bsd authentication + Add a BSD authentication method to allow use of the BSD Authentication service for PostgreSQL client authentication (Marisa Emerson) @@ -1300,6 +1302,16 @@ and many others in the same vein + + Raise the maximum allowed value + of to 24 hours (Simon Riggs) + + + + + @@ -1346,9 +1358,9 @@ and many others in the same vein Making a distinction between these settings is no longer useful, - and is part of a planned future simplification of replication - setup. The old names are still accepted but are converted - internally. + and merging them is a step towards a planned future simplification + of replication setup. The old names are still accepted but are + converted to replica internally. @@ -1375,7 +1387,7 @@ and many others in the same vein --> Allow the server's SSL key file to have group read - access if owned by root (Christoph Berg) + access if it is owned by root (Christoph Berg) @@ -1616,7 +1628,7 @@ XXX this is pending backpatch, may need to remove Previously, such cases failed if the same target column was mentioned more than once, e.g., INSERT INTO tab (x[1], - x[2]) VALUES .... + x[2]) VALUES (...). @@ -1797,9 +1809,9 @@ XXX this is pending backpatch, may need to remove 2016-03-23 [473b93287] Support CREATE ACCESS METHOD --> - Introduce CREATE ACCESS METHOD to allow extensions - to create index access methods (Alexander Korotkov, Petr - Jelínek) + Introduce CREATE + ACCESS METHOD to allow extensions to create index access + methods (Alexander Korotkov, Petr Jelínek) @@ -1912,7 +1924,7 @@ XXX this is pending backpatch, may need to remove Formerly, many security-sensitive functions contained hard-wired checks that would throw an error if they were called by a - non-superuser role. This forced the use of superuser roles for + non-superuser. This forced the use of superuser roles for some relatively pedestrian tasks. The hard-wired error checks are now gone in favor of making initdb revoke the default public EXECUTE privilege on these functions. @@ -1931,6 +1943,11 @@ XXX this is pending backpatch, may need to remove that can be used to grant access to what were previously superuser-only functions (Stephen Frost) + + + Currently the only such role is pg_signal_backend, + but more are expected to be added in future. + @@ -2211,7 +2228,7 @@ XXX this is pending backpatch, may need to remove Allow ts_stat() and tsvector_update_trigger() to operate on values that are of types binary-compatible with the - expected argument type, not only that argument type; for example + expected argument type, not just exactly that type; for example allow citext where text is expected (Teodor Sigaev) @@ -2623,10 +2640,26 @@ This commit is also listed under psql and PL/pgSQL + + Allow pg_dump to dump non-extension-owned objects + that are within an extension-owned schema + (Martín Marqués) + + + + Previously such objects were ignored because they were mistakenly + assumed to belong to the extension owning their schema. + + + + + - In pg_dump, include the table name in object + In pg_dump output, include the table name in object tags for object types that are only uniquely named per-table (for example, triggers) (Peter Eisentraut) @@ -2912,6 +2945,7 @@ This commit is also listed under libpq and PL/pgSQL Allow changing the selection probabilities (weights) for scripts @@ -3011,6 +3045,7 @@ This commit is also listed under libpq and PL/pgSQL Speed up initdb by using just one @@ -3420,6 +3455,16 @@ This commit is also listed under libpq and PL/pgSQL + + + + Support OpenSSL 1.1.0 (Andreas Karlsson, Heikki Linnakangas) + + + -- cgit v1.2.3 From da6c4f6ca88df346573bdada2aa2544510bf167e Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 25 Sep 2016 15:40:57 -0400 Subject: Refer to OS X as "macOS", except for the port name which is still "darwin". We weren't terribly consistent about whether to call Apple's OS "OS X" or "Mac OS X", and the former is probably confusing to people who aren't Apple users. Now that Apple has rebranded it "macOS", follow their lead to establish a consistent naming pattern. Also, avoid the use of the ancient project name "Darwin", except as the port code name which does not seem desirable to change. (In short, this patch touches documentation and comments, but no actual code.) I didn't touch contrib/start-scripts/osx/, either. I suspect those are obsolete and due for a rewrite, anyway. I dithered about whether to apply this edit to old release notes, but those were responsible for quite a lot of the inconsistencies, so I ended up changing them too. Anyway, Apple's being ahistorical about this, so why shouldn't we be? --- config/c-library.m4 | 4 ++-- configure | 8 ++++---- configure.in | 12 ++++++------ doc/src/sgml/client-auth.sgml | 2 +- doc/src/sgml/dfunc.sgml | 4 ++-- doc/src/sgml/docguide.sgml | 2 +- doc/src/sgml/install-windows.sgml | 2 +- doc/src/sgml/installation.sgml | 6 +++--- doc/src/sgml/monitoring.sgml | 2 +- doc/src/sgml/release-7.4.sgml | 14 +++++++------- doc/src/sgml/release-8.0.sgml | 10 +++++----- doc/src/sgml/release-8.1.sgml | 10 +++++----- doc/src/sgml/release-8.2.sgml | 4 ++-- doc/src/sgml/release-8.3.sgml | 4 ++-- doc/src/sgml/release-8.4.sgml | 6 +++--- doc/src/sgml/release-9.0.sgml | 8 ++++---- doc/src/sgml/release-9.1.sgml | 8 ++++---- doc/src/sgml/release-9.2.sgml | 6 +++--- doc/src/sgml/release-9.3.sgml | 10 +++++----- doc/src/sgml/release-9.4.sgml | 6 +++--- doc/src/sgml/release-9.5.sgml | 2 +- doc/src/sgml/release-old.sgml | 2 +- doc/src/sgml/runtime.sgml | 18 +++++++++--------- doc/src/sgml/uuid-ossp.sgml | 2 +- doc/src/sgml/wal.sgml | 2 +- src/Makefile.shlib | 2 +- src/backend/Makefile | 4 ++-- src/backend/port/dynloader/darwin.c | 2 +- src/backend/postmaster/postmaster.c | 2 +- src/backend/utils/adt/varlena.c | 4 ++-- src/backend/utils/misc/ps_status.c | 4 ++-- src/backend/utils/probes.d | 2 +- src/bin/psql/input.c | 2 +- src/include/port/darwin.h | 2 +- src/interfaces/libpq/fe-connect.c | 2 +- src/port/README | 2 +- src/port/chklocale.c | 2 +- src/template/darwin | 11 +++++++---- src/test/regress/pg_regress.c | 4 ++-- src/tools/find_typedef | 2 +- 40 files changed, 102 insertions(+), 99 deletions(-) (limited to 'doc/src') diff --git a/config/c-library.m4 b/config/c-library.m4 index 50d068d3fb..d330b0cf95 100644 --- a/config/c-library.m4 +++ b/config/c-library.m4 @@ -292,8 +292,8 @@ AC_MSG_RESULT([$pgac_cv_snprintf_size_t_support]) # PGAC_TYPE_LOCALE_T # ------------------ -# Check for the locale_t type and find the right header file. Mac OS -# X needs xlocale.h; standard is locale.h, but glibc also has an +# Check for the locale_t type and find the right header file. macOS +# needs xlocale.h; standard is locale.h, but glibc also has an # xlocale.h file that we should not use. # AC_DEFUN([PGAC_TYPE_LOCALE_T], diff --git a/configure b/configure index 5fc8c442a2..55c771a11e 100755 --- a/configure +++ b/configure @@ -7658,9 +7658,9 @@ $as_echo "${python_libspec} ${python_additional_libs}" >&6; } if test "$python_enable_shared" != 1; then if test "$PORTNAME" = darwin; then - # OS X does supply a .dylib even though Py_ENABLE_SHARED does + # macOS does supply a .dylib even though Py_ENABLE_SHARED does # not get set. The file detection logic below doesn't succeed - # on older OS X versions, so make it explicit. + # on older macOS versions, so make it explicit. python_enable_shared=1 elif test "$PORTNAME" = win32; then # Windows also needs an explicit override. @@ -10120,7 +10120,7 @@ else fi elif test "$with_uuid" = e2fs ; then - # On OS X, the UUID functions are in libc + # On macOS, the UUID functions are in libc ac_fn_c_check_func "$LINENO" "uuid_generate" "ac_cv_func_uuid_generate" if test "x$ac_cv_func_uuid_generate" = xyes; then : UUID_LIBS="" @@ -12672,7 +12672,7 @@ cat >>confdefs.h <<_ACEOF #define HAVE_DECL_STRLCPY $ac_have_decl _ACEOF -# This is probably only present on Darwin, but may as well check always +# This is probably only present on macOS, but may as well check always ac_fn_c_check_decl "$LINENO" "F_FULLFSYNC" "ac_cv_have_decl_F_FULLFSYNC" "#include " if test "x$ac_cv_have_decl_F_FULLFSYNC" = xyes; then : diff --git a/configure.in b/configure.in index 96d865de9f..9850d993ff 100644 --- a/configure.in +++ b/configure.in @@ -943,9 +943,9 @@ if test "$with_python" = yes; then if test "$python_enable_shared" != 1; then if test "$PORTNAME" = darwin; then - # OS X does supply a .dylib even though Py_ENABLE_SHARED does + # macOS does supply a .dylib even though Py_ENABLE_SHARED does # not get set. The file detection logic below doesn't succeed - # on older OS X versions, so make it explicit. + # on older macOS versions, so make it explicit. python_enable_shared=1 elif test "$PORTNAME" = win32; then # Windows also needs an explicit override. @@ -1182,7 +1182,7 @@ if test "$with_uuid" = bsd ; then [UUID_LIBS=""], [AC_MSG_ERROR([BSD UUID functions are not present])]) elif test "$with_uuid" = e2fs ; then - # On OS X, the UUID functions are in libc + # On macOS, the UUID functions are in libc AC_CHECK_FUNC(uuid_generate, [UUID_LIBS=""], [AC_CHECK_LIB(uuid, uuid_generate, @@ -1425,8 +1425,8 @@ esac if test "$PORTNAME" != "win32"; then AC_SYS_LARGEFILE dnl Autoconf 2.69's AC_SYS_LARGEFILE believes it's a good idea to #define - dnl _DARWIN_USE_64_BIT_INODE, but it isn't: on OS X 10.5 that activates a - dnl bug that causes readdir() to sometimes return EINVAL. On later OS X + dnl _DARWIN_USE_64_BIT_INODE, but it isn't: on macOS 10.5 that activates a + dnl bug that causes readdir() to sometimes return EINVAL. On later macOS dnl versions where the feature actually works, it's on by default anyway. AH_VERBATIM([_DARWIN_USE_64_BIT_INODE],[]) fi @@ -1479,7 +1479,7 @@ fi AC_CHECK_DECLS(fdatasync, [], [], [#include ]) AC_CHECK_DECLS([strlcat, strlcpy]) -# This is probably only present on Darwin, but may as well check always +# This is probably only present on macOS, but may as well check always AC_CHECK_DECLS(F_FULLFSYNC, [], [], [#include ]) HAVE_IPV6=no diff --git a/doc/src/sgml/client-auth.sgml b/doc/src/sgml/client-auth.sgml index ca262d9452..a0d97ffbac 100644 --- a/doc/src/sgml/client-auth.sgml +++ b/doc/src/sgml/client-auth.sgml @@ -1306,7 +1306,7 @@ omicron bryanh guest1 socket parameter, or similar mechanisms. Currently that includes Linux, most flavors of BSD including - OS X, + macOS, and Solaris. diff --git a/doc/src/sgml/dfunc.sgml b/doc/src/sgml/dfunc.sgml index 5a368f6df0..ba2684cc3c 100644 --- a/doc/src/sgml/dfunc.sgml +++ b/doc/src/sgml/dfunc.sgml @@ -127,8 +127,8 @@ cc -shared -o foo.so foo.o - OS X - OS Xshared library + macOS + macOSshared library diff --git a/doc/src/sgml/docguide.sgml b/doc/src/sgml/docguide.sgml index 6f896b565f..48828aff37 100644 --- a/doc/src/sgml/docguide.sgml +++ b/doc/src/sgml/docguide.sgml @@ -275,7 +275,7 @@ apt-get install docbook docbook-dsssl docbook-xsl libxml2-utils openjade1.3 open - OS X + macOS If you use MacPorts, the following will get you set up: diff --git a/doc/src/sgml/install-windows.sgml b/doc/src/sgml/install-windows.sgml index 50116f315d..20fc47ae5f 100644 --- a/doc/src/sgml/install-windows.sgml +++ b/doc/src/sgml/install-windows.sgml @@ -51,7 +51,7 @@ MinGW-w64. These tools can also be used to cross-compile for 32 bit and 64 bit Windows targets on other hosts, such as Linux and - Darwin. + macOS. Cygwin is not recommended for running a production server, and it should only be used for running on older versions of Windows where diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml index f6de18ed2d..5ee28fcf85 100644 --- a/doc/src/sgml/installation.sgml +++ b/doc/src/sgml/installation.sgml @@ -874,7 +874,7 @@ su - postgres Build with Bonjour support. This requires Bonjour support - in your operating system. Recommended on OS X. + in your operating system. Recommended on macOS. @@ -900,7 +900,7 @@ su - postgres @@ -2010,7 +2010,7 @@ kill `cat /usr/local/pgsql/data/postmaster.pid` PostgreSQL can be expected to work on these operating systems: Linux (all recent distributions), Windows (Win2000 SP4 and later), - FreeBSD, OpenBSD, NetBSD, OS X, AIX, HP/UX, Solaris, + FreeBSD, OpenBSD, NetBSD, macOS, AIX, HP/UX, Solaris, and UnixWare. Other Unix-like systems may also work but are not currently being tested. In most cases, all CPU architectures supported by a given operating system will work. Look in diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 077642878e..f400785721 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -2739,7 +2739,7 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid, Currently, the DTrace utility is supported, which, at the time of this writing, is available - on Solaris, OS X, FreeBSD, NetBSD, and Oracle Linux. The + on Solaris, macOS, FreeBSD, NetBSD, and Oracle Linux. The SystemTap project for Linux provides a DTrace equivalent and can also be used. Supporting other dynamic tracing utilities is theoretically possible by changing the definitions for diff --git a/doc/src/sgml/release-7.4.sgml b/doc/src/sgml/release-7.4.sgml index 5a4c52d4c2..e42be5b89d 100644 --- a/doc/src/sgml/release-7.4.sgml +++ b/doc/src/sgml/release-7.4.sgml @@ -268,7 +268,7 @@ - This behavior has been observed on BSD-derived kernels including OS X. + This behavior has been observed on BSD-derived kernels including macOS. It resulted in an entirely-misleading startup failure complaining that the shared memory request size was too large. @@ -2437,7 +2437,7 @@ aggregate plan Pretty-print UNION queries correctly Make psql handle \r\n newlines properly in COPY IN pg_dump handled ACLs with grant options incorrectly -Fix thread support for OS X and Solaris +Fix thread support for macOS and Solaris Updated JDBC driver (build 215) with various fixes ECPG fixes Translation updates (various contributors) @@ -2627,7 +2627,7 @@ memory error during COPY IN TABLE AS from tables without OIDs Fix problems with alter_table regression test during parallel testing -Fix problems with hitting open file limit, especially on OS X (Tom) +Fix problems with hitting open file limit, especially on macOS (Tom) Partial fix for Turkish-locale issues initdb will succeed now in Turkish locale, but there are still some inconveniences associated with the i/I problem. @@ -3256,7 +3256,7 @@ DROP SCHEMA information_schema CASCADE; - Enable PAM for Mac OS X (Aaron Hillegass) + Enable PAM for macOS (Aaron Hillegass) Make B-tree indexes fully WAL-safe (Tom) @@ -3539,9 +3539,9 @@ DROP SCHEMA information_schema CASCADE; - Add Mac OS X Rendezvous server support (Chris Campbell) + Add macOS Rendezvous server support (Chris Campbell) - This allows Mac OS X hosts to query the network for available + This allows macOS hosts to query the network for available PostgreSQL servers. @@ -4561,7 +4561,7 @@ DROP SCHEMA information_schema CASCADE; Fix locking code for s390x CPU (64-bit) (Tom) Allow OpenBSD to use local ident credentials (William Ahern) Make query plan trees read-only to executor (Tom) - Add Darwin startup scripts (David Wheeler) + Add macOS startup scripts (David Wheeler) Allow libpq to compile with Borland C++ compiler (Lester Godwin, Karl Waclawek) Use our own version of getopt_long() if needed (Peter) Convert administration scripts to C (Peter) diff --git a/doc/src/sgml/release-8.0.sgml b/doc/src/sgml/release-8.0.sgml index 299c34e0f0..becd5090cc 100644 --- a/doc/src/sgml/release-8.0.sgml +++ b/doc/src/sgml/release-8.0.sgml @@ -345,7 +345,7 @@ - This behavior has been observed on BSD-derived kernels including OS X. + This behavior has been observed on BSD-derived kernels including macOS. It resulted in an entirely-misleading startup failure complaining that the shared memory request size was too large. @@ -1715,7 +1715,7 @@ While this could theoretically happen anywhere, no standard build of - Perl did things this way ... until Mac OS X 10.5. + Perl did things this way ... until macOS 10.5. @@ -2449,7 +2449,7 @@ Win32 to match the backend (Andrew) (Bruce) Fix pgxs -L library path -specification for Win32, Cygwin, OS X, AIX (Bruce) +specification for Win32, Cygwin, macOS, AIX (Bruce) Check that SID is enabled while checking for Win32 admin privileges (Magnus) @@ -5224,7 +5224,7 @@ typedefs (Michael) - Improvements to the Mac OS X startup scripts (Ray A.) + Improvements to the macOS startup scripts (Ray A.) @@ -5328,7 +5328,7 @@ typedefs (Michael) - Make libpq and ECPG build as proper shared libraries on OS X (Tom) + Make libpq and ECPG build as proper shared libraries on macOS (Tom) diff --git a/doc/src/sgml/release-8.1.sgml b/doc/src/sgml/release-8.1.sgml index 0cb5587e9b..05b07ade99 100644 --- a/doc/src/sgml/release-8.1.sgml +++ b/doc/src/sgml/release-8.1.sgml @@ -572,7 +572,7 @@ - This behavior has been observed on BSD-derived kernels including OS X. + This behavior has been observed on BSD-derived kernels including macOS. It resulted in an entirely-misleading startup failure complaining that the shared memory request size was too large. @@ -2188,7 +2188,7 @@ While this could theoretically happen anywhere, no standard build of - Perl did things this way ... until Mac OS X 10.5. + Perl did things this way ... until macOS 10.5. @@ -2730,7 +2730,7 @@ - Fix for Darwin (OS X) compilation (Tom) + Fix for macOS (Darwin) compilation (Tom) @@ -3104,7 +3104,7 @@ Win32 to match the backend (Andrew) (Bruce) Fix pgxs -L library path -specification for Win32, Cygwin, OS X, AIX (Bruce) +specification for Win32, Cygwin, macOS, AIX (Bruce) Check that SID is enabled while checking for Win32 admin privileges (Magnus) @@ -5225,7 +5225,7 @@ SELECT CURRENT_TIMESTAMP AT TIME ZONE 'Europe/London'; Add support for fsync_writethrough on - Darwin (Chris Campbell) + macOS (Chris Campbell) diff --git a/doc/src/sgml/release-8.2.sgml b/doc/src/sgml/release-8.2.sgml index 7f6a74bac9..2d21728cf7 100644 --- a/doc/src/sgml/release-8.2.sgml +++ b/doc/src/sgml/release-8.2.sgml @@ -1487,7 +1487,7 @@ - This behavior has been observed on BSD-derived kernels including OS X. + This behavior has been observed on BSD-derived kernels including macOS. It resulted in an entirely-misleading startup failure complaining that the shared memory request size was too large. @@ -3765,7 +3765,7 @@ While this could theoretically happen anywhere, no standard build of - Perl did things this way ... until Mac OS X 10.5. + Perl did things this way ... until macOS 10.5. diff --git a/doc/src/sgml/release-8.3.sgml b/doc/src/sgml/release-8.3.sgml index b56edb0102..b1b5d4875c 100644 --- a/doc/src/sgml/release-8.3.sgml +++ b/doc/src/sgml/release-8.3.sgml @@ -3075,7 +3075,7 @@ - This behavior has been observed on BSD-derived kernels including OS X. + This behavior has been observed on BSD-derived kernels including macOS. It resulted in an entirely-misleading startup failure complaining that the shared memory request size was too large. @@ -8396,7 +8396,7 @@ current_date < 2017-11-17 Use SYSV semaphores rather than POSIX on Darwin - >= 6.0, i.e., OS X 10.2 and up (Chris Marcellino) + >= 6.0, i.e., macOS 10.2 and up (Chris Marcellino) diff --git a/doc/src/sgml/release-8.4.sgml b/doc/src/sgml/release-8.4.sgml index 8b16c9e9d3..0d0478855e 100644 --- a/doc/src/sgml/release-8.4.sgml +++ b/doc/src/sgml/release-8.4.sgml @@ -240,7 +240,7 @@ - Fix linking of libpython on OS X (Tom Lane) + Fix linking of libpython on macOS (Tom Lane) @@ -5334,7 +5334,7 @@ - This behavior has been observed on BSD-derived kernels including OS X. + This behavior has been observed on BSD-derived kernels including macOS. It resulted in an entirely-misleading startup failure complaining that the shared memory request size was too large. @@ -9764,7 +9764,7 @@ WITH w AS (SELECT * FROM foo) SELECT * FROM w, bar ... FOR UPDATE - Enable DTrace support on Mac OS X + Enable DTrace support on macOS Leopard and other non-Solaris platforms (Robert Lor) diff --git a/doc/src/sgml/release-9.0.sgml b/doc/src/sgml/release-9.0.sgml index 61dce9fd78..2238b53745 100644 --- a/doc/src/sgml/release-9.0.sgml +++ b/doc/src/sgml/release-9.0.sgml @@ -1541,7 +1541,7 @@ - Warn if OS X's setlocale() starts an unwanted extra + Warn if macOS's setlocale() starts an unwanted extra thread inside the postmaster (Noah Misch) @@ -2093,7 +2093,7 @@ - Fix linking of libpython on OS X (Tom Lane) + Fix linking of libpython on macOS (Tom Lane) @@ -5895,7 +5895,7 @@ - Fix incorrect quoting of log file name in Mac OS X start script + Fix incorrect quoting of log file name in macOS start script (Sidar Lopez) @@ -10745,7 +10745,7 @@ if TG_OP = 'INSERT' and NEW.col1 = ... then - Bonjour support now requires OS X 10.3 or later. + Bonjour support now requires macOS 10.3 or later. The older API has been deprecated by Apple. diff --git a/doc/src/sgml/release-9.1.sgml b/doc/src/sgml/release-9.1.sgml index a66ca0d5b3..26b709056f 100644 --- a/doc/src/sgml/release-9.1.sgml +++ b/doc/src/sgml/release-9.1.sgml @@ -599,7 +599,7 @@ Branch: REL9_1_STABLE [354b3a3ac] 2016-06-19 14:01:17 -0400 This dodges a portability problem on FreeBSD-derived platforms - (including OS X). + (including macOS). @@ -2937,7 +2937,7 @@ Branch: REL9_0_STABLE [9d6af7367] 2015-08-15 11:02:34 -0400 - Warn if OS X's setlocale() starts an unwanted extra + Warn if macOS's setlocale() starts an unwanted extra thread inside the postmaster (Noah Misch) @@ -3574,7 +3574,7 @@ Branch: REL9_0_STABLE [9d6af7367] 2015-08-15 11:02:34 -0400 - Fix linking of libpython on OS X (Tom Lane) + Fix linking of libpython on macOS (Tom Lane) @@ -8443,7 +8443,7 @@ Branch: REL9_0_STABLE [9d6af7367] 2015-08-15 11:02:34 -0400 - Fix incorrect quoting of log file name in Mac OS X start script + Fix incorrect quoting of log file name in macOS start script (Sidar Lopez) diff --git a/doc/src/sgml/release-9.2.sgml b/doc/src/sgml/release-9.2.sgml index c801f98c3f..0f6e3d127f 100644 --- a/doc/src/sgml/release-9.2.sgml +++ b/doc/src/sgml/release-9.2.sgml @@ -629,7 +629,7 @@ This dodges a portability problem on FreeBSD-derived platforms - (including OS X). + (including macOS). @@ -3190,7 +3190,7 @@ Branch: REL9_2_STABLE [6b700301c] 2015-02-17 16:03:00 +0100 - Warn if OS X's setlocale() starts an unwanted extra + Warn if macOS's setlocale() starts an unwanted extra thread inside the postmaster (Noah Misch) @@ -3899,7 +3899,7 @@ Branch: REL9_2_STABLE [6b700301c] 2015-02-17 16:03:00 +0100 - Fix linking of libpython on OS X (Tom Lane) + Fix linking of libpython on macOS (Tom Lane) diff --git a/doc/src/sgml/release-9.3.sgml b/doc/src/sgml/release-9.3.sgml index c75f1109e1..e321f4b31c 100644 --- a/doc/src/sgml/release-9.3.sgml +++ b/doc/src/sgml/release-9.3.sgml @@ -812,7 +812,7 @@ Branch: REL9_2_STABLE [37f30b251] 2016-04-18 13:19:52 -0400 This dodges a portability problem on FreeBSD-derived platforms - (including OS X). + (including macOS). @@ -3021,7 +3021,7 @@ Branch: REL9_0_STABLE [4dddf8552] 2015-05-21 20:41:55 -0400 - Silence some build warnings on OS X (Tom Lane) + Silence some build warnings on macOS (Tom Lane) @@ -4092,7 +4092,7 @@ Branch: REL9_0_STABLE [2e4946169] 2015-01-07 22:46:20 -0500 - Warn if OS X's setlocale() starts an unwanted extra + Warn if macOS's setlocale() starts an unwanted extra thread inside the postmaster (Noah Misch) @@ -5743,7 +5743,7 @@ Branch: REL8_4_STABLE [ae41bb4be] 2014-05-30 18:18:32 -0400 - Fix linking of libpython on OS X (Tom Lane) + Fix linking of libpython on macOS (Tom Lane) @@ -10710,7 +10710,7 @@ ALTER EXTENSION hstore UPDATE; Add instructions for setting - up the documentation tool chain on Mac OS X + up the documentation tool chain on macOS (Peter Eisentraut) diff --git a/doc/src/sgml/release-9.4.sgml b/doc/src/sgml/release-9.4.sgml index 443c772846..51896924c9 100644 --- a/doc/src/sgml/release-9.4.sgml +++ b/doc/src/sgml/release-9.4.sgml @@ -929,7 +929,7 @@ Branch: REL9_1_STABLE [de887cc8a] 2016-05-25 19:39:49 -0400 This dodges a portability problem on FreeBSD-derived platforms - (including OS X). + (including macOS). @@ -5254,7 +5254,7 @@ Branch: REL9_3_STABLE [6347bdb31] 2015-04-05 13:01:55 -0400 - Silence some build warnings on OS X (Tom Lane) + Silence some build warnings on macOS (Tom Lane) @@ -5813,7 +5813,7 @@ Branch: REL9_0_STABLE [2e4946169] 2015-01-07 22:46:20 -0500 - Warn if OS X's setlocale() starts an unwanted extra + Warn if macOS's setlocale() starts an unwanted extra thread inside the postmaster (Noah Misch) diff --git a/doc/src/sgml/release-9.5.sgml b/doc/src/sgml/release-9.5.sgml index fa3537de10..c3f0f7051e 100644 --- a/doc/src/sgml/release-9.5.sgml +++ b/doc/src/sgml/release-9.5.sgml @@ -2023,7 +2023,7 @@ Branch: REL9_1_STABLE [e56acbe2a] 2016-02-10 19:30:12 -0500 This dodges a portability problem on FreeBSD-derived platforms - (including OS X). + (including macOS). diff --git a/doc/src/sgml/release-old.sgml b/doc/src/sgml/release-old.sgml index ec8e43f6ea..cd9b3db35a 100644 --- a/doc/src/sgml/release-old.sgml +++ b/doc/src/sgml/release-old.sgml @@ -3299,7 +3299,7 @@ New BeOS port (David Reid, Cyril Velter) Add proofreader's changes to docs (Addison-Wesley, Bruce) New Alpha spinlock code (Adriaan Joubert, Compaq) UnixWare port overhaul (Peter E) -New Darwin/Mac OS X port (Peter Bierman, Bruce Hartzler) +New macOS (Darwin) port (Peter Bierman, Bruce Hartzler) New FreeBSD Alpha port (Alfred) Overhaul shared memory segments (Tom) Add IBM S/390 support (Neale Ferguson) diff --git a/doc/src/sgml/runtime.sgml b/doc/src/sgml/runtime.sgml index ef0139c365..88ec120841 100644 --- a/doc/src/sgml/runtime.sgml +++ b/doc/src/sgml/runtime.sgml @@ -1006,12 +1006,12 @@ option SEMMAP=256 - OS X - OS XIPC configuration + macOS + macOSIPC configuration - The recommended method for configuring shared memory in OS X + The recommended method for configuring shared memory in macOS is to create a file named /etc/sysctl.conf, containing variable assignments such as: @@ -1021,13 +1021,13 @@ kern.sysv.shmmni=32 kern.sysv.shmseg=8 kern.sysv.shmall=1024 - Note that in some OS X versions, + Note that in some macOS versions, all five shared-memory parameters must be set in /etc/sysctl.conf, else the values will be ignored. - Beware that recent releases of OS X ignore attempts to set + Beware that recent releases of macOS ignore attempts to set SHMMAX to a value that isn't an exact multiple of 4096. @@ -1036,7 +1036,7 @@ kern.sysv.shmall=1024 - In older OS X versions, you will need to reboot to have changes in the + In older macOS versions, you will need to reboot to have changes in the shared memory parameters take effect. As of 10.5 it is possible to change all but SHMMNI on the fly, using sysctl. But it's still best to set up your preferred @@ -1045,7 +1045,7 @@ kern.sysv.shmall=1024 - The file /etc/sysctl.conf is only honored in OS X + The file /etc/sysctl.conf is only honored in macOS 10.3.9 and later. If you are running a previous 10.3.x release, you must edit the file /etc/rc and change the values in the following commands: @@ -1057,12 +1057,12 @@ sysctl -w kern.sysv.shmseg sysctl -w kern.sysv.shmall Note that - /etc/rc is usually overwritten by OS X system updates, + /etc/rc is usually overwritten by macOS system updates, so you should expect to have to redo these edits after each update. - In OS X 10.2 and earlier, instead edit these commands in the file + In macOS 10.2 and earlier, instead edit these commands in the file /System/Library/StartupItems/SystemTuning/SystemTuning. diff --git a/doc/src/sgml/uuid-ossp.sgml b/doc/src/sgml/uuid-ossp.sgml index e275febe4e..227d4a839c 100644 --- a/doc/src/sgml/uuid-ossp.sgml +++ b/doc/src/sgml/uuid-ossp.sgml @@ -169,7 +169,7 @@ SELECT uuid_generate_v3(uuid_ns_url(), 'http://www.postgresql.org'); platforms. uuid-ossp can now be built without the OSSP library on some platforms. On FreeBSD, NetBSD, and some other BSD-derived platforms, suitable UUID creation functions are included in the - core libc library. On Linux, OS X, and some other + core libc library. On Linux, macOS, and some other platforms, suitable functions are provided in the libuuid library, which originally came from the e2fsprogs project (though on modern Linux it is considered part diff --git a/doc/src/sgml/wal.sgml b/doc/src/sgml/wal.sgml index 9ae6547721..fe3b588c72 100644 --- a/doc/src/sgml/wal.sgml +++ b/doc/src/sgml/wal.sgml @@ -115,7 +115,7 @@ - On OS X, write caching can be prevented by + On macOS, write caching can be prevented by setting wal_sync_method to fsync_writethrough. diff --git a/src/Makefile.shlib b/src/Makefile.shlib index 924d21f443..de93f41639 100644 --- a/src/Makefile.shlib +++ b/src/Makefile.shlib @@ -323,7 +323,7 @@ endif endif # shlib_major # Where possible, restrict the symbols exported by the library to just the -# official list, so as to avoid unintentional ABI changes. On recent Darwin +# official list, so as to avoid unintentional ABI changes. On recent macOS # this also quiets multiply-defined-symbol warnings in programs that use # libpgport along with libpq. ifneq (,$(SHLIB_EXPORTS)) diff --git a/src/backend/Makefile b/src/backend/Makefile index 3b08defe2b..4946d37929 100644 --- a/src/backend/Makefile +++ b/src/backend/Makefile @@ -26,8 +26,8 @@ include $(srcdir)/common.mk # As of 1/2010: # The probes.o file is necessary for dtrace support on Solaris, and on recent # versions of systemtap. (Older systemtap releases just produce an empty -# file, but that's okay.) However, OS X's dtrace doesn't use it and doesn't -# even recognize the -G option. So, build probes.o except on Darwin. +# file, but that's okay.) However, macOS's dtrace doesn't use it and doesn't +# even recognize the -G option. So, build probes.o except on macOS. # This might need adjustment as other platforms add dtrace support. ifneq ($(PORTNAME), darwin) ifeq ($(enable_dtrace), yes) diff --git a/src/backend/port/dynloader/darwin.c b/src/backend/port/dynloader/darwin.c index a83c614f4f..7b6b48d14a 100644 --- a/src/backend/port/dynloader/darwin.c +++ b/src/backend/port/dynloader/darwin.c @@ -1,5 +1,5 @@ /* - * Dynamic loading support for Darwin + * Dynamic loading support for macOS (Darwin) * * If dlopen() is available (Darwin 10.3 and later), we just use it. * Otherwise we emulate it with the older, now deprecated, NSLinkModule API. diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 40995580af..0c0a609735 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -1273,7 +1273,7 @@ PostmasterMain(int argc, char *argv[]) #ifdef HAVE_PTHREAD_IS_THREADED_NP /* - * On Darwin, libintl replaces setlocale() with a version that calls + * On macOS, libintl replaces setlocale() with a version that calls * CFLocaleCopyCurrent() when its second argument is "" and every relevant * environment variable is unset or empty. CFLocaleCopyCurrent() makes * the process multithreaded. The postmaster calls sigprocmask() and diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c index 582d3e460b..260a5aac49 100644 --- a/src/backend/utils/adt/varlena.c +++ b/src/backend/utils/adt/varlena.c @@ -1844,8 +1844,8 @@ varstr_sortsupport(SortSupport ssup, Oid collid, bool bpchar) * Even apart from the risk of broken locales, it's possible that there * are platforms where the use of abbreviated keys should be disabled at * compile time. Having only 4 byte datums could make worst-case - * performance drastically more likely, for example. Moreover, Darwin's - * strxfrm() implementations is known to not effectively concentrate a + * performance drastically more likely, for example. Moreover, macOS's + * strxfrm() implementation is known to not effectively concentrate a * significant amount of entropy from the original string in earlier * transformed blobs. It's possible that other supported platforms are * similarly encumbered. So, if we ever get past disabling this diff --git a/src/backend/utils/misc/ps_status.c b/src/backend/utils/misc/ps_status.c index c50be8aab6..a889b170c8 100644 --- a/src/backend/utils/misc/ps_status.c +++ b/src/backend/utils/misc/ps_status.c @@ -223,8 +223,8 @@ save_ps_display_args(int argc, char **argv) #if defined(__darwin__) /* - * Darwin (and perhaps other NeXT-derived platforms?) has a static - * copy of the argv pointer, which we may fix like so: + * macOS (and perhaps other NeXT-derived platforms?) has a static copy + * of the argv pointer, which we may fix like so: */ *_NSGetArgv() = new_argv; #endif diff --git a/src/backend/utils/probes.d b/src/backend/utils/probes.d index 976774e795..2f92dfa9ad 100644 --- a/src/backend/utils/probes.d +++ b/src/backend/utils/probes.d @@ -12,7 +12,7 @@ * Typedefs used in PostgreSQL. * * NOTE: Do not use system-provided typedefs (e.g. uintptr_t, uint32_t, etc) - * in probe definitions, as they cause compilation errors on Mac OS X 10.5. + * in probe definitions, as they cause compilation errors on macOS 10.5. */ #define LocalTransactionId unsigned int #define LWLockMode int diff --git a/src/bin/psql/input.c b/src/bin/psql/input.c index 2359b11dcd..a7d017a2d5 100644 --- a/src/bin/psql/input.c +++ b/src/bin/psql/input.c @@ -411,7 +411,7 @@ saveHistory(char *fname, int max_lines) /* * Suppressing the write attempt when HISTFILE is set to /dev/null may - * look like a negligible optimization, but it's necessary on e.g. Darwin, + * look like a negligible optimization, but it's necessary on e.g. macOS, * where write_history will fail because it tries to chmod the target * file. */ diff --git a/src/include/port/darwin.h b/src/include/port/darwin.h index 29c4b91d8c..15fb69d6db 100644 --- a/src/include/port/darwin.h +++ b/src/include/port/darwin.h @@ -2,7 +2,7 @@ #define __darwin__ 1 -#if HAVE_DECL_F_FULLFSYNC /* not present before OS X 10.3 */ +#if HAVE_DECL_F_FULLFSYNC /* not present before macOS 10.3 */ #define HAVE_FSYNC_WRITETHROUGH #endif diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index 9668b52103..f3a9e5a83f 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -1252,7 +1252,7 @@ setKeepalivesIdle(PGconn *conn) } #else #ifdef TCP_KEEPALIVE - /* Darwin uses TCP_KEEPALIVE rather than TCP_KEEPIDLE */ + /* macOS uses TCP_KEEPALIVE rather than TCP_KEEPIDLE */ if (setsockopt(conn->sock, IPPROTO_TCP, TCP_KEEPALIVE, (char *) &idle, sizeof(idle)) < 0) { diff --git a/src/port/README b/src/port/README index 58fb32d9f9..4ae96da015 100644 --- a/src/port/README +++ b/src/port/README @@ -28,5 +28,5 @@ applications. from libpgport are linked first. This avoids having applications dependent on symbols that are _used_ by libpq, but not intended to be exported by libpq. libpq's libpgport usage changes over time, so such a -dependency is a problem. Win32, Linux, and Darwin use an export list to +dependency is a problem. Windows, Linux, and macOS use an export list to control the symbols exported by libpq. diff --git a/src/port/chklocale.c b/src/port/chklocale.c index 3c0ef6a253..915821a4e9 100644 --- a/src/port/chklocale.c +++ b/src/port/chklocale.c @@ -395,7 +395,7 @@ pg_get_encoding_from_locale(const char *ctype, bool write_message) #ifdef __darwin__ /* - * Current OS X has many locales that report an empty string for CODESET, + * Current macOS has many locales that report an empty string for CODESET, * but they all seem to actually use UTF-8. */ if (strlen(sys) == 0) diff --git a/src/template/darwin b/src/template/darwin index 542f706b0f..ea6d3b0b04 100644 --- a/src/template/darwin +++ b/src/template/darwin @@ -1,9 +1,12 @@ # src/template/darwin -# Select appropriate semaphore support. Darwin 6.0 (Mac OS X 10.2) and up -# support System V semaphores; before that we have to use POSIX semaphores, -# which are less good for our purposes because they eat a file descriptor -# per backend per max_connection slot. +# Note: Darwin is the original code name for macOS, also known as OS X. +# We still use "darwin" as the port name, partly because config.guess does. + +# Select appropriate semaphore support. Darwin 6.0 (macOS 10.2) and up +# support System V semaphores; before that we have to use named POSIX +# semaphores, which are less good for our purposes because they eat a +# file descriptor per backend per max_connection slot. case $host_os in darwin[015].*) USE_NAMED_POSIX_SEMAPHORES=1 diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c index 1154d4c300..b28cb0b1e1 100644 --- a/src/test/regress/pg_regress.c +++ b/src/test/regress/pg_regress.c @@ -738,9 +738,9 @@ initialize_environment(void) /* * Most platforms have adopted the POSIX locale as their * implementation-defined default locale. Exceptions include native - * Windows, Darwin with --enable-nls, and Cygwin with --enable-nls. + * Windows, macOS with --enable-nls, and Cygwin with --enable-nls. * (Use of --enable-nls matters because libintl replaces setlocale().) - * Also, PostgreSQL does not support Darwin with locale environment + * Also, PostgreSQL does not support macOS with locale environment * variables unset; see PostmasterMain(). */ #if defined(WIN32) || defined(__CYGWIN__) || defined(__darwin__) diff --git a/src/tools/find_typedef b/src/tools/find_typedef index fee0fb5152..24e9b76651 100755 --- a/src/tools/find_typedef +++ b/src/tools/find_typedef @@ -13,7 +13,7 @@ # find both .o files and executables. Therefore, ignore error messages about # unsuitable files being fed to objdump. # -# This is known to work on Linux and on some BSDen, including Mac OS X. +# This is known to work on Linux and on some BSDen, including macOS. # # Caution: on the platforms we use, this only prints typedefs that are used # to declare at least one variable or struct field. If you have say -- cgit v1.2.3 From a4afb2b5c0b409bb175c20104b2ae9d47cf71be6 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 26 Sep 2016 11:50:35 -0400 Subject: Document has_type_privilege(). Evidently an oversight in commit 729205571. Back-patch to 9.2 where privileges for types were introduced. Report: <20160922173517.8214.88959@wrigleys.postgresql.org> --- doc/src/sgml/func.sgml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) (limited to 'doc/src') diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 3cc69bbffd..67de029c6a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -15897,6 +15897,21 @@ SET search_path TO schema , schema, .. boolean does current user have privilege for tablespace + + has_type_privilege(user, + type, + privilege) + + boolean + does user have privilege for type + + + has_type_privilege(type, + privilege) + + boolean + does current user have privilege for type + pg_has_role(user, role, @@ -15955,6 +15970,9 @@ SET search_path TO schema , schema, .. has_tablespace_privilege + + has_type_privilege + pg_has_role @@ -16109,6 +16127,18 @@ SELECT has_function_privilege('joeuser', 'myfunc(int, text)', 'execute'); CREATE. + + has_type_privilege checks whether a user + can access a type in a particular way. + Its argument possibilities + are analogous to has_table_privilege. + When specifying a type by a text string rather than by OID, + the allowed input is the same as for the regtype data type + (see ). + The desired access privilege type must evaluate to + USAGE. + + pg_has_role checks whether a user can access a role in a particular way. -- cgit v1.2.3 From fdc9186f7ed1ead827509584f3b763f8dc332c43 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 26 Sep 2016 14:52:44 -0400 Subject: Replace the built-in GIN array opclasses with a single polymorphic opclass. We had thirty different GIN array opclasses sharing the same operators and support functions. That still didn't cover all the built-in types, nor did it cover arrays of extension-added types. What we want is a single polymorphic opclass for "anyarray". There were two missing features needed to make this possible: 1. We have to be able to declare the index storage type as ANYELEMENT when the opclass is declared to index ANYARRAY. This just takes a few more lines in index_create(). Although this currently seems of use only for GIN, there's no reason to make index_create() restrict it to that. 2. We have to be able to identify the proper GIN compare function for the index storage type. This patch proceeds by making the compare function optional in GIN opclass definitions, and specifying that the default btree comparison function for the index storage type will be looked up when the opclass omits it. Again, that seems pretty generically useful. Since the comparison function lookup is done in initGinState(), making use of the second feature adds an additional cache lookup to GIN index access setup. It seems unlikely that that would be very noticeable given the other costs involved, but maybe at some point we should consider making GinState data persist longer than it now does --- we could keep it in the index relcache entry, perhaps. Rather fortuitously, we don't seem to need to do anything to get this change to play nice with dump/reload or pg_upgrade scenarios: the new opclass definition is automatically selected to replace existing index definitions, and the on-disk data remains compatible. Also, if a user has created a custom opclass definition for a non-builtin type, this doesn't break that, since CREATE INDEX will prefer an exact match to opcintype over a match to ANYARRAY. However, if there's anyone out there with handwritten DDL that explicitly specifies _bool_ops or one of the other replaced opclass names, they'll need to adjust that. Tom Lane, reviewed by Enrique Meneses Discussion: <14436.1470940379@sss.pgh.pa.us> --- doc/src/sgml/gin.sgml | 349 ++++------------------------------- doc/src/sgml/indices.sgml | 5 +- doc/src/sgml/ref/create_opclass.sgml | 5 + doc/src/sgml/xindex.sgml | 2 +- src/backend/access/gin/ginutil.c | 32 +++- src/backend/access/gin/ginvalidate.c | 2 +- src/backend/catalog/index.c | 19 +- src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_amop.h | 3 +- src/include/catalog/pg_amproc.h | 154 +--------------- src/include/catalog/pg_opclass.h | 31 +--- 11 files changed, 101 insertions(+), 503 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/gin.sgml b/doc/src/sgml/gin.sgml index 05d92eb975..7c2321ec3c 100644 --- a/doc/src/sgml/gin.sgml +++ b/doc/src/sgml/gin.sgml @@ -85,298 +85,8 @@ - _abstime_ops - abstime[] - - && - <@ - = - @> - - - - _bit_ops - bit[] - - && - <@ - = - @> - - - - _bool_ops - boolean[] - - && - <@ - = - @> - - - - _bpchar_ops - character[] - - && - <@ - = - @> - - - - _bytea_ops - bytea[] - - && - <@ - = - @> - - - - _char_ops - "char"[] - - && - <@ - = - @> - - - - _cidr_ops - cidr[] - - && - <@ - = - @> - - - - _date_ops - date[] - - && - <@ - = - @> - - - - _float4_ops - float4[] - - && - <@ - = - @> - - - - _float8_ops - float8[] - - && - <@ - = - @> - - - - _inet_ops - inet[] - - && - <@ - = - @> - - - - _int2_ops - smallint[] - - && - <@ - = - @> - - - - _int4_ops - integer[] - - && - <@ - = - @> - - - - _int8_ops - bigint[] - - && - <@ - = - @> - - - - _interval_ops - interval[] - - && - <@ - = - @> - - - - _macaddr_ops - macaddr[] - - && - <@ - = - @> - - - - _money_ops - money[] - - && - <@ - = - @> - - - - _name_ops - name[] - - && - <@ - = - @> - - - - _numeric_ops - numeric[] - - && - <@ - = - @> - - - - _oid_ops - oid[] - - && - <@ - = - @> - - - - _oidvector_ops - oidvector[] - - && - <@ - = - @> - - - - _reltime_ops - reltime[] - - && - <@ - = - @> - - - - _text_ops - text[] - - && - <@ - = - @> - - - - _time_ops - time[] - - && - <@ - = - @> - - - - _timestamp_ops - timestamp[] - - && - <@ - = - @> - - - - _timestamptz_ops - timestamp with time zone[] - - && - <@ - = - @> - - - - _timetz_ops - time with time zone[] - - && - <@ - = - @> - - - - _tinterval_ops - tinterval[] - - && - <@ - = - @> - - - - _varbit_ops - bit varying[] - - && - <@ - = - @> - - - - _varchar_ops - character varying[] + array_ops + anyarray && <@ @@ -441,22 +151,10 @@ - There are three methods that an operator class for + There are two methods that an operator class for GIN must provide: - - - int compare(Datum a, Datum b) - - - Compares two keys (not indexed items!) and returns an integer less than - zero, zero, or greater than zero, indicating whether the first key is - less than, equal to, or greater than the second. Null keys are never - passed to this function. - - - - + Datum *extractValue(Datum itemValue, int32 *nkeys, bool **nullFlags) @@ -645,7 +343,38 @@ + + + + In addition, GIN must have a way to sort the key values stored in the index. + The operator class can define the sort ordering by specifying a comparison + method: + + + int compare(Datum a, Datum b) + + + Compares two keys (not indexed items!) and returns an integer less than + zero, zero, or greater than zero, indicating whether the first key is + less than, equal to, or greater than the second. Null keys are never + passed to this function. + + + + + + Alternatively, if the operator class does not provide a compare + method, GIN will look up the default btree operator class for the index + key data type, and use its comparison function. It is recommended to + specify the comparison function in a GIN operator class that is meant for + just one data type, as looking up the btree operator class costs a few + cycles. However, polymorphic GIN operator classes (such + as array_ops) typically cannot specify a single comparison + function. + + + Optionally, an operator class for GIN can supply the following method: @@ -900,11 +629,9 @@ Examples - The PostgreSQL source distribution includes - GIN operator classes for tsvector and - for one-dimensional arrays of all internal types. Prefix searching in - tsvector is implemented using the GIN partial match - feature. + The core PostgreSQL distribution + includes the GIN operator classes previously shown in + . The following contrib modules also contain GIN operator classes: diff --git a/doc/src/sgml/indices.sgml b/doc/src/sgml/indices.sgml index 46f8e55ca9..271c135519 100644 --- a/doc/src/sgml/indices.sgml +++ b/doc/src/sgml/indices.sgml @@ -315,9 +315,8 @@ SELECT * FROM places ORDER BY location <-> point '(101,456)' LIMIT 10; operators with which a GIN index can be used vary depending on the indexing strategy. As an example, the standard distribution of - PostgreSQL includes GIN operator classes - for one-dimensional arrays, which support indexed - queries using these operators: + PostgreSQL includes a GIN operator class + for arrays, which supports indexed queries using these operators: <@ diff --git a/doc/src/sgml/ref/create_opclass.sgml b/doc/src/sgml/ref/create_opclass.sgml index 7b9d55d38d..829d8f2fff 100644 --- a/doc/src/sgml/ref/create_opclass.sgml +++ b/doc/src/sgml/ref/create_opclass.sgml @@ -235,6 +235,11 @@ CREATE OPERATOR CLASS name [ DEFAUL (currently GiST, GIN and BRIN) allow it to be different. The STORAGE clause must be omitted unless the index method allows a different type to be used. + If the column data_type is specified + as anyarray, the storage_type + can be declared as anyelement to indicate that the index + entries are members of the element type belonging to the actual array + type that each particular index is created for. diff --git a/doc/src/sgml/xindex.sgml b/doc/src/sgml/xindex.sgml index f0b711e2ce..333a36c456 100644 --- a/doc/src/sgml/xindex.sgml +++ b/doc/src/sgml/xindex.sgml @@ -288,7 +288,7 @@ have a fixed set of strategies either. Instead the support routines of each operator class interpret the strategy numbers according to the operator class's definition. As an example, the strategy numbers used by - the built-in operator classes for arrays are shown in + the built-in operator class for arrays are shown in . diff --git a/src/backend/access/gin/ginutil.c b/src/backend/access/gin/ginutil.c index d9146488c4..f07eedc0fa 100644 --- a/src/backend/access/gin/ginutil.c +++ b/src/backend/access/gin/ginutil.c @@ -22,7 +22,9 @@ #include "miscadmin.h" #include "storage/indexfsm.h" #include "storage/lmgr.h" +#include "utils/builtins.h" #include "utils/index_selfuncs.h" +#include "utils/typcache.h" /* @@ -105,9 +107,33 @@ initGinState(GinState *state, Relation index) origTupdesc->attrs[i]->attcollation); } - fmgr_info_copy(&(state->compareFn[i]), - index_getprocinfo(index, i + 1, GIN_COMPARE_PROC), - CurrentMemoryContext); + /* + * If the compare proc isn't specified in the opclass definition, look + * up the index key type's default btree comparator. + */ + if (index_getprocid(index, i + 1, GIN_COMPARE_PROC) != InvalidOid) + { + fmgr_info_copy(&(state->compareFn[i]), + index_getprocinfo(index, i + 1, GIN_COMPARE_PROC), + CurrentMemoryContext); + } + else + { + TypeCacheEntry *typentry; + + typentry = lookup_type_cache(origTupdesc->attrs[i]->atttypid, + TYPECACHE_CMP_PROC_FINFO); + if (!OidIsValid(typentry->cmp_proc_finfo.fn_oid)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FUNCTION), + errmsg("could not identify a comparison function for type %s", + format_type_be(origTupdesc->attrs[i]->atttypid)))); + fmgr_info_copy(&(state->compareFn[i]), + &(typentry->cmp_proc_finfo), + CurrentMemoryContext); + } + + /* Opclass must always provide extract procs */ fmgr_info_copy(&(state->extractValueFn[i]), index_getprocinfo(index, i + 1, GIN_EXTRACTVALUE_PROC), CurrentMemoryContext); diff --git a/src/backend/access/gin/ginvalidate.c b/src/backend/access/gin/ginvalidate.c index 032508387d..02196e0f12 100644 --- a/src/backend/access/gin/ginvalidate.c +++ b/src/backend/access/gin/ginvalidate.c @@ -237,7 +237,7 @@ ginvalidate(Oid opclassoid) if (opclassgroup && (opclassgroup->functionset & (((uint64) 1) << i)) != 0) continue; /* got it */ - if (i == GIN_COMPARE_PARTIAL_PROC) + if (i == GIN_COMPARE_PROC || i == GIN_COMPARE_PARTIAL_PROC) continue; /* optional method */ if (i == GIN_CONSISTENT_PROC || i == GIN_TRICONSISTENT_PROC) continue; /* don't need both, see check below loop */ diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index b0b43cf02d..08b646d8f3 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -437,11 +437,28 @@ ConstructTupleDescriptor(Relation heapRelation, keyType = opclassTup->opckeytype; else keyType = amroutine->amkeytype; + + /* + * If keytype is specified as ANYELEMENT, and opcintype is ANYARRAY, + * then the attribute type must be an array (else it'd not have + * matched this opclass); use its element type. + */ + if (keyType == ANYELEMENTOID && opclassTup->opcintype == ANYARRAYOID) + { + keyType = get_base_element_type(to->atttypid); + if (!OidIsValid(keyType)) + elog(ERROR, "could not get element type of array type %u", + to->atttypid); + } + ReleaseSysCache(tuple); + /* + * If a key type different from the heap value is specified, update + * the type-related fields in the index tupdesc. + */ if (OidIsValid(keyType) && keyType != to->atttypid) { - /* index value and heap value have different types */ tuple = SearchSysCache1(TYPEOID, ObjectIdGetDatum(keyType)); if (!HeapTupleIsValid(tuple)) elog(ERROR, "cache lookup failed for type %u", keyType); diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index ef691c5721..3fdd0d6129 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201609131 +#define CATALOG_VERSION_NO 201609261 #endif diff --git a/src/include/catalog/pg_amop.h b/src/include/catalog/pg_amop.h index 917ed46b71..15b629029f 100644 --- a/src/include/catalog/pg_amop.h +++ b/src/include/catalog/pg_amop.h @@ -673,8 +673,7 @@ DATA(insert ( 2595 718 718 14 s 2864 783 0 )); DATA(insert ( 2595 718 600 15 o 3291 783 1970 )); /* - * gin array_ops (these anyarray operators are used with all the opclasses - * of the family) + * gin array_ops */ DATA(insert ( 2745 2277 2277 1 s 2750 2742 0 )); DATA(insert ( 2745 2277 2277 2 s 2751 2742 0 )); diff --git a/src/include/catalog/pg_amproc.h b/src/include/catalog/pg_amproc.h index 0cbb416392..1b654d5be4 100644 --- a/src/include/catalog/pg_amproc.h +++ b/src/include/catalog/pg_amproc.h @@ -255,156 +255,10 @@ DATA(insert ( 3550 869 869 9 3573 )); /* gin */ -DATA(insert ( 2745 1007 1007 1 351 )); -DATA(insert ( 2745 1007 1007 2 2743 )); -DATA(insert ( 2745 1007 1007 3 2774 )); -DATA(insert ( 2745 1007 1007 4 2744 )); -DATA(insert ( 2745 1007 1007 6 3920 )); -DATA(insert ( 2745 1009 1009 1 360 )); -DATA(insert ( 2745 1009 1009 2 2743 )); -DATA(insert ( 2745 1009 1009 3 2774 )); -DATA(insert ( 2745 1009 1009 4 2744 )); -DATA(insert ( 2745 1009 1009 6 3920 )); -DATA(insert ( 2745 1015 1015 1 360 )); -DATA(insert ( 2745 1015 1015 2 2743 )); -DATA(insert ( 2745 1015 1015 3 2774 )); -DATA(insert ( 2745 1015 1015 4 2744 )); -DATA(insert ( 2745 1015 1015 6 3920 )); -DATA(insert ( 2745 1023 1023 1 357 )); -DATA(insert ( 2745 1023 1023 2 2743 )); -DATA(insert ( 2745 1023 1023 3 2774 )); -DATA(insert ( 2745 1023 1023 4 2744 )); -DATA(insert ( 2745 1023 1023 6 3920 )); -DATA(insert ( 2745 1561 1561 1 1596 )); -DATA(insert ( 2745 1561 1561 2 2743 )); -DATA(insert ( 2745 1561 1561 3 2774 )); -DATA(insert ( 2745 1561 1561 4 2744 )); -DATA(insert ( 2745 1561 1561 6 3920 )); -DATA(insert ( 2745 1000 1000 1 1693 )); -DATA(insert ( 2745 1000 1000 2 2743 )); -DATA(insert ( 2745 1000 1000 3 2774 )); -DATA(insert ( 2745 1000 1000 4 2744 )); -DATA(insert ( 2745 1000 1000 6 3920 )); -DATA(insert ( 2745 1014 1014 1 1078 )); -DATA(insert ( 2745 1014 1014 2 2743 )); -DATA(insert ( 2745 1014 1014 3 2774 )); -DATA(insert ( 2745 1014 1014 4 2744 )); -DATA(insert ( 2745 1014 1014 6 3920 )); -DATA(insert ( 2745 1001 1001 1 1954 )); -DATA(insert ( 2745 1001 1001 2 2743 )); -DATA(insert ( 2745 1001 1001 3 2774 )); -DATA(insert ( 2745 1001 1001 4 2744 )); -DATA(insert ( 2745 1001 1001 6 3920 )); -DATA(insert ( 2745 1002 1002 1 358 )); -DATA(insert ( 2745 1002 1002 2 2743 )); -DATA(insert ( 2745 1002 1002 3 2774 )); -DATA(insert ( 2745 1002 1002 4 2744 )); -DATA(insert ( 2745 1002 1002 6 3920 )); -DATA(insert ( 2745 1182 1182 1 1092 )); -DATA(insert ( 2745 1182 1182 2 2743 )); -DATA(insert ( 2745 1182 1182 3 2774 )); -DATA(insert ( 2745 1182 1182 4 2744 )); -DATA(insert ( 2745 1182 1182 6 3920 )); -DATA(insert ( 2745 1021 1021 1 354 )); -DATA(insert ( 2745 1021 1021 2 2743 )); -DATA(insert ( 2745 1021 1021 3 2774 )); -DATA(insert ( 2745 1021 1021 4 2744 )); -DATA(insert ( 2745 1021 1021 6 3920 )); -DATA(insert ( 2745 1022 1022 1 355 )); -DATA(insert ( 2745 1022 1022 2 2743 )); -DATA(insert ( 2745 1022 1022 3 2774 )); -DATA(insert ( 2745 1022 1022 4 2744 )); -DATA(insert ( 2745 1022 1022 6 3920 )); -DATA(insert ( 2745 1041 1041 1 926 )); -DATA(insert ( 2745 1041 1041 2 2743 )); -DATA(insert ( 2745 1041 1041 3 2774 )); -DATA(insert ( 2745 1041 1041 4 2744 )); -DATA(insert ( 2745 1041 1041 6 3920 )); -DATA(insert ( 2745 651 651 1 926 )); -DATA(insert ( 2745 651 651 2 2743 )); -DATA(insert ( 2745 651 651 3 2774 )); -DATA(insert ( 2745 651 651 4 2744 )); -DATA(insert ( 2745 651 651 6 3920 )); -DATA(insert ( 2745 1005 1005 1 350 )); -DATA(insert ( 2745 1005 1005 2 2743 )); -DATA(insert ( 2745 1005 1005 3 2774 )); -DATA(insert ( 2745 1005 1005 4 2744 )); -DATA(insert ( 2745 1005 1005 6 3920 )); -DATA(insert ( 2745 1016 1016 1 842 )); -DATA(insert ( 2745 1016 1016 2 2743 )); -DATA(insert ( 2745 1016 1016 3 2774 )); -DATA(insert ( 2745 1016 1016 4 2744 )); -DATA(insert ( 2745 1016 1016 6 3920 )); -DATA(insert ( 2745 1187 1187 1 1315 )); -DATA(insert ( 2745 1187 1187 2 2743 )); -DATA(insert ( 2745 1187 1187 3 2774 )); -DATA(insert ( 2745 1187 1187 4 2744 )); -DATA(insert ( 2745 1187 1187 6 3920 )); -DATA(insert ( 2745 1040 1040 1 836 )); -DATA(insert ( 2745 1040 1040 2 2743 )); -DATA(insert ( 2745 1040 1040 3 2774 )); -DATA(insert ( 2745 1040 1040 4 2744 )); -DATA(insert ( 2745 1040 1040 6 3920 )); -DATA(insert ( 2745 1003 1003 1 359 )); -DATA(insert ( 2745 1003 1003 2 2743 )); -DATA(insert ( 2745 1003 1003 3 2774 )); -DATA(insert ( 2745 1003 1003 4 2744 )); -DATA(insert ( 2745 1003 1003 6 3920 )); -DATA(insert ( 2745 1231 1231 1 1769 )); -DATA(insert ( 2745 1231 1231 2 2743 )); -DATA(insert ( 2745 1231 1231 3 2774 )); -DATA(insert ( 2745 1231 1231 4 2744 )); -DATA(insert ( 2745 1231 1231 6 3920 )); -DATA(insert ( 2745 1028 1028 1 356 )); -DATA(insert ( 2745 1028 1028 2 2743 )); -DATA(insert ( 2745 1028 1028 3 2774 )); -DATA(insert ( 2745 1028 1028 4 2744 )); -DATA(insert ( 2745 1028 1028 6 3920 )); -DATA(insert ( 2745 1013 1013 1 404 )); -DATA(insert ( 2745 1013 1013 2 2743 )); -DATA(insert ( 2745 1013 1013 3 2774 )); -DATA(insert ( 2745 1013 1013 4 2744 )); -DATA(insert ( 2745 1013 1013 6 3920 )); -DATA(insert ( 2745 1183 1183 1 1107 )); -DATA(insert ( 2745 1183 1183 2 2743 )); -DATA(insert ( 2745 1183 1183 3 2774 )); -DATA(insert ( 2745 1183 1183 4 2744 )); -DATA(insert ( 2745 1183 1183 6 3920 )); -DATA(insert ( 2745 1185 1185 1 1314 )); -DATA(insert ( 2745 1185 1185 2 2743 )); -DATA(insert ( 2745 1185 1185 3 2774 )); -DATA(insert ( 2745 1185 1185 4 2744 )); -DATA(insert ( 2745 1185 1185 6 3920 )); -DATA(insert ( 2745 1270 1270 1 1358 )); -DATA(insert ( 2745 1270 1270 2 2743 )); -DATA(insert ( 2745 1270 1270 3 2774 )); -DATA(insert ( 2745 1270 1270 4 2744 )); -DATA(insert ( 2745 1270 1270 6 3920 )); -DATA(insert ( 2745 1563 1563 1 1672 )); -DATA(insert ( 2745 1563 1563 2 2743 )); -DATA(insert ( 2745 1563 1563 3 2774 )); -DATA(insert ( 2745 1563 1563 4 2744 )); -DATA(insert ( 2745 1563 1563 6 3920 )); -DATA(insert ( 2745 1115 1115 1 2045 )); -DATA(insert ( 2745 1115 1115 2 2743 )); -DATA(insert ( 2745 1115 1115 3 2774 )); -DATA(insert ( 2745 1115 1115 4 2744 )); -DATA(insert ( 2745 1115 1115 6 3920 )); -DATA(insert ( 2745 791 791 1 377 )); -DATA(insert ( 2745 791 791 2 2743 )); -DATA(insert ( 2745 791 791 3 2774 )); -DATA(insert ( 2745 791 791 4 2744 )); -DATA(insert ( 2745 791 791 6 3920 )); -DATA(insert ( 2745 1024 1024 1 380 )); -DATA(insert ( 2745 1024 1024 2 2743 )); -DATA(insert ( 2745 1024 1024 3 2774 )); -DATA(insert ( 2745 1024 1024 4 2744 )); -DATA(insert ( 2745 1024 1024 6 3920 )); -DATA(insert ( 2745 1025 1025 1 381 )); -DATA(insert ( 2745 1025 1025 2 2743 )); -DATA(insert ( 2745 1025 1025 3 2774 )); -DATA(insert ( 2745 1025 1025 4 2744 )); -DATA(insert ( 2745 1025 1025 6 3920 )); +DATA(insert ( 2745 2277 2277 2 2743 )); +DATA(insert ( 2745 2277 2277 3 2774 )); +DATA(insert ( 2745 2277 2277 4 2744 )); +DATA(insert ( 2745 2277 2277 6 3920 )); DATA(insert ( 3659 3614 3614 1 3724 )); DATA(insert ( 3659 3614 3614 2 3656 )); DATA(insert ( 3659 3614 3614 3 3657 )); diff --git a/src/include/catalog/pg_opclass.h b/src/include/catalog/pg_opclass.h index f40b06112b..5900cdc5b0 100644 --- a/src/include/catalog/pg_opclass.h +++ b/src/include/catalog/pg_opclass.h @@ -184,36 +184,7 @@ DATA(insert ( 783 box_ops PGNSP PGUID 2593 603 t 0 )); DATA(insert ( 783 point_ops PGNSP PGUID 1029 600 t 603 )); DATA(insert ( 783 poly_ops PGNSP PGUID 2594 604 t 603 )); DATA(insert ( 783 circle_ops PGNSP PGUID 2595 718 t 603 )); -DATA(insert ( 2742 _int4_ops PGNSP PGUID 2745 1007 t 23 )); -DATA(insert ( 2742 _text_ops PGNSP PGUID 2745 1009 t 25 )); -DATA(insert ( 2742 _abstime_ops PGNSP PGUID 2745 1023 t 702 )); -DATA(insert ( 2742 _bit_ops PGNSP PGUID 2745 1561 t 1560 )); -DATA(insert ( 2742 _bool_ops PGNSP PGUID 2745 1000 t 16 )); -DATA(insert ( 2742 _bpchar_ops PGNSP PGUID 2745 1014 t 1042 )); -DATA(insert ( 2742 _bytea_ops PGNSP PGUID 2745 1001 t 17 )); -DATA(insert ( 2742 _char_ops PGNSP PGUID 2745 1002 t 18 )); -DATA(insert ( 2742 _cidr_ops PGNSP PGUID 2745 651 t 650 )); -DATA(insert ( 2742 _date_ops PGNSP PGUID 2745 1182 t 1082 )); -DATA(insert ( 2742 _float4_ops PGNSP PGUID 2745 1021 t 700 )); -DATA(insert ( 2742 _float8_ops PGNSP PGUID 2745 1022 t 701 )); -DATA(insert ( 2742 _inet_ops PGNSP PGUID 2745 1041 t 869 )); -DATA(insert ( 2742 _int2_ops PGNSP PGUID 2745 1005 t 21 )); -DATA(insert ( 2742 _int8_ops PGNSP PGUID 2745 1016 t 20 )); -DATA(insert ( 2742 _interval_ops PGNSP PGUID 2745 1187 t 1186 )); -DATA(insert ( 2742 _macaddr_ops PGNSP PGUID 2745 1040 t 829 )); -DATA(insert ( 2742 _name_ops PGNSP PGUID 2745 1003 t 19 )); -DATA(insert ( 2742 _numeric_ops PGNSP PGUID 2745 1231 t 1700 )); -DATA(insert ( 2742 _oid_ops PGNSP PGUID 2745 1028 t 26 )); -DATA(insert ( 2742 _oidvector_ops PGNSP PGUID 2745 1013 t 30 )); -DATA(insert ( 2742 _time_ops PGNSP PGUID 2745 1183 t 1083 )); -DATA(insert ( 2742 _timestamptz_ops PGNSP PGUID 2745 1185 t 1184 )); -DATA(insert ( 2742 _timetz_ops PGNSP PGUID 2745 1270 t 1266 )); -DATA(insert ( 2742 _varbit_ops PGNSP PGUID 2745 1563 t 1562 )); -DATA(insert ( 2742 _varchar_ops PGNSP PGUID 2745 1015 t 1043 )); -DATA(insert ( 2742 _timestamp_ops PGNSP PGUID 2745 1115 t 1114 )); -DATA(insert ( 2742 _money_ops PGNSP PGUID 2745 791 t 790 )); -DATA(insert ( 2742 _reltime_ops PGNSP PGUID 2745 1024 t 703 )); -DATA(insert ( 2742 _tinterval_ops PGNSP PGUID 2745 1025 t 704 )); +DATA(insert ( 2742 array_ops PGNSP PGUID 2745 2277 t 2283 )); DATA(insert ( 403 uuid_ops PGNSP PGUID 2968 2950 t 0 )); DATA(insert ( 405 uuid_ops PGNSP PGUID 2969 2950 t 0 )); DATA(insert ( 403 pg_lsn_ops PGNSP PGUID 3253 3220 t 0 )); -- cgit v1.2.3 From babe05bc2b781eb3eb84a18d7010d08277e2e399 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 28 Sep 2016 12:22:44 +0300 Subject: Turn password_encryption GUC into an enum. This makes the parameter easier to extend, to support other password-based authentication protocols than MD5. (SCRAM is being worked on.) The GUC still accepts on/off as aliases for "md5" and "plain", although we may want to remove those once we actually add support for another password hash type. Michael Paquier, reviewed by David Steele, with some further edits by me. Discussion: --- doc/src/sgml/config.sgml | 17 ++++++----- src/backend/commands/user.c | 18 +++++------ src/backend/utils/misc/guc.c | 44 +++++++++++++++++++-------- src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/commands/user.h | 15 +++++++-- 5 files changed, 62 insertions(+), 34 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index a848a7edd1..e826c19698 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -1163,21 +1163,22 @@ include_dir 'conf.d' - password_encryption (boolean) + password_encryption (enum) password_encryption configuration parameter - When a password is specified in or - - without writing either ENCRYPTED or - UNENCRYPTED, this parameter determines whether the - password is to be encrypted. The default is on - (encrypt the password). + When a password is specified in or + without writing either ENCRYPTED + or UNENCRYPTED, this parameter determines whether the + password is to be encrypted. The default value is md5, which + stores the password as an MD5 hash. Setting this to plain stores + it in plaintext. on and off are also accepted, as + aliases for md5 and plain, respectively. + diff --git a/src/backend/commands/user.c b/src/backend/commands/user.c index 4027c89b14..adc6b99b21 100644 --- a/src/backend/commands/user.c +++ b/src/backend/commands/user.c @@ -44,7 +44,7 @@ Oid binary_upgrade_next_pg_authid_oid = InvalidOid; /* GUC parameter */ -extern bool Password_encryption; +int Password_encryption = PASSWORD_TYPE_MD5; /* Hook to check passwords in CreateRole() and AlterRole() */ check_password_hook_type check_password_hook = NULL; @@ -80,7 +80,7 @@ CreateRole(ParseState *pstate, CreateRoleStmt *stmt) ListCell *item; ListCell *option; char *password = NULL; /* user password */ - bool encrypt_password = Password_encryption; /* encrypt password? */ + int password_type = Password_encryption; char encrypted_password[MD5_PASSWD_LEN + 1]; bool issuper = false; /* Make the user a superuser? */ bool inherit = true; /* Auto inherit privileges? */ @@ -140,9 +140,9 @@ CreateRole(ParseState *pstate, CreateRoleStmt *stmt) parser_errposition(pstate, defel->location))); dpassword = defel; if (strcmp(defel->defname, "encryptedPassword") == 0) - encrypt_password = true; + password_type = PASSWORD_TYPE_MD5; else if (strcmp(defel->defname, "unencryptedPassword") == 0) - encrypt_password = false; + password_type = PASSWORD_TYPE_PLAINTEXT; } else if (strcmp(defel->defname, "sysid") == 0) { @@ -393,7 +393,7 @@ CreateRole(ParseState *pstate, CreateRoleStmt *stmt) if (password) { - if (!encrypt_password || isMD5(password)) + if (password_type == PASSWORD_TYPE_PLAINTEXT || isMD5(password)) new_record[Anum_pg_authid_rolpassword - 1] = CStringGetTextDatum(password); else @@ -505,7 +505,7 @@ AlterRole(AlterRoleStmt *stmt) ListCell *option; char *rolename = NULL; char *password = NULL; /* user password */ - bool encrypt_password = Password_encryption; /* encrypt password? */ + int password_type = Password_encryption; char encrypted_password[MD5_PASSWD_LEN + 1]; int issuper = -1; /* Make the user a superuser? */ int inherit = -1; /* Auto inherit privileges? */ @@ -550,9 +550,9 @@ AlterRole(AlterRoleStmt *stmt) errmsg("conflicting or redundant options"))); dpassword = defel; if (strcmp(defel->defname, "encryptedPassword") == 0) - encrypt_password = true; + password_type = PASSWORD_TYPE_MD5; else if (strcmp(defel->defname, "unencryptedPassword") == 0) - encrypt_password = false; + password_type = PASSWORD_TYPE_PLAINTEXT; } else if (strcmp(defel->defname, "superuser") == 0) { @@ -804,7 +804,7 @@ AlterRole(AlterRoleStmt *stmt) /* password */ if (password) { - if (!encrypt_password || isMD5(password)) + if (password_type == PASSWORD_TYPE_PLAINTEXT || isMD5(password)) new_record[Anum_pg_authid_rolpassword - 1] = CStringGetTextDatum(password); else diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index ce4eef950b..cced814d6a 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -34,6 +34,7 @@ #include "catalog/namespace.h" #include "commands/async.h" #include "commands/prepare.h" +#include "commands/user.h" #include "commands/vacuum.h" #include "commands/variable.h" #include "commands/trigger.h" @@ -393,6 +394,24 @@ static const struct config_enum_entry force_parallel_mode_options[] = { {NULL, 0, false} }; +/* + * password_encryption used to be a boolean, so accept all the likely + * variants of "on" and "off", too. + */ +static const struct config_enum_entry password_encryption_options[] = { + {"plain", PASSWORD_TYPE_PLAINTEXT, false}, + {"md5", PASSWORD_TYPE_MD5, false}, + {"off", PASSWORD_TYPE_PLAINTEXT, false}, + {"on", PASSWORD_TYPE_MD5, false}, + {"true", PASSWORD_TYPE_MD5, true}, + {"false", PASSWORD_TYPE_PLAINTEXT, true}, + {"yes", PASSWORD_TYPE_MD5, true}, + {"no", PASSWORD_TYPE_PLAINTEXT, true}, + {"1", PASSWORD_TYPE_MD5, true}, + {"0", PASSWORD_TYPE_PLAINTEXT, true}, + {NULL, 0, false} +}; + /* * Options for enum values stored in other modules */ @@ -423,8 +442,6 @@ bool check_function_bodies = true; bool default_with_oids = false; bool SQL_inheritance = true; -bool Password_encryption = true; - int log_min_error_statement = ERROR; int log_min_messages = WARNING; int client_min_messages = NOTICE; @@ -1313,17 +1330,6 @@ static struct config_bool ConfigureNamesBool[] = true, NULL, NULL, NULL }, - { - {"password_encryption", PGC_USERSET, CONN_AUTH_SECURITY, - gettext_noop("Encrypt passwords."), - gettext_noop("When a password is specified in CREATE USER or " - "ALTER USER without writing either ENCRYPTED or UNENCRYPTED, " - "this parameter determines whether the password is to be encrypted.") - }, - &Password_encryption, - true, - NULL, NULL, NULL - }, { {"transform_null_equals", PGC_USERSET, COMPAT_OPTIONS_CLIENT, gettext_noop("Treats \"expr=NULL\" as \"expr IS NULL\"."), @@ -3810,6 +3816,18 @@ static struct config_enum ConfigureNamesEnum[] = NULL, NULL, NULL }, + { + {"password_encryption", PGC_USERSET, CONN_AUTH_SECURITY, + gettext_noop("Encrypt passwords."), + gettext_noop("When a password is specified in CREATE USER or " + "ALTER USER without writing either ENCRYPTED or UNENCRYPTED, " + "this parameter determines whether the password is to be encrypted.") + }, + &Password_encryption, + PASSWORD_TYPE_MD5, password_encryption_options, + NULL, NULL, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, 0, NULL, NULL, NULL, NULL diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index b1c3aea9ee..05b1373594 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -85,7 +85,7 @@ #ssl_key_file = 'server.key' # (change requires restart) #ssl_ca_file = '' # (change requires restart) #ssl_crl_file = '' # (change requires restart) -#password_encryption = on +#password_encryption = md5 # md5 or plain #db_user_namespace = off #row_security = on diff --git a/src/include/commands/user.h b/src/include/commands/user.h index 1f0cfcc86f..102c2a5861 100644 --- a/src/include/commands/user.h +++ b/src/include/commands/user.h @@ -16,10 +16,19 @@ #include "parser/parse_node.h" -/* Hook to check passwords in CreateRole() and AlterRole() */ -#define PASSWORD_TYPE_PLAINTEXT 0 -#define PASSWORD_TYPE_MD5 1 +/* + * Types of password, for Password_encryption GUC and the password_type + * argument of the check-password hook. + */ +typedef enum PasswordType +{ + PASSWORD_TYPE_PLAINTEXT = 0, + PASSWORD_TYPE_MD5 +} PasswordType; +extern int Password_encryption; /* GUC */ + +/* Hook to check passwords in CreateRole() and AlterRole() */ typedef void (*check_password_hook_type) (const char *username, const char *password, int password_type, Datum validuntil_time, bool validuntil_null); extern PGDLLIMPORT check_password_hook_type check_password_hook; -- cgit v1.2.3 From d3cd36a133d96ad5578b6c10279b55fd5b538093 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 28 Sep 2016 14:36:04 -0400 Subject: Make to_timestamp() and to_date() range-check fields of their input. Historically, something like to_date('2009-06-40','YYYY-MM-DD') would return '2009-07-10' because there was no prohibition on out-of-range month or day numbers. This has been widely panned, and it also turns out that Oracle throws an error in such cases. Since these functions are nominally Oracle-compatibility features, let's change that. There's no particular restriction on year (modulo the fact that the scanner may not believe that more than 4 digits are year digits, a matter to be addressed separately if at all). But we now check month, day, hour, minute, second, and fractional-second fields, as well as day-of-year and second-of-day fields if those are used. Currently, no checks are made on ISO-8601-style week numbers or day numbers; it's not very clear what the appropriate rules would be there, and they're probably so little used that it's not worth sweating over. Artur Zakirov, reviewed by Amul Sul, further adjustments by me Discussion: <1873520224.1784572.1465833145330.JavaMail.yahoo@mail.yahoo.com> See-Also: <57786490.9010201@wars-nicht.de> --- doc/src/sgml/func.sgml | 85 +++++++++++++++++------------- src/backend/utils/adt/formatting.c | 94 +++++++++++++++++++++++---------- src/test/regress/expected/horology.out | 96 +++++++++++++++++++++++++++++++++- src/test/regress/sql/horology.sql | 30 ++++++++++- 4 files changed, 239 insertions(+), 66 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 67de029c6a..a58835082b 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -5832,6 +5832,17 @@ SELECT regexp_match('abc01234xyz', '(?:(.*?)(\d+)(.*)){1,1}'); + + + to_timestamp and to_date + exist to handle input formats that cannot be converted by + simple casting. For most standard date/time formats, simply casting the + source string to the required data type works, and is much easier. + Similarly, to_number is unnecessary for standard numeric + representations. + + + In a to_char output template string, there are certain patterns that are recognized and replaced with appropriately-formatted @@ -6038,7 +6049,7 @@ SELECT regexp_match('abc01234xyz', '(?:(.*?)(\d+)(.*)){1,1}'); Q - quarter (ignored by to_date and to_timestamp) + quarter RM @@ -6156,20 +6167,6 @@ SELECT regexp_match('abc01234xyz', '(?:(.*?)(\d+)(.*)){1,1}'); - - - to_timestamp and to_date - exist to handle input formats that cannot be converted by - simple casting. These functions interpret input liberally, - with minimal error checking. While they produce valid output, - the conversion can yield unexpected results. For example, - input to these functions is not restricted by normal ranges, - thus to_date('20096040','YYYYMMDD') returns - 2014-01-17 rather than causing an error. - Casting does not have this behavior. - - - Ordinary text is allowed in to_char @@ -6195,7 +6192,8 @@ SELECT regexp_match('abc01234xyz', '(?:(.*?)(\d+)(.*)){1,1}'); - If the year format specification is less than four digits, e.g. + In to_timestamp and to_date, + if the year format specification is less than four digits, e.g. YYY, and the supplied year is less than four digits, the year will be adjusted to be nearest to the year 2020, e.g. 95 becomes 1995. @@ -6204,8 +6202,9 @@ SELECT regexp_match('abc01234xyz', '(?:(.*?)(\d+)(.*)){1,1}'); - The YYYY conversion from string to timestamp or - date has a restriction when processing years with more than 4 digits. You must + In to_timestamp and to_date, + the YYYY conversion has a restriction when + processing years with more than 4 digits. You must use some non-digit character or template after YYYY, otherwise the year is always interpreted as 4 digits. For example (with the year 20000): @@ -6219,12 +6218,12 @@ SELECT regexp_match('abc01234xyz', '(?:(.*?)(\d+)(.*)){1,1}'); - In conversions from string to timestamp or - date, the CC (century) field is ignored + In to_timestamp and to_date, + the CC (century) field is accepted but ignored if there is a YYY, YYYY or Y,YYY field. If CC is used with - YY or Y then the year is computed - as the year in the specified century. If the century is + YY or Y then the result is + computed as that year in the specified century. If the century is specified but the year is not, the first year of the century is assumed. @@ -6232,9 +6231,19 @@ SELECT regexp_match('abc01234xyz', '(?:(.*?)(\d+)(.*)){1,1}'); - An ISO 8601 week-numbering date (as distinct from a Gregorian date) - can be specified to to_timestamp and - to_date in one of two ways: + In to_timestamp and to_date, + weekday names or numbers (DAY, D, + and related field types) are accepted but are ignored for purposes of + computing the result. The same is true for quarter + (Q) fields. + + + + + + In to_timestamp and to_date, + an ISO 8601 week-numbering date (as distinct from a Gregorian date) + can be specified in one of two ways: @@ -6276,23 +6285,24 @@ SELECT regexp_match('abc01234xyz', '(?:(.*?)(\d+)(.*)){1,1}'); - In a conversion from string to timestamp, millisecond + In to_timestamp, millisecond (MS) or microsecond (US) - values are used as the + fields are used as the seconds digits after the decimal point. For example - to_timestamp('12:3', 'SS:MS') is not 3 milliseconds, - but 300, because the conversion counts it as 12 + 0.3 seconds. - This means for the format SS:MS, the input values - 12:3, 12:30, and 12:300 specify the - same number of milliseconds. To get three milliseconds, one must use - 12:003, which the conversion counts as + to_timestamp('12.3', 'SS.MS') is not 3 milliseconds, + but 300, because the conversion treats it as 12 + 0.3 seconds. + So, for the format SS.MS, the input values + 12.3, 12.30, + and 12.300 specify the + same number of milliseconds. To get three milliseconds, one must write + 12.003, which the conversion treats as 12 + 0.003 = 12.003 seconds. Here is a more complex example: - to_timestamp('15:12:02.020.001230', 'HH:MI:SS.MS.US') + to_timestamp('15:12:02.020.001230', 'HH24:MI:SS.MS.US') is 15 hours, 12 minutes, and 2 seconds + 20 milliseconds + 1230 microseconds = 2.021230 seconds. @@ -6310,9 +6320,10 @@ SELECT regexp_match('abc01234xyz', '(?:(.*?)(\d+)(.*)){1,1}'); to_char(interval) formats HH and - HH12 as shown on a 12-hour clock, i.e. zero hours - and 36 hours output as 12, while HH24 - outputs the full hour value, which can exceed 23 for intervals. + HH12 as shown on a 12-hour clock, for example zero hours + and 36 hours both output as 12, while HH24 + outputs the full hour value, which can exceed 23 in + an interval value. diff --git a/src/backend/utils/adt/formatting.c b/src/backend/utils/adt/formatting.c index bbd97dc84b..d2d23d31ff 100644 --- a/src/backend/utils/adt/formatting.c +++ b/src/backend/utils/adt/formatting.c @@ -3553,9 +3553,6 @@ to_date(PG_FUNCTION_ARGS) * * The TmFromChar is then analysed and converted into the final results in * struct 'tm' and 'fsec'. - * - * This function does very little error checking, e.g. - * to_timestamp('20096040','YYYYMMDD') works */ static void do_to_timestamp(text *date_txt, text *fmt, @@ -3564,30 +3561,35 @@ do_to_timestamp(text *date_txt, text *fmt, FormatNode *format; TmFromChar tmfc; int fmt_len; + char *date_str; + int fmask; + + date_str = text_to_cstring(date_txt); ZERO_tmfc(&tmfc); ZERO_tm(tm); *fsec = 0; + fmask = 0; /* bit mask for ValidateDate() */ fmt_len = VARSIZE_ANY_EXHDR(fmt); if (fmt_len) { char *fmt_str; - char *date_str; bool incache; fmt_str = text_to_cstring(fmt); - /* - * Allocate new memory if format picture is bigger than static cache - * and not use cache (call parser always) - */ if (fmt_len > DCH_CACHE_SIZE) { - format = (FormatNode *) palloc((fmt_len + 1) * sizeof(FormatNode)); + /* + * Allocate new memory if format picture is bigger than static + * cache and not use cache (call parser always) + */ incache = FALSE; + format = (FormatNode *) palloc((fmt_len + 1) * sizeof(FormatNode)); + parse_format(format, fmt_str, DCH_keywords, DCH_suff, DCH_index, DCH_TYPE, NULL); @@ -3604,33 +3606,27 @@ do_to_timestamp(text *date_txt, text *fmt, if ((ent = DCH_cache_search(fmt_str)) == NULL) { - ent = DCH_cache_getnew(fmt_str); - /* * Not in the cache, must run parser and save a new * format-picture to the cache. */ + ent = DCH_cache_getnew(fmt_str); + parse_format(ent->format, fmt_str, DCH_keywords, DCH_suff, DCH_index, DCH_TYPE, NULL); (ent->format + fmt_len)->type = NODE_TYPE_END; /* Paranoia? */ -#ifdef DEBUG_TO_FROM_CHAR - /* dump_node(ent->format, fmt_len); */ - /* dump_index(DCH_keywords, DCH_index); */ -#endif } format = ent->format; } #ifdef DEBUG_TO_FROM_CHAR /* dump_node(format, fmt_len); */ + /* dump_index(DCH_keywords, DCH_index); */ #endif - date_str = text_to_cstring(date_txt); - DCH_from_char(format, date_str, &tmfc); - pfree(date_str); pfree(fmt_str); if (!incache) pfree(format); @@ -3639,8 +3635,7 @@ do_to_timestamp(text *date_txt, text *fmt, DEBUG_TMFC(&tmfc); /* - * Convert values that user define for FROM_CHAR (to_date/to_timestamp) to - * standard 'tm' + * Convert to_date/to_timestamp input fields to standard 'tm' */ if (tmfc.ssss) { @@ -3696,19 +3691,23 @@ do_to_timestamp(text *date_txt, text *fmt, tm->tm_year = (tmfc.cc + 1) * 100 - tm->tm_year + 1; } else + { /* find century year for dates ending in "00" */ tm->tm_year = tmfc.cc * 100 + ((tmfc.cc >= 0) ? 0 : 1); + } } else - /* If a 4-digit year is provided, we use that and ignore CC. */ { + /* If a 4-digit year is provided, we use that and ignore CC. */ tm->tm_year = tmfc.year; if (tmfc.bc && tm->tm_year > 0) tm->tm_year = -(tm->tm_year - 1); } + fmask |= DTK_M(YEAR); } - else if (tmfc.cc) /* use first year of century */ + else if (tmfc.cc) { + /* use first year of century */ if (tmfc.bc) tmfc.cc = -tmfc.cc; if (tmfc.cc >= 0) @@ -3717,10 +3716,14 @@ do_to_timestamp(text *date_txt, text *fmt, else /* +1 because year == 599 is 600 BC */ tm->tm_year = tmfc.cc * 100 + 1; + fmask |= DTK_M(YEAR); } if (tmfc.j) + { j2date(tmfc.j, &tm->tm_year, &tm->tm_mon, &tm->tm_mday); + fmask |= DTK_DATE_M; + } if (tmfc.ww) { @@ -3734,6 +3737,7 @@ do_to_timestamp(text *date_txt, text *fmt, isoweekdate2date(tmfc.ww, tmfc.d, &tm->tm_year, &tm->tm_mon, &tm->tm_mday); else isoweek2date(tmfc.ww, &tm->tm_year, &tm->tm_mon, &tm->tm_mday); + fmask |= DTK_DATE_M; } else tmfc.ddd = (tmfc.ww - 1) * 7 + 1; @@ -3741,14 +3745,16 @@ do_to_timestamp(text *date_txt, text *fmt, if (tmfc.w) tmfc.dd = (tmfc.w - 1) * 7 + 1; - if (tmfc.d) - tm->tm_wday = tmfc.d - 1; /* convert to native numbering */ if (tmfc.dd) + { tm->tm_mday = tmfc.dd; - if (tmfc.ddd) - tm->tm_yday = tmfc.ddd; + fmask |= DTK_M(DAY); + } if (tmfc.mm) + { tm->tm_mon = tmfc.mm; + fmask |= DTK_M(MONTH); + } if (tmfc.ddd && (tm->tm_mon <= 1 || tm->tm_mday <= 1)) { @@ -3771,6 +3777,7 @@ do_to_timestamp(text *date_txt, text *fmt, j0 = isoweek2j(tm->tm_year, 1) - 1; j2date(j0 + tmfc.ddd, &tm->tm_year, &tm->tm_mon, &tm->tm_mday); + fmask |= DTK_DATE_M; } else { @@ -3785,7 +3792,7 @@ do_to_timestamp(text *date_txt, text *fmt, for (i = 1; i <= MONTHS_PER_YEAR; i++) { - if (tmfc.ddd < y[i]) + if (tmfc.ddd <= y[i]) break; } if (tm->tm_mon <= 1) @@ -3793,6 +3800,8 @@ do_to_timestamp(text *date_txt, text *fmt, if (tm->tm_mday <= 1) tm->tm_mday = tmfc.ddd - y[i - 1]; + + fmask |= DTK_M(MONTH) | DTK_M(DAY); } } @@ -3808,7 +3817,38 @@ do_to_timestamp(text *date_txt, text *fmt, *fsec += (double) tmfc.us / 1000000; #endif + /* Range-check date fields according to bit mask computed above */ + if (fmask != 0) + { + /* We already dealt with AD/BC, so pass isjulian = true */ + int dterr = ValidateDate(fmask, true, false, false, tm); + + if (dterr != 0) + { + /* + * Force the error to be DTERR_FIELD_OVERFLOW even if ValidateDate + * said DTERR_MD_FIELD_OVERFLOW, because we don't want to print an + * irrelevant hint about datestyle. + */ + DateTimeParseError(DTERR_FIELD_OVERFLOW, date_str, "timestamp"); + } + } + + /* Range-check time fields too */ + if (tm->tm_hour < 0 || tm->tm_hour >= HOURS_PER_DAY || + tm->tm_min < 0 || tm->tm_min >= MINS_PER_HOUR || + tm->tm_sec < 0 || tm->tm_sec >= SECS_PER_MINUTE || +#ifdef HAVE_INT64_TIMESTAMP + *fsec < INT64CONST(0) || *fsec >= USECS_PER_SEC +#else + *fsec < 0 || *fsec >= 1 +#endif + ) + DateTimeParseError(DTERR_FIELD_OVERFLOW, date_str, "timestamp"); + DEBUG_TM(tm); + + pfree(date_str); } diff --git a/src/test/regress/expected/horology.out b/src/test/regress/expected/horology.out index 1fe02be093..f9d12e0f8a 100644 --- a/src/test/regress/expected/horology.out +++ b/src/test/regress/expected/horology.out @@ -2822,6 +2822,18 @@ SELECT to_timestamp('20000-1116', 'YYYY-MMDD'); Thu Nov 16 00:00:00 20000 PST (1 row) +SELECT to_timestamp('1997 AD 11 16', 'YYYY BC MM DD'); + to_timestamp +------------------------------ + Sun Nov 16 00:00:00 1997 PST +(1 row) + +SELECT to_timestamp('1997 BC 11 16', 'YYYY BC MM DD'); + to_timestamp +--------------------------------- + Tue Nov 16 00:00:00 1997 PST BC +(1 row) + SELECT to_timestamp('9-1116', 'Y-MMDD'); to_timestamp ------------------------------ @@ -2906,6 +2918,18 @@ SELECT to_timestamp(' 20050302', 'YYYYMMDD'); Wed Mar 02 00:00:00 2005 PST (1 row) +SELECT to_timestamp('2011-12-18 11:38 AM', 'YYYY-MM-DD HH12:MI PM'); + to_timestamp +------------------------------ + Sun Dec 18 11:38:00 2011 PST +(1 row) + +SELECT to_timestamp('2011-12-18 11:38 PM', 'YYYY-MM-DD HH12:MI PM'); + to_timestamp +------------------------------ + Sun Dec 18 23:38:00 2011 PST +(1 row) + -- -- Check handling of multiple spaces in format and/or input -- @@ -2982,7 +3006,7 @@ SELECT to_date('2011 12 18', 'YYYY MM DD'); (1 row) -- --- Check errors for some incorrect usages of to_timestamp() +-- Check errors for some incorrect usages of to_timestamp() and to_date() -- -- Mixture of date conventions (ISO week and Gregorian): SELECT to_timestamp('2005527', 'YYYYIWID'); @@ -3010,6 +3034,76 @@ DETAIL: Value must be an integer. SELECT to_timestamp('10000000000', 'FMYYYY'); ERROR: value for "YYYY" in source string is out of range DETAIL: Value must be in the range -2147483648 to 2147483647. +-- Out-of-range and not-quite-out-of-range fields: +SELECT to_timestamp('2016-06-13 25:00:00', 'YYYY-MM-DD HH24:MI:SS'); +ERROR: date/time field value out of range: "2016-06-13 25:00:00" +SELECT to_timestamp('2016-06-13 15:60:00', 'YYYY-MM-DD HH24:MI:SS'); +ERROR: date/time field value out of range: "2016-06-13 15:60:00" +SELECT to_timestamp('2016-06-13 15:50:60', 'YYYY-MM-DD HH24:MI:SS'); +ERROR: date/time field value out of range: "2016-06-13 15:50:60" +SELECT to_timestamp('2016-06-13 15:50:55', 'YYYY-MM-DD HH24:MI:SS'); -- ok + to_timestamp +------------------------------ + Mon Jun 13 15:50:55 2016 PDT +(1 row) + +SELECT to_timestamp('2016-06-13 15:50:55', 'YYYY-MM-DD HH:MI:SS'); +ERROR: hour "15" is invalid for the 12-hour clock +HINT: Use the 24-hour clock, or give an hour between 1 and 12. +SELECT to_timestamp('2016-13-01 15:50:55', 'YYYY-MM-DD HH24:MI:SS'); +ERROR: date/time field value out of range: "2016-13-01 15:50:55" +SELECT to_timestamp('2016-02-30 15:50:55', 'YYYY-MM-DD HH24:MI:SS'); +ERROR: date/time field value out of range: "2016-02-30 15:50:55" +SELECT to_timestamp('2016-02-29 15:50:55', 'YYYY-MM-DD HH24:MI:SS'); -- ok + to_timestamp +------------------------------ + Mon Feb 29 15:50:55 2016 PST +(1 row) + +SELECT to_timestamp('2015-02-29 15:50:55', 'YYYY-MM-DD HH24:MI:SS'); +ERROR: date/time field value out of range: "2015-02-29 15:50:55" +SELECT to_timestamp('2015-02-11 86000', 'YYYY-MM-DD SSSS'); -- ok + to_timestamp +------------------------------ + Wed Feb 11 23:53:20 2015 PST +(1 row) + +SELECT to_timestamp('2015-02-11 86400', 'YYYY-MM-DD SSSS'); +ERROR: date/time field value out of range: "2015-02-11 86400" +SELECT to_date('2016-13-10', 'YYYY-MM-DD'); +ERROR: date/time field value out of range: "2016-13-10" +SELECT to_date('2016-02-30', 'YYYY-MM-DD'); +ERROR: date/time field value out of range: "2016-02-30" +SELECT to_date('2016-02-29', 'YYYY-MM-DD'); -- ok + to_date +------------ + 02-29-2016 +(1 row) + +SELECT to_date('2015-02-29', 'YYYY-MM-DD'); +ERROR: date/time field value out of range: "2015-02-29" +SELECT to_date('2015 365', 'YYYY DDD'); -- ok + to_date +------------ + 12-31-2015 +(1 row) + +SELECT to_date('2015 366', 'YYYY DDD'); +ERROR: date/time field value out of range: "2015 366" +SELECT to_date('2016 365', 'YYYY DDD'); -- ok + to_date +------------ + 12-30-2016 +(1 row) + +SELECT to_date('2016 366', 'YYYY DDD'); -- ok + to_date +------------ + 12-31-2016 +(1 row) + +SELECT to_date('2016 367', 'YYYY DDD'); +ERROR: date/time field value out of range: "2016 367" -- -- Check behavior with SQL-style fixed-GMT-offset time zone (cf bug #8572) -- diff --git a/src/test/regress/sql/horology.sql b/src/test/regress/sql/horology.sql index c81437ba35..a7bc9dcfc4 100644 --- a/src/test/regress/sql/horology.sql +++ b/src/test/regress/sql/horology.sql @@ -412,6 +412,9 @@ SELECT to_timestamp('19971116', 'YYYYMMDD'); SELECT to_timestamp('20000-1116', 'YYYY-MMDD'); +SELECT to_timestamp('1997 AD 11 16', 'YYYY BC MM DD'); +SELECT to_timestamp('1997 BC 11 16', 'YYYY BC MM DD'); + SELECT to_timestamp('9-1116', 'Y-MMDD'); SELECT to_timestamp('95-1116', 'YY-MMDD'); @@ -440,6 +443,9 @@ SELECT to_timestamp(' 2005 03 02', 'YYYYMMDD'); SELECT to_timestamp(' 20050302', 'YYYYMMDD'); +SELECT to_timestamp('2011-12-18 11:38 AM', 'YYYY-MM-DD HH12:MI PM'); +SELECT to_timestamp('2011-12-18 11:38 PM', 'YYYY-MM-DD HH12:MI PM'); + -- -- Check handling of multiple spaces in format and/or input -- @@ -461,7 +467,7 @@ SELECT to_date('2011 12 18', 'YYYY MM DD'); SELECT to_date('2011 12 18', 'YYYY MM DD'); -- --- Check errors for some incorrect usages of to_timestamp() +-- Check errors for some incorrect usages of to_timestamp() and to_date() -- -- Mixture of date conventions (ISO week and Gregorian): @@ -482,6 +488,28 @@ SELECT to_timestamp('199711xy', 'YYYYMMDD'); -- Input that doesn't fit in an int: SELECT to_timestamp('10000000000', 'FMYYYY'); +-- Out-of-range and not-quite-out-of-range fields: +SELECT to_timestamp('2016-06-13 25:00:00', 'YYYY-MM-DD HH24:MI:SS'); +SELECT to_timestamp('2016-06-13 15:60:00', 'YYYY-MM-DD HH24:MI:SS'); +SELECT to_timestamp('2016-06-13 15:50:60', 'YYYY-MM-DD HH24:MI:SS'); +SELECT to_timestamp('2016-06-13 15:50:55', 'YYYY-MM-DD HH24:MI:SS'); -- ok +SELECT to_timestamp('2016-06-13 15:50:55', 'YYYY-MM-DD HH:MI:SS'); +SELECT to_timestamp('2016-13-01 15:50:55', 'YYYY-MM-DD HH24:MI:SS'); +SELECT to_timestamp('2016-02-30 15:50:55', 'YYYY-MM-DD HH24:MI:SS'); +SELECT to_timestamp('2016-02-29 15:50:55', 'YYYY-MM-DD HH24:MI:SS'); -- ok +SELECT to_timestamp('2015-02-29 15:50:55', 'YYYY-MM-DD HH24:MI:SS'); +SELECT to_timestamp('2015-02-11 86000', 'YYYY-MM-DD SSSS'); -- ok +SELECT to_timestamp('2015-02-11 86400', 'YYYY-MM-DD SSSS'); +SELECT to_date('2016-13-10', 'YYYY-MM-DD'); +SELECT to_date('2016-02-30', 'YYYY-MM-DD'); +SELECT to_date('2016-02-29', 'YYYY-MM-DD'); -- ok +SELECT to_date('2015-02-29', 'YYYY-MM-DD'); +SELECT to_date('2015 365', 'YYYY DDD'); -- ok +SELECT to_date('2015 366', 'YYYY DDD'); +SELECT to_date('2016 365', 'YYYY DDD'); -- ok +SELECT to_date('2016 366', 'YYYY DDD'); -- ok +SELECT to_date('2016 367', 'YYYY DDD'); + -- -- Check behavior with SQL-style fixed-GMT-offset time zone (cf bug #8572) -- -- cgit v1.2.3 From 6ad8ac6026287e3ccbc4d606b6ab6116ccc0eec8 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 28 Sep 2016 12:00:00 -0400 Subject: Exclude additional directories in pg_basebackup The list of files and directories that pg_basebackup excludes from the backup was somewhat incomplete and unorganized. Change that with having the exclusion driven from tables. Clean up some code around it. Also document the exclusions in more detail so that users of pg_start_backup can make use of it as well. The contents of these directories are now excluded from the backup: pg_dynshmem, pg_notify, pg_serial, pg_snapshots, pg_subtrans Also fix a bug that a pg_repl_slot or pg_stat_tmp being a symlink would cause a corrupt tar header to be created. Now such symlinks are included in the backup as empty directories. Bug found by Ashutosh Sharma . From: David Steele Reviewed-by: Michael Paquier --- doc/src/sgml/backup.sgml | 16 ++ doc/src/sgml/protocol.sgml | 13 +- doc/src/sgml/ref/pg_basebackup.sgml | 10 +- src/backend/replication/basebackup.c | 260 +++++++++++++++++---------- src/bin/pg_basebackup/t/010_pg_basebackup.pl | 44 ++++- 5 files changed, 243 insertions(+), 100 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml index 0f09d82d65..95d0ff3149 100644 --- a/doc/src/sgml/backup.sgml +++ b/doc/src/sgml/backup.sgml @@ -1089,6 +1089,22 @@ SELECT pg_stop_backup(); the new master comes on line. + + The contents of the directories pg_dynshmem/, + pg_notify/, pg_serial/, + pg_snapshots/, pg_stat_tmp/, + and pg_subtrans/ (but not the directories themselves) can be + omitted from the backup as they will be initialized on postmaster startup. + If is set and is under the data + directory then the contents of that directory can also be omitted. + + + + Any file or directory beginning with pgsql_tmp can be + omitted from the backup. These files are removed on postmaster start and + the directories will be recreated as needed. + + The backup label file includes the label string you gave to pg_start_backup, diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index 68b0941029..3384e73448 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -2069,7 +2069,9 @@ The commands accepted in walsender mode are: - various temporary files created during the operation of the PostgreSQL server + Various temporary files and directories created during the operation + of the PostgreSQL server, such as any file or directory beginning + with pgsql_tmp. @@ -2082,13 +2084,18 @@ The commands accepted in walsender mode are: - pg_replslot is copied as an empty directory. + pg_dynshmem, pg_notify, + pg_replslot, pg_serial, + pg_snapshots, pg_stat_tmp, and + pg_subtrans are copied as empty directories (even if + they are symbolic links). Files other than regular files and directories, such as symbolic - links and special device files, are skipped. (Symbolic links + links (other than for the directories listed above) and special + device files, are skipped. (Symbolic links in pg_tblspc are maintained.) diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml index 9f1eae12d8..fe557ed002 100644 --- a/doc/src/sgml/ref/pg_basebackup.sgml +++ b/doc/src/sgml/ref/pg_basebackup.sgml @@ -610,10 +610,12 @@ PostgreSQL documentation The backup will include all files in the data directory and tablespaces, including the configuration files and any additional files placed in the - directory by third parties. But only regular files and directories are - copied. Symbolic links (other than those used for tablespaces) and special - device files are skipped. (See for - the precise details.) + directory by third parties, except certain temporary files managed by + PostgreSQL. But only regular files and directories are copied, except that + symbolic links used for tablespaces are preserved. Symbolic links pointing + to certain directories known to PostgreSQL are copied as empty directories. + Other symbolic links and special device files are skipped. + See for the precise details. diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c index da9b7a6f0d..1eabaef492 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c @@ -30,6 +30,7 @@ #include "replication/basebackup.h" #include "replication/walsender.h" #include "replication/walsender_private.h" +#include "storage/dsm_impl.h" #include "storage/fd.h" #include "storage/ipc.h" #include "utils/builtins.h" @@ -55,8 +56,10 @@ static int64 sendDir(char *path, int basepathlen, bool sizeonly, static bool sendFile(char *readfilename, char *tarfilename, struct stat * statbuf, bool missing_ok); static void sendFileWithContent(const char *filename, const char *content); -static void _tarWriteHeader(const char *filename, const char *linktarget, - struct stat * statbuf); +static int64 _tarWriteHeader(const char *filename, const char *linktarget, + struct stat * statbuf, bool sizeonly); +static int64 _tarWriteDir(const char *pathbuf, int basepathlen, struct stat *statbuf, + bool sizeonly); static void send_int8_string(StringInfoData *buf, int64 intval); static void SendBackupHeader(List *tablespaces); static void base_backup_cleanup(int code, Datum arg); @@ -94,6 +97,73 @@ static int64 elapsed_min_unit; /* The last check of the transfer rate. */ static int64 throttled_last; +/* + * The contents of these directories are removed or recreated during server + * start so they are not included in backups. The directories themselves are + * kept and included as empty to preserve access permissions. + */ +static const char *excludeDirContents[] = +{ + /* + * Skip temporary statistics files. PG_STAT_TMP_DIR must be skipped even + * when stats_temp_directory is set because PGSS_TEXT_FILE is always created + * there. + */ + PG_STAT_TMP_DIR, + + /* + * It is generally not useful to backup the contents of this directory even + * if the intention is to restore to another master. See backup.sgml for a + * more detailed description. + */ + "pg_replslot", + + /* Contents removed on startup, see dsm_cleanup_for_mmap(). */ + PG_DYNSHMEM_DIR, + + /* Contents removed on startup, see AsyncShmemInit(). */ + "pg_notify", + + /* + * Old contents are loaded for possible debugging but are not required for + * normal operation, see OldSerXidInit(). + */ + "pg_serial", + + /* Contents removed on startup, see DeleteAllExportedSnapshotFiles(). */ + "pg_snapshots", + + /* Contents zeroed on startup, see StartupSUBTRANS(). */ + "pg_subtrans", + + /* end of list */ + NULL +}; + +/* + * List of files excluded from backups. + */ +static const char *excludeFiles[] = +{ + /* Skip auto conf temporary file. */ + PG_AUTOCONF_FILENAME ".tmp", + + /* + * If there's a backup_label or tablespace_map file, it belongs to a + * backup started by the user with pg_start_backup(). It is *not* correct + * for this backup. Our backup_label/tablespace_map is injected into the + * tar separately. + */ + BACKUP_LABEL_FILE, + TABLESPACE_MAP, + + "postmaster.pid", + "postmaster.opts", + + /* end of list */ + NULL +}; + /* * Called when ERROR or FATAL happens in perform_base_backup() after * we have started the backup - make sure we end it! @@ -415,7 +485,7 @@ perform_base_backup(basebackup_options *opt, DIR *tblspcdir) } /* send the WAL file itself */ - _tarWriteHeader(pathbuf, NULL, &statbuf); + _tarWriteHeader(pathbuf, NULL, &statbuf, false); while ((cnt = fread(buf, 1, Min(sizeof(buf), XLogSegSize - len), fp)) > 0) { @@ -807,7 +877,7 @@ sendFileWithContent(const char *filename, const char *content) statbuf.st_mode = S_IRUSR | S_IWUSR; statbuf.st_size = len; - _tarWriteHeader(filename, NULL, &statbuf); + _tarWriteHeader(filename, NULL, &statbuf, false); /* Send the contents as a CopyData message */ pq_putmessage('d', content, len); @@ -858,9 +928,9 @@ sendTablespace(char *path, bool sizeonly) /* If the tablespace went away while scanning, it's no error. */ return 0; } - if (!sizeonly) - _tarWriteHeader(TABLESPACE_VERSION_DIRECTORY, NULL, &statbuf); - size = 512; /* Size of the header just added */ + + size = _tarWriteHeader(TABLESPACE_VERSION_DIRECTORY, NULL, &statbuf, + sizeonly); /* Send all the files in the tablespace version directory */ size += sendDir(pathbuf, strlen(path), sizeonly, NIL, true); @@ -893,6 +963,9 @@ sendDir(char *path, int basepathlen, bool sizeonly, List *tablespaces, dir = AllocateDir(path); while ((de = ReadDir(dir, path)) != NULL) { + int excludeIdx; + bool excludeFound; + /* Skip special stuff */ if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue; @@ -903,24 +976,6 @@ sendDir(char *path, int basepathlen, bool sizeonly, List *tablespaces, strlen(PG_TEMP_FILE_PREFIX)) == 0) continue; - /* skip auto conf temporary file */ - if (strncmp(de->d_name, - PG_AUTOCONF_FILENAME ".tmp", - sizeof(PG_AUTOCONF_FILENAME) + 4) == 0) - continue; - - /* - * If there's a backup_label or tablespace_map file, it belongs to a - * backup started by the user with pg_start_backup(). It is *not* - * correct for this backup, our backup_label/tablespace_map is - * injected into the tar separately. - */ - if (strcmp(de->d_name, BACKUP_LABEL_FILE) == 0) - continue; - - if (strcmp(de->d_name, TABLESPACE_MAP) == 0) - continue; - /* * Check if the postmaster has signaled us to exit, and abort with an * error in that case. The error handler further up will call @@ -938,13 +993,23 @@ sendDir(char *path, int basepathlen, bool sizeonly, List *tablespaces, "and should not be used. " "Try taking another online backup."))); - snprintf(pathbuf, MAXPGPATH, "%s/%s", path, de->d_name); + /* Scan for files that should be excluded */ + excludeFound = false; + for (excludeIdx = 0; excludeFiles[excludeIdx] != NULL; excludeIdx++) + { + if (strcmp(de->d_name, excludeFiles[excludeIdx]) == 0) + { + elog(DEBUG1, "file \"%s\" excluded from backup", de->d_name); + excludeFound = true; + break; + } + } - /* Skip postmaster.pid and postmaster.opts in the data directory */ - if (strcmp(pathbuf, "./postmaster.pid") == 0 || - strcmp(pathbuf, "./postmaster.opts") == 0) + if (excludeFound) continue; + snprintf(pathbuf, MAXPGPATH, "%s/%s", path, de->d_name); + /* Skip pg_control here to back up it last */ if (strcmp(pathbuf, "./global/pg_control") == 0) continue; @@ -957,33 +1022,34 @@ sendDir(char *path, int basepathlen, bool sizeonly, List *tablespaces, errmsg("could not stat file or directory \"%s\": %m", pathbuf))); - /* If the file went away while scanning, it's no error. */ + /* If the file went away while scanning, it's not an error. */ continue; } - /* - * Skip temporary statistics files. PG_STAT_TMP_DIR must be skipped - * even when stats_temp_directory is set because PGSS_TEXT_FILE is - * always created there. - */ - if ((statrelpath != NULL && strcmp(pathbuf, statrelpath) == 0) || - strncmp(de->d_name, PG_STAT_TMP_DIR, strlen(PG_STAT_TMP_DIR)) == 0) + /* Scan for directories whose contents should be excluded */ + excludeFound = false; + for (excludeIdx = 0; excludeDirContents[excludeIdx] != NULL; excludeIdx++) { - if (!sizeonly) - _tarWriteHeader(pathbuf + basepathlen + 1, NULL, &statbuf); - size += 512; - continue; + if (strcmp(de->d_name, excludeDirContents[excludeIdx]) == 0) + { + elog(DEBUG1, "contents of directory \"%s\" excluded from backup", de->d_name); + size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly); + excludeFound = true; + break; + } } + if (excludeFound) + continue; + /* - * Skip pg_replslot, not useful to copy. But include it as an empty - * directory anyway, so we get permissions right. + * Exclude contents of directory specified by statrelpath if not set + * to the default (pg_stat_tmp) which is caught in the loop above. */ - if (strcmp(de->d_name, "pg_replslot") == 0) + if (statrelpath != NULL && strcmp(pathbuf, statrelpath) == 0) { - if (!sizeonly) - _tarWriteHeader(pathbuf + basepathlen + 1, NULL, &statbuf); - size += 512; /* Size of the header just added */ + elog(DEBUG1, "contents of directory \"%s\" excluded from backup", statrelpath); + size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly); continue; } @@ -994,26 +1060,15 @@ sendDir(char *path, int basepathlen, bool sizeonly, List *tablespaces, */ if (strcmp(pathbuf, "./pg_xlog") == 0) { - if (!sizeonly) - { - /* If pg_xlog is a symlink, write it as a directory anyway */ -#ifndef WIN32 - if (S_ISLNK(statbuf.st_mode)) -#else - if (pgwin32_is_junction(pathbuf)) -#endif - statbuf.st_mode = S_IFDIR | S_IRWXU; - _tarWriteHeader(pathbuf + basepathlen + 1, NULL, &statbuf); - } - size += 512; /* Size of the header just added */ + /* If pg_xlog is a symlink, write it as a directory anyway */ + size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly); /* * Also send archive_status directory (by hackishly reusing * statbuf from above ...). */ - if (!sizeonly) - _tarWriteHeader("./pg_xlog/archive_status", NULL, &statbuf); - size += 512; /* Size of the header just added */ + size += _tarWriteHeader("./pg_xlog/archive_status", NULL, &statbuf, + sizeonly); continue; /* don't recurse into pg_xlog */ } @@ -1044,9 +1099,8 @@ sendDir(char *path, int basepathlen, bool sizeonly, List *tablespaces, pathbuf))); linkpath[rllen] = '\0'; - if (!sizeonly) - _tarWriteHeader(pathbuf + basepathlen + 1, linkpath, &statbuf); - size += 512; /* Size of the header just added */ + size += _tarWriteHeader(pathbuf + basepathlen + 1, linkpath, + &statbuf, sizeonly); #else /* @@ -1069,9 +1123,8 @@ sendDir(char *path, int basepathlen, bool sizeonly, List *tablespaces, * Store a directory entry in the tar file so we can get the * permissions right. */ - if (!sizeonly) - _tarWriteHeader(pathbuf + basepathlen + 1, NULL, &statbuf); - size += 512; /* Size of the header just added */ + size += _tarWriteHeader(pathbuf + basepathlen + 1, NULL, &statbuf, + sizeonly); /* * Call ourselves recursively for a directory, unless it happens @@ -1162,7 +1215,7 @@ sendFile(char *readfilename, char *tarfilename, struct stat * statbuf, errmsg("could not open file \"%s\": %m", readfilename))); } - _tarWriteHeader(tarfilename, NULL, statbuf); + _tarWriteHeader(tarfilename, NULL, statbuf, false); while ((cnt = fread(buf, 1, Min(sizeof(buf), statbuf->st_size - len), fp)) > 0) { @@ -1215,36 +1268,61 @@ sendFile(char *readfilename, char *tarfilename, struct stat * statbuf, } -static void +static int64 _tarWriteHeader(const char *filename, const char *linktarget, - struct stat * statbuf) + struct stat * statbuf, bool sizeonly) { char h[512]; enum tarError rc; - rc = tarCreateHeader(h, filename, linktarget, statbuf->st_size, - statbuf->st_mode, statbuf->st_uid, statbuf->st_gid, - statbuf->st_mtime); - - switch (rc) + if (!sizeonly) { - case TAR_OK: - break; - case TAR_NAME_TOO_LONG: - ereport(ERROR, - (errmsg("file name too long for tar format: \"%s\"", - filename))); - break; - case TAR_SYMLINK_TOO_LONG: - ereport(ERROR, - (errmsg("symbolic link target too long for tar format: file name \"%s\", target \"%s\"", - filename, linktarget))); - break; - default: - elog(ERROR, "unrecognized tar error: %d", rc); + rc = tarCreateHeader(h, filename, linktarget, statbuf->st_size, + statbuf->st_mode, statbuf->st_uid, statbuf->st_gid, + statbuf->st_mtime); + + switch (rc) + { + case TAR_OK: + break; + case TAR_NAME_TOO_LONG: + ereport(ERROR, + (errmsg("file name too long for tar format: \"%s\"", + filename))); + break; + case TAR_SYMLINK_TOO_LONG: + ereport(ERROR, + (errmsg("symbolic link target too long for tar format: " + "file name \"%s\", target \"%s\"", + filename, linktarget))); + break; + default: + elog(ERROR, "unrecognized tar error: %d", rc); + } + + pq_putmessage('d', h, sizeof(h)); } - pq_putmessage('d', h, 512); + return sizeof(h); +} + +/* + * Write tar header for a directory. If the entry in statbuf is a link then + * write it as a directory anyway. + */ +static int64 +_tarWriteDir(const char *pathbuf, int basepathlen, struct stat *statbuf, + bool sizeonly) +{ + /* If symlink, write it as a directory anyway */ +#ifndef WIN32 + if (S_ISLNK(statbuf->st_mode)) +#else + if (pgwin32_is_junction(pathbuf)) +#endif + statbuf->st_mode = S_IFDIR | S_IRWXU; + + return _tarWriteHeader(pathbuf + basepathlen + 1, NULL, statbuf, sizeonly); } /* diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/010_pg_basebackup.pl index fd9857d67b..a52bd4e124 100644 --- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl +++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl @@ -4,7 +4,7 @@ use Cwd; use Config; use PostgresNode; use TestLib; -use Test::More tests => 54; +use Test::More tests => 67; program_help_ok('pg_basebackup'); program_version_ok('pg_basebackup'); @@ -55,15 +55,43 @@ print CONF "wal_level = replica\n"; close CONF; $node->restart; +# Write some files to test that they are not copied. +foreach my $filename (qw(backup_label tablespace_map postgresql.auto.conf.tmp)) +{ + open FILE, ">>$pgdata/$filename"; + print FILE "DONOTCOPY"; + close FILE; +} + $node->command_ok([ 'pg_basebackup', '-D', "$tempdir/backup" ], 'pg_basebackup runs'); ok(-f "$tempdir/backup/PG_VERSION", 'backup was created'); +# Only archive_status directory should be copied in pg_xlog/. is_deeply( [ sort(slurp_dir("$tempdir/backup/pg_xlog/")) ], [ sort qw(. .. archive_status) ], 'no WAL files copied'); +# Contents of these directories should not be copied. +foreach my $dirname (qw(pg_dynshmem pg_notify pg_replslot pg_serial pg_snapshots pg_stat_tmp pg_subtrans)) +{ + is_deeply( + [ sort(slurp_dir("$tempdir/backup/$dirname/")) ], + [ sort qw(. ..) ], + "contents of $dirname/ not copied"); +} + +# These files should not be copied. +foreach my $filename (qw(postgresql.auto.conf.tmp postmaster.opts postmaster.pid tablespace_map)) +{ + ok(! -f "$tempdir/backup/$filename", "$filename not copied"); +} + +# Make sure existing backup_label was ignored. +isnt(slurp_file("$tempdir/backup/backup_label"), 'DONOTCOPY', + 'existing backup_label not copied'); + $node->command_ok( [ 'pg_basebackup', '-D', "$tempdir/backup2", '--xlogdir', "$tempdir/xlog2" ], @@ -110,7 +138,17 @@ unlink "$pgdata/$superlongname"; # skip on Windows. SKIP: { - skip "symlinks not supported on Windows", 10 if ($windows_os); + skip "symlinks not supported on Windows", 11 if ($windows_os); + + # Move pg_replslot out of $pgdata and create a symlink to it. + $node->stop; + + rename("$pgdata/pg_replslot", "$tempdir/pg_replslot") + or BAIL_OUT "could not move $pgdata/pg_replslot"; + symlink("$tempdir/pg_replslot", "$pgdata/pg_replslot") + or BAIL_OUT "could not symlink to $pgdata/pg_replslot"; + + $node->start; # Create a temporary directory in the system location and symlink it # to our physical temp location. That way we can use shorter names @@ -148,6 +186,8 @@ SKIP: "tablespace symlink was updated"); closedir $dh; + ok(-d "$tempdir/backup1/pg_replslot", 'pg_replslot symlink copied as directory'); + mkdir "$tempdir/tbl=spc2"; $node->safe_psql('postgres', "DROP TABLE test1;"); $node->safe_psql('postgres', "DROP TABLESPACE tblspc1;"); -- cgit v1.2.3 From 6ed2d8584cc680a2d6898480de74a57cd96176b5 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 29 Sep 2016 12:00:00 -0400 Subject: pg_basebackup: Add --nosync option This is useful for testing, similar to initdb's --nosync. From: Michael Paquier --- doc/src/sgml/ref/pg_basebackup.sgml | 15 +++++++++++++++ src/bin/pg_basebackup/pg_basebackup.c | 28 +++++++++++++++++++--------- src/bin/pg_basebackup/pg_receivexlog.c | 1 + src/bin/pg_basebackup/receivelog.c | 34 ++++++++++++++++++---------------- src/bin/pg_basebackup/receivelog.h | 4 +++- 5 files changed, 56 insertions(+), 26 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml index fe557ed002..55e913f70d 100644 --- a/doc/src/sgml/ref/pg_basebackup.sgml +++ b/doc/src/sgml/ref/pg_basebackup.sgml @@ -438,6 +438,21 @@ PostgreSQL documentation + + + + + + By default, pg_basebackup will wait for all files + to be written safely to disk. This option causes + pg_basebackup to return without waiting, which is + faster, but means that a subsequent operating system crash can leave + the base backup corrupt. Generally, this option is useful for testing + but should not be used when creating a production installation. + + + + diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index cd7d095103..0f5d9d6a87 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -69,6 +69,7 @@ static bool includewal = false; static bool streamwal = false; static bool fastcheckpoint = false; static bool writerecoveryconf = false; +static bool do_sync = true; static int standby_message_timeout = 10 * 1000; /* 10 sec = default */ static pg_time_t last_progress_report = 0; static int32 maxrate = 0; /* no limit by default */ @@ -329,6 +330,7 @@ usage(void) " set fast or spread checkpointing\n")); printf(_(" -l, --label=LABEL set backup label\n")); printf(_(" -n, --noclean do not clean up after errors\n")); + printf(_(" -N, --nosync do not wait for changes to be written safely to disk\n")); printf(_(" -P, --progress show progress information\n")); printf(_(" -v, --verbose output verbose messages\n")); printf(_(" -V, --version output version information, then exit\n")); @@ -460,6 +462,7 @@ LogStreamerMain(logstreamer_param *param) stream.stream_stop = reached_end_position; stream.standby_message_timeout = standby_message_timeout; stream.synchronous = false; + stream.do_sync = do_sync; stream.mark_done = true; stream.basedir = param->xlogdir; stream.partial_suffix = NULL; @@ -1199,7 +1202,7 @@ ReceiveTarFile(PGconn *conn, PGresult *res, int rownum) PQfreemem(copybuf); /* sync the resulting tar file, errors are not considered fatal */ - if (strcmp(basedir, "-") != 0) + if (do_sync && strcmp(basedir, "-") != 0) (void) fsync_fname(filename, false, progname); } @@ -1967,14 +1970,17 @@ BaseBackup(void) * all the data of the base directory is synced, taking into account * all the tablespaces. Errors are not considered fatal. */ - if (format == 't') + if (do_sync) { - if (strcmp(basedir, "-") != 0) - (void) fsync_fname(basedir, true, progname); - } - else - { - (void) fsync_pgdata(basedir, progname); + if (format == 't') + { + if (strcmp(basedir, "-") != 0) + (void) fsync_fname(basedir, true, progname); + } + else + { + (void) fsync_pgdata(basedir, progname); + } } if (verbose) @@ -2001,6 +2007,7 @@ main(int argc, char **argv) {"compress", required_argument, NULL, 'Z'}, {"label", required_argument, NULL, 'l'}, {"noclean", no_argument, NULL, 'n'}, + {"nosync", no_argument, NULL, 'N'}, {"dbname", required_argument, NULL, 'd'}, {"host", required_argument, NULL, 'h'}, {"port", required_argument, NULL, 'p'}, @@ -2037,7 +2044,7 @@ main(int argc, char **argv) atexit(cleanup_directories_atexit); - while ((c = getopt_long(argc, argv, "D:F:r:RT:xX:l:nzZ:d:c:h:p:U:s:S:wWvP", + while ((c = getopt_long(argc, argv, "D:F:r:RT:xX:l:nNzZ:d:c:h:p:U:s:S:wWvP", long_options, &option_index)) != -1) { switch (c) @@ -2115,6 +2122,9 @@ main(int argc, char **argv) case 'n': noclean = true; break; + case 'N': + do_sync = false; + break; case 'z': #ifdef HAVE_LIBZ compresslevel = Z_DEFAULT_COMPRESSION; diff --git a/src/bin/pg_basebackup/pg_receivexlog.c b/src/bin/pg_basebackup/pg_receivexlog.c index 7f7ee9dc9b..a58a251a59 100644 --- a/src/bin/pg_basebackup/pg_receivexlog.c +++ b/src/bin/pg_basebackup/pg_receivexlog.c @@ -336,6 +336,7 @@ StreamLog(void) stream.stream_stop = stop_streaming; stream.standby_message_timeout = standby_message_timeout; stream.synchronous = synchronous; + stream.do_sync = true; stream.mark_done = false; stream.basedir = basedir; stream.partial_suffix = ".partial"; diff --git a/src/bin/pg_basebackup/receivelog.c b/src/bin/pg_basebackup/receivelog.c index 6b78a60f27..8f29d19114 100644 --- a/src/bin/pg_basebackup/receivelog.c +++ b/src/bin/pg_basebackup/receivelog.c @@ -41,8 +41,8 @@ static PGresult *HandleCopyStream(PGconn *conn, StreamCtl *stream, XLogRecPtr *stoppos); static int CopyStreamPoll(PGconn *conn, long timeout_ms); static int CopyStreamReceive(PGconn *conn, long timeout, char **buffer); -static bool ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len, - XLogRecPtr blockpos, int64 *last_status); +static bool ProcessKeepaliveMsg(PGconn *conn, StreamCtl *stream, char *copybuf, + int len, XLogRecPtr blockpos, int64 *last_status); static bool ProcessXLogDataMsg(PGconn *conn, StreamCtl *stream, char *copybuf, int len, XLogRecPtr *blockpos); static PGresult *HandleEndOfCopyStream(PGconn *conn, StreamCtl *stream, char *copybuf, @@ -56,7 +56,7 @@ static bool ReadEndOfStreamingResult(PGresult *res, XLogRecPtr *startpos, uint32 *timeline); static bool -mark_file_as_archived(const char *basedir, const char *fname) +mark_file_as_archived(const char *basedir, const char *fname, bool do_sync) { int fd; static char tmppath[MAXPGPATH]; @@ -74,10 +74,10 @@ mark_file_as_archived(const char *basedir, const char *fname) close(fd); - if (fsync_fname(tmppath, false, progname) != 0) + if (do_sync && fsync_fname(tmppath, false, progname) != 0) return false; - if (fsync_parent_path(tmppath, progname) != 0) + if (do_sync && fsync_parent_path(tmppath, progname) != 0) return false; return true; @@ -134,9 +134,9 @@ open_walfile(StreamCtl *stream, XLogRecPtr startpoint) * fsync, in case of a previous crash between padding and fsyncing the * file. */ - if (fsync_fname(fn, false, progname) != 0) + if (stream->do_sync && fsync_fname(fn, false, progname) != 0) return false; - if (fsync_parent_path(fn, progname) != 0) + if (stream->do_sync && fsync_parent_path(fn, progname) != 0) return false; return true; @@ -173,9 +173,9 @@ open_walfile(StreamCtl *stream, XLogRecPtr startpoint) * using synchronous mode, where the file is modified and fsynced * in-place, without a directory fsync. */ - if (fsync_fname(fn, false, progname) != 0) + if (stream->do_sync && fsync_fname(fn, false, progname) != 0) return false; - if (fsync_parent_path(fn, progname) != 0) + if (stream->do_sync && fsync_parent_path(fn, progname) != 0) return false; if (lseek(f, SEEK_SET, 0) != 0) @@ -212,7 +212,7 @@ close_walfile(StreamCtl *stream, XLogRecPtr pos) return false; } - if (fsync(walfile) != 0) + if (stream->do_sync && fsync(walfile) != 0) { fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"), progname, current_walfile_name, strerror(errno)); @@ -258,7 +258,8 @@ close_walfile(StreamCtl *stream, XLogRecPtr pos) if (currpos == XLOG_SEG_SIZE && stream->mark_done) { /* writes error message if failed */ - if (!mark_file_as_archived(stream->basedir, current_walfile_name)) + if (!mark_file_as_archived(stream->basedir, current_walfile_name, + stream->do_sync)) return false; } @@ -378,7 +379,8 @@ writeTimeLineHistoryFile(StreamCtl *stream, char *filename, char *content) if (stream->mark_done) { /* writes error message if failed */ - if (!mark_file_as_archived(stream->basedir, histfname)) + if (!mark_file_as_archived(stream->basedir, histfname, + stream->do_sync)) return false; } @@ -836,7 +838,7 @@ HandleCopyStream(PGconn *conn, StreamCtl *stream, */ if (stream->synchronous && lastFlushPosition < blockpos && walfile != -1) { - if (fsync(walfile) != 0) + if (stream->do_sync && fsync(walfile) != 0) { fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"), progname, current_walfile_name, strerror(errno)); @@ -890,7 +892,7 @@ HandleCopyStream(PGconn *conn, StreamCtl *stream, /* Check the message type. */ if (copybuf[0] == 'k') { - if (!ProcessKeepaliveMsg(conn, copybuf, r, blockpos, + if (!ProcessKeepaliveMsg(conn, stream, copybuf, r, blockpos, &last_status)) goto error; } @@ -1043,7 +1045,7 @@ CopyStreamReceive(PGconn *conn, long timeout, char **buffer) * Process the keepalive message. */ static bool -ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len, +ProcessKeepaliveMsg(PGconn *conn, StreamCtl *stream, char *copybuf, int len, XLogRecPtr blockpos, int64 *last_status) { int pos; @@ -1079,7 +1081,7 @@ ProcessKeepaliveMsg(PGconn *conn, char *copybuf, int len, * data has been successfully replicated or not, at the normal * shutdown of the server. */ - if (fsync(walfile) != 0) + if (stream->do_sync && fsync(walfile) != 0) { fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"), progname, current_walfile_name, strerror(errno)); diff --git a/src/bin/pg_basebackup/receivelog.h b/src/bin/pg_basebackup/receivelog.h index 554ff8b5b2..7a3bbc5080 100644 --- a/src/bin/pg_basebackup/receivelog.h +++ b/src/bin/pg_basebackup/receivelog.h @@ -34,8 +34,10 @@ typedef struct StreamCtl * timeline */ int standby_message_timeout; /* Send status messages this * often */ - bool synchronous; /* Flush data on write */ + bool synchronous; /* Flush immediately WAL data on write */ bool mark_done; /* Mark segment as done in generated archive */ + bool do_sync; /* Flush to disk to ensure consistent state + * of data */ stream_stop_callback stream_stop; /* Stop streaming when returns true */ -- cgit v1.2.3 From 8e91e12bc3af85ba2287866669268f6825d2cc03 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 29 Sep 2016 13:32:27 -0400 Subject: Allow contrib/file_fdw to read from a program, like COPY FROM PROGRAM. This patch just exposes COPY's FROM PROGRAM option in contrib/file_fdw. There don't seem to be any security issues with that that are any worse than what already exist with file_fdw and COPY; as in the existing cases, only superusers are allowed to control what gets executed. A regression test case might be nice here, but choosing a 100% portable command to run is hard. (We haven't got a test for COPY FROM PROGRAM itself, either.) Corey Huinker and Adam Gomaa, reviewed by Amit Langote Discussion: --- contrib/file_fdw/file_fdw.c | 131 +++++++++++++++++++++----------- contrib/file_fdw/output/file_fdw.source | 6 +- doc/src/sgml/file-fdw.sgml | 66 +++++++++++----- 3 files changed, 137 insertions(+), 66 deletions(-) (limited to 'doc/src') diff --git a/contrib/file_fdw/file_fdw.c b/contrib/file_fdw/file_fdw.c index b471991318..d325f53467 100644 --- a/contrib/file_fdw/file_fdw.c +++ b/contrib/file_fdw/file_fdw.c @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * file_fdw.c - * foreign-data wrapper for server-side flat files. + * foreign-data wrapper for server-side flat files (or programs). * * Copyright (c) 2010-2016, PostgreSQL Global Development Group * @@ -57,8 +57,9 @@ struct FileFdwOption * fileGetOptions(), which currently doesn't bother to look at user mappings. */ static const struct FileFdwOption valid_options[] = { - /* File options */ + /* Data source options */ {"filename", ForeignTableRelationId}, + {"program", ForeignTableRelationId}, /* Format options */ /* oids option is not supported */ @@ -85,10 +86,12 @@ static const struct FileFdwOption valid_options[] = { */ typedef struct FileFdwPlanState { - char *filename; /* file to read */ - List *options; /* merged COPY options, excluding filename */ + char *filename; /* file or program to read from */ + bool is_program; /* true if filename represents an OS command */ + List *options; /* merged COPY options, excluding filename and + * is_program */ BlockNumber pages; /* estimate of file's physical size */ - double ntuples; /* estimate of number of rows in file */ + double ntuples; /* estimate of number of data rows */ } FileFdwPlanState; /* @@ -96,9 +99,11 @@ typedef struct FileFdwPlanState */ typedef struct FileFdwExecutionState { - char *filename; /* file to read */ - List *options; /* merged COPY options, excluding filename */ - CopyState cstate; /* state of reading file */ + char *filename; /* file or program to read from */ + bool is_program; /* true if filename represents an OS command */ + List *options; /* merged COPY options, excluding filename and + * is_program */ + CopyState cstate; /* COPY execution state */ } FileFdwExecutionState; /* @@ -139,7 +144,9 @@ static bool fileIsForeignScanParallelSafe(PlannerInfo *root, RelOptInfo *rel, */ static bool is_valid_option(const char *option, Oid context); static void fileGetOptions(Oid foreigntableid, - char **filename, List **other_options); + char **filename, + bool *is_program, + List **other_options); static List *get_file_fdw_attribute_options(Oid relid); static bool check_selective_binary_conversion(RelOptInfo *baserel, Oid foreigntableid, @@ -196,16 +203,16 @@ file_fdw_validator(PG_FUNCTION_ARGS) /* * Only superusers are allowed to set options of a file_fdw foreign table. - * This is because the filename is one of those options, and we don't want - * non-superusers to be able to determine which file gets read. + * This is because we don't want non-superusers to be able to control + * which file gets read or which program gets executed. * * Putting this sort of permissions check in a validator is a bit of a * crock, but there doesn't seem to be any other place that can enforce * the check more cleanly. * - * Note that the valid_options[] array disallows setting filename at any - * options level other than foreign table --- otherwise there'd still be a - * security hole. + * Note that the valid_options[] array disallows setting filename and + * program at any options level other than foreign table --- otherwise + * there'd still be a security hole. */ if (catalog == ForeignTableRelationId && !superuser()) ereport(ERROR, @@ -247,11 +254,11 @@ file_fdw_validator(PG_FUNCTION_ARGS) } /* - * Separate out filename and column-specific options, since + * Separate out filename, program, and column-specific options, since * ProcessCopyOptions won't accept them. */ - - if (strcmp(def->defname, "filename") == 0) + if (strcmp(def->defname, "filename") == 0 || + strcmp(def->defname, "program") == 0) { if (filename) ereport(ERROR, @@ -296,12 +303,13 @@ file_fdw_validator(PG_FUNCTION_ARGS) ProcessCopyOptions(NULL, NULL, true, other_options); /* - * Filename option is required for file_fdw foreign tables. + * Either filename or program option is required for file_fdw foreign + * tables. */ if (catalog == ForeignTableRelationId && filename == NULL) ereport(ERROR, (errcode(ERRCODE_FDW_DYNAMIC_PARAMETER_VALUE_NEEDED), - errmsg("filename is required for file_fdw foreign tables"))); + errmsg("either filename or program is required for file_fdw foreign tables"))); PG_RETURN_VOID(); } @@ -326,12 +334,12 @@ is_valid_option(const char *option, Oid context) /* * Fetch the options for a file_fdw foreign table. * - * We have to separate out "filename" from the other options because - * it must not appear in the options list passed to the core COPY code. + * We have to separate out filename/program from the other options because + * those must not appear in the options list passed to the core COPY code. */ static void fileGetOptions(Oid foreigntableid, - char **filename, List **other_options) + char **filename, bool *is_program, List **other_options) { ForeignTable *table; ForeignServer *server; @@ -359,9 +367,11 @@ fileGetOptions(Oid foreigntableid, options = list_concat(options, get_file_fdw_attribute_options(foreigntableid)); /* - * Separate out the filename. + * Separate out the filename or program option (we assume there is only + * one). */ *filename = NULL; + *is_program = false; prev = NULL; foreach(lc, options) { @@ -373,15 +383,22 @@ fileGetOptions(Oid foreigntableid, options = list_delete_cell(options, lc, prev); break; } + else if (strcmp(def->defname, "program") == 0) + { + *filename = defGetString(def); + *is_program = true; + options = list_delete_cell(options, lc, prev); + break; + } prev = lc; } /* - * The validator should have checked that a filename was included in the - * options, but check again, just in case. + * The validator should have checked that filename or program was included + * in the options, but check again, just in case. */ if (*filename == NULL) - elog(ERROR, "filename is required for file_fdw foreign tables"); + elog(ERROR, "either filename or program is required for file_fdw foreign tables"); *other_options = options; } @@ -475,12 +492,15 @@ fileGetForeignRelSize(PlannerInfo *root, FileFdwPlanState *fdw_private; /* - * Fetch options. We only need filename at this point, but we might as - * well get everything and not need to re-fetch it later in planning. + * Fetch options. We only need filename (or program) at this point, but + * we might as well get everything and not need to re-fetch it later in + * planning. */ fdw_private = (FileFdwPlanState *) palloc(sizeof(FileFdwPlanState)); fileGetOptions(foreigntableid, - &fdw_private->filename, &fdw_private->options); + &fdw_private->filename, + &fdw_private->is_program, + &fdw_private->options); baserel->fdw_private = (void *) fdw_private; /* Estimate relation size */ @@ -583,20 +603,25 @@ static void fileExplainForeignScan(ForeignScanState *node, ExplainState *es) { char *filename; + bool is_program; List *options; - /* Fetch options --- we only need filename at this point */ + /* Fetch options --- we only need filename and is_program at this point */ fileGetOptions(RelationGetRelid(node->ss.ss_currentRelation), - &filename, &options); + &filename, &is_program, &options); - ExplainPropertyText("Foreign File", filename, es); + if (is_program) + ExplainPropertyText("Foreign Program", filename, es); + else + ExplainPropertyText("Foreign File", filename, es); /* Suppress file size if we're not showing cost details */ if (es->costs) { struct stat stat_buf; - if (stat(filename, &stat_buf) == 0) + if (!is_program && + stat(filename, &stat_buf) == 0) ExplainPropertyLong("Foreign File Size", (long) stat_buf.st_size, es); } @@ -611,6 +636,7 @@ fileBeginForeignScan(ForeignScanState *node, int eflags) { ForeignScan *plan = (ForeignScan *) node->ss.ps.plan; char *filename; + bool is_program; List *options; CopyState cstate; FileFdwExecutionState *festate; @@ -623,7 +649,7 @@ fileBeginForeignScan(ForeignScanState *node, int eflags) /* Fetch options of foreign table */ fileGetOptions(RelationGetRelid(node->ss.ss_currentRelation), - &filename, &options); + &filename, &is_program, &options); /* Add any options from the plan (currently only convert_selectively) */ options = list_concat(options, plan->fdw_private); @@ -635,7 +661,7 @@ fileBeginForeignScan(ForeignScanState *node, int eflags) cstate = BeginCopyFrom(NULL, node->ss.ss_currentRelation, filename, - false, + is_program, NIL, options); @@ -645,6 +671,7 @@ fileBeginForeignScan(ForeignScanState *node, int eflags) */ festate = (FileFdwExecutionState *) palloc(sizeof(FileFdwExecutionState)); festate->filename = filename; + festate->is_program = is_program; festate->options = options; festate->cstate = cstate; @@ -709,7 +736,7 @@ fileReScanForeignScan(ForeignScanState *node) festate->cstate = BeginCopyFrom(NULL, node->ss.ss_currentRelation, festate->filename, - false, + festate->is_program, NIL, festate->options); } @@ -738,11 +765,22 @@ fileAnalyzeForeignTable(Relation relation, BlockNumber *totalpages) { char *filename; + bool is_program; List *options; struct stat stat_buf; /* Fetch options of foreign table */ - fileGetOptions(RelationGetRelid(relation), &filename, &options); + fileGetOptions(RelationGetRelid(relation), &filename, &is_program, &options); + + /* + * If this is a program instead of a file, just return false to skip + * analyzing the table. We could run the program and collect stats on + * whatever it currently returns, but it seems likely that in such cases + * the output would be too volatile for the stats to be useful. Maybe + * there should be an option to enable doing this? + */ + if (is_program) + return false; /* * Get size of the file. (XXX if we fail here, would it be better to just @@ -769,8 +807,8 @@ fileAnalyzeForeignTable(Relation relation, /* * fileIsForeignScanParallelSafe - * Reading a file in a parallel worker should work just the same as - * reading it in the leader, so mark scans safe. + * Reading a file, or external program, in a parallel worker should work + * just the same as reading it in the leader, so mark scans safe. */ static bool fileIsForeignScanParallelSafe(PlannerInfo *root, RelOptInfo *rel, @@ -916,9 +954,10 @@ estimate_size(PlannerInfo *root, RelOptInfo *baserel, /* * Get size of the file. It might not be there at plan time, though, in - * which case we have to use a default estimate. + * which case we have to use a default estimate. We also have to fall + * back to the default if using a program as the input. */ - if (stat(fdw_private->filename, &stat_buf) < 0) + if (fdw_private->is_program || stat(fdw_private->filename, &stat_buf) < 0) stat_buf.st_size = 10 * BLCKSZ; /* @@ -1000,6 +1039,11 @@ estimate_costs(PlannerInfo *root, RelOptInfo *baserel, * that I/O costs are equivalent to a regular table file of the same size. * However, we take per-tuple CPU costs as 10x of a seqscan, to account * for the cost of parsing records. + * + * In the case of a program source, this calculation is even more divorced + * from reality, but we have no good alternative; and it's not clear that + * the numbers we produce here matter much anyway, since there's only one + * access path for the rel. */ run_cost += seq_page_cost * pages; @@ -1036,6 +1080,7 @@ file_acquire_sample_rows(Relation onerel, int elevel, bool *nulls; bool found; char *filename; + bool is_program; List *options; CopyState cstate; ErrorContextCallback errcallback; @@ -1050,12 +1095,12 @@ file_acquire_sample_rows(Relation onerel, int elevel, nulls = (bool *) palloc(tupDesc->natts * sizeof(bool)); /* Fetch options of foreign table */ - fileGetOptions(RelationGetRelid(onerel), &filename, &options); + fileGetOptions(RelationGetRelid(onerel), &filename, &is_program, &options); /* * Create CopyState from FDW options. */ - cstate = BeginCopyFrom(NULL, onerel, filename, false, NIL, options); + cstate = BeginCopyFrom(NULL, onerel, filename, is_program, NIL, options); /* * Use per-tuple memory context to prevent leak of memory used to read diff --git a/contrib/file_fdw/output/file_fdw.source b/contrib/file_fdw/output/file_fdw.source index 6fa54409b9..01e2690a82 100644 --- a/contrib/file_fdw/output/file_fdw.source +++ b/contrib/file_fdw/output/file_fdw.source @@ -76,7 +76,7 @@ CREATE FOREIGN TABLE tbl () SERVER file_server OPTIONS (format 'csv', null ' '); -- ERROR ERROR: COPY null representation cannot use newline or carriage return CREATE FOREIGN TABLE tbl () SERVER file_server; -- ERROR -ERROR: filename is required for file_fdw foreign tables +ERROR: either filename or program is required for file_fdw foreign tables CREATE FOREIGN TABLE agg_text ( a int2 CHECK (a >= 0), b float4 @@ -132,7 +132,7 @@ ERROR: invalid option "force_not_null" HINT: There are no valid options in this context. CREATE FOREIGN TABLE tbl () SERVER file_server OPTIONS (force_not_null '*'); -- ERROR ERROR: invalid option "force_not_null" -HINT: Valid options in this context are: filename, format, header, delimiter, quote, escape, null, encoding +HINT: Valid options in this context are: filename, program, format, header, delimiter, quote, escape, null, encoding -- force_null is not allowed to be specified at any foreign object level: ALTER FOREIGN DATA WRAPPER file_fdw OPTIONS (ADD force_null '*'); -- ERROR ERROR: invalid option "force_null" @@ -145,7 +145,7 @@ ERROR: invalid option "force_null" HINT: There are no valid options in this context. CREATE FOREIGN TABLE tbl () SERVER file_server OPTIONS (force_null '*'); -- ERROR ERROR: invalid option "force_null" -HINT: Valid options in this context are: filename, format, header, delimiter, quote, escape, null, encoding +HINT: Valid options in this context are: filename, program, format, header, delimiter, quote, escape, null, encoding -- basic query tests SELECT * FROM agg_text WHERE b > 10.0 ORDER BY a; a | b diff --git a/doc/src/sgml/file-fdw.sgml b/doc/src/sgml/file-fdw.sgml index d3b39aa120..309a303e03 100644 --- a/doc/src/sgml/file-fdw.sgml +++ b/doc/src/sgml/file-fdw.sgml @@ -10,10 +10,11 @@ The file_fdw module provides the foreign-data wrapper file_fdw, which can be used to access data - files in the server's file system. Data files must be in a format + files in the server's file system, or to execute programs on the server + and read their output. The data file or program output must be in a format that can be read by COPY FROM; see for details. - Access to such data files is currently read-only. + Access to data files is currently read-only. @@ -27,7 +28,22 @@ - Specifies the file to be read. Required. Must be an absolute path name. + Specifies the file to be read. Must be an absolute path name. + Either filename or program must be + specified, but not both. + + + + + + program + + + + Specifies the command to be executed. The standard output of this + command will be read as though COPY FROM PROGRAM were used. + Either program or filename must be + specified, but not both. @@ -37,7 +53,7 @@ - Specifies the file's format, + Specifies the data format, the same as COPY's FORMAT option. @@ -48,7 +64,7 @@ - Specifies whether the file has a header line, + Specifies whether the data has a header line, the same as COPY's HEADER option. @@ -59,7 +75,7 @@ - Specifies the file's delimiter character, + Specifies the data delimiter character, the same as COPY's DELIMITER option. @@ -70,7 +86,7 @@ - Specifies the file's quote character, + Specifies the data quote character, the same as COPY's QUOTE option. @@ -81,7 +97,7 @@ - Specifies the file's escape character, + Specifies the data escape character, the same as COPY's ESCAPE option. @@ -92,7 +108,7 @@ - Specifies the file's null string, + Specifies the data null string, the same as COPY's NULL option. @@ -103,7 +119,7 @@ - Specifies the file's encoding, + Specifies the data encoding, the same as COPY's ENCODING option. @@ -112,11 +128,11 @@ - Note that while COPY allows options such as OIDS and HEADER - to be specified without a corresponding value, the foreign data wrapper + Note that while COPY allows options such as HEADER + to be specified without a corresponding value, the foreign table option syntax requires a value to be present in all cases. To activate - COPY options normally supplied without a value, you can - instead pass the value TRUE. + COPY options typically written without a value, you can pass + the value TRUE, since all such options are Booleans. @@ -133,7 +149,7 @@ This is a Boolean option. If true, it specifies that values of the column should not be matched against the null string (that is, the - file-level null option). This has the same effect + table-level null option). This has the same effect as listing the column in COPY's FORCE_NOT_NULL option. @@ -171,14 +187,24 @@ Changing table-level options requires superuser privileges, for security - reasons: only a superuser should be able to determine which file is read. - In principle non-superusers could be allowed to change the other options, - but that's not supported at present. + reasons: only a superuser should be able to control which file is read + or which program is run. In principle non-superusers could be allowed to + change the other options, but that's not supported at present. + + + + When specifying the program option, keep in mind that the option + string is executed by the shell. If you need to pass any arguments to the + command that come from an untrusted source, you must be careful to strip or + escape any characters that might have special meaning to the shell. + For security reasons, it is best to use a fixed command string, or at least + avoid passing any user input in it. For a foreign table using file_fdw, EXPLAIN shows - the name of the file to be read. Unless COSTS OFF is + the name of the file to be read or program to be run. + For a file, unless COSTS OFF is specified, the file size (in bytes) is shown as well. @@ -186,7 +212,7 @@ Create a Foreign Table for PostgreSQL CSV Logs - One of the obvious uses for the file_fdw is to make + One of the obvious uses for file_fdw is to make the PostgreSQL activity log available as a table for querying. To do this, first you must be logging to a CSV file, which here we will call pglog.csv. First, install file_fdw -- cgit v1.2.3 From fd321a1dfd64d30bf1652ea6b39b654304f68ae4 Mon Sep 17 00:00:00 2001 From: Stephen Frost Date: Thu, 29 Sep 2016 22:13:38 -0400 Subject: Remove superuser checks in pgstattuple Now that we track initial privileges on extension objects and changes to those permissions, we can drop the superuser() checks from the various functions which are part of the pgstattuple extension and rely on the GRANT system to control access to those functions. Since a pg_upgrade will preserve the version of the extension which existed prior to the upgrade, we can't simply modify the existing functions but instead need to create new functions which remove the checks and update the SQL-level functions to use the new functions (and to REVOKE EXECUTE rights on those functions from PUBLIC). Thanks to Tom and Andres for adding support for extensions to follow update paths (see: 40b449a), allowing this patch to be much smaller since no new base version script needed to be included. Approach suggested by Noah. Reviewed by Michael Paquier. --- contrib/pgstattuple/Makefile | 7 +- contrib/pgstattuple/pgstatapprox.c | 39 ++++++-- contrib/pgstattuple/pgstatindex.c | 122 ++++++++++++++++++++++++-- contrib/pgstattuple/pgstattuple--1.4--1.5.sql | 111 +++++++++++++++++++++++ contrib/pgstattuple/pgstattuple.c | 41 +++++++++ contrib/pgstattuple/pgstattuple.control | 2 +- doc/src/sgml/pgstattuple.sgml | 8 ++ 7 files changed, 315 insertions(+), 15 deletions(-) create mode 100644 contrib/pgstattuple/pgstattuple--1.4--1.5.sql (limited to 'doc/src') diff --git a/contrib/pgstattuple/Makefile b/contrib/pgstattuple/Makefile index e732680dea..294077d4c1 100644 --- a/contrib/pgstattuple/Makefile +++ b/contrib/pgstattuple/Makefile @@ -4,9 +4,10 @@ MODULE_big = pgstattuple OBJS = pgstattuple.o pgstatindex.o pgstatapprox.o $(WIN32RES) EXTENSION = pgstattuple -DATA = pgstattuple--1.4.sql pgstattuple--1.3--1.4.sql \ - pgstattuple--1.2--1.3.sql pgstattuple--1.1--1.2.sql \ - pgstattuple--1.0--1.1.sql pgstattuple--unpackaged--1.0.sql +DATA = pgstattuple--1.4.sql pgstattuple--1.4--1.5.sql \ + pgstattuple--1.3--1.4.sql pgstattuple--1.2--1.3.sql \ + pgstattuple--1.1--1.2.sql pgstattuple--1.0--1.1.sql \ + pgstattuple--unpackaged--1.0.sql PGFILEDESC = "pgstattuple - tuple-level statistics" REGRESS = pgstattuple diff --git a/contrib/pgstattuple/pgstatapprox.c b/contrib/pgstattuple/pgstatapprox.c index a49ff543d2..f524fc4e30 100644 --- a/contrib/pgstattuple/pgstatapprox.c +++ b/contrib/pgstattuple/pgstatapprox.c @@ -29,6 +29,9 @@ #include "commands/vacuum.h" PG_FUNCTION_INFO_V1(pgstattuple_approx); +PG_FUNCTION_INFO_V1(pgstattuple_approx_v1_5); + +Datum pgstattuple_approx_internal(Oid relid, FunctionCallInfo fcinfo); typedef struct output_type { @@ -204,11 +207,42 @@ statapprox_heap(Relation rel, output_type *stat) /* * Returns estimated live/dead tuple statistics for the given relid. + * + * The superuser() check here must be kept as the library might be upgraded + * without the extension being upgraded, meaning that in pre-1.5 installations + * these functions could be called by any user. */ Datum pgstattuple_approx(PG_FUNCTION_ARGS) { Oid relid = PG_GETARG_OID(0); + + if (!superuser()) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + (errmsg("must be superuser to use pgstattuple functions")))); + + PG_RETURN_DATUM(pgstattuple_approx_internal(relid, fcinfo)); +} + +/* + * As of pgstattuple version 1.5, we no longer need to check if the user + * is a superuser because we REVOKE EXECUTE on the SQL function from PUBLIC. + * Users can then grant access to it based on their policies. + * + * Otherwise identical to pgstattuple_approx (above). + */ +Datum +pgstattuple_approx_v1_5(PG_FUNCTION_ARGS) +{ + Oid relid = PG_GETARG_OID(0); + + PG_RETURN_DATUM(pgstattuple_approx_internal(relid, fcinfo)); +} + +Datum +pgstattuple_approx_internal(Oid relid, FunctionCallInfo fcinfo) +{ Relation rel; output_type stat = {0}; TupleDesc tupdesc; @@ -217,11 +251,6 @@ pgstattuple_approx(PG_FUNCTION_ARGS) HeapTuple ret; int i = 0; - if (!superuser()) - ereport(ERROR, - (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - (errmsg("must be superuser to use pgstattuple functions")))); - if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) elog(ERROR, "return type must be a row type"); diff --git a/contrib/pgstattuple/pgstatindex.c b/contrib/pgstattuple/pgstatindex.c index 6084589e07..d9a722ac6b 100644 --- a/contrib/pgstattuple/pgstatindex.c +++ b/contrib/pgstattuple/pgstatindex.c @@ -54,6 +54,14 @@ PG_FUNCTION_INFO_V1(pg_relpages); PG_FUNCTION_INFO_V1(pg_relpagesbyid); PG_FUNCTION_INFO_V1(pgstatginindex); +PG_FUNCTION_INFO_V1(pgstatindex_v1_5); +PG_FUNCTION_INFO_V1(pgstatindexbyid_v1_5); +PG_FUNCTION_INFO_V1(pg_relpages_v1_5); +PG_FUNCTION_INFO_V1(pg_relpagesbyid_v1_5); +PG_FUNCTION_INFO_V1(pgstatginindex_v1_5); + +Datum pgstatginindex_internal(Oid relid, FunctionCallInfo fcinfo); + #define IS_INDEX(r) ((r)->rd_rel->relkind == RELKIND_INDEX) #define IS_BTREE(r) ((r)->rd_rel->relam == BTREE_AM_OID) #define IS_GIN(r) ((r)->rd_rel->relam == GIN_AM_OID) @@ -99,6 +107,10 @@ static Datum pgstatindex_impl(Relation rel, FunctionCallInfo fcinfo); * pgstatindex() * * Usage: SELECT * FROM pgstatindex('t1_pkey'); + * + * The superuser() check here must be kept as the library might be upgraded + * without the extension being upgraded, meaning that in pre-1.5 installations + * these functions could be called by any user. * ------------------------------------------------------ */ Datum @@ -119,6 +131,31 @@ pgstatindex(PG_FUNCTION_ARGS) PG_RETURN_DATUM(pgstatindex_impl(rel, fcinfo)); } +/* + * As of pgstattuple version 1.5, we no longer need to check if the user + * is a superuser because we REVOKE EXECUTE on the function from PUBLIC. + * Users can then grant access to it based on their policies. + * + * Otherwise identical to pgstatindex (above). + */ +Datum +pgstatindex_v1_5(PG_FUNCTION_ARGS) +{ + text *relname = PG_GETARG_TEXT_P(0); + Relation rel; + RangeVar *relrv; + + relrv = makeRangeVarFromNameList(textToQualifiedNameList(relname)); + rel = relation_openrv(relrv, AccessShareLock); + + PG_RETURN_DATUM(pgstatindex_impl(rel, fcinfo)); +} + +/* + * The superuser() check here must be kept as the library might be upgraded + * without the extension being upgraded, meaning that in pre-1.5 installations + * these functions could be called by any user. + */ Datum pgstatindexbyid(PG_FUNCTION_ARGS) { @@ -135,6 +172,18 @@ pgstatindexbyid(PG_FUNCTION_ARGS) PG_RETURN_DATUM(pgstatindex_impl(rel, fcinfo)); } +/* No need for superuser checks in v1.5, see above */ +Datum +pgstatindexbyid_v1_5(PG_FUNCTION_ARGS) +{ + Oid relid = PG_GETARG_OID(0); + Relation rel; + + rel = relation_open(relid, AccessShareLock); + + PG_RETURN_DATUM(pgstatindex_impl(rel, fcinfo)); +} + static Datum pgstatindex_impl(Relation rel, FunctionCallInfo fcinfo) { @@ -292,6 +341,8 @@ pgstatindex_impl(Relation rel, FunctionCallInfo fcinfo) * * Usage: SELECT pg_relpages('t1'); * SELECT pg_relpages('t1_pkey'); + * + * Must keep superuser() check, see above. * -------------------------------------------------------- */ Datum @@ -319,6 +370,28 @@ pg_relpages(PG_FUNCTION_ARGS) PG_RETURN_INT64(relpages); } +/* No need for superuser checks in v1.5, see above */ +Datum +pg_relpages_v1_5(PG_FUNCTION_ARGS) +{ + text *relname = PG_GETARG_TEXT_P(0); + int64 relpages; + Relation rel; + RangeVar *relrv; + + relrv = makeRangeVarFromNameList(textToQualifiedNameList(relname)); + rel = relation_openrv(relrv, AccessShareLock); + + /* note: this will work OK on non-local temp tables */ + + relpages = RelationGetNumberOfBlocks(rel); + + relation_close(rel, AccessShareLock); + + PG_RETURN_INT64(relpages); +} + +/* Must keep superuser() check, see above. */ Datum pg_relpagesbyid(PG_FUNCTION_ARGS) { @@ -342,16 +415,58 @@ pg_relpagesbyid(PG_FUNCTION_ARGS) PG_RETURN_INT64(relpages); } +/* No need for superuser checks in v1.5, see above */ +Datum +pg_relpagesbyid_v1_5(PG_FUNCTION_ARGS) +{ + Oid relid = PG_GETARG_OID(0); + int64 relpages; + Relation rel; + + rel = relation_open(relid, AccessShareLock); + + /* note: this will work OK on non-local temp tables */ + + relpages = RelationGetNumberOfBlocks(rel); + + relation_close(rel, AccessShareLock); + + PG_RETURN_INT64(relpages); +} + /* ------------------------------------------------------ * pgstatginindex() * * Usage: SELECT * FROM pgstatginindex('ginindex'); + * + * Must keep superuser() check, see above. * ------------------------------------------------------ */ Datum pgstatginindex(PG_FUNCTION_ARGS) { Oid relid = PG_GETARG_OID(0); + + if (!superuser()) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + (errmsg("must be superuser to use pgstattuple functions")))); + + PG_RETURN_DATUM(pgstatginindex_internal(relid, fcinfo)); +} + +/* No need for superuser checks in v1.5, see above */ +Datum +pgstatginindex_v1_5(PG_FUNCTION_ARGS) +{ + Oid relid = PG_GETARG_OID(0); + + PG_RETURN_DATUM(pgstatginindex_internal(relid, fcinfo)); +} + +Datum +pgstatginindex_internal(Oid relid, FunctionCallInfo fcinfo) +{ Relation rel; Buffer buffer; Page page; @@ -363,11 +478,6 @@ pgstatginindex(PG_FUNCTION_ARGS) bool nulls[3] = {false, false, false}; Datum result; - if (!superuser()) - ereport(ERROR, - (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - (errmsg("must be superuser to use pgstattuple functions")))); - rel = relation_open(relid, AccessShareLock); if (!IS_INDEX(rel) || !IS_GIN(rel)) @@ -415,5 +525,5 @@ pgstatginindex(PG_FUNCTION_ARGS) tuple = heap_form_tuple(tupleDesc, values, nulls); result = HeapTupleGetDatum(tuple); - PG_RETURN_DATUM(result); + return (result); } diff --git a/contrib/pgstattuple/pgstattuple--1.4--1.5.sql b/contrib/pgstattuple/pgstattuple--1.4--1.5.sql new file mode 100644 index 0000000000..65d7f19c2a --- /dev/null +++ b/contrib/pgstattuple/pgstattuple--1.4--1.5.sql @@ -0,0 +1,111 @@ +/* contrib/pgstattuple/pgstattuple--1.4--1.5.sql */ + +-- complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION pgstattuple UPDATE TO '1.5'" to load this file. \quit + +CREATE OR REPLACE FUNCTION pgstattuple(IN relname text, + OUT table_len BIGINT, -- physical table length in bytes + OUT tuple_count BIGINT, -- number of live tuples + OUT tuple_len BIGINT, -- total tuples length in bytes + OUT tuple_percent FLOAT8, -- live tuples in % + OUT dead_tuple_count BIGINT, -- number of dead tuples + OUT dead_tuple_len BIGINT, -- total dead tuples length in bytes + OUT dead_tuple_percent FLOAT8, -- dead tuples in % + OUT free_space BIGINT, -- free space in bytes + OUT free_percent FLOAT8) -- free space in % +AS 'MODULE_PATHNAME', 'pgstattuple_v1_5' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pgstattuple(text) FROM PUBLIC; + +CREATE OR REPLACE FUNCTION pgstatindex(IN relname text, + OUT version INT, + OUT tree_level INT, + OUT index_size BIGINT, + OUT root_block_no BIGINT, + OUT internal_pages BIGINT, + OUT leaf_pages BIGINT, + OUT empty_pages BIGINT, + OUT deleted_pages BIGINT, + OUT avg_leaf_density FLOAT8, + OUT leaf_fragmentation FLOAT8) +AS 'MODULE_PATHNAME', 'pgstatindex_v1_5' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pgstatindex(text) FROM PUBLIC; + +CREATE OR REPLACE FUNCTION pg_relpages(IN relname text) +RETURNS BIGINT +AS 'MODULE_PATHNAME', 'pg_relpages_v1_5' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pg_relpages(text) FROM PUBLIC; + +/* New stuff in 1.1 begins here */ + +CREATE OR REPLACE FUNCTION pgstatginindex(IN relname regclass, + OUT version INT4, + OUT pending_pages INT4, + OUT pending_tuples BIGINT) +AS 'MODULE_PATHNAME', 'pgstatginindex_v1_5' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pgstatginindex(regclass) FROM PUBLIC; + +/* New stuff in 1.2 begins here */ + +CREATE OR REPLACE FUNCTION pgstattuple(IN reloid regclass, + OUT table_len BIGINT, -- physical table length in bytes + OUT tuple_count BIGINT, -- number of live tuples + OUT tuple_len BIGINT, -- total tuples length in bytes + OUT tuple_percent FLOAT8, -- live tuples in % + OUT dead_tuple_count BIGINT, -- number of dead tuples + OUT dead_tuple_len BIGINT, -- total dead tuples length in bytes + OUT dead_tuple_percent FLOAT8, -- dead tuples in % + OUT free_space BIGINT, -- free space in bytes + OUT free_percent FLOAT8) -- free space in % +AS 'MODULE_PATHNAME', 'pgstattuplebyid_v1_5' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pgstattuple(regclass) FROM PUBLIC; + +CREATE OR REPLACE FUNCTION pgstatindex(IN relname regclass, + OUT version INT, + OUT tree_level INT, + OUT index_size BIGINT, + OUT root_block_no BIGINT, + OUT internal_pages BIGINT, + OUT leaf_pages BIGINT, + OUT empty_pages BIGINT, + OUT deleted_pages BIGINT, + OUT avg_leaf_density FLOAT8, + OUT leaf_fragmentation FLOAT8) +AS 'MODULE_PATHNAME', 'pgstatindexbyid_v1_5' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pgstatindex(regclass) FROM PUBLIC; + +CREATE OR REPLACE FUNCTION pg_relpages(IN relname regclass) +RETURNS BIGINT +AS 'MODULE_PATHNAME', 'pg_relpagesbyid_v1_5' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pg_relpages(regclass) FROM PUBLIC; + +/* New stuff in 1.3 begins here */ + +CREATE OR REPLACE FUNCTION pgstattuple_approx(IN reloid regclass, + OUT table_len BIGINT, -- physical table length in bytes + OUT scanned_percent FLOAT8, -- what percentage of the table's pages was scanned + OUT approx_tuple_count BIGINT, -- estimated number of live tuples + OUT approx_tuple_len BIGINT, -- estimated total length in bytes of live tuples + OUT approx_tuple_percent FLOAT8, -- live tuples in % (based on estimate) + OUT dead_tuple_count BIGINT, -- exact number of dead tuples + OUT dead_tuple_len BIGINT, -- exact total length in bytes of dead tuples + OUT dead_tuple_percent FLOAT8, -- dead tuples in % (based on estimate) + OUT approx_free_space BIGINT, -- estimated free space in bytes + OUT approx_free_percent FLOAT8) -- free space in % (based on estimate) +AS 'MODULE_PATHNAME', 'pgstattuple_approx_v1_5' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pgstattuple_approx(regclass) FROM PUBLIC; diff --git a/contrib/pgstattuple/pgstattuple.c b/contrib/pgstattuple/pgstattuple.c index c1122b496a..68b07aaf26 100644 --- a/contrib/pgstattuple/pgstattuple.c +++ b/contrib/pgstattuple/pgstattuple.c @@ -40,7 +40,9 @@ PG_MODULE_MAGIC; PG_FUNCTION_INFO_V1(pgstattuple); +PG_FUNCTION_INFO_V1(pgstattuple_v1_5); PG_FUNCTION_INFO_V1(pgstattuplebyid); +PG_FUNCTION_INFO_V1(pgstattuplebyid_v1_5); /* * struct pgstattuple_type @@ -152,6 +154,10 @@ build_pgstattuple_type(pgstattuple_type *stat, FunctionCallInfo fcinfo) * * C FUNCTION definition * pgstattuple(text) returns pgstattuple_type + * + * The superuser() check here must be kept as the library might be upgraded + * without the extension being upgraded, meaning that in pre-1.5 installations + * these functions could be called by any user. * ---------- */ @@ -174,6 +180,28 @@ pgstattuple(PG_FUNCTION_ARGS) PG_RETURN_DATUM(pgstat_relation(rel, fcinfo)); } +/* + * As of pgstattuple version 1.5, we no longer need to check if the user + * is a superuser because we REVOKE EXECUTE on the function from PUBLIC. + * Users can then grant access to it based on their policies. + * + * Otherwise identical to pgstattuple (above). + */ +Datum +pgstattuple_v1_5(PG_FUNCTION_ARGS) +{ + text *relname = PG_GETARG_TEXT_P(0); + RangeVar *relrv; + Relation rel; + + /* open relation */ + relrv = makeRangeVarFromNameList(textToQualifiedNameList(relname)); + rel = relation_openrv(relrv, AccessShareLock); + + PG_RETURN_DATUM(pgstat_relation(rel, fcinfo)); +} + +/* Must keep superuser() check, see above. */ Datum pgstattuplebyid(PG_FUNCTION_ARGS) { @@ -191,6 +219,19 @@ pgstattuplebyid(PG_FUNCTION_ARGS) PG_RETURN_DATUM(pgstat_relation(rel, fcinfo)); } +/* Remove superuser() check for 1.5 version, see above */ +Datum +pgstattuplebyid_v1_5(PG_FUNCTION_ARGS) +{ + Oid relid = PG_GETARG_OID(0); + Relation rel; + + /* open relation */ + rel = relation_open(relid, AccessShareLock); + + PG_RETURN_DATUM(pgstat_relation(rel, fcinfo)); +} + /* * pgstat_relation */ diff --git a/contrib/pgstattuple/pgstattuple.control b/contrib/pgstattuple/pgstattuple.control index fa328fd664..6af40757b2 100644 --- a/contrib/pgstattuple/pgstattuple.control +++ b/contrib/pgstattuple/pgstattuple.control @@ -1,5 +1,5 @@ # pgstattuple extension comment = 'show tuple-level statistics' -default_version = '1.4' +default_version = '1.5' module_pathname = '$libdir/pgstattuple' relocatable = true diff --git a/doc/src/sgml/pgstattuple.sgml b/doc/src/sgml/pgstattuple.sgml index 61340bedbc..9ada5d209a 100644 --- a/doc/src/sgml/pgstattuple.sgml +++ b/doc/src/sgml/pgstattuple.sgml @@ -12,6 +12,14 @@ obtain tuple-level statistics. + + As these functions return detailed page-level information, only the superuser + has EXECUTE privileges on them upon installation. After the functions have + been installed, users may issue GRANT commands to change + the privileges on the functions to allow non-superusers to execute them. See + the description of the command for specifics. + + Functions -- cgit v1.2.3 From 33596edf09516a7cab65914e16cfd6adf9fc55d1 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 1 Oct 2016 15:32:53 -0400 Subject: Copy-editing for contrib/pg_visibility documentation. Add omitted names for some function parameters. Fix some minor grammatical issues. --- doc/src/sgml/pgvisibility.sgml | 87 +++++++++++++++++++++--------------------- 1 file changed, 44 insertions(+), 43 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/pgvisibility.sgml b/doc/src/sgml/pgvisibility.sgml index d764eff9a0..fd486696fc 100644 --- a/doc/src/sgml/pgvisibility.sgml +++ b/doc/src/sgml/pgvisibility.sgml @@ -9,31 +9,33 @@ The pg_visibility module provides a means for examining the - visibility map (VM) and page-level visibility information. It also - provides functions to check the integrity of the visibility map and to + visibility map (VM) and page-level visibility information of a table. + It also provides functions to check the integrity of a visibility map and to force it to be rebuilt. Three different bits are used to store information about page-level visibility. The all-visible bit in the visibility map indicates that every - tuple on a given page of a relation is visible to every current transaction. - The all-frozen bit in the visibility map indicates that every tuple on the - page is frozen; that is, no future vacuum will need to modify the page - until such time as a tuple is inserted, updated, deleted, or locked on - that page. The page-level PD_ALL_VISIBLE bit has the + tuple in the corresponding page of the relation is visible to every current + and future transaction. The all-frozen bit in the visibility map indicates + that every tuple in the page is frozen; that is, no future vacuum will need + to modify the page until such time as a tuple is inserted, updated, deleted, + or locked on that page. + The page header's PD_ALL_VISIBLE bit has the same meaning as the all-visible bit in the visibility map, but is stored - within the data page itself rather than a separate data structure. These - will normally agree, but the page-level bit can sometimes be set while the - visibility map bit is clear after a crash recovery; or they can disagree - because of a change which occurs after pg_visibility examines - the visibility map and before it examines the data page. Any event which - causes data corruption can also cause these bits to disagree. + within the data page itself rather than in a separate data structure. + These two bits will normally agree, but the page's all-visible bit can + sometimes be set while the visibility map bit is clear after a crash + recovery. The reported values can also disagree because of a change that + occurs after pg_visibility examines the visibility map and + before it examines the data page. Any event that causes data corruption + can also cause these bits to disagree. - Functions which display information about PD_ALL_VISIBLE - are much more costly than those which only consult the visibility map, + Functions that display information about PD_ALL_VISIBLE bits + are much more costly than those that only consult the visibility map, because they must read the relation's data blocks rather than only the (much smaller) visibility map. Functions that check the relation's data blocks are similarly expensive. @@ -44,7 +46,7 @@ - pg_visibility_map(regclass, blkno bigint, all_visible OUT boolean, all_frozen OUT boolean) returns record + pg_visibility_map(relation regclass, blkno bigint, all_visible OUT boolean, all_frozen OUT boolean) returns record Returns the all-visible and all-frozen bits in the visibility map for @@ -54,40 +56,40 @@ - pg_visibility(regclass, blkno bigint, all_visible OUT boolean, all_frozen OUT boolean, pd_all_visible OUT boolean) returns record + pg_visibility(relation regclass, blkno bigint, all_visible OUT boolean, all_frozen OUT boolean, pd_all_visible OUT boolean) returns record Returns the all-visible and all-frozen bits in the visibility map for the given block of the given relation, plus the - PD_ALL_VISIBLE bit for that block. + PD_ALL_VISIBLE bit of that block. - pg_visibility_map(regclass, blkno OUT bigint, all_visible OUT boolean, all_frozen OUT boolean) returns setof record + pg_visibility_map(relation regclass, blkno OUT bigint, all_visible OUT boolean, all_frozen OUT boolean) returns setof record Returns the all-visible and all-frozen bits in the visibility map for - each block the given relation. + each block of the given relation. - pg_visibility(regclass, blkno OUT bigint, all_visible OUT boolean, all_frozen OUT boolean, pd_all_visible OUT boolean) returns setof record + pg_visibility(relation regclass, blkno OUT bigint, all_visible OUT boolean, all_frozen OUT boolean, pd_all_visible OUT boolean) returns setof record Returns the all-visible and all-frozen bits in the visibility map for - each block the given relation, plus the PD_ALL_VISIBLE - bit for each block. + each block of the given relation, plus the PD_ALL_VISIBLE + bit of each block. - pg_visibility_map_summary(regclass, all_visible OUT bigint, all_frozen OUT bigint) returns record + pg_visibility_map_summary(relation regclass, all_visible OUT bigint, all_frozen OUT bigint) returns record @@ -96,50 +98,49 @@ - + - pg_check_frozen(regclass, t_ctid OUT tid) returns setof tid + pg_check_frozen(relation regclass, t_ctid OUT tid) returns setof tid - Returns the TIDs of non-frozen tuples present in pages marked all-frozen + Returns the TIDs of non-frozen tuples stored in pages marked all-frozen in the visibility map. If this function returns a non-empty set of - TIDs, the database is corrupt. + TIDs, the visibility map is corrupt. - - - pg_check_visible(regclass, t_ctid OUT tid) returns setof tid + + + pg_check_visible(relation regclass, t_ctid OUT tid) returns setof tid - Returns the TIDs of tuples which are not all-visible despite the fact - that the pages which contain them are marked as all-visible in the - visibility map. If this function returns a non-empty set of TIDs, the - database is corrupt. + Returns the TIDs of non-all-visible tuples stored in pages marked + all-visible in the visibility map. If this function returns a non-empty + set of TIDs, the visibility map is corrupt. - pg_truncate_visibility_map(regclass) returns void + pg_truncate_visibility_map(relation regclass) returns void - Truncates the visibility map for the given relation. This function - is only expected to be useful if you suspect that the visibility map - for the indicated relation is corrupt and wish to rebuild it. The first - VACUUM executed on the given relation after this function - is executed will scan every page in the relation and rebuild the - visibility map. + Truncates the visibility map for the given relation. This function is + useful if you believe that the visibility map for the relation is + corrupt and wish to force rebuilding it. The first VACUUM + executed on the given relation after this function is executed will scan + every page in the relation and rebuild the visibility map. (Until that + is done, queries will treat the visibility map as containing all zeroes.) - By default, these functions are not publicly executable. + By default, these functions are executable only by superusers. -- cgit v1.2.3 From e8bdee2770ff52aab208bc6f8965a4a01979d0aa Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 2 Oct 2016 14:31:28 -0400 Subject: Add ALTER EXTENSION ADD/DROP ACCESS METHOD, and use it in pg_upgrade. Without this, an extension containing an access method is not properly dumped/restored during pg_upgrade --- the AM ends up not being a member of the extension after upgrading. Another oversight in commit 473b93287, reported by Andrew Dunstan. Report: --- doc/src/sgml/ref/alter_extension.sgml | 1 + src/backend/parser/gram.y | 11 ++++++++++- src/bin/pg_dump/pg_dump.c | 3 +++ 3 files changed, 14 insertions(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/alter_extension.sgml b/doc/src/sgml/ref/alter_extension.sgml index 7141ee352e..de6d6dca16 100644 --- a/doc/src/sgml/ref/alter_extension.sgml +++ b/doc/src/sgml/ref/alter_extension.sgml @@ -30,6 +30,7 @@ ALTER EXTENSION name DROP where member_object is: + ACCESS METHOD object_name | AGGREGATE aggregate_name ( aggregate_signature ) | CAST (source_type AS target_type) | COLLATION object_name | diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 1526c73a1c..5547fc8658 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -3931,7 +3931,16 @@ alter_extension_opt_item: *****************************************************************************/ AlterExtensionContentsStmt: - ALTER EXTENSION name add_drop AGGREGATE func_name aggr_args + ALTER EXTENSION name add_drop ACCESS METHOD name + { + AlterExtensionContentsStmt *n = makeNode(AlterExtensionContentsStmt); + n->extname = $3; + n->action = $4; + n->objtype = OBJECT_ACCESS_METHOD; + n->objname = list_make1(makeString($7)); + $$ = (Node *)n; + } + | ALTER EXTENSION name add_drop AGGREGATE func_name aggr_args { AlterExtensionContentsStmt *n = makeNode(AlterExtensionContentsStmt); n->extname = $3; diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 51b8a1a622..299e88788e 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -12505,6 +12505,9 @@ dumpAccessMethod(Archive *fout, AccessMethodInfo *aminfo) appendPQExpBuffer(labelq, "ACCESS METHOD %s", qamname); + if (dopt->binary_upgrade) + binary_upgrade_extension_member(q, &aminfo->dobj, labelq->data); + if (aminfo->dobj.dump & DUMP_COMPONENT_DEFINITION) ArchiveEntry(fout, aminfo->dobj.catId, aminfo->dobj.dumpId, aminfo->dobj.name, -- cgit v1.2.3 From 6f3bd98ebfc008cbd676da777bb0b2376c4c4bfa Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Tue, 4 Oct 2016 10:50:13 -0400 Subject: Extend framework from commit 53be0b1ad to report latch waits. WaitLatch, WaitLatchOrSocket, and WaitEventSetWait now taken an additional wait_event_info parameter; legal values are defined in pgstat.h. This makes it possible to uniquely identify every point in the core code where we are waiting for a latch; extensions can pass WAIT_EXTENSION. Because latches were the major wait primitive not previously covered by this patch, it is now possible to see information in pg_stat_activity on a large number of important wait events not previously addressed, such as ClientRead, ClientWrite, and SyncRep. Unfortunately, many of the wait events added by this patch will fail to appear in pg_stat_activity because they're only used in background processes which don't currently appear in pg_stat_activity. We should fix this either by creating a separate view for such information, or else by deciding to include them in pg_stat_activity after all. Michael Paquier and Robert Haas, reviewed by Alexander Korotkov and Thomas Munro. --- contrib/postgres_fdw/connection.c | 3 +- doc/src/sgml/monitoring.sgml | 169 ++++++++++++++++++++++++ src/backend/access/transam/parallel.c | 4 +- src/backend/access/transam/xlog.c | 7 +- src/backend/executor/nodeGather.c | 3 +- src/backend/libpq/be-secure-openssl.c | 4 +- src/backend/libpq/be-secure.c | 7 +- src/backend/libpq/pqmq.c | 4 +- src/backend/postmaster/autovacuum.c | 3 +- src/backend/postmaster/bgworker.c | 7 +- src/backend/postmaster/bgwriter.c | 5 +- src/backend/postmaster/checkpointer.c | 3 +- src/backend/postmaster/pgarch.c | 3 +- src/backend/postmaster/pgstat.c | 236 ++++++++++++++++++++++++++++++++-- src/backend/postmaster/syslogger.c | 4 +- src/backend/postmaster/walwriter.c | 3 +- src/backend/replication/basebackup.c | 3 +- src/backend/replication/syncrep.c | 4 +- src/backend/replication/walreceiver.c | 7 +- src/backend/replication/walsender.c | 9 +- src/backend/storage/buffer/bufmgr.c | 7 +- src/backend/storage/ipc/latch.c | 18 ++- src/backend/storage/ipc/shm_mq.c | 7 +- src/backend/storage/ipc/standby.c | 5 +- src/backend/storage/lmgr/lock.c | 3 - src/backend/storage/lmgr/lwlock.c | 6 +- src/backend/storage/lmgr/predicate.c | 3 +- src/backend/storage/lmgr/proc.c | 8 +- src/backend/utils/adt/misc.c | 4 +- src/include/pgstat.h | 99 ++++++++++++-- src/include/storage/latch.h | 9 +- src/include/storage/lwlock.h | 2 +- src/include/storage/proc.h | 2 +- src/test/modules/test_shm_mq/setup.c | 3 +- src/test/modules/test_shm_mq/test.c | 3 +- 35 files changed, 584 insertions(+), 83 deletions(-) (limited to 'doc/src') diff --git a/contrib/postgres_fdw/connection.c b/contrib/postgres_fdw/connection.c index 8ca1c1c898..9badfe6a7d 100644 --- a/contrib/postgres_fdw/connection.c +++ b/contrib/postgres_fdw/connection.c @@ -17,6 +17,7 @@ #include "access/xact.h" #include "mb/pg_wchar.h" #include "miscadmin.h" +#include "pgstat.h" #include "storage/latch.h" #include "utils/hsearch.h" #include "utils/memutils.h" @@ -496,7 +497,7 @@ pgfdw_get_result(PGconn *conn, const char *query) wc = WaitLatchOrSocket(MyLatch, WL_LATCH_SET | WL_SOCKET_READABLE, PQsocket(conn), - -1L); + -1L, WAIT_EXTENSION); ResetLatch(MyLatch); CHECK_FOR_INTERRUPTS(); diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index f400785721..3de489e2f0 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -679,6 +679,42 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser buffer in question. + + + Activity: The server process is idle. This is used by + system processes waiting for activity in their main processing loop. + wait_event will identify the specific wait point. + + + + + Extension: The server process is waiting for activity + in an extension module. This category is useful for modules to + track custom waiting points. + + + + + Client: The server process is waiting for some activity + on a socket from user applications, and that the server expects + something to happen that is independent from its internal processes. + wait_event will identify the specific wait point. + + + + + IPC: The server process is waiting for some activity + from another process in the server. wait_event will + identify the specific wait point. + + + + + Timeout: The server process is waiting for a timeout + to expire. wait_event will identify the specific wait + point. + + @@ -1085,6 +1121,139 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser BufferPinWaiting to acquire a pin on a buffer. + + Activity + ArchiverMain + Waiting in main loop of the archiver process. + + + AutoVacuumMain + Waiting in main loop of autovacuum launcher process. + + + BgWriterHibernate + Waiting in background writer process, hibernating. + + + BgWriterMain + Waiting in main loop of background writer process background worker. + + + CheckpointerMain + Waiting in main loop of checkpointer process. + + + PgStatMain + Waiting in main loop of the statistics collector process. + + + RecoveryWalAll + Waiting for WAL from any kind of source (local, archive or stream) at recovery. + + + RecoveryWalStream + Waiting for WAL from a stream at recovery. + + + SysLoggerMain + Waiting in main loop of syslogger process. + + + WalReceiverMain + Waiting in main loop of WAL receiver process. + + + WalSenderMain + Waiting in main loop of WAL sender process. + + + WalWriterMain + Waiting in main loop of WAL writer process. + + + Client + ClientRead + Waiting to read data from the client. + + + ClientWrite + Waiting to write data from the client. + + + SSLOpenServer + Waiting for SSL while attempting connection. + + + WalReceiverWaitStart + Waiting for startup process to send initial data for streaming replication. + + + WalSenderWaitForWAL + Waiting for WAL to be flushed in WAL sender process. + + + WalSenderWriteData + Waiting for any activity when processing replies from WAL receiver in WAL sender process. + + + Extension + Extension + Waiting in an extension. + + + IPC + BgWorkerShutdown + Waiting for background worker to shut down. + + + BgWorkerStartup + Waiting for background worker to start up. + + + ExecuteGather + Waiting for activity from child process when executing Gather node. + + + MessageQueueInternal + Waiting for other process to be attached in shared message queue. + + + MessageQueuePutMessage + Waiting to write a protoocol message to a shared message queue. + + + MessageQueueReceive + Waiting to receive bytes from a shared message queue. + + + MessageQueueSend + Waiting to send bytes to a shared message queue. + + + ParallelFinish + Waiting for parallel workers to finish computing. + + + SafeSnapshot + Waiting for a snapshot for a READ ONLY DEFERRABLE transaction. + + + SyncRep + Waiting for confirmation from remote server during synchronous replication. + + + Timeout + BaseBackupThrottle + Waiting during base backup when throttling activity. + + + PgSleep + Waiting in process that called pg_sleep. + + + RecoveryApplyDelay + Waiting to apply WAL at recovery because it is delayed. +
diff --git a/src/backend/access/transam/parallel.c b/src/backend/access/transam/parallel.c index cde0ed300f..59dc3949d8 100644 --- a/src/backend/access/transam/parallel.c +++ b/src/backend/access/transam/parallel.c @@ -24,6 +24,7 @@ #include "libpq/pqmq.h" #include "miscadmin.h" #include "optimizer/planmain.h" +#include "pgstat.h" #include "storage/ipc.h" #include "storage/sinval.h" #include "storage/spin.h" @@ -540,7 +541,8 @@ WaitForParallelWorkersToFinish(ParallelContext *pcxt) if (!anyone_alive) break; - WaitLatch(&MyProc->procLatch, WL_LATCH_SET, -1); + WaitLatch(&MyProc->procLatch, WL_LATCH_SET, -1, + WAIT_EVENT_PARALLEL_FINISH); ResetLatch(&MyProc->procLatch); } diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index c1b9a97147..08c87f91be 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -5827,7 +5827,8 @@ recoveryApplyDelay(XLogReaderState *record) WaitLatch(&XLogCtl->recoveryWakeupLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, - secs * 1000L + microsecs / 1000); + secs * 1000L + microsecs / 1000, + WAIT_EVENT_RECOVERY_APPLY_DELAY); } return true; } @@ -11387,7 +11388,7 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, WaitLatch(&XLogCtl->recoveryWakeupLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, - wait_time); + wait_time, WAIT_EVENT_RECOVERY_WAL_STREAM); ResetLatch(&XLogCtl->recoveryWakeupLatch); now = GetCurrentTimestamp(); } @@ -11550,7 +11551,7 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, */ WaitLatch(&XLogCtl->recoveryWakeupLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, - 5000L); + 5000L, WAIT_EVENT_RECOVERY_WAL_ALL); ResetLatch(&XLogCtl->recoveryWakeupLatch); break; } diff --git a/src/backend/executor/nodeGather.c b/src/backend/executor/nodeGather.c index 438d1b24fc..880ca62397 100644 --- a/src/backend/executor/nodeGather.c +++ b/src/backend/executor/nodeGather.c @@ -38,6 +38,7 @@ #include "executor/nodeSubplan.h" #include "executor/tqueue.h" #include "miscadmin.h" +#include "pgstat.h" #include "utils/memutils.h" #include "utils/rel.h" @@ -387,7 +388,7 @@ gather_readnext(GatherState *gatherstate) return NULL; /* Nothing to do except wait for developments. */ - WaitLatch(MyLatch, WL_LATCH_SET, 0); + WaitLatch(MyLatch, WL_LATCH_SET, 0, WAIT_EVENT_EXECUTE_GATHER); ResetLatch(MyLatch); nvisited = 0; } diff --git a/src/backend/libpq/be-secure-openssl.c b/src/backend/libpq/be-secure-openssl.c index fedb02cd82..668f217bba 100644 --- a/src/backend/libpq/be-secure-openssl.c +++ b/src/backend/libpq/be-secure-openssl.c @@ -60,6 +60,7 @@ #include "libpq/libpq.h" #include "miscadmin.h" +#include "pgstat.h" #include "storage/latch.h" #include "tcop/tcopprot.h" #include "utils/memutils.h" @@ -419,7 +420,8 @@ aloop: else waitfor = WL_SOCKET_WRITEABLE; - WaitLatchOrSocket(MyLatch, waitfor, port->sock, 0); + WaitLatchOrSocket(MyLatch, waitfor, port->sock, 0, + WAIT_EVENT_SSL_OPEN_SERVER); goto aloop; case SSL_ERROR_SYSCALL: if (r < 0) diff --git a/src/backend/libpq/be-secure.c b/src/backend/libpq/be-secure.c index cdd07d577b..b267507de9 100644 --- a/src/backend/libpq/be-secure.c +++ b/src/backend/libpq/be-secure.c @@ -33,6 +33,7 @@ #include "libpq/libpq.h" #include "miscadmin.h" +#include "pgstat.h" #include "tcop/tcopprot.h" #include "utils/memutils.h" #include "storage/ipc.h" @@ -146,7 +147,8 @@ retry: ModifyWaitEvent(FeBeWaitSet, 0, waitfor, NULL); - WaitEventSetWait(FeBeWaitSet, -1 /* no timeout */ , &event, 1); + WaitEventSetWait(FeBeWaitSet, -1 /* no timeout */ , &event, 1, + WAIT_EVENT_CLIENT_READ); /* * If the postmaster has died, it's not safe to continue running, @@ -247,7 +249,8 @@ retry: ModifyWaitEvent(FeBeWaitSet, 0, waitfor, NULL); - WaitEventSetWait(FeBeWaitSet, -1 /* no timeout */ , &event, 1); + WaitEventSetWait(FeBeWaitSet, -1 /* no timeout */ , &event, 1, + WAIT_EVENT_CLIENT_WRITE); /* See comments in secure_read. */ if (event.events & WL_POSTMASTER_DEATH) diff --git a/src/backend/libpq/pqmq.c b/src/backend/libpq/pqmq.c index bfe66c6c44..f93ccae148 100644 --- a/src/backend/libpq/pqmq.c +++ b/src/backend/libpq/pqmq.c @@ -17,6 +17,7 @@ #include "libpq/pqformat.h" #include "libpq/pqmq.h" #include "miscadmin.h" +#include "pgstat.h" #include "tcop/tcopprot.h" #include "utils/builtins.h" @@ -171,7 +172,8 @@ mq_putmessage(char msgtype, const char *s, size_t len) if (result != SHM_MQ_WOULD_BLOCK) break; - WaitLatch(&MyProc->procLatch, WL_LATCH_SET, 0); + WaitLatch(&MyProc->procLatch, WL_LATCH_SET, 0, + WAIT_EVENT_MQ_PUT_MESSAGE); ResetLatch(&MyProc->procLatch); CHECK_FOR_INTERRUPTS(); } diff --git a/src/backend/postmaster/autovacuum.c b/src/backend/postmaster/autovacuum.c index 1a92ca1deb..e3a6911fba 100644 --- a/src/backend/postmaster/autovacuum.c +++ b/src/backend/postmaster/autovacuum.c @@ -598,7 +598,8 @@ AutoVacLauncherMain(int argc, char *argv[]) */ rc = WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, - (nap.tv_sec * 1000L) + (nap.tv_usec / 1000L)); + (nap.tv_sec * 1000L) + (nap.tv_usec / 1000L), + WAIT_EVENT_AUTOVACUUM_MAIN); ResetLatch(MyLatch); diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c index 699c934240..028a9eed2d 100644 --- a/src/backend/postmaster/bgworker.c +++ b/src/backend/postmaster/bgworker.c @@ -18,6 +18,7 @@ #include "libpq/pqsignal.h" #include "postmaster/bgworker_internals.h" #include "postmaster/postmaster.h" +#include "pgstat.h" #include "storage/barrier.h" #include "storage/dsm.h" #include "storage/ipc.h" @@ -969,7 +970,8 @@ WaitForBackgroundWorkerStartup(BackgroundWorkerHandle *handle, pid_t *pidp) break; rc = WaitLatch(MyLatch, - WL_LATCH_SET | WL_POSTMASTER_DEATH, 0); + WL_LATCH_SET | WL_POSTMASTER_DEATH, 0, + WAIT_EVENT_BGWORKER_STARTUP); if (rc & WL_POSTMASTER_DEATH) { @@ -1008,7 +1010,8 @@ WaitForBackgroundWorkerShutdown(BackgroundWorkerHandle *handle) break; rc = WaitLatch(&MyProc->procLatch, - WL_LATCH_SET | WL_POSTMASTER_DEATH, 0); + WL_LATCH_SET | WL_POSTMASTER_DEATH, 0, + WAIT_EVENT_BGWORKER_SHUTDOWN); if (rc & WL_POSTMASTER_DEATH) { diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c index 10020349a2..c3f33561da 100644 --- a/src/backend/postmaster/bgwriter.c +++ b/src/backend/postmaster/bgwriter.c @@ -345,7 +345,7 @@ BackgroundWriterMain(void) */ rc = WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, - BgWriterDelay /* ms */ ); + BgWriterDelay /* ms */, WAIT_EVENT_BGWRITER_MAIN); /* * If no latch event and BgBufferSync says nothing's happening, extend @@ -372,7 +372,8 @@ BackgroundWriterMain(void) /* Sleep ... */ rc = WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, - BgWriterDelay * HIBERNATE_FACTOR); + BgWriterDelay * HIBERNATE_FACTOR, + WAIT_EVENT_BGWRITER_HIBERNATE); /* Reset the notification request in case we timed out */ StrategyNotifyBgWriter(-1); } diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c index d702a4864d..397267c6b7 100644 --- a/src/backend/postmaster/checkpointer.c +++ b/src/backend/postmaster/checkpointer.c @@ -556,7 +556,8 @@ CheckpointerMain(void) rc = WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, - cur_timeout * 1000L /* convert to ms */ ); + cur_timeout * 1000L /* convert to ms */, + WAIT_EVENT_CHECKPOINTER_MAIN); /* * Emergency bailout if postmaster has died. This is to avoid the diff --git a/src/backend/postmaster/pgarch.c b/src/backend/postmaster/pgarch.c index 1aa6466d67..62783d9259 100644 --- a/src/backend/postmaster/pgarch.c +++ b/src/backend/postmaster/pgarch.c @@ -390,7 +390,8 @@ pgarch_MainLoop(void) rc = WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, - timeout * 1000L); + timeout * 1000L, + WAIT_EVENT_ARCHIVER_MAIN); if (rc & WL_TIMEOUT) wakened = true; } diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index 96578dcedb..8c9d06fdaa 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -276,6 +276,11 @@ static PgStat_TableStatus *get_tabstat_entry(Oid rel_id, bool isshared); static void pgstat_setup_memcxt(void); +static const char *pgstat_get_wait_activity(WaitEventActivity w); +static const char *pgstat_get_wait_client(WaitEventClient w); +static const char *pgstat_get_wait_ipc(WaitEventIPC w); +static const char *pgstat_get_wait_timeout(WaitEventTimeout w); + static void pgstat_setheader(PgStat_MsgHdr *hdr, StatMsgType mtype); static void pgstat_send(void *msg, int len); @@ -3131,15 +3136,14 @@ pgstat_read_current_status(void) const char * pgstat_get_wait_event_type(uint32 wait_event_info) { - uint8 classId; + uint32 classId; const char *event_type; /* report process as not waiting. */ if (wait_event_info == 0) return NULL; - wait_event_info = wait_event_info >> 24; - classId = wait_event_info & 0XFF; + classId = wait_event_info & 0xFF000000; switch (classId) { @@ -3155,6 +3159,21 @@ pgstat_get_wait_event_type(uint32 wait_event_info) case WAIT_BUFFER_PIN: event_type = "BufferPin"; break; + case WAIT_ACTIVITY: + event_type = "Activity"; + break; + case WAIT_CLIENT: + event_type = "Client"; + break; + case WAIT_EXTENSION: + event_type = "Extension"; + break; + case WAIT_IPC: + event_type = "IPC"; + break; + case WAIT_TIMEOUT: + event_type = "Timeout"; + break; default: event_type = "???"; break; @@ -3172,7 +3191,7 @@ pgstat_get_wait_event_type(uint32 wait_event_info) const char * pgstat_get_wait_event(uint32 wait_event_info) { - uint8 classId; + uint32 classId; uint16 eventId; const char *event_name; @@ -3180,9 +3199,8 @@ pgstat_get_wait_event(uint32 wait_event_info) if (wait_event_info == 0) return NULL; - eventId = wait_event_info & ((1 << 24) - 1); - wait_event_info = wait_event_info >> 24; - classId = wait_event_info & 0XFF; + classId = wait_event_info & 0xFF000000; + eventId = wait_event_info & 0x0000FFFF; switch (classId) { @@ -3196,6 +3214,37 @@ pgstat_get_wait_event(uint32 wait_event_info) case WAIT_BUFFER_PIN: event_name = "BufferPin"; break; + case WAIT_ACTIVITY: + { + WaitEventActivity w = (WaitEventActivity) wait_event_info; + + event_name = pgstat_get_wait_activity(w); + break; + } + case WAIT_CLIENT: + { + WaitEventClient w = (WaitEventClient) wait_event_info; + + event_name = pgstat_get_wait_client(w); + break; + } + case WAIT_EXTENSION: + event_name = "Extension"; + break; + case WAIT_IPC: + { + WaitEventIPC w = (WaitEventIPC) wait_event_info; + + event_name = pgstat_get_wait_ipc(w); + break; + } + case WAIT_TIMEOUT: + { + WaitEventTimeout w = (WaitEventTimeout) wait_event_info; + + event_name = pgstat_get_wait_timeout(w); + break; + } default: event_name = "unknown wait event"; break; @@ -3204,6 +3253,175 @@ pgstat_get_wait_event(uint32 wait_event_info) return event_name; } +/* ---------- + * pgstat_get_wait_activity() - + * + * Convert WaitEventActivity to string. + * ---------- + */ +static const char * +pgstat_get_wait_activity(WaitEventActivity w) +{ + const char *event_name = "unknown wait event"; + + switch (w) + { + case WAIT_EVENT_ARCHIVER_MAIN: + event_name = "ArchiverMain"; + break; + case WAIT_EVENT_AUTOVACUUM_MAIN: + event_name = "AutoVacuumMain"; + break; + case WAIT_EVENT_BGWRITER_HIBERNATE: + event_name = "BgWriterHibernate"; + break; + case WAIT_EVENT_BGWRITER_MAIN: + event_name = "BgWriterMain"; + break; + case WAIT_EVENT_CHECKPOINTER_MAIN: + event_name = "CheckpointerMain"; + break; + case WAIT_EVENT_PGSTAT_MAIN: + event_name = "PgStatMain"; + break; + case WAIT_EVENT_RECOVERY_WAL_ALL: + event_name = "RecoveryWalAll"; + break; + case WAIT_EVENT_RECOVERY_WAL_STREAM: + event_name = "RecoveryWalStream"; + break; + case WAIT_EVENT_SYSLOGGER_MAIN: + event_name = "SysLoggerMain"; + break; + case WAIT_EVENT_WAL_RECEIVER_MAIN: + event_name = "WalReceiverMain"; + break; + case WAIT_EVENT_WAL_SENDER_MAIN: + event_name = "WalSenderMain"; + break; + case WAIT_EVENT_WAL_WRITER_MAIN: + event_name = "WalWriterMain"; + break; + /* no default case, so that compiler will warn */ + } + + return event_name; +} + +/* ---------- + * pgstat_get_wait_client() - + * + * Convert WaitEventClient to string. + * ---------- + */ +static const char * +pgstat_get_wait_client(WaitEventClient w) +{ + const char *event_name = "unknown wait event"; + + switch (w) + { + case WAIT_EVENT_CLIENT_READ: + event_name = "ClientRead"; + break; + case WAIT_EVENT_CLIENT_WRITE: + event_name = "ClientWrite"; + break; + case WAIT_EVENT_SSL_OPEN_SERVER: + event_name = "SSLOpenServer"; + break; + case WAIT_EVENT_WAL_RECEIVER_WAIT_START: + event_name = "WalReceiverWaitStart"; + break; + case WAIT_EVENT_WAL_SENDER_WAIT_WAL: + event_name = "WalSenderWaitForWAL"; + break; + case WAIT_EVENT_WAL_SENDER_WRITE_DATA: + event_name = "WalSenderWriteData"; + break; + /* no default case, so that compiler will warn */ + } + + return event_name; +} + +/* ---------- + * pgstat_get_wait_ipc() - + * + * Convert WaitEventIPC to string. + * ---------- + */ +static const char * +pgstat_get_wait_ipc(WaitEventIPC w) +{ + const char *event_name = "unknown wait event"; + + switch (w) + { + case WAIT_EVENT_BGWORKER_SHUTDOWN: + event_name = "BgWorkerShutdown"; + break; + case WAIT_EVENT_BGWORKER_STARTUP: + event_name = "BgWorkerStartup"; + break; + case WAIT_EVENT_EXECUTE_GATHER: + event_name = "ExecuteGather"; + break; + case WAIT_EVENT_MQ_INTERNAL: + event_name = "MessageQueueInternal"; + break; + case WAIT_EVENT_MQ_PUT_MESSAGE: + event_name = "MessageQueuePutMessage"; + break; + case WAIT_EVENT_MQ_RECEIVE: + event_name = "MessageQueueReceive"; + break; + case WAIT_EVENT_MQ_SEND: + event_name = "MessageQueueSend"; + break; + case WAIT_EVENT_PARALLEL_FINISH: + event_name = "ParallelFinish"; + break; + case WAIT_EVENT_SAFE_SNAPSHOT: + event_name = "SafeSnapshot"; + break; + case WAIT_EVENT_SYNC_REP: + event_name = "SyncRep"; + break; + /* no default case, so that compiler will warn */ + } + + return event_name; +} + +/* ---------- + * pgstat_get_wait_timeout() - + * + * Convert WaitEventTimeout to string. + * ---------- + */ +static const char * +pgstat_get_wait_timeout(WaitEventTimeout w) +{ + const char *event_name = "unknown wait event"; + + switch (w) + { + case WAIT_EVENT_BASE_BACKUP_THROTTLE: + event_name = "BaseBackupThrottle"; + break; + case WAIT_EVENT_PG_SLEEP: + event_name = "PgSleep"; + break; + case WAIT_EVENT_RECOVERY_APPLY_DELAY: + event_name = "RecoveryApplyDelay"; + break; + /* no default case, so that compiler will warn */ + } + + return event_name; +} + /* ---------- * pgstat_get_backend_current_activity() - * @@ -3684,8 +3902,8 @@ PgstatCollectorMain(int argc, char *argv[]) #ifndef WIN32 wr = WaitLatchOrSocket(MyLatch, WL_LATCH_SET | WL_POSTMASTER_DEATH | WL_SOCKET_READABLE, - pgStatSock, - -1L); + pgStatSock, -1L, + WAIT_EVENT_PGSTAT_MAIN); #else /* diff --git a/src/backend/postmaster/syslogger.c b/src/backend/postmaster/syslogger.c index e7e488a236..af7136760a 100644 --- a/src/backend/postmaster/syslogger.c +++ b/src/backend/postmaster/syslogger.c @@ -35,6 +35,7 @@ #include "libpq/pqsignal.h" #include "miscadmin.h" #include "nodes/pg_list.h" +#include "pgstat.h" #include "pgtime.h" #include "postmaster/fork_process.h" #include "postmaster/postmaster.h" @@ -424,7 +425,8 @@ SysLoggerMain(int argc, char *argv[]) rc = WaitLatchOrSocket(MyLatch, WL_LATCH_SET | WL_SOCKET_READABLE | cur_flags, syslogPipe[0], - cur_timeout); + cur_timeout, + WAIT_EVENT_SYSLOGGER_MAIN); if (rc & WL_SOCKET_READABLE) { diff --git a/src/backend/postmaster/walwriter.c b/src/backend/postmaster/walwriter.c index 11ec56aebb..67dcff63b1 100644 --- a/src/backend/postmaster/walwriter.c +++ b/src/backend/postmaster/walwriter.c @@ -290,7 +290,8 @@ WalWriterMain(void) rc = WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, - cur_timeout); + cur_timeout, + WAIT_EVENT_WAL_WRITER_MAIN); /* * Emergency bailout if postmaster has died. This is to avoid the diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c index 1eabaef492..fa75930c9f 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c @@ -1364,7 +1364,8 @@ throttle(size_t increment) */ wait_result = WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, - (long) (sleep / 1000)); + (long) (sleep / 1000), + WAIT_EVENT_BASE_BACKUP_THROTTLE); if (wait_result & WL_LATCH_SET) CHECK_FOR_INTERRUPTS(); diff --git a/src/backend/replication/syncrep.c b/src/backend/replication/syncrep.c index b442d061ec..ac29f567c3 100644 --- a/src/backend/replication/syncrep.c +++ b/src/backend/replication/syncrep.c @@ -61,6 +61,7 @@ #include "access/xact.h" #include "miscadmin.h" +#include "pgstat.h" #include "replication/syncrep.h" #include "replication/walsender.h" #include "replication/walsender_private.h" @@ -258,7 +259,8 @@ SyncRepWaitForLSN(XLogRecPtr lsn, bool commit) * Wait on latch. Any condition that should wake us up will set the * latch, so no need for timeout. */ - WaitLatch(MyLatch, WL_LATCH_SET | WL_POSTMASTER_DEATH, -1); + WaitLatch(MyLatch, WL_LATCH_SET | WL_POSTMASTER_DEATH, -1, + WAIT_EVENT_SYNC_REP); } /* diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index 413ee3a5c1..eed6effeeb 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -55,6 +55,7 @@ #include "libpq/pqformat.h" #include "libpq/pqsignal.h" #include "miscadmin.h" +#include "pgstat.h" #include "replication/walreceiver.h" #include "replication/walsender.h" #include "storage/ipc.h" @@ -486,7 +487,8 @@ WalReceiverMain(void) WL_POSTMASTER_DEATH | WL_SOCKET_READABLE | WL_TIMEOUT | WL_LATCH_SET, wait_fd, - NAPTIME_PER_CYCLE); + NAPTIME_PER_CYCLE, + WAIT_EVENT_WAL_RECEIVER_MAIN); if (rc & WL_LATCH_SET) { ResetLatch(&walrcv->latch); @@ -685,7 +687,8 @@ WalRcvWaitForStartPosition(XLogRecPtr *startpoint, TimeLineID *startpointTLI) } SpinLockRelease(&walrcv->mutex); - WaitLatch(&walrcv->latch, WL_LATCH_SET | WL_POSTMASTER_DEATH, 0); + WaitLatch(&walrcv->latch, WL_LATCH_SET | WL_POSTMASTER_DEATH, 0, + WAIT_EVENT_WAL_RECEIVER_WAIT_START); } if (update_process_title) diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index c7743da034..0f3ced250c 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -1146,7 +1146,8 @@ WalSndWriteData(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId xid, /* Sleep until something happens or we time out */ WaitLatchOrSocket(MyLatch, wakeEvents, - MyProcPort->sock, sleeptime); + MyProcPort->sock, sleeptime, + WAIT_EVENT_WAL_SENDER_WRITE_DATA); } /* reactivate latch so WalSndLoop knows to continue */ @@ -1272,7 +1273,8 @@ WalSndWaitForWal(XLogRecPtr loc) /* Sleep until something happens or we time out */ WaitLatchOrSocket(MyLatch, wakeEvents, - MyProcPort->sock, sleeptime); + MyProcPort->sock, sleeptime, + WAIT_EVENT_WAL_SENDER_WAIT_WAL); } /* reactivate latch so WalSndLoop knows to continue */ @@ -1924,7 +1926,8 @@ WalSndLoop(WalSndSendDataCallback send_data) /* Sleep until something happens or we time out */ WaitLatchOrSocket(MyLatch, wakeEvents, - MyProcPort->sock, sleeptime); + MyProcPort->sock, sleeptime, + WAIT_EVENT_WAL_SENDER_MAIN); } } return; diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 90804a3c53..91dc24c301 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -3635,9 +3635,6 @@ LockBufferForCleanup(Buffer buffer) UnlockBufHdr(bufHdr, buf_state); LockBuffer(buffer, BUFFER_LOCK_UNLOCK); - /* Report the wait */ - pgstat_report_wait_start(WAIT_BUFFER_PIN, 0); - /* Wait to be signaled by UnpinBuffer() */ if (InHotStandby) { @@ -3649,9 +3646,7 @@ LockBufferForCleanup(Buffer buffer) SetStartupBufferPinWaitBufId(-1); } else - ProcWaitForSignal(); - - pgstat_report_wait_end(); + ProcWaitForSignal(WAIT_BUFFER_PIN); /* * Remove flag marking us as waiter. Normally this will not be set diff --git a/src/backend/storage/ipc/latch.c b/src/backend/storage/ipc/latch.c index 9def8a12d3..8488f944de 100644 --- a/src/backend/storage/ipc/latch.c +++ b/src/backend/storage/ipc/latch.c @@ -55,6 +55,7 @@ #endif #include "miscadmin.h" +#include "pgstat.h" #include "portability/instr_time.h" #include "postmaster/postmaster.h" #include "storage/barrier.h" @@ -297,9 +298,11 @@ DisownLatch(volatile Latch *latch) * we return all of them in one call, but we will return at least one. */ int -WaitLatch(volatile Latch *latch, int wakeEvents, long timeout) +WaitLatch(volatile Latch *latch, int wakeEvents, long timeout, + uint32 wait_event_info) { - return WaitLatchOrSocket(latch, wakeEvents, PGINVALID_SOCKET, timeout); + return WaitLatchOrSocket(latch, wakeEvents, PGINVALID_SOCKET, timeout, + wait_event_info); } /* @@ -316,7 +319,7 @@ WaitLatch(volatile Latch *latch, int wakeEvents, long timeout) */ int WaitLatchOrSocket(volatile Latch *latch, int wakeEvents, pgsocket sock, - long timeout) + long timeout, uint32 wait_event_info) { int ret = 0; int rc; @@ -344,7 +347,7 @@ WaitLatchOrSocket(volatile Latch *latch, int wakeEvents, pgsocket sock, AddWaitEventToSet(set, ev, sock, NULL, NULL); } - rc = WaitEventSetWait(set, timeout, &event, 1); + rc = WaitEventSetWait(set, timeout, &event, 1, wait_event_info); if (rc == 0) ret |= WL_TIMEOUT; @@ -863,7 +866,8 @@ WaitEventAdjustWin32(WaitEventSet *set, WaitEvent *event) */ int WaitEventSetWait(WaitEventSet *set, long timeout, - WaitEvent *occurred_events, int nevents) + WaitEvent *occurred_events, int nevents, + uint32 wait_event_info) { int returned_events = 0; instr_time start_time; @@ -883,6 +887,8 @@ WaitEventSetWait(WaitEventSet *set, long timeout, cur_timeout = timeout; } + pgstat_report_wait_start(wait_event_info); + #ifndef WIN32 waiting = true; #else @@ -960,6 +966,8 @@ WaitEventSetWait(WaitEventSet *set, long timeout, waiting = false; #endif + pgstat_report_wait_end(); + return returned_events; } diff --git a/src/backend/storage/ipc/shm_mq.c b/src/backend/storage/ipc/shm_mq.c index 5b32782022..bfb67038ad 100644 --- a/src/backend/storage/ipc/shm_mq.c +++ b/src/backend/storage/ipc/shm_mq.c @@ -19,6 +19,7 @@ #include "postgres.h" #include "miscadmin.h" +#include "pgstat.h" #include "postmaster/bgworker.h" #include "storage/procsignal.h" #include "storage/shm_mq.h" @@ -894,7 +895,7 @@ shm_mq_send_bytes(shm_mq_handle *mqh, Size nbytes, const void *data, * at top of loop, because setting an already-set latch is much * cheaper than setting one that has been reset. */ - WaitLatch(MyLatch, WL_LATCH_SET, 0); + WaitLatch(MyLatch, WL_LATCH_SET, 0, WAIT_EVENT_MQ_SEND); /* Reset the latch so we don't spin. */ ResetLatch(MyLatch); @@ -991,7 +992,7 @@ shm_mq_receive_bytes(shm_mq *mq, Size bytes_needed, bool nowait, * loop, because setting an already-set latch is much cheaper than * setting one that has been reset. */ - WaitLatch(MyLatch, WL_LATCH_SET, 0); + WaitLatch(MyLatch, WL_LATCH_SET, 0, WAIT_EVENT_MQ_RECEIVE); /* Reset the latch so we don't spin. */ ResetLatch(MyLatch); @@ -1090,7 +1091,7 @@ shm_mq_wait_internal(volatile shm_mq *mq, PGPROC *volatile * ptr, } /* Wait to be signalled. */ - WaitLatch(MyLatch, WL_LATCH_SET, 0); + WaitLatch(MyLatch, WL_LATCH_SET, 0, WAIT_EVENT_MQ_INTERNAL); /* Reset the latch so we don't spin. */ ResetLatch(MyLatch); diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c index 547f1a88fe..fb887b3230 100644 --- a/src/backend/storage/ipc/standby.c +++ b/src/backend/storage/ipc/standby.c @@ -22,6 +22,7 @@ #include "access/xlog.h" #include "access/xloginsert.h" #include "miscadmin.h" +#include "pgstat.h" #include "storage/bufmgr.h" #include "storage/lmgr.h" #include "storage/proc.h" @@ -389,7 +390,7 @@ ResolveRecoveryConflictWithLock(LOCKTAG locktag) } /* Wait to be signaled by the release of the Relation Lock */ - ProcWaitForSignal(); + ProcWaitForSignal(WAIT_LOCK | locktag.locktag_type); /* * Clear any timeout requests established above. We assume here that the @@ -469,7 +470,7 @@ ResolveRecoveryConflictWithBufferPin(void) } /* Wait to be signaled by UnpinBuffer() */ - ProcWaitForSignal(); + ProcWaitForSignal(WAIT_BUFFER_PIN); /* * Clear any timeout requests established above. We assume here that the diff --git a/src/backend/storage/lmgr/lock.c b/src/backend/storage/lmgr/lock.c index dba3809e74..71a4dd4736 100644 --- a/src/backend/storage/lmgr/lock.c +++ b/src/backend/storage/lmgr/lock.c @@ -1676,7 +1676,6 @@ WaitOnLock(LOCALLOCK *locallock, ResourceOwner owner) set_ps_display(new_status, false); new_status[len] = '\0'; /* truncate off " waiting" */ } - pgstat_report_wait_start(WAIT_LOCK, locallock->tag.lock.locktag_type); awaitedLock = locallock; awaitedOwner = owner; @@ -1724,7 +1723,6 @@ WaitOnLock(LOCALLOCK *locallock, ResourceOwner owner) /* In this path, awaitedLock remains set until LockErrorCleanup */ /* Report change to non-waiting status */ - pgstat_report_wait_end(); if (update_process_title) { set_ps_display(new_status, false); @@ -1739,7 +1737,6 @@ WaitOnLock(LOCALLOCK *locallock, ResourceOwner owner) awaitedLock = NULL; /* Report change to non-waiting status */ - pgstat_report_wait_end(); if (update_process_title) { set_ps_display(new_status, false); diff --git a/src/backend/storage/lmgr/lwlock.c b/src/backend/storage/lmgr/lwlock.c index 9d08de75ae..a90b54ac86 100644 --- a/src/backend/storage/lmgr/lwlock.c +++ b/src/backend/storage/lmgr/lwlock.c @@ -732,9 +732,9 @@ LWLockReportWaitStart(LWLock *lock) int lockId = T_ID(lock); if (lock->tranche == 0) - pgstat_report_wait_start(WAIT_LWLOCK_NAMED, (uint16) lockId); + pgstat_report_wait_start(WAIT_LWLOCK_NAMED | (uint16) lockId); else - pgstat_report_wait_start(WAIT_LWLOCK_TRANCHE, lock->tranche); + pgstat_report_wait_start(WAIT_LWLOCK_TRANCHE | lock->tranche); } /* @@ -750,7 +750,7 @@ LWLockReportWaitEnd(void) * Return an identifier for an LWLock based on the wait class and event. */ const char * -GetLWLockIdentifier(uint8 classId, uint16 eventId) +GetLWLockIdentifier(uint32 classId, uint16 eventId) { if (classId == WAIT_LWLOCK_NAMED) return MainLWLockNames[eventId]; diff --git a/src/backend/storage/lmgr/predicate.c b/src/backend/storage/lmgr/predicate.c index 4064b2033c..24ed21b487 100644 --- a/src/backend/storage/lmgr/predicate.c +++ b/src/backend/storage/lmgr/predicate.c @@ -192,6 +192,7 @@ #include "access/xact.h" #include "access/xlog.h" #include "miscadmin.h" +#include "pgstat.h" #include "storage/bufmgr.h" #include "storage/predicate.h" #include "storage/predicate_internals.h" @@ -1518,7 +1519,7 @@ GetSafeSnapshot(Snapshot origSnapshot) SxactIsROUnsafe(MySerializableXact))) { LWLockRelease(SerializableXactHashLock); - ProcWaitForSignal(); + ProcWaitForSignal(WAIT_EVENT_SAFE_SNAPSHOT); LWLockAcquire(SerializableXactHashLock, LW_EXCLUSIVE); } MySerializableXact->flags &= ~SXACT_FLAG_DEFERRABLE_WAITING; diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c index 33e7023656..dd76094bcd 100644 --- a/src/backend/storage/lmgr/proc.c +++ b/src/backend/storage/lmgr/proc.c @@ -39,6 +39,7 @@ #include "access/twophase.h" #include "access/xact.h" #include "miscadmin.h" +#include "pgstat.h" #include "postmaster/autovacuum.h" #include "replication/slot.h" #include "replication/syncrep.h" @@ -1212,7 +1213,8 @@ ProcSleep(LOCALLOCK *locallock, LockMethod lockMethodTable) } else { - WaitLatch(MyLatch, WL_LATCH_SET, 0); + WaitLatch(MyLatch, WL_LATCH_SET, 0, + WAIT_LOCK | locallock->tag.lock.locktag_type); ResetLatch(MyLatch); /* check for deadlocks first, as that's probably log-worthy */ if (got_deadlock_timeout) @@ -1722,9 +1724,9 @@ CheckDeadLockAlert(void) * wait again if not. */ void -ProcWaitForSignal(void) +ProcWaitForSignal(uint32 wait_event_info) { - WaitLatch(MyLatch, WL_LATCH_SET, 0); + WaitLatch(MyLatch, WL_LATCH_SET, 0, wait_event_info); ResetLatch(MyLatch); CHECK_FOR_INTERRUPTS(); } diff --git a/src/backend/utils/adt/misc.c b/src/backend/utils/adt/misc.c index 5e705e9308..0da051a2f2 100644 --- a/src/backend/utils/adt/misc.c +++ b/src/backend/utils/adt/misc.c @@ -29,6 +29,7 @@ #include "common/keywords.h" #include "funcapi.h" #include "miscadmin.h" +#include "pgstat.h" #include "parser/scansup.h" #include "postmaster/syslogger.h" #include "rewrite/rewriteHandler.h" @@ -560,7 +561,8 @@ pg_sleep(PG_FUNCTION_ARGS) (void) WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT, - delay_ms); + delay_ms, + WAIT_EVENT_PG_SLEEP); ResetLatch(MyLatch); } diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 0c98c59e72..b530c01984 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -715,15 +715,91 @@ typedef enum BackendState * Wait Classes * ---------- */ -typedef enum WaitClass +#define WAIT_LWLOCK_NAMED 0x01000000U +#define WAIT_LWLOCK_TRANCHE 0x02000000U +#define WAIT_LOCK 0x03000000U +#define WAIT_BUFFER_PIN 0x04000000U +#define WAIT_ACTIVITY 0x05000000U +#define WAIT_CLIENT 0x06000000U +#define WAIT_EXTENSION 0x07000000U +#define WAIT_IPC 0x08000000U +#define WAIT_TIMEOUT 0x09000000U + +/* ---------- + * Wait Events - Activity + * + * Use this category when a process is waiting because it has no work to do, + * unless the "Client" or "Timeout" category describes the situation better. + * Typically, this should only be used for background processes. + * ---------- + */ +typedef enum { - WAIT_UNDEFINED, - WAIT_LWLOCK_NAMED, - WAIT_LWLOCK_TRANCHE, - WAIT_LOCK, - WAIT_BUFFER_PIN -} WaitClass; + WAIT_EVENT_ARCHIVER_MAIN = WAIT_ACTIVITY, + WAIT_EVENT_AUTOVACUUM_MAIN, + WAIT_EVENT_BGWRITER_HIBERNATE, + WAIT_EVENT_BGWRITER_MAIN, + WAIT_EVENT_CHECKPOINTER_MAIN, + WAIT_EVENT_PGSTAT_MAIN, + WAIT_EVENT_RECOVERY_WAL_ALL, + WAIT_EVENT_RECOVERY_WAL_STREAM, + WAIT_EVENT_SYSLOGGER_MAIN, + WAIT_EVENT_WAL_RECEIVER_MAIN, + WAIT_EVENT_WAL_SENDER_MAIN, + WAIT_EVENT_WAL_WRITER_MAIN, +} WaitEventActivity; +/* ---------- + * Wait Events - Client + * + * Use this category when a process is waiting to send data to or receive data + * from the frontend process to which it is connected. This is never used for + * a background process, which has no client connection. + * ---------- + */ +typedef enum +{ + WAIT_EVENT_CLIENT_READ = WAIT_CLIENT, + WAIT_EVENT_CLIENT_WRITE, + WAIT_EVENT_SSL_OPEN_SERVER, + WAIT_EVENT_WAL_RECEIVER_WAIT_START, + WAIT_EVENT_WAL_SENDER_WAIT_WAL, + WAIT_EVENT_WAL_SENDER_WRITE_DATA, +} WaitEventClient; + +/* ---------- + * Wait Events - IPC + * + * Use this category when a process cannot complete the work it is doing because + * it is waiting for a notification from another process. + * ---------- + */ +typedef enum +{ + WAIT_EVENT_BGWORKER_SHUTDOWN = WAIT_IPC, + WAIT_EVENT_BGWORKER_STARTUP, + WAIT_EVENT_EXECUTE_GATHER, + WAIT_EVENT_MQ_INTERNAL, + WAIT_EVENT_MQ_PUT_MESSAGE, + WAIT_EVENT_MQ_RECEIVE, + WAIT_EVENT_MQ_SEND, + WAIT_EVENT_PARALLEL_FINISH, + WAIT_EVENT_SAFE_SNAPSHOT, + WAIT_EVENT_SYNC_REP +} WaitEventIPC; + +/* ---------- + * Wait Events - Timeout + * + * Use this category when a process is waiting for a timeout to expire. + * ---------- + */ +typedef enum +{ + WAIT_EVENT_BASE_BACKUP_THROTTLE = WAIT_TIMEOUT, + WAIT_EVENT_PG_SLEEP, + WAIT_EVENT_RECOVERY_APPLY_DELAY +} WaitEventTimeout; /* ---------- * Command type for progress reporting purposes @@ -1018,23 +1094,18 @@ extern void pgstat_initstats(Relation rel); * ---------- */ static inline void -pgstat_report_wait_start(uint8 classId, uint16 eventId) +pgstat_report_wait_start(uint32 wait_event_info) { volatile PGPROC *proc = MyProc; - uint32 wait_event_val; if (!pgstat_track_activities || !proc) return; - wait_event_val = classId; - wait_event_val <<= 24; - wait_event_val |= eventId; - /* * Since this is a four-byte field which is always read and written as * four-bytes, updates are atomic. */ - proc->wait_event_info = wait_event_val; + proc->wait_event_info = wait_event_info; } /* ---------- diff --git a/src/include/storage/latch.h b/src/include/storage/latch.h index 5179ecc0db..e96e88f2fa 100644 --- a/src/include/storage/latch.h +++ b/src/include/storage/latch.h @@ -155,10 +155,13 @@ extern int AddWaitEventToSet(WaitEventSet *set, uint32 events, pgsocket fd, Latch *latch, void *user_data); extern void ModifyWaitEvent(WaitEventSet *set, int pos, uint32 events, Latch *latch); -extern int WaitEventSetWait(WaitEventSet *set, long timeout, WaitEvent *occurred_events, int nevents); -extern int WaitLatch(volatile Latch *latch, int wakeEvents, long timeout); +extern int WaitEventSetWait(WaitEventSet *set, long timeout, + WaitEvent *occurred_events, int nevents, + uint32 wait_event_info); +extern int WaitLatch(volatile Latch *latch, int wakeEvents, long timeout, + uint32 wait_event_info); extern int WaitLatchOrSocket(volatile Latch *latch, int wakeEvents, - pgsocket sock, long timeout); + pgsocket sock, long timeout, uint32 wait_event_info); /* * Unix implementation uses SIGUSR1 for inter-process signaling. diff --git a/src/include/storage/lwlock.h b/src/include/storage/lwlock.h index 18931eb046..9a2d86975c 100644 --- a/src/include/storage/lwlock.h +++ b/src/include/storage/lwlock.h @@ -184,7 +184,7 @@ extern Size LWLockShmemSize(void); extern void CreateLWLocks(void); extern void InitLWLockAccess(void); -extern const char *GetLWLockIdentifier(uint8 classId, uint16 eventId); +extern const char *GetLWLockIdentifier(uint32 classId, uint16 eventId); /* * Extensions (or core code) can obtain an LWLocks by calling diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h index f576f052df..7dc8dac6d1 100644 --- a/src/include/storage/proc.h +++ b/src/include/storage/proc.h @@ -291,7 +291,7 @@ extern void CheckDeadLockAlert(void); extern bool IsWaitingForLock(void); extern void LockErrorCleanup(void); -extern void ProcWaitForSignal(void); +extern void ProcWaitForSignal(uint32 wait_event_info); extern void ProcSendSignal(int pid); extern void BecomeLockGroupLeader(void); diff --git a/src/test/modules/test_shm_mq/setup.c b/src/test/modules/test_shm_mq/setup.c index 143df4eb65..cb86f3c37d 100644 --- a/src/test/modules/test_shm_mq/setup.c +++ b/src/test/modules/test_shm_mq/setup.c @@ -16,6 +16,7 @@ #include "postgres.h" #include "miscadmin.h" +#include "pgstat.h" #include "postmaster/bgworker.h" #include "storage/procsignal.h" #include "storage/shm_toc.h" @@ -279,7 +280,7 @@ wait_for_workers_to_become_ready(worker_state *wstate, } /* Wait to be signalled. */ - WaitLatch(MyLatch, WL_LATCH_SET, 0); + WaitLatch(MyLatch, WL_LATCH_SET, 0, WAIT_EXTENSION); /* Reset the latch so we don't spin. */ ResetLatch(MyLatch); diff --git a/src/test/modules/test_shm_mq/test.c b/src/test/modules/test_shm_mq/test.c index dd34bc7e7f..bf11137a96 100644 --- a/src/test/modules/test_shm_mq/test.c +++ b/src/test/modules/test_shm_mq/test.c @@ -15,6 +15,7 @@ #include "fmgr.h" #include "miscadmin.h" +#include "pgstat.h" #include "test_shm_mq.h" @@ -230,7 +231,7 @@ test_shm_mq_pipelined(PG_FUNCTION_ARGS) * have read or written data and therefore there may now be work * for us to do. */ - WaitLatch(MyLatch, WL_LATCH_SET, 0); + WaitLatch(MyLatch, WL_LATCH_SET, 0, WAIT_EXTENSION); ResetLatch(MyLatch); CHECK_FOR_INTERRUPTS(); } -- cgit v1.2.3 From 3d21f08bccd316c3850a1943c1ee1e381dab1588 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 10 Oct 2016 15:11:33 -0400 Subject: Update user docs for switch to POSIX semaphores. Since commit ecb0d20a9 hasn't crashed and burned, here's the promised docs update for it. In addition to explaining that Linux and FreeBSD ports now use POSIX semaphores, I did some wordsmithing on pre-existing wording; in particular trying to clarify which SysV parameters need to be set with an eye to total usage across all applications. --- doc/src/sgml/runtime.sgml | 83 ++++++++++++++++++++++++++++------------------- 1 file changed, 50 insertions(+), 33 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/runtime.sgml b/doc/src/sgml/runtime.sgml index 88ec120841..a8efb3d006 100644 --- a/doc/src/sgml/runtime.sgml +++ b/doc/src/sgml/runtime.sgml @@ -605,27 +605,47 @@ psql: could not connect to server: No such file or directory - Shared memory and semaphores are collectively referred to as - System V - IPC (together with message queues, which are not - relevant for PostgreSQL). Except on - Windows, where PostgreSQL - provides its own replacement implementation of these facilities, these - facilities are required in order to run - PostgreSQL. + PostgreSQL requires the operating system to provide + inter-process communication (IPC) features, specifically + shared memory and semaphores. Unix-derived systems typically provide + System V IPC, + POSIX IPC, or both. + Windows has its own implementation of + these features and is not discussed here. The complete lack of these facilities is usually manifested by an - Illegal system call error upon server start. In - that case there is no alternative but to reconfigure your + Illegal system call error upon server + start. In that case there is no alternative but to reconfigure your kernel. PostgreSQL won't work without them. This situation is rare, however, among modern operating systems. - When PostgreSQL exceeds one of the various hard - IPC limits, the server will refuse to start and + Upon starting the server, PostgreSQL normally allocates + a very small amount of System V shared memory, as well as a much larger + amount of POSIX (mmap) shared memory. + In addition a significant number of semaphores, which can be either + System V or POSIX style, are created at server startup. Currently, + POSIX semaphores are used on Linux and FreeBSD systems while other + platforms use System V semaphores. + + + + + Prior to PostgreSQL 9.3, only System V shared memory + was used, so the amount of System V shared memory required to start the + server was much larger. If you are running an older version of the + server, please consult the documentation for your server version. + + + + + System V IPC features are typically constrained by + system-wide allocation limits. + When PostgreSQL exceeds one of these limits, + the server will refuse to start and should leave an instructive error message describing the problem and what to do about it. (See also .) The relevant kernel @@ -634,15 +654,6 @@ psql: could not connect to server: No such file or directory them, however, vary. Suggestions for some platforms are given below. - - - Prior to PostgreSQL 9.3, the amount of System V shared - memory required to start the server was much larger. If you are running - an older version of the server, please consult the documentation for - your server version. - - - <systemitem class="osname">System V</> <acronym>IPC</> Parameters @@ -651,7 +662,7 @@ psql: could not connect to server: No such file or directory Name Description - Reasonable values + Values needed to run one PostgreSQL instance @@ -659,7 +670,7 @@ psql: could not connect to server: No such file or directory SHMMAX Maximum size of shared memory segment (bytes) - at least 1kB (more if running many copies of the server) + at least 1kB, but the default is usually much higher @@ -671,7 +682,9 @@ psql: could not connect to server: No such file or directory SHMALL Total amount of shared memory available (bytes or pages) - if bytes, same as SHMMAX; if pages, ceil(SHMMAX/PAGE_SIZE) + same as SHMMAX if bytes, + or ceil(SHMMAX/PAGE_SIZE) if pages, + plus room for other applications @@ -689,7 +702,7 @@ psql: could not connect to server: No such file or directory SEMMNI Maximum number of semaphore identifiers (i.e., sets) - at least ceil((max_connections + autovacuum_max_workers + max_worker_processes + 5) / 16) + at least ceil((max_connections + autovacuum_max_workers + max_worker_processes + 5) / 16) plus room for other applications @@ -725,9 +738,8 @@ psql: could not connect to server: No such file or directory (typically 48 bytes, on 64-bit platforms) for each copy of the server. On most modern operating systems, this amount can easily be allocated. However, if you are running many copies of the server, or if other - applications are also using System V shared memory, it may be necessary - to increase SHMMAX, the maximum size in bytes of a shared - memory segment, or SHMALL, the total amount of System V shared + applications are also using System V shared memory, it may be necessary to + increase SHMALL, which is the total amount of System V shared memory system-wide. Note that SHMALL is measured in pages rather than bytes on many systems. @@ -742,6 +754,7 @@ psql: could not connect to server: No such file or directory + When using System V semaphores, PostgreSQL uses one semaphore per allowed connection (), allowed autovacuum worker process () and allowed background @@ -779,15 +792,19 @@ psql: could not connect to server: No such file or directory - The SEMMSL parameter, which determines how many - semaphores can be in a set, must be at least 17 for + Various other settings related to semaphore undo, such as + SEMMNU and SEMUME, do not affect PostgreSQL. - Various other settings related to semaphore undo, such as - SEMMNU and SEMUME, do not affect - PostgreSQL. + When using POSIX semaphores, the number of semaphores needed is the + same as for System V, that is one semaphore per allowed connection + (), allowed autovacuum worker process + () and allowed background + process (). + On the platforms where this option is preferred, there is no specific + kernel limit on the number of POSIX semaphores. -- cgit v1.2.3 From e34318725ca5b274efd6f57ea7460e89f4dca9f9 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 11 Oct 2016 10:08:45 -0400 Subject: Improve documentation for CREATE RECURSIVE VIEW. It was perhaps not entirely clear that internal self-references shouldn't be schema-qualified even if the view name is written with a schema. Spell it out. Discussion: <871sznz69m.fsf@metapensiero.it> --- doc/src/sgml/ref/create_view.sgml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/create_view.sgml b/doc/src/sgml/ref/create_view.sgml index ede1698051..8641e1925e 100644 --- a/doc/src/sgml/ref/create_view.sgml +++ b/doc/src/sgml/ref/create_view.sgml @@ -87,13 +87,13 @@ CREATE [ OR REPLACE ] [ TEMP | TEMPORARY ] [ RECURSIVE ] VIEW Creates a recursive view. The syntax -CREATE RECURSIVE VIEW name (columns) AS SELECT ...; +CREATE RECURSIVE VIEW [ schema . ] view_name (column_names) AS SELECT ...; is equivalent to -CREATE VIEW name AS WITH RECURSIVE name (columns) AS (SELECT ...) SELECT columns FROM name; +CREATE VIEW [ schema . ] view_name AS WITH RECURSIVE view_name (column_names) AS (SELECT ...) SELECT column_names FROM view_name; - A view column list must be specified for a recursive view. + A view column name list must be specified for a recursive view. @@ -462,11 +462,16 @@ CREATE VIEW comedies AS Create a recursive view consisting of the numbers from 1 to 100: -CREATE RECURSIVE VIEW nums_1_100 (n) AS +CREATE RECURSIVE VIEW public.nums_1_100 (n) AS VALUES (1) UNION ALL SELECT n+1 FROM nums_1_100 WHERE n < 100; - + + Notice that although the recursive view's name is schema-qualified in this + CREATE, its internal self-reference is not schema-qualified. + This is because the implicitly-created CTE's name cannot be + schema-qualified. + -- cgit v1.2.3 From c7e56811fa38cbc39efd6bdd4bb45f2f0444803e Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 11 Oct 2016 10:33:59 -0400 Subject: Docs: grammatical fix. Fix poor grammar introduced in 741ccd501. --- doc/src/sgml/client-auth.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/client-auth.sgml b/doc/src/sgml/client-auth.sgml index a0d97ffbac..960f5b5871 100644 --- a/doc/src/sgml/client-auth.sgml +++ b/doc/src/sgml/client-auth.sgml @@ -711,7 +711,7 @@ local db1,db2,@demodbs all md5 When using an external authentication system such as Ident or GSSAPI, the name of the operating system user that initiated the connection - might not be the same as the database user that is to be connect as. + might not be the same as the database user (role) that is to be used. In this case, a user name map can be applied to map the operating system user name to a database user. To use user name mapping, specify map=map-name -- cgit v1.2.3 From 2b860f52ed1b1784cdf3f03886805f5bf250ea74 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 11 Oct 2016 11:26:04 -0400 Subject: Remove "sco" and "unixware" ports. SCO OpenServer and SCO UnixWare are more or less dead platforms. We have never had a buildfarm member testing the "sco" port, and the last "unixware" member was last heard from in 2012, so it's fair to doubt that the code even compiles anymore on either one. Remove both ports. We can always undo this if someone shows up with an interest in maintaining and testing these platforms. Discussion: <17177.1476136994@sss.pgh.pa.us> --- configure | 37 -------- configure.in | 14 --- doc/src/sgml/Makefile | 6 +- doc/src/sgml/dfunc.sgml | 26 ------ doc/src/sgml/installation.sgml | 164 ++-------------------------------- doc/src/sgml/runtime.sgml | 58 ------------ src/Makefile.global.in | 1 - src/Makefile.shlib | 24 ----- src/backend/libpq/pqcomm.c | 10 --- src/backend/port/dynloader/sco.c | 7 -- src/backend/port/dynloader/sco.h | 46 ---------- src/backend/port/dynloader/unixware.c | 7 -- src/backend/port/dynloader/unixware.h | 49 ---------- src/include/port/sco.h | 7 -- src/include/port/unixware.h | 11 --- src/include/storage/s_lock.h | 23 ----- src/makefiles/Makefile.sco | 13 --- src/makefiles/Makefile.unixware | 35 -------- src/port/getrusage.c | 1 - src/template/sco | 1 - src/template/unixware | 41 --------- 21 files changed, 6 insertions(+), 575 deletions(-) delete mode 100644 src/backend/port/dynloader/sco.c delete mode 100644 src/backend/port/dynloader/sco.h delete mode 100644 src/backend/port/dynloader/unixware.c delete mode 100644 src/backend/port/dynloader/unixware.h delete mode 100644 src/include/port/sco.h delete mode 100644 src/include/port/unixware.h delete mode 100644 src/makefiles/Makefile.sco delete mode 100644 src/makefiles/Makefile.unixware delete mode 100644 src/template/sco delete mode 100644 src/template/unixware (limited to 'doc/src') diff --git a/configure b/configure index 1d94256a9e..f4f2f8b7ce 100755 --- a/configure +++ b/configure @@ -694,7 +694,6 @@ STRIP_SHARED_LIB STRIP_STATIC_LIB STRIP RANLIB -ld_R_works with_gnu_ld LD LDFLAGS_SL @@ -2867,9 +2866,7 @@ dragonfly*) template=netbsd ;; mingw*) template=win32 ;; netbsd*) template=netbsd ;; openbsd*) template=openbsd ;; - sco*) template=sco ;; solaris*) template=solaris ;; - sysv5*) template=unixware ;; esac if test x"$template" = x"" ; then @@ -6382,40 +6379,6 @@ with_gnu_ld=$ac_cv_prog_gnu_ld -case $host_os in sysv5*) - { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether ld -R works" >&5 -$as_echo_n "checking whether ld -R works... " >&6; } -if ${pgac_cv_prog_ld_R+:} false; then : - $as_echo_n "(cached) " >&6 -else - - pgac_save_LDFLAGS=$LDFLAGS; LDFLAGS="$LDFLAGS -Wl,-R/usr/lib" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -int -main () -{ - - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - pgac_cv_prog_ld_R=yes -else - pgac_cv_prog_ld_R=no -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext - LDFLAGS=$pgac_save_LDFLAGS - -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_prog_ld_R" >&5 -$as_echo "$pgac_cv_prog_ld_R" >&6; } - ld_R_works=$pgac_cv_prog_ld_R - -esac if test -n "$ac_tool_prefix"; then # Extract the first word of "${ac_tool_prefix}ranlib", so it can be a program name with args. set dummy ${ac_tool_prefix}ranlib; ac_word=$2 diff --git a/configure.in b/configure.in index 9ace625029..9f7611caeb 100644 --- a/configure.in +++ b/configure.in @@ -69,9 +69,7 @@ dragonfly*) template=netbsd ;; mingw*) template=win32 ;; netbsd*) template=netbsd ;; openbsd*) template=openbsd ;; - sco*) template=sco ;; solaris*) template=solaris ;; - sysv5*) template=unixware ;; esac if test x"$template" = x"" ; then @@ -871,18 +869,6 @@ AC_ARG_VAR(LDFLAGS_SL, [extra linker flags for linking shared libraries only]) PGAC_PROG_LD AC_SUBST(LD) AC_SUBST(with_gnu_ld) -case $host_os in sysv5*) - AC_CACHE_CHECK([whether ld -R works], [pgac_cv_prog_ld_R], - [ - pgac_save_LDFLAGS=$LDFLAGS; LDFLAGS="$LDFLAGS -Wl,-R/usr/lib" - AC_LINK_IFELSE([AC_LANG_PROGRAM([], [])], - [pgac_cv_prog_ld_R=yes], - [pgac_cv_prog_ld_R=no]) - LDFLAGS=$pgac_save_LDFLAGS - ]) - ld_R_works=$pgac_cv_prog_ld_R - AC_SUBST(ld_R_works) -esac AC_PROG_RANLIB PGAC_CHECK_STRIP AC_CHECK_TOOL(AR, ar, ar) diff --git a/doc/src/sgml/Makefile b/doc/src/sgml/Makefile index 24b895f3c3..5df2f04dd6 100644 --- a/doc/src/sgml/Makefile +++ b/doc/src/sgml/Makefile @@ -326,11 +326,7 @@ check: postgres.sgml $(ALMOSTALLSGML) check-tabs ## Install ## -install: install-html - -ifneq ($(PORTNAME), sco) -install: install-man -endif +install: install-html install-man installdirs: $(MKDIR_P) '$(DESTDIR)$(htmldir)'/html $(addprefix '$(DESTDIR)$(mandir)'/man, 1 3 $(sqlmansectnum)) diff --git a/doc/src/sgml/dfunc.sgml b/doc/src/sgml/dfunc.sgml index ba2684cc3c..6a4b7d6e97 100644 --- a/doc/src/sgml/dfunc.sgml +++ b/doc/src/sgml/dfunc.sgml @@ -200,32 +200,6 @@ cc -G -o foo.so foo.o gcc -fpic -c foo.c gcc -G -o foo.so foo.o - - - - - - - - UnixWare - UnixWareshared library - - - - The compiler flag to create PIC is with the SCO compiler and - with GCC. To link shared libraries, - the compiler option is with the SCO compiler - and with - GCC. - -cc -K PIC -c foo.c -cc -G -o foo.so foo.o - - or - -gcc -fpic -c foo.c -gcc -shared -o foo.so foo.o diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml index 5ee28fcf85..883e575946 100644 --- a/doc/src/sgml/installation.sgml +++ b/doc/src/sgml/installation.sgml @@ -2000,8 +2000,8 @@ kill `cat /usr/local/pgsql/data/postmaster.pid` In general, PostgreSQL can be expected to work on these CPU architectures: x86, x86_64, IA64, PowerPC, - PowerPC 64, S/390, S/390x, Sparc, Sparc 64, ARM, MIPS, MIPSEL, M68K, - and PA-RISC. Code support exists for M32R and VAX, but these + PowerPC 64, S/390, S/390x, Sparc, Sparc 64, ARM, MIPS, MIPSEL, + and PA-RISC. Code support exists for M68K, M32R, and VAX, but these architectures are not known to have been tested recently. It is often possible to build on an unsupported CPU type by configuring with , but performance will be poor. @@ -2010,11 +2010,11 @@ kill `cat /usr/local/pgsql/data/postmaster.pid` PostgreSQL can be expected to work on these operating systems: Linux (all recent distributions), Windows (Win2000 SP4 and later), - FreeBSD, OpenBSD, NetBSD, macOS, AIX, HP/UX, Solaris, - and UnixWare. Other Unix-like systems may also work but are not currently + FreeBSD, OpenBSD, NetBSD, macOS, AIX, HP/UX, and Solaris. + Other Unix-like systems may also work but are not currently being tested. In most cases, all CPU architectures supported by a given operating system will work. Look in - the below to see if + below to see if there is information specific to your operating system, particularly if using an older system. @@ -2639,160 +2639,6 @@ PHSS_30849 s700_800 u2comp/be/plugin library Patch - - SCO OpenServer and SCO UnixWare - - - SCO - installation on - - - - UnixWare - installation on - - - - PostgreSQL can be built on SCO UnixWare 7 and SCO OpenServer 5. - On OpenServer, you can use either the OpenServer Development Kit - or the Universal Development Kit. However, some tweaking may be - needed, as described below. - - - - Skunkware - - - You should locate your copy of the SCO Skunkware CD. The - Skunkware CD is included with UnixWare 7 and current versions of - OpenServer 5. Skunkware includes ready-to-install versions of - many popular programs that are available on the Internet. For - example, gzip, gunzip, GNU Make, Flex, and Bison are all - included. For UnixWare 7.1, this CD is now labeled "Open License - Software Supplement". If you do not have this CD, the software - on it is available - from . - - - - Skunkware has different versions for UnixWare and OpenServer. - Make sure you install the correct version for your operating - system, except as noted below. - - - - On UnixWare 7.1.3 and beyond, the GCC compiler is included on the - UDK CD as is GNU Make. - - - - - GNU Make - - - You need to use the GNU Make program, which is on the Skunkware - CD. By default, it installs - as /usr/local/bin/make. - - - - As of UnixWare 7.1.3 and above, the GNU Make program is the - OSTK portion of the UDK CD, and is - in /usr/gnu/bin/gmake. - - - - - Readline - - - The Readline library is on the Skunkware CD. But it is not - included on the UnixWare 7.1 Skunkware CD. If you have the - UnixWare 7.0.0 or 7.0.1 Skunkware CDs, you can install it from - there. Otherwise, - try . - - - - By default, Readline installs into /usr/local/lib and - /usr/local/include. However, the - PostgreSQL configure program will not find it - there without help. If you installed Readline, then use the - following options to configure: - -./configure --with-libraries=/usr/local/lib --with-includes=/usr/local/include - - - - - - Using the UDK on OpenServer - - - If you are using the new Universal Development Kit (UDK) compiler - on OpenServer, you need to specify the locations of the UDK - libraries: - -./configure --with-libraries=/udk/usr/lib --with-includes=/udk/usr/include - - Putting these together with the Readline options from above: - -./configure --with-libraries="/udk/usr/lib /usr/local/lib" --with-includes="/udk/usr/include /usr/local/include" - - - - - - Reading the PostgreSQL Man Pages - - - By default, the PostgreSQL man pages are installed into - /usr/local/pgsql/share/man. By default, UnixWare - does not look there for man pages. To be able to read them you - need to modify the - MANPATH variable - in /etc/default/man, for example: - -MANPATH=/usr/lib/scohelp/%L/man:/usr/dt/man:/usr/man:/usr/share/man:scohelp:/usr/local/man:/usr/local/pgsql/share/man - - - - - On OpenServer, some extra research needs to be invested to make - the man pages usable, because the man system is a bit different - from other platforms. Currently, PostgreSQL will not install - them at all. - - - - - C99 Issues with the 7.1.1b Feature Supplement - - - For compilers earlier than the one released with OpenUNIX 8.0.0 - (UnixWare 7.1.2), including the 7.1.1b Feature Supplement, you - may need to specify - in CFLAGS or the CC - environment variable. The indication of this is an error in - compiling tuplesort.c referencing inline - functions. Apparently there was a change in the 7.1.2(8.0.0) - compiler and beyond. - - - - - Threading on UnixWare - - - For threading, youmust use - on all libpq-using programs. libpq - uses pthread_* calls, which are only - available with the - - - - Solaris diff --git a/doc/src/sgml/runtime.sgml b/doc/src/sgml/runtime.sgml index a8efb3d006..6ae62b4d2b 100644 --- a/doc/src/sgml/runtime.sgml +++ b/doc/src/sgml/runtime.sgml @@ -1086,34 +1086,6 @@ sysctl -w kern.sysv.shmall - - SCO OpenServer - SCO OpenServerIPC configuration - - - - In the default configuration, only 512 kB of shared memory per - segment is allowed. To increase the setting, first change to the - directory /etc/conf/cf.d. To display the current value of - SHMMAX, run: - -./configure -y SHMMAX - - To set a new value for SHMMAX, run: - -./configure SHMMAX=value - - where value is the new value you want to use - (in bytes). After setting SHMMAX, rebuild the kernel: - -./link_unix - - and reboot. - - - - - Solaris 2.6 to 2.9 (Solaris 6 to Solaris 9) @@ -1189,36 +1161,6 @@ project.max-msg-ids=(priv,4096,deny) - - - UnixWare - UnixWareIPC configuration - - - - On UnixWare 7, the maximum size for shared - memory segments is 512 kB in the default configuration. - To display the current value of SHMMAX, run: - -/etc/conf/bin/idtune -g SHMMAX - - which displays the current, default, minimum, and maximum - values. To set a new value for SHMMAX, - run: - -/etc/conf/bin/idtune SHMMAX value - - where value is the new value you want to use - (in bytes). After setting SHMMAX, rebuild the - kernel: - -/etc/conf/bin/idbuild -B - - and reboot. - - - - diff --git a/src/Makefile.global.in b/src/Makefile.global.in index e1e2c0adaf..ea61eb518f 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -265,7 +265,6 @@ UUID_LIBS = @UUID_LIBS@ UUID_EXTRA_OBJS = @UUID_EXTRA_OBJS@ LD = @LD@ with_gnu_ld = @with_gnu_ld@ -ld_R_works = @ld_R_works@ # We want -L for libpgport.a and libpgcommon.a to be first in LDFLAGS. We # also need LDFLAGS to be a "recursively expanded" variable, else adjustments diff --git a/src/Makefile.shlib b/src/Makefile.shlib index 87c80c5d01..358d90837c 100644 --- a/src/Makefile.shlib +++ b/src/Makefile.shlib @@ -236,30 +236,6 @@ ifeq ($(PORTNAME), solaris) endif endif -ifeq ($(PORTNAME), sco) - ifeq ($(GCC), yes) - LINK.shared = $(CC) -shared - else - LINK.shared = $(CC) -G - endif - LINK.shared += -Wl,-z,text - ifdef soname - LINK.shared += -Wl,-h,$(soname) - endif -endif - -ifeq ($(PORTNAME), unixware) - ifeq ($(GCC), yes) - LINK.shared = $(CC) -shared - else - LINK.shared = $(CC) -G - endif - LINK.shared += -Wl,-z,text - ifdef soname - LINK.shared += -Wl,-h,$(soname) - endif -endif - ifeq ($(PORTNAME), cygwin) LINK.shared = $(CC) -shared ifdef SO_MAJOR_VERSION diff --git a/src/backend/libpq/pqcomm.c b/src/backend/libpq/pqcomm.c index bae96bf18f..affa9bb7ab 100644 --- a/src/backend/libpq/pqcomm.c +++ b/src/backend/libpq/pqcomm.c @@ -683,16 +683,6 @@ StreamConnection(pgsocket server_fd, Port *port) return STATUS_ERROR; } -#ifdef SCO_ACCEPT_BUG - - /* - * UnixWare 7+ and OpenServer 5.0.4 are known to have this bug, but it - * shouldn't hurt to catch it for all versions of those platforms. - */ - if (port->raddr.addr.ss_family == 0) - port->raddr.addr.ss_family = AF_UNIX; -#endif - /* fill in the server (local) address */ port->laddr.salen = sizeof(port->laddr.addr); if (getsockname(port->sock, diff --git a/src/backend/port/dynloader/sco.c b/src/backend/port/dynloader/sco.c deleted file mode 100644 index 1e24f494ac..0000000000 --- a/src/backend/port/dynloader/sco.c +++ /dev/null @@ -1,7 +0,0 @@ -/* - * src/backend/port/dynloader/sco.c - * - * Dummy file used for nothing at this point - * - * see sco.h - */ diff --git a/src/backend/port/dynloader/sco.h b/src/backend/port/dynloader/sco.h deleted file mode 100644 index 86f2383729..0000000000 --- a/src/backend/port/dynloader/sco.h +++ /dev/null @@ -1,46 +0,0 @@ -/*------------------------------------------------------------------------- - * - * sco.h - * port-specific prototypes for SCO 3.2v5.2 - * - * - * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * src/backend/port/dynloader/sco.h - * - *------------------------------------------------------------------------- - */ -#ifndef PORT_PROTOS_H -#define PORT_PROTOS_H - -#include -#include "utils/dynamic_loader.h" /* pgrminclude ignore */ - -/* - * Dynamic Loader on SCO 3.2v5.0.2 - * - * this dynamic loader uses the system dynamic loading interface for shared - * libraries (ie. dlopen/dlsym/dlclose). The user must specify a shared - * library as the file to be dynamically loaded. - */ - -/* - * In some older systems, the RTLD_NOW flag isn't defined and the mode - * argument to dlopen must always be 1. The RTLD_GLOBAL flag is wanted - * if available, but it doesn't exist everywhere. - * If it doesn't exist, set it to 0 so it has no effect. - */ -#ifndef RTLD_NOW -#define RTLD_NOW 1 -#endif -#ifndef RTLD_GLOBAL -#define RTLD_GLOBAL 0 -#endif - -#define pg_dlopen(f) dlopen((f), RTLD_NOW | RTLD_GLOBAL) -#define pg_dlsym dlsym -#define pg_dlclose dlclose -#define pg_dlerror dlerror - -#endif /* PORT_PROTOS_H */ diff --git a/src/backend/port/dynloader/unixware.c b/src/backend/port/dynloader/unixware.c deleted file mode 100644 index afb36dfe99..0000000000 --- a/src/backend/port/dynloader/unixware.c +++ /dev/null @@ -1,7 +0,0 @@ -/* - * src/backend/port/dynloader/unixware.c - * - * Dummy file used for nothing at this point - * - * see unixware.h - */ diff --git a/src/backend/port/dynloader/unixware.h b/src/backend/port/dynloader/unixware.h deleted file mode 100644 index 130a9a25d5..0000000000 --- a/src/backend/port/dynloader/unixware.h +++ /dev/null @@ -1,49 +0,0 @@ -/* - * src/backend/port/dynloader/unixware.h - * - *------------------------------------------------------------------------- - * - * unixware.h - * port-specific prototypes for Intel x86/UNIXWARE 7 - * - * - * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * unixware.h,v 1.2 1995/03/17 06:40:18 andrew Exp - * - *------------------------------------------------------------------------- - */ -#ifndef PORT_PROTOS_H -#define PORT_PROTOS_H - -#include -#include "utils/dynamic_loader.h" /* pgrminclude ignore */ - -/* - * Dynamic Loader on UnixWare. - * - * this dynamic loader uses the system dynamic loading interface for shared - * libraries (ie. dlopen/dlsym/dlclose). The user must specify a shared - * library as the file to be dynamically loaded. - */ - -/* - * In some older systems, the RTLD_NOW flag isn't defined and the mode - * argument to dlopen must always be 1. The RTLD_GLOBAL flag is wanted - * if available, but it doesn't exist everywhere. - * If it doesn't exist, set it to 0 so it has no effect. - */ -#ifndef RTLD_NOW -#define RTLD_NOW 1 -#endif -#ifndef RTLD_GLOBAL -#define RTLD_GLOBAL 0 -#endif - -#define pg_dlopen(f) dlopen((f), RTLD_NOW | RTLD_GLOBAL) -#define pg_dlsym dlsym -#define pg_dlclose dlclose -#define pg_dlerror dlerror - -#endif /* PORT_PROTOS_H */ diff --git a/src/include/port/sco.h b/src/include/port/sco.h deleted file mode 100644 index 30811450c9..0000000000 --- a/src/include/port/sco.h +++ /dev/null @@ -1,7 +0,0 @@ -/* - * src/include/port/sco.h - * - * see src/backend/libpq/pqcomm.c */ -#define SCO_ACCEPT_BUG - -#define USE_UNIVEL_CC diff --git a/src/include/port/unixware.h b/src/include/port/unixware.h deleted file mode 100644 index e068820957..0000000000 --- a/src/include/port/unixware.h +++ /dev/null @@ -1,11 +0,0 @@ -/* - * src/include/port/unixware.h - * - * see src/backend/libpq/pqcomm.c */ -#define SCO_ACCEPT_BUG - -/*************************************** - * Define this if you are compiling with - * the native UNIXWARE C compiler. - ***************************************/ -#define USE_UNIVEL_CC diff --git a/src/include/storage/s_lock.h b/src/include/storage/s_lock.h index 7aad2de43d..3fe29cede6 100644 --- a/src/include/storage/s_lock.h +++ b/src/include/storage/s_lock.h @@ -706,29 +706,6 @@ typedef unsigned char slock_t; #if !defined(HAS_TEST_AND_SET) /* We didn't trigger above, let's try here */ -#if defined(USE_UNIVEL_CC) /* Unixware compiler */ -#define HAS_TEST_AND_SET - -typedef unsigned char slock_t; - -#define TAS(lock) tas(lock) - -asm int -tas(volatile slock_t *s_lock) -{ -/* UNIVEL wants %mem in column 1, so we don't pgindent this file */ -%mem s_lock - pushl %ebx - movl s_lock, %ebx - movl $255, %eax - lock - xchgb %al, (%ebx) - popl %ebx -} - -#endif /* defined(USE_UNIVEL_CC) */ - - #if defined(__hppa) || defined(__hppa__) /* HP PA-RISC, GCC and HP compilers */ /* * HP's PA-RISC diff --git a/src/makefiles/Makefile.sco b/src/makefiles/Makefile.sco deleted file mode 100644 index 993861570a..0000000000 --- a/src/makefiles/Makefile.sco +++ /dev/null @@ -1,13 +0,0 @@ -AROPT = cr -export_dynamic = -Wl,-Bexport - -DLSUFFIX = .so -ifeq ($(GCC), yes) -CFLAGS_SL = -fpic -else -CFLAGS_SL = -K PIC -endif - -# Rule for building a shared library from a single .o file -%.so: %.o - $(LD) -G -Bdynamic -o $@ $< diff --git a/src/makefiles/Makefile.unixware b/src/makefiles/Makefile.unixware deleted file mode 100644 index a52717b268..0000000000 --- a/src/makefiles/Makefile.unixware +++ /dev/null @@ -1,35 +0,0 @@ -AROPT = crs -ifeq ($(with_gnu_ld), yes) - export_dynamic = -Wl,-E -else - export_dynamic = -Wl,-Bexport -endif - -ifeq ($(ld_R_works), yes) -ifeq ($(with_gnu_ld), yes) - rpath = -Wl,-rpath,'$(rpathdir)' -else - rpath = -Wl,-R'$(rpathdir)' -endif -endif - -# Unixware needs threads for everything that uses libpq -CFLAGS += $(PTHREAD_CFLAGS) - -DLSUFFIX = .so -ifeq ($(GCC), yes) -CFLAGS_SL = -fpic -else -CFLAGS_SL = -K PIC -endif -ifeq ($(GCC), yes) -SO_FLAGS = -shared -else -SO_FLAGS = -G -endif - -# Rule for building a shared library from a single .o file -%.so: %.o - $(CC) $(CFLAGS) $(LDFLAGS) $(LDFLAGS_SL) $(SO_FLAGS) -o $@ $< - -sqlmansect = 5sql diff --git a/src/port/getrusage.c b/src/port/getrusage.c index a6f1ef2681..d24af92339 100644 --- a/src/port/getrusage.c +++ b/src/port/getrusage.c @@ -18,7 +18,6 @@ #include "rusagestub.h" /* This code works on: - * sco * solaris_i386 * solaris_sparc * hpux 9.* diff --git a/src/template/sco b/src/template/sco deleted file mode 100644 index 9a736da8be..0000000000 --- a/src/template/sco +++ /dev/null @@ -1 +0,0 @@ -CC="$CC -b elf" diff --git a/src/template/unixware b/src/template/unixware deleted file mode 100644 index d08fca1e6b..0000000000 --- a/src/template/unixware +++ /dev/null @@ -1,41 +0,0 @@ -if test "$GCC" != yes; then - # The -Kno_host is for a bug in the compiler. See -hackers - # discussion on 7-8/Aug/2003. - cat >conftest.c <<__EOF__ -extern char *strcpy(char *, const char *); - -static void f(char *p, int n){ - strcpy(p+n,""); -} -void g(void){ - f(0, 0); -} -__EOF__ - - # Debugging and optimization are mutually exclusive - if test "$enable_debug" != yes; then - CFLAGS="-O" - fi - if $CC -c -O -Kinline conftest.c >conftest.err 2>&1; then - CFLAGS="$CFLAGS -Kinline" - else - CFLAGS="$CFLAGS -Kinline,no_host" - fi - rm -f conftest.* - - PTHREAD_CFLAGS="-Kpthread" - -# The effect of doing threading for the backend does not work -# because of a threading bug that appears in the regression tests: -# -# in make check, the plpgsql test (plpgsql.sql) -# set statement_timeout to 1000; -# select blockme(); -# reset statement_timeout; -# -# per report from Olivier PRENANT - -fi - -# Unixware's ldap library reportedly needs these too -EXTRA_LDAP_LIBS="-llber -lresolv" -- cgit v1.2.3 From 64f3524e2c8deebc02808aa5ebdfa17859473add Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 12 Oct 2016 12:19:56 -0400 Subject: Remove pg_dump/pg_dumpall support for dumping from pre-8.0 servers. The need for dumping from such ancient servers has decreased to about nil in the field, so let's remove all the code that catered to it. Aside from removing a lot of boilerplate variant queries, this allows us to not have to cope with servers that don't have (a) schemas or (b) pg_depend. That means we can get rid of assorted squishy code around that. There may be some nonobvious additional simplifications possible, but this patch already removes about 1500 lines of code. I did not remove the ability for pg_restore to read custom-format archives generated by these old versions (and light testing says that that does still work). If you have an old server, you probably also have a pg_dump that will work with it; but you have an old custom-format backup file, that might be all you have. It'd be possible at this point to remove fmtQualifiedId()'s version argument, but I refrained since that would affect code outside pg_dump. Discussion: <2661.1475849167@sss.pgh.pa.us> --- doc/src/sgml/ref/pg_dump.sgml | 7 +- src/bin/pg_dump/dumputils.c | 50 +- src/bin/pg_dump/pg_backup_archiver.c | 2 +- src/bin/pg_dump/pg_dump.c | 1761 ++++------------------------------ src/bin/pg_dump/pg_dump.h | 1 - src/bin/pg_dump/pg_dump_sort.c | 96 +- src/bin/pg_dump/pg_dumpall.c | 110 +-- 7 files changed, 250 insertions(+), 1777 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml index be1b684082..371a61427d 100644 --- a/doc/src/sgml/ref/pg_dump.sgml +++ b/doc/src/sgml/ref/pg_dump.sgml @@ -758,10 +758,9 @@ PostgreSQL documentation the dump. Instead fail if unable to lock a table within the specified timeout. The timeout may be specified in any of the formats accepted by SET - statement_timeout. (Allowed values vary depending on the server + statement_timeout. (Allowed formats vary depending on the server version you are dumping from, but an integer number of milliseconds - is accepted by all versions since 7.3. This option is ignored when - dumping from a pre-7.3 server.) + is accepted by all versions.) @@ -1172,7 +1171,7 @@ CREATE DATABASE foo WITH TEMPLATE template0; PostgreSQL server versions newer than pg_dump's version. pg_dump can also dump from PostgreSQL servers older than its own version. - (Currently, servers back to version 7.0 are supported.) + (Currently, servers back to version 8.0 are supported.) However, pg_dump cannot dump from PostgreSQL servers newer than its own major version; it will refuse to even try, rather than risk making an invalid dump. diff --git a/src/bin/pg_dump/dumputils.c b/src/bin/pg_dump/dumputils.c index cd1e8c4a68..0d5166891e 100644 --- a/src/bin/pg_dump/dumputils.c +++ b/src/bin/pg_dump/dumputils.c @@ -18,8 +18,6 @@ #include "fe_utils/string_utils.h" -#define supports_grant_options(version) ((version) >= 70400) - static bool parseAclItem(const char *item, const char *type, const char *name, const char *subname, int remoteVersion, PQExpBuffer grantee, PQExpBuffer grantor, @@ -246,11 +244,9 @@ buildACLCommands(const char *name, const char *subname, /* * For the owner, the default privilege level is ALL WITH - * GRANT OPTION (only ALL prior to 7.4). + * GRANT OPTION. */ - if (supports_grant_options(remoteVersion) - ? strcmp(privswgo->data, "ALL") != 0 - : strcmp(privs->data, "ALL") != 0) + if (strcmp(privswgo->data, "ALL") != 0) { appendPQExpBuffer(firstsql, "%sREVOKE ALL", prefix); if (subname) @@ -403,16 +399,19 @@ buildDefaultACLCommands(const char *type, const char *nspname, * username=privilegecodes/grantor * or * group groupname=privilegecodes/grantor - * (the /grantor part will not be present if pre-7.4 database). + * (the "group" case occurs only with servers before 8.1). + * + * Returns true on success, false on parse error. On success, the components + * of the string are returned in the PQExpBuffer parameters. * * The returned grantee string will be the dequoted username or groupname - * (preceded with "group " in the latter case). The returned grantor is - * the dequoted grantor name or empty. Privilege characters are decoded - * and split between privileges with grant option (privswgo) and without - * (privs). + * (preceded with "group " in the latter case). Note that a grant to PUBLIC + * is represented by an empty grantee string. The returned grantor is the + * dequoted grantor name. Privilege characters are decoded and split between + * privileges with grant option (privswgo) and without (privs). * - * Note: for cross-version compatibility, it's important to use ALL when - * appropriate. + * Note: for cross-version compatibility, it's important to use ALL to + * represent the privilege sets whenever appropriate. */ static bool parseAclItem(const char *item, const char *type, @@ -439,7 +438,7 @@ parseAclItem(const char *item, const char *type, return false; } - /* grantor may be listed after / */ + /* grantor should appear after / */ slpos = strchr(eqpos + 1, '/'); if (slpos) { @@ -452,7 +451,10 @@ parseAclItem(const char *item, const char *type, } } else - resetPQExpBuffer(grantor); + { + free(buf); + return false; + } /* privilege codes */ #define CONVERT_PRIV(code, keywd) \ @@ -490,29 +492,19 @@ do { \ { /* table only */ CONVERT_PRIV('a', "INSERT"); - if (remoteVersion >= 70200) - CONVERT_PRIV('x', "REFERENCES"); + CONVERT_PRIV('x', "REFERENCES"); /* rest are not applicable to columns */ if (subname == NULL) { - if (remoteVersion >= 70200) - { - CONVERT_PRIV('d', "DELETE"); - CONVERT_PRIV('t', "TRIGGER"); - } + CONVERT_PRIV('d', "DELETE"); + CONVERT_PRIV('t', "TRIGGER"); if (remoteVersion >= 80400) CONVERT_PRIV('D', "TRUNCATE"); } } /* UPDATE */ - if (remoteVersion >= 70200 || - strcmp(type, "SEQUENCE") == 0 || - strcmp(type, "SEQUENCES") == 0) - CONVERT_PRIV('w', "UPDATE"); - else - /* 7.0 and 7.1 have a simpler worldview */ - CONVERT_PRIV('w', "UPDATE,DELETE"); + CONVERT_PRIV('w', "UPDATE"); } else if (strcmp(type, "FUNCTION") == 0 || strcmp(type, "FUNCTIONS") == 0) diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index bba8b6ca9f..e237b4a9c9 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -388,7 +388,7 @@ RestoreArchive(Archive *AHX) * target. */ AHX->minRemoteVersion = 0; - AHX->maxRemoteVersion = 999999; + AHX->maxRemoteVersion = 9999999; ConnectDatabase(AHX, ropt->dbname, ropt->pghost, ropt->pgport, ropt->username, diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index fde7f59c3d..c821f3b0ee 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -96,9 +96,6 @@ bool g_verbose; /* User wants verbose narration of our /* subquery used to convert user ID (eg, datdba) to user name */ static const char *username_subquery; -/* obsolete as of 7.3: */ -static Oid g_last_builtin_oid; /* value of the last builtin oid */ - /* The specified names/patterns should to match at least one entity */ static int strict_names = 0; @@ -142,7 +139,7 @@ static void expand_table_name_patterns(Archive *fout, SimpleStringList *patterns, SimpleOidList *oids, bool strict_names); -static NamespaceInfo *findNamespace(Archive *fout, Oid nsoid, Oid objoid); +static NamespaceInfo *findNamespace(Archive *fout, Oid nsoid); static void dumpTableData(Archive *fout, TableDataInfo *tdinfo); static void refreshMatViewData(Archive *fout, TableDataInfo *tdinfo); static void guessConstraintInheritance(TableInfo *tblinfo, int numTables); @@ -236,11 +233,8 @@ static char *convertRegProcReference(Archive *fout, const char *proc); static char *convertOperatorReference(Archive *fout, const char *opr); static char *convertTSFunction(Archive *fout, Oid funcOid); -static Oid findLastBuiltinOid_V71(Archive *fout, const char *); -static Oid findLastBuiltinOid_V70(Archive *fout); static void selectSourceSchema(Archive *fout, const char *schemaName); static char *getFormattedTypeName(Archive *fout, Oid oid, OidOptions opts); -static char *myFormatType(const char *typname, int32 typmod); static void getBlobs(Archive *fout); static void dumpBlob(Archive *fout, BlobInfo *binfo); static int dumpBlobs(Archive *fout, void *arg); @@ -633,10 +627,10 @@ main(int argc, char **argv) fout->verbose = g_verbose; /* - * We allow the server to be back to 7.0, and up to any minor release of + * We allow the server to be back to 8.0, and up to any minor release of * our own major version. (See also version check in pg_dumpall.c.) */ - fout->minRemoteVersion = 70000; + fout->minRemoteVersion = 80000; fout->maxRemoteVersion = (PG_VERSION_NUM / 100) * 100 + 99; fout->numWorkers = numWorkers; @@ -665,10 +659,8 @@ main(int argc, char **argv) /* Select the appropriate subquery to convert user IDs to names */ if (fout->remoteVersion >= 80100) username_subquery = "SELECT rolname FROM pg_catalog.pg_roles WHERE oid ="; - else if (fout->remoteVersion >= 70300) - username_subquery = "SELECT usename FROM pg_catalog.pg_user WHERE usesysid ="; else - username_subquery = "SELECT usename FROM pg_user WHERE usesysid ="; + username_subquery = "SELECT usename FROM pg_catalog.pg_user WHERE usesysid ="; /* check the version for the synchronized snapshots feature */ if (numWorkers > 1 && fout->remoteVersion < 90200 @@ -683,18 +675,6 @@ main(int argc, char **argv) exit_horribly(NULL, "Exported snapshots are not supported by this server version.\n"); - /* Find the last built-in OID, if needed */ - if (fout->remoteVersion < 70300) - { - if (fout->remoteVersion >= 70100) - g_last_builtin_oid = findLastBuiltinOid_V71(fout, - PQdb(GetConnection(fout))); - else - g_last_builtin_oid = findLastBuiltinOid_V70(fout); - if (g_verbose) - write_msg(NULL, "last built-in OID is %u\n", g_last_builtin_oid); - } - /* Expand schema selection patterns into OID lists */ if (schema_include_patterns.head != NULL) { @@ -774,16 +754,11 @@ main(int argc, char **argv) /* * Sort the objects into a safe dump order (no forward references). * - * In 7.3 or later, we can rely on dependency information to help us - * determine a safe order, so the initial sort is mostly for cosmetic - * purposes: we sort by name to ensure that logically identical schemas - * will dump identically. Before 7.3 we don't have dependencies and we - * use OID ordering as an (unreliable) guide to creation order. + * We rely on dependency information to help us determine a safe order, so + * the initial sort is mostly for cosmetic purposes: we sort by name to + * ensure that logically identical schemas will dump identically. */ - if (fout->remoteVersion >= 70300) - sortDumpableObjectsByTypeName(dobjs, numObjs); - else - sortDumpableObjectsByTypeOid(dobjs, numObjs); + sortDumpableObjectsByTypeName(dobjs, numObjs); /* If we do a parallel dump, we want the largest tables to go first */ if (archiveFormat == archDirectory && numWorkers > 1) @@ -1000,12 +975,12 @@ setup_connection(Archive *AH, const char *dumpencoding, ExecuteSqlStatement(AH, "SET INTERVALSTYLE = POSTGRES"); /* - * If supported, set extra_float_digits so that we can dump float data - * exactly (given correctly implemented float I/O code, anyway) + * Set extra_float_digits so that we can dump float data exactly (given + * correctly implemented float I/O code, anyway) */ if (AH->remoteVersion >= 90000) ExecuteSqlStatement(AH, "SET extra_float_digits TO 3"); - else if (AH->remoteVersion >= 70400) + else ExecuteSqlStatement(AH, "SET extra_float_digits TO 2"); /* @@ -1018,8 +993,7 @@ setup_connection(Archive *AH, const char *dumpencoding, /* * Disable timeouts if supported. */ - if (AH->remoteVersion >= 70300) - ExecuteSqlStatement(AH, "SET statement_timeout = 0"); + ExecuteSqlStatement(AH, "SET statement_timeout = 0"); if (AH->remoteVersion >= 90300) ExecuteSqlStatement(AH, "SET lock_timeout = 0"); if (AH->remoteVersion >= 90600) @@ -1065,16 +1039,12 @@ setup_connection(Archive *AH, const char *dumpencoding, "SET TRANSACTION ISOLATION LEVEL " "REPEATABLE READ, READ ONLY"); } - else if (AH->remoteVersion >= 70400) + else { - /* note: comma was not accepted in SET TRANSACTION before 8.0 */ ExecuteSqlStatement(AH, "SET TRANSACTION ISOLATION LEVEL " - "SERIALIZABLE READ ONLY"); + "SERIALIZABLE, READ ONLY"); } - else - ExecuteSqlStatement(AH, - "SET TRANSACTION ISOLATION LEVEL SERIALIZABLE"); /* * If user specified a snapshot to use, select that. In a parallel dump @@ -1190,9 +1160,6 @@ expand_schema_name_patterns(Archive *fout, if (patterns->head == NULL) return; /* nothing to do */ - if (fout->remoteVersion < 70300) - exit_horribly(NULL, "server version must be at least 7.3 to use schema selection switches\n"); - query = createPQExpBuffer(); /* @@ -1661,15 +1628,12 @@ dumpTableData_copy(Archive *fout, void *dcontext) selectSourceSchema(fout, tbinfo->dobj.namespace->dobj.name); /* - * If possible, specify the column list explicitly so that we have no - * possibility of retrieving data in the wrong column order. (The default - * column ordering of COPY will not be what we want in certain corner - * cases involving ADD COLUMN and inheritance.) + * Specify the column list explicitly so that we have no possibility of + * retrieving data in the wrong column order. (The default column + * ordering of COPY will not be what we want in certain corner cases + * involving ADD COLUMN and inheritance.) */ - if (fout->remoteVersion >= 70300) - column_list = fmtCopyColumnList(tbinfo, clistBuf); - else - column_list = ""; /* can't select columns in COPY */ + column_list = fmtCopyColumnList(tbinfo, clistBuf); if (oids && hasoids) { @@ -1829,22 +1793,11 @@ dumpTableData_insert(Archive *fout, void *dcontext) */ selectSourceSchema(fout, tbinfo->dobj.namespace->dobj.name); - if (fout->remoteVersion >= 70100) - { - appendPQExpBuffer(q, "DECLARE _pg_dump_cursor CURSOR FOR " - "SELECT * FROM ONLY %s", - fmtQualifiedId(fout->remoteVersion, - tbinfo->dobj.namespace->dobj.name, - classname)); - } - else - { - appendPQExpBuffer(q, "DECLARE _pg_dump_cursor CURSOR FOR " - "SELECT * FROM %s", - fmtQualifiedId(fout->remoteVersion, - tbinfo->dobj.namespace->dobj.name, - classname)); - } + appendPQExpBuffer(q, "DECLARE _pg_dump_cursor CURSOR FOR " + "SELECT * FROM ONLY %s", + fmtQualifiedId(fout->remoteVersion, + tbinfo->dobj.namespace->dobj.name, + classname)); if (tdinfo->filtercond) appendPQExpBuffer(q, " %s", tdinfo->filtercond); @@ -2480,7 +2433,7 @@ dumpDatabase(Archive *fout) username_subquery); appendStringLiteralAH(dbQry, datname, fout); } - else if (fout->remoteVersion >= 80000) + else { appendPQExpBuffer(dbQry, "SELECT tableoid, oid, " "(%s datdba) AS dba, " @@ -2492,34 +2445,6 @@ dumpDatabase(Archive *fout) username_subquery); appendStringLiteralAH(dbQry, datname, fout); } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBuffer(dbQry, "SELECT tableoid, oid, " - "(%s datdba) AS dba, " - "pg_encoding_to_char(encoding) AS encoding, " - "NULL AS datcollate, NULL AS datctype, " - "0 AS datfrozenxid, 0 AS datminmxid, " - "NULL AS tablespace " - "FROM pg_database " - "WHERE datname = ", - username_subquery); - appendStringLiteralAH(dbQry, datname, fout); - } - else - { - appendPQExpBuffer(dbQry, "SELECT " - "(SELECT oid FROM pg_class WHERE relname = 'pg_database') AS tableoid, " - "oid, " - "(%s datdba) AS dba, " - "pg_encoding_to_char(encoding) AS encoding, " - "NULL AS datcollate, NULL AS datctype, " - "0 AS datfrozenxid, 0 AS datminmxid, " - "NULL AS tablespace " - "FROM pg_database " - "WHERE datname = ", - username_subquery); - appendStringLiteralAH(dbQry, datname, fout); - } res = ExecuteSqlQueryForSingleRow(fout, dbQry->data); @@ -2879,19 +2804,13 @@ getBlobs(Archive *fout) "NULL AS initrlomacl " " FROM pg_largeobject_metadata", username_subquery); - else if (fout->remoteVersion >= 70100) + else appendPQExpBufferStr(blobQry, "SELECT DISTINCT loid AS oid, " "NULL::name AS rolname, NULL::oid AS lomacl, " "NULL::oid AS rlomacl, NULL::oid AS initlomacl, " "NULL::oid AS initrlomacl " " FROM pg_largeobject"); - else - appendPQExpBufferStr(blobQry, - "SELECT oid, NULL AS rolname, NULL AS lomacl, " - "NULL AS rlomacl, NULL AS initlomacl, " - "NULL AS initrlomacl " - " FROM pg_class WHERE relkind = 'l'"); res = ExecuteSqlQuery(fout, blobQry->data, PGRES_TUPLES_OK); @@ -3031,10 +2950,8 @@ dumpBlobs(Archive *fout, void *arg) */ if (fout->remoteVersion >= 90000) blobQry = "DECLARE bloboid CURSOR FOR SELECT oid FROM pg_largeobject_metadata"; - else if (fout->remoteVersion >= 70100) - blobQry = "DECLARE bloboid CURSOR FOR SELECT DISTINCT loid FROM pg_largeobject"; else - blobQry = "DECLARE bloboid CURSOR FOR SELECT oid FROM pg_class WHERE relkind = 'l'"; + blobQry = "DECLARE bloboid CURSOR FOR SELECT DISTINCT loid FROM pg_largeobject"; ExecuteSqlStatement(fout, blobQry); @@ -3536,45 +3453,6 @@ getNamespaces(Archive *fout, int *numNamespaces) int i_initnspacl; int i_initrnspacl; - /* - * Before 7.3, there are no real namespaces; create two dummy entries, one - * for user stuff and one for system stuff. - */ - if (fout->remoteVersion < 70300) - { - nsinfo = (NamespaceInfo *) pg_malloc(2 * sizeof(NamespaceInfo)); - - nsinfo[0].dobj.objType = DO_NAMESPACE; - nsinfo[0].dobj.catId.tableoid = 0; - nsinfo[0].dobj.catId.oid = 0; - AssignDumpId(&nsinfo[0].dobj); - nsinfo[0].dobj.name = pg_strdup("public"); - nsinfo[0].rolname = pg_strdup(""); - nsinfo[0].nspacl = pg_strdup(""); - nsinfo[0].rnspacl = pg_strdup(""); - nsinfo[0].initnspacl = pg_strdup(""); - nsinfo[0].initrnspacl = pg_strdup(""); - - selectDumpableNamespace(&nsinfo[0], fout); - - nsinfo[1].dobj.objType = DO_NAMESPACE; - nsinfo[1].dobj.catId.tableoid = 0; - nsinfo[1].dobj.catId.oid = 1; - AssignDumpId(&nsinfo[1].dobj); - nsinfo[1].dobj.name = pg_strdup("pg_catalog"); - nsinfo[1].rolname = pg_strdup(""); - nsinfo[1].nspacl = pg_strdup(""); - nsinfo[1].rnspacl = pg_strdup(""); - nsinfo[1].initnspacl = pg_strdup(""); - nsinfo[1].initrnspacl = pg_strdup(""); - - selectDumpableNamespace(&nsinfo[1], fout); - - *numNamespaces = 2; - - return nsinfo; - } - query = createPQExpBuffer(); /* Make sure we are in proper schema */ @@ -3684,37 +3562,16 @@ getNamespaces(Archive *fout, int *numNamespaces) /* * findNamespace: - * given a namespace OID and an object OID, look up the info read by - * getNamespaces - * - * NB: for pre-7.3 source database, we use object OID to guess whether it's - * a system object or not. In 7.3 and later there is no guessing, and we - * don't use objoid at all. + * given a namespace OID, look up the info read by getNamespaces */ static NamespaceInfo * -findNamespace(Archive *fout, Oid nsoid, Oid objoid) +findNamespace(Archive *fout, Oid nsoid) { NamespaceInfo *nsinfo; - if (fout->remoteVersion >= 70300) - { - nsinfo = findNamespaceByOid(nsoid); - } - else - { - /* This code depends on the dummy objects set up by getNamespaces. */ - Oid i; - - if (objoid > g_last_builtin_oid) - i = 0; /* user object */ - else - i = 1; /* system object */ - nsinfo = findNamespaceByOid(i); - } - + nsinfo = findNamespaceByOid(nsoid); if (nsinfo == NULL) exit_horribly(NULL, "schema with OID %u does not exist\n", nsoid); - return nsinfo; } @@ -3932,7 +3789,7 @@ getTypes(Archive *fout, int *numTypes) "FROM pg_type", username_subquery); } - else if (fout->remoteVersion >= 70300) + else { appendPQExpBuffer(query, "SELECT tableoid, oid, typname, " "typnamespace, NULL AS typacl, NULL as rtypacl, " @@ -3947,38 +3804,6 @@ getTypes(Archive *fout, int *numTypes) "FROM pg_type", username_subquery); } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBuffer(query, "SELECT tableoid, oid, typname, " - "0::oid AS typnamespace, NULL AS typacl, NULL as rtypacl, " - "NULL AS inittypacl, NULL AS initrtypacl, " - "(%s typowner) AS rolname, " - "typinput::oid AS typinput, " - "typoutput::oid AS typoutput, typelem, typrelid, " - "CASE WHEN typrelid = 0 THEN ' '::\"char\" " - "ELSE (SELECT relkind FROM pg_class WHERE oid = typrelid) END AS typrelkind, " - "typtype, typisdefined, " - "typname[0] = '_' AND typelem != 0 AS isarray " - "FROM pg_type", - username_subquery); - } - else - { - appendPQExpBuffer(query, "SELECT " - "(SELECT oid FROM pg_class WHERE relname = 'pg_type') AS tableoid, " - "oid, typname, " - "0::oid AS typnamespace, NULL AS typacl, NULL as rtypacl, " - "NULL AS inittypacl, NULL AS initrtypacl, " - "(%s typowner) AS rolname, " - "typinput::oid AS typinput, " - "typoutput::oid AS typoutput, typelem, typrelid, " - "CASE WHEN typrelid = 0 THEN ' '::\"char\" " - "ELSE (SELECT relkind FROM pg_class WHERE oid = typrelid) END AS typrelkind, " - "typtype, typisdefined, " - "typname[0] = '_' AND typelem != 0 AS isarray " - "FROM pg_type", - username_subquery); - } res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -4013,8 +3838,7 @@ getTypes(Archive *fout, int *numTypes) tyinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_typname)); tyinfo[i].dobj.namespace = findNamespace(fout, - atooid(PQgetvalue(res, i, i_typnamespace)), - tyinfo[i].dobj.catId.oid); + atooid(PQgetvalue(res, i, i_typnamespace))); tyinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); tyinfo[i].typacl = pg_strdup(PQgetvalue(res, i, i_typacl)); tyinfo[i].rtypacl = pg_strdup(PQgetvalue(res, i, i_rtypacl)); @@ -4083,48 +3907,6 @@ getTypes(Archive *fout, int *numTypes) * this is taken care of while sorting dependencies. */ stinfo->dobj.dump = DUMP_COMPONENT_NONE; - - /* - * However, if dumping from pre-7.3, there will be no dependency - * info so we have to fake it here. We only need to worry about - * typinput and typoutput since the other functions only exist - * post-7.3. - */ - if (fout->remoteVersion < 70300) - { - Oid typinput; - Oid typoutput; - FuncInfo *funcInfo; - - typinput = atooid(PQgetvalue(res, i, i_typinput)); - typoutput = atooid(PQgetvalue(res, i, i_typoutput)); - - funcInfo = findFuncByOid(typinput); - if (funcInfo && funcInfo->dobj.dump & DUMP_COMPONENT_DEFINITION) - { - /* base type depends on function */ - addObjectDependency(&tyinfo[i].dobj, - funcInfo->dobj.dumpId); - /* function depends on shell type */ - addObjectDependency(&funcInfo->dobj, - stinfo->dobj.dumpId); - /* mark shell type as to be dumped */ - stinfo->dobj.dump = DUMP_COMPONENT_ALL; - } - - funcInfo = findFuncByOid(typoutput); - if (funcInfo && funcInfo->dobj.dump & DUMP_COMPONENT_DEFINITION) - { - /* base type depends on function */ - addObjectDependency(&tyinfo[i].dobj, - funcInfo->dobj.dumpId); - /* function depends on shell type */ - addObjectDependency(&funcInfo->dobj, - stinfo->dobj.dumpId); - /* mark shell type as to be dumped */ - stinfo->dobj.dump = DUMP_COMPONENT_ALL; - } - } } if (strlen(tyinfo[i].rolname) == 0) @@ -4172,38 +3954,13 @@ getOperators(Archive *fout, int *numOprs) /* Make sure we are in proper schema */ selectSourceSchema(fout, "pg_catalog"); - if (fout->remoteVersion >= 70300) - { - appendPQExpBuffer(query, "SELECT tableoid, oid, oprname, " - "oprnamespace, " - "(%s oprowner) AS rolname, " - "oprkind, " - "oprcode::oid AS oprcode " - "FROM pg_operator", - username_subquery); - } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBuffer(query, "SELECT tableoid, oid, oprname, " - "0::oid AS oprnamespace, " - "(%s oprowner) AS rolname, " - "oprkind, " - "oprcode::oid AS oprcode " - "FROM pg_operator", - username_subquery); - } - else - { - appendPQExpBuffer(query, "SELECT " - "(SELECT oid FROM pg_class WHERE relname = 'pg_operator') AS tableoid, " - "oid, oprname, " - "0::oid AS oprnamespace, " - "(%s oprowner) AS rolname, " - "oprkind, " - "oprcode::oid AS oprcode " - "FROM pg_operator", - username_subquery); - } + appendPQExpBuffer(query, "SELECT tableoid, oid, oprname, " + "oprnamespace, " + "(%s oprowner) AS rolname, " + "oprkind, " + "oprcode::oid AS oprcode " + "FROM pg_operator", + username_subquery); res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -4229,8 +3986,7 @@ getOperators(Archive *fout, int *numOprs) oprinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oprname)); oprinfo[i].dobj.namespace = findNamespace(fout, - atooid(PQgetvalue(res, i, i_oprnamespace)), - oprinfo[i].dobj.catId.oid); + atooid(PQgetvalue(res, i, i_oprnamespace))); oprinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); oprinfo[i].oprkind = (PQgetvalue(res, i, i_oprkind))[0]; oprinfo[i].oprcode = atooid(PQgetvalue(res, i, i_oprcode)); @@ -4319,8 +4075,7 @@ getCollations(Archive *fout, int *numCollations) collinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_collname)); collinfo[i].dobj.namespace = findNamespace(fout, - atooid(PQgetvalue(res, i, i_collnamespace)), - collinfo[i].dobj.catId.oid); + atooid(PQgetvalue(res, i, i_collnamespace))); collinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); /* Decide whether we want to dump it */ @@ -4358,13 +4113,6 @@ getConversions(Archive *fout, int *numConversions) int i_connamespace; int i_rolname; - /* Conversions didn't exist pre-7.3 */ - if (fout->remoteVersion < 70300) - { - *numConversions = 0; - return NULL; - } - query = createPQExpBuffer(); /* @@ -4403,8 +4151,7 @@ getConversions(Archive *fout, int *numConversions) convinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_conname)); convinfo[i].dobj.namespace = findNamespace(fout, - atooid(PQgetvalue(res, i, i_connamespace)), - convinfo[i].dobj.catId.oid); + atooid(PQgetvalue(res, i, i_connamespace))); convinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); /* Decide whether we want to dump it */ @@ -4527,30 +4274,11 @@ getOpclasses(Archive *fout, int *numOpclasses) /* Make sure we are in proper schema */ selectSourceSchema(fout, "pg_catalog"); - if (fout->remoteVersion >= 70300) - { - appendPQExpBuffer(query, "SELECT tableoid, oid, opcname, " - "opcnamespace, " - "(%s opcowner) AS rolname " - "FROM pg_opclass", - username_subquery); - } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBufferStr(query, "SELECT tableoid, oid, opcname, " - "0::oid AS opcnamespace, " - "''::name AS rolname " - "FROM pg_opclass"); - } - else - { - appendPQExpBufferStr(query, "SELECT " - "(SELECT oid FROM pg_class WHERE relname = 'pg_opclass') AS tableoid, " - "oid, opcname, " - "0::oid AS opcnamespace, " - "''::name AS rolname " - "FROM pg_opclass"); - } + appendPQExpBuffer(query, "SELECT tableoid, oid, opcname, " + "opcnamespace, " + "(%s opcowner) AS rolname " + "FROM pg_opclass", + username_subquery); res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -4574,8 +4302,7 @@ getOpclasses(Archive *fout, int *numOpclasses) opcinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_opcname)); opcinfo[i].dobj.namespace = findNamespace(fout, - atooid(PQgetvalue(res, i, i_opcnamespace)), - opcinfo[i].dobj.catId.oid); + atooid(PQgetvalue(res, i, i_opcnamespace))); opcinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); /* Decide whether we want to dump it */ @@ -4584,12 +4311,9 @@ getOpclasses(Archive *fout, int *numOpclasses) /* Op Classes do not currently have ACLs. */ opcinfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL; - if (fout->remoteVersion >= 70300) - { - if (strlen(opcinfo[i].rolname) == 0) - write_msg(NULL, "WARNING: owner of operator class \"%s\" appears to be invalid\n", - opcinfo[i].dobj.name); - } + if (strlen(opcinfo[i].rolname) == 0) + write_msg(NULL, "WARNING: owner of operator class \"%s\" appears to be invalid\n", + opcinfo[i].dobj.name); } PQclear(res); @@ -4665,8 +4389,7 @@ getOpfamilies(Archive *fout, int *numOpfamilies) opfinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_opfname)); opfinfo[i].dobj.namespace = findNamespace(fout, - atooid(PQgetvalue(res, i, i_opfnamespace)), - opfinfo[i].dobj.catId.oid); + atooid(PQgetvalue(res, i, i_opfnamespace))); opfinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); /* Decide whether we want to dump it */ @@ -4675,12 +4398,9 @@ getOpfamilies(Archive *fout, int *numOpfamilies) /* Extensions do not currently have ACLs. */ opfinfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL; - if (fout->remoteVersion >= 70300) - { - if (strlen(opfinfo[i].rolname) == 0) - write_msg(NULL, "WARNING: owner of operator family \"%s\" appears to be invalid\n", - opfinfo[i].dobj.name); - } + if (strlen(opfinfo[i].rolname) == 0) + write_msg(NULL, "WARNING: owner of operator family \"%s\" appears to be invalid\n", + opfinfo[i].dobj.name); } PQclear(res); @@ -4798,7 +4518,7 @@ getAggregates(Archive *fout, int *numAggs) "deptype = 'e')"); appendPQExpBufferChar(query, ')'); } - else if (fout->remoteVersion >= 70300) + else { appendPQExpBuffer(query, "SELECT tableoid, oid, proname AS aggname, " "pronamespace AS aggnamespace, " @@ -4814,38 +4534,6 @@ getAggregates(Archive *fout, int *numAggs) "(SELECT oid FROM pg_namespace WHERE nspname = 'pg_catalog')", username_subquery); } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBuffer(query, "SELECT tableoid, oid, aggname, " - "0::oid AS aggnamespace, " - "CASE WHEN aggbasetype = 0 THEN 0 ELSE 1 END AS pronargs, " - "aggbasetype AS proargtypes, " - "(%s aggowner) AS rolname, " - "NULL AS aggacl, " - "NULL AS raggacl, " - "NULL AS initaggacl, NULL AS initraggacl " - "FROM pg_aggregate " - "where oid > '%u'::oid", - username_subquery, - g_last_builtin_oid); - } - else - { - appendPQExpBuffer(query, "SELECT " - "(SELECT oid FROM pg_class WHERE relname = 'pg_aggregate') AS tableoid, " - "oid, aggname, " - "0::oid AS aggnamespace, " - "CASE WHEN aggbasetype = 0 THEN 0 ELSE 1 END AS pronargs, " - "aggbasetype AS proargtypes, " - "(%s aggowner) AS rolname, " - "NULL AS aggacl, " - "NULL AS raggacl, " - "NULL AS initaggacl, NULL AS initraggacl " - "FROM pg_aggregate " - "where oid > '%u'::oid", - username_subquery, - g_last_builtin_oid); - } res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -4875,8 +4563,7 @@ getAggregates(Archive *fout, int *numAggs) agginfo[i].aggfn.dobj.name = pg_strdup(PQgetvalue(res, i, i_aggname)); agginfo[i].aggfn.dobj.namespace = findNamespace(fout, - atooid(PQgetvalue(res, i, i_aggnamespace)), - agginfo[i].aggfn.dobj.catId.oid); + atooid(PQgetvalue(res, i, i_aggnamespace))); agginfo[i].aggfn.rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); if (strlen(agginfo[i].aggfn.rolname) == 0) write_msg(NULL, "WARNING: owner of aggregate function \"%s\" appears to be invalid\n", @@ -4893,13 +4580,9 @@ getAggregates(Archive *fout, int *numAggs) else { agginfo[i].aggfn.argtypes = (Oid *) pg_malloc(agginfo[i].aggfn.nargs * sizeof(Oid)); - if (fout->remoteVersion >= 70300) - parseOidArray(PQgetvalue(res, i, i_proargtypes), - agginfo[i].aggfn.argtypes, - agginfo[i].aggfn.nargs); - else - /* it's just aggbasetype */ - agginfo[i].aggfn.argtypes[0] = atooid(PQgetvalue(res, i, i_proargtypes)); + parseOidArray(PQgetvalue(res, i, i_proargtypes), + agginfo[i].aggfn.argtypes, + agginfo[i].aggfn.nargs); } /* Decide whether we want to dump it */ @@ -5025,7 +4708,7 @@ getFuncs(Archive *fout, int *numFuncs) destroyPQExpBuffer(initacl_subquery); destroyPQExpBuffer(initracl_subquery); } - else if (fout->remoteVersion >= 70300) + else { appendPQExpBuffer(query, "SELECT tableoid, oid, proname, prolang, " @@ -5056,39 +4739,6 @@ getFuncs(Archive *fout, int *numFuncs) "deptype = 'e')"); appendPQExpBufferChar(query, ')'); } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBuffer(query, - "SELECT tableoid, oid, proname, prolang, " - "pronargs, proargtypes, prorettype, " - "NULL AS proacl, " - "NULL AS rproacl, " - "NULL as initproacl, NULL AS initrproacl, " - "0::oid AS pronamespace, " - "(%s proowner) AS rolname " - "FROM pg_proc " - "WHERE pg_proc.oid > '%u'::oid", - username_subquery, - g_last_builtin_oid); - } - else - { - appendPQExpBuffer(query, - "SELECT " - "(SELECT oid FROM pg_class " - " WHERE relname = 'pg_proc') AS tableoid, " - "oid, proname, prolang, " - "pronargs, proargtypes, prorettype, " - "NULL AS proacl, " - "NULL AS rproacl, " - "NULL as initproacl, NULL AS initrproacl, " - "0::oid AS pronamespace, " - "(%s proowner) AS rolname " - "FROM pg_proc " - "where pg_proc.oid > '%u'::oid", - username_subquery, - g_last_builtin_oid); - } res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -5121,8 +4771,7 @@ getFuncs(Archive *fout, int *numFuncs) finfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_proname)); finfo[i].dobj.namespace = findNamespace(fout, - atooid(PQgetvalue(res, i, i_pronamespace)), - finfo[i].dobj.catId.oid); + atooid(PQgetvalue(res, i, i_pronamespace))); finfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); finfo[i].lang = atooid(PQgetvalue(res, i, i_prolang)); finfo[i].prorettype = atooid(PQgetvalue(res, i, i_prorettype)); @@ -5645,7 +5294,7 @@ getTables(Archive *fout, int *numTables) RELKIND_RELATION, RELKIND_SEQUENCE, RELKIND_VIEW, RELKIND_COMPOSITE_TYPE); } - else if (fout->remoteVersion >= 80000) + else { /* * Left join to pick up dependency info linking sequences to their @@ -5686,153 +5335,6 @@ getTables(Archive *fout, int *numTables) RELKIND_RELATION, RELKIND_SEQUENCE, RELKIND_VIEW, RELKIND_COMPOSITE_TYPE); } - else if (fout->remoteVersion >= 70300) - { - /* - * Left join to pick up dependency info linking sequences to their - * owning column, if any - */ - appendPQExpBuffer(query, - "SELECT c.tableoid, c.oid, relname, " - "relacl, NULL as rrelacl, " - "NULL AS initrelacl, NULL AS initrrelacl, " - "relkind, relnamespace, " - "(%s relowner) AS rolname, " - "relchecks, (reltriggers <> 0) AS relhastriggers, " - "relhasindex, relhasrules, relhasoids, " - "'f'::bool AS relrowsecurity, " - "'f'::bool AS relforcerowsecurity, " - "0 AS relfrozenxid, 0 AS relminmxid," - "0 AS toid, " - "0 AS tfrozenxid, 0 AS tminmxid," - "'p' AS relpersistence, 't' as relispopulated, " - "'d' AS relreplident, relpages, " - "NULL AS reloftype, " - "d.refobjid AS owning_tab, " - "d.refobjsubid AS owning_col, " - "NULL AS reltablespace, " - "NULL AS reloptions, " - "NULL AS toast_reloptions, " - "NULL AS changed_acl " - "FROM pg_class c " - "LEFT JOIN pg_depend d ON " - "(c.relkind = '%c' AND " - "d.classid = c.tableoid AND d.objid = c.oid AND " - "d.objsubid = 0 AND " - "d.refclassid = c.tableoid AND d.deptype = 'i') " - "WHERE relkind IN ('%c', '%c', '%c', '%c') " - "ORDER BY c.oid", - username_subquery, - RELKIND_SEQUENCE, - RELKIND_RELATION, RELKIND_SEQUENCE, - RELKIND_VIEW, RELKIND_COMPOSITE_TYPE); - } - else if (fout->remoteVersion >= 70200) - { - appendPQExpBuffer(query, - "SELECT tableoid, oid, relname, relacl, " - "NULL as rrelacl, " - "NULL AS initrelacl, NULL AS initrrelacl, " - "relkind, " - "0::oid AS relnamespace, " - "(%s relowner) AS rolname, " - "relchecks, (reltriggers <> 0) AS relhastriggers, " - "relhasindex, relhasrules, relhasoids, " - "'f'::bool AS relrowsecurity, " - "'f'::bool AS relforcerowsecurity, " - "0 AS relfrozenxid, 0 AS relminmxid," - "0 AS toid, " - "0 AS tfrozenxid, 0 AS tminmxid," - "'p' AS relpersistence, 't' as relispopulated, " - "'d' AS relreplident, relpages, " - "NULL AS reloftype, " - "NULL::oid AS owning_tab, " - "NULL::int4 AS owning_col, " - "NULL AS reltablespace, " - "NULL AS reloptions, " - "NULL AS toast_reloptions, " - "NULL AS changed_acl " - "FROM pg_class " - "WHERE relkind IN ('%c', '%c', '%c') " - "ORDER BY oid", - username_subquery, - RELKIND_RELATION, RELKIND_SEQUENCE, RELKIND_VIEW); - } - else if (fout->remoteVersion >= 70100) - { - /* all tables have oids in 7.1 */ - appendPQExpBuffer(query, - "SELECT tableoid, oid, relname, relacl, " - "NULL as rrelacl, " - "NULL AS initrelacl, NULL AS initrrelacl, " - "relkind, " - "0::oid AS relnamespace, " - "(%s relowner) AS rolname, " - "relchecks, (reltriggers <> 0) AS relhastriggers, " - "relhasindex, relhasrules, " - "'t'::bool AS relhasoids, " - "'f'::bool AS relrowsecurity, " - "'f'::bool AS relforcerowsecurity, " - "0 AS relfrozenxid, 0 AS relminmxid," - "0 AS toid, " - "0 AS tfrozenxid, 0 AS tminmxid," - "'p' AS relpersistence, 't' as relispopulated, " - "'d' AS relreplident, relpages, " - "NULL AS reloftype, " - "NULL::oid AS owning_tab, " - "NULL::int4 AS owning_col, " - "NULL AS reltablespace, " - "NULL AS reloptions, " - "NULL AS toast_reloptions, " - "NULL AS changed_acl " - "FROM pg_class " - "WHERE relkind IN ('%c', '%c', '%c') " - "ORDER BY oid", - username_subquery, - RELKIND_RELATION, RELKIND_SEQUENCE, RELKIND_VIEW); - } - else - { - /* - * Before 7.1, view relkind was not set to 'v', so we must check if we - * have a view by looking for a rule in pg_rewrite. - */ - appendPQExpBuffer(query, - "SELECT " - "(SELECT oid FROM pg_class WHERE relname = 'pg_class') AS tableoid, " - "oid, relname, relacl, NULL as rrelacl, " - "NULL AS initrelacl, NULL AS initrrelacl, " - "CASE WHEN relhasrules and relkind = 'r' " - " and EXISTS(SELECT rulename FROM pg_rewrite r WHERE " - " r.ev_class = c.oid AND r.ev_type = '1') " - "THEN '%c'::\"char\" " - "ELSE relkind END AS relkind," - "0::oid AS relnamespace, " - "(%s relowner) AS rolname, " - "relchecks, (reltriggers <> 0) AS relhastriggers, " - "relhasindex, relhasrules, " - "'t'::bool AS relhasoids, " - "'f'::bool AS relrowsecurity, " - "'f'::bool AS relforcerowsecurity, " - "0 AS relfrozenxid, 0 AS relminmxid," - "0 AS toid, " - "0 AS tfrozenxid, 0 AS tminmxid," - "'p' AS relpersistence, 't' as relispopulated, " - "'d' AS relreplident, 0 AS relpages, " - "NULL AS reloftype, " - "NULL::oid AS owning_tab, " - "NULL::int4 AS owning_col, " - "NULL AS reltablespace, " - "NULL AS reloptions, " - "NULL AS toast_reloptions, " - "NULL AS changed_acl " - "FROM pg_class c " - "WHERE relkind IN ('%c', '%c') " - "ORDER BY oid", - RELKIND_VIEW, - username_subquery, - RELKIND_RELATION, RELKIND_SEQUENCE); - } res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -5886,7 +5388,7 @@ getTables(Archive *fout, int *numTables) i_reloftype = PQfnumber(res, "reloftype"); i_changed_acl = PQfnumber(res, "changed_acl"); - if (dopt->lockWaitTimeout && fout->remoteVersion >= 70300) + if (dopt->lockWaitTimeout) { /* * Arrange to fail instead of waiting forever for a table lock. @@ -5910,8 +5412,7 @@ getTables(Archive *fout, int *numTables) tblinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_relname)); tblinfo[i].dobj.namespace = findNamespace(fout, - atooid(PQgetvalue(res, i, i_relnamespace)), - tblinfo[i].dobj.catId.oid); + atooid(PQgetvalue(res, i, i_relnamespace))); tblinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); tblinfo[i].relacl = pg_strdup(PQgetvalue(res, i, i_relacl)); tblinfo[i].rrelacl = pg_strdup(PQgetvalue(res, i, i_rrelacl)); @@ -6017,7 +5518,7 @@ getTables(Archive *fout, int *numTables) tblinfo[i].dobj.name); } - if (dopt->lockWaitTimeout && fout->remoteVersion >= 70300) + if (dopt->lockWaitTimeout) { ExecuteSqlStatement(fout, "SET statement_timeout = 0"); } @@ -6290,7 +5791,7 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) "ORDER BY indexname", tbinfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 80000) + else { appendPQExpBuffer(query, "SELECT t.tableoid, t.oid, " @@ -6319,87 +5820,6 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) "ORDER BY indexname", tbinfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 70300) - { - appendPQExpBuffer(query, - "SELECT t.tableoid, t.oid, " - "t.relname AS indexname, " - "pg_catalog.pg_get_indexdef(i.indexrelid) AS indexdef, " - "t.relnatts AS indnkeys, " - "i.indkey, i.indisclustered, " - "false AS indisreplident, t.relpages, " - "c.contype, c.conname, " - "c.condeferrable, c.condeferred, " - "c.tableoid AS contableoid, " - "c.oid AS conoid, " - "null AS condef, " - "NULL AS tablespace, " - "null AS indreloptions " - "FROM pg_catalog.pg_index i " - "JOIN pg_catalog.pg_class t ON (t.oid = i.indexrelid) " - "LEFT JOIN pg_catalog.pg_depend d " - "ON (d.classid = t.tableoid " - "AND d.objid = t.oid " - "AND d.deptype = 'i') " - "LEFT JOIN pg_catalog.pg_constraint c " - "ON (d.refclassid = c.tableoid " - "AND d.refobjid = c.oid) " - "WHERE i.indrelid = '%u'::pg_catalog.oid " - "ORDER BY indexname", - tbinfo->dobj.catId.oid); - } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBuffer(query, - "SELECT t.tableoid, t.oid, " - "t.relname AS indexname, " - "pg_get_indexdef(i.indexrelid) AS indexdef, " - "t.relnatts AS indnkeys, " - "i.indkey, false AS indisclustered, " - "false AS indisreplident, t.relpages, " - "CASE WHEN i.indisprimary THEN 'p'::char " - "ELSE '0'::char END AS contype, " - "t.relname AS conname, " - "false AS condeferrable, " - "false AS condeferred, " - "0::oid AS contableoid, " - "t.oid AS conoid, " - "null AS condef, " - "NULL AS tablespace, " - "null AS indreloptions " - "FROM pg_index i, pg_class t " - "WHERE t.oid = i.indexrelid " - "AND i.indrelid = '%u'::oid " - "ORDER BY indexname", - tbinfo->dobj.catId.oid); - } - else - { - appendPQExpBuffer(query, - "SELECT " - "(SELECT oid FROM pg_class WHERE relname = 'pg_class') AS tableoid, " - "t.oid, " - "t.relname AS indexname, " - "pg_get_indexdef(i.indexrelid) AS indexdef, " - "t.relnatts AS indnkeys, " - "i.indkey, false AS indisclustered, " - "false AS indisreplident, t.relpages, " - "CASE WHEN i.indisprimary THEN 'p'::char " - "ELSE '0'::char END AS contype, " - "t.relname AS conname, " - "false AS condeferrable, " - "false AS condeferred, " - "0::oid AS contableoid, " - "t.oid AS conoid, " - "null AS condef, " - "NULL AS tablespace, " - "null AS indreloptions " - "FROM pg_index i, pg_class t " - "WHERE t.oid = i.indexrelid " - "AND i.indrelid = '%u'::oid " - "ORDER BY indexname", - tbinfo->dobj.catId.oid); - } res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -6442,19 +5862,9 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) indxinfo[j].indnkeys = atoi(PQgetvalue(res, j, i_indnkeys)); indxinfo[j].tablespace = pg_strdup(PQgetvalue(res, j, i_tablespace)); indxinfo[j].indreloptions = pg_strdup(PQgetvalue(res, j, i_indreloptions)); - - /* - * In pre-7.4 releases, indkeys may contain more entries than - * indnkeys says (since indnkeys will be 1 for a functional - * index). We don't actually care about this case since we don't - * examine indkeys except for indexes associated with PRIMARY and - * UNIQUE constraints, which are never functional indexes. But we - * have to allocate enough space to keep parseOidArray from - * complaining. - */ - indxinfo[j].indkeys = (Oid *) pg_malloc(INDEX_MAX_KEYS * sizeof(Oid)); + indxinfo[j].indkeys = (Oid *) pg_malloc(indxinfo[j].indnkeys * sizeof(Oid)); parseOidArray(PQgetvalue(res, j, i_indkey), - indxinfo[j].indkeys, INDEX_MAX_KEYS); + indxinfo[j].indkeys, indxinfo[j].indnkeys); indxinfo[j].indisclustered = (PQgetvalue(res, j, i_indisclustered)[0] == 't'); indxinfo[j].indisreplident = (PQgetvalue(res, j, i_indisreplident)[0] == 't'); indxinfo[j].relpages = atoi(PQgetvalue(res, j, i_relpages)); @@ -6465,9 +5875,6 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) /* * If we found a constraint matching the index, create an * entry for it. - * - * In a pre-7.3 database, we take this path iff the index was - * marked indisprimary. */ constrinfo[j].dobj.objType = DO_CONSTRAINT; constrinfo[j].dobj.catId.tableoid = atooid(PQgetvalue(res, j, i_contableoid)); @@ -6490,10 +5897,6 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) constrinfo[j].separate = true; indxinfo[j].indexconstraint = constrinfo[j].dobj.dumpId; - - /* If pre-7.3 DB, better make sure table comes first */ - addObjectDependency(&constrinfo[j].dobj, - tbinfo->dobj.dumpId); } else { @@ -6532,10 +5935,6 @@ getConstraints(Archive *fout, TableInfo tblinfo[], int numTables) i_condef; int ntups; - /* pg_constraint was created in 7.3, so nothing to do if older */ - if (fout->remoteVersion < 70300) - return; - query = createPQExpBuffer(); for (i = 0; i < numTables; i++) @@ -6621,10 +6020,6 @@ getDomainConstraints(Archive *fout, TypeInfo *tyinfo) i_consrc; int ntups; - /* pg_constraint was created in 7.3, so nothing to do if older */ - if (fout->remoteVersion < 70300) - return; - /* * select appropriate schema to ensure names in constraint are properly * qualified @@ -6642,17 +6037,9 @@ getDomainConstraints(Archive *fout, TypeInfo *tyinfo) "ORDER BY conname", tyinfo->dobj.catId.oid); - else if (fout->remoteVersion >= 70400) - appendPQExpBuffer(query, "SELECT tableoid, oid, conname, " - "pg_catalog.pg_get_constraintdef(oid) AS consrc, " - "true as convalidated " - "FROM pg_catalog.pg_constraint " - "WHERE contypid = '%u'::pg_catalog.oid " - "ORDER BY conname", - tyinfo->dobj.catId.oid); else appendPQExpBuffer(query, "SELECT tableoid, oid, conname, " - "'CHECK (' || consrc || ')' AS consrc, " + "pg_catalog.pg_get_constraintdef(oid) AS consrc, " "true as convalidated " "FROM pg_catalog.pg_constraint " "WHERE contypid = '%u'::pg_catalog.oid " @@ -6745,20 +6132,10 @@ getRules(Archive *fout, int *numRules) "FROM pg_rewrite " "ORDER BY oid"); } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBufferStr(query, "SELECT " - "tableoid, oid, rulename, " - "ev_class AS ruletable, ev_type, is_instead, " - "'O'::char AS ev_enabled " - "FROM pg_rewrite " - "ORDER BY oid"); - } else { appendPQExpBufferStr(query, "SELECT " - "(SELECT oid FROM pg_class WHERE relname = 'pg_rewrite') AS tableoid, " - "oid, rulename, " + "tableoid, oid, rulename, " "ev_class AS ruletable, ev_type, is_instead, " "'O'::char AS ev_enabled " "FROM pg_rewrite " @@ -6931,7 +6308,7 @@ getTriggers(Archive *fout, TableInfo tblinfo[], int numTables) "AND tgconstraint = 0", tbinfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 70300) + else { /* * We ignore triggers that are tied to a foreign-key constraint, @@ -6954,34 +6331,7 @@ getTriggers(Archive *fout, TableInfo tblinfo[], int numTables) " WHERE d.classid = t.tableoid AND d.objid = t.oid AND d.deptype = 'i' AND c.contype = 'f'))", tbinfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBuffer(query, - "SELECT tgname, tgfoid::regproc AS tgfname, " - "tgtype, tgnargs, tgargs, tgenabled, " - "tgisconstraint, tgconstrname, tgdeferrable, " - "tgconstrrelid, tginitdeferred, tableoid, oid, " - "(SELECT relname FROM pg_class WHERE oid = tgconstrrelid) " - " AS tgconstrrelname " - "FROM pg_trigger " - "WHERE tgrelid = '%u'::oid", - tbinfo->dobj.catId.oid); - } - else - { - appendPQExpBuffer(query, - "SELECT tgname, tgfoid::regproc AS tgfname, " - "tgtype, tgnargs, tgargs, tgenabled, " - "tgisconstraint, tgconstrname, tgdeferrable, " - "tgconstrrelid, tginitdeferred, " - "(SELECT oid FROM pg_class WHERE relname = 'pg_trigger') AS tableoid, " - "oid, " - "(SELECT relname FROM pg_class WHERE oid = tgconstrrelid) " - " AS tgconstrrelname " - "FROM pg_trigger " - "WHERE tgrelid = '%u'::oid", - tbinfo->dobj.catId.oid); - } + res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); ntups = PQntuples(res); @@ -7278,7 +6628,7 @@ getProcLangs(Archive *fout, int *numProcLangs) "ORDER BY oid", username_subquery); } - else if (fout->remoteVersion >= 70400) + else { /* Languages are owned by the bootstrap superuser, sysid 1 */ appendPQExpBuffer(query, "SELECT tableoid, oid, " @@ -7292,45 +6642,6 @@ getProcLangs(Archive *fout, int *numProcLangs) "ORDER BY oid", username_subquery); } - else if (fout->remoteVersion >= 70300) - { - /* No clear notion of an owner at all before 7.4 ... */ - appendPQExpBuffer(query, "SELECT tableoid, oid, " - "lanname, lanpltrusted, lanplcallfoid, " - "0 AS laninline, lanvalidator, lanacl, " - "NULL AS rlanacl, " - "NULL AS initlanacl, NULL AS initrlanacl, " - "NULL AS lanowner " - "FROM pg_language " - "WHERE lanispl " - "ORDER BY oid"); - } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBuffer(query, "SELECT tableoid, oid, " - "lanname, lanpltrusted, lanplcallfoid, " - "0 AS laninline, 0 AS lanvalidator, NULL AS lanacl, " - "NULL AS rlanacl, " - "NULL AS initlanacl, NULL AS initrlanacl, " - "NULL AS lanowner " - "FROM pg_language " - "WHERE lanispl " - "ORDER BY oid"); - } - else - { - appendPQExpBuffer(query, "SELECT " - "(SELECT oid FROM pg_class WHERE relname = 'pg_language') AS tableoid, " - "oid, " - "lanname, lanpltrusted, lanplcallfoid, " - "0 AS laninline, 0 AS lanvalidator, NULL AS lanacl, " - "NULL AS rlanacl, " - "NULL AS initlanacl, NULL AS initrlanacl, " - "NULL AS lanowner " - "FROM pg_language " - "WHERE lanispl " - "ORDER BY oid"); - } res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -7379,20 +6690,6 @@ getProcLangs(Archive *fout, int *numProcLangs) PQgetisnull(res, i, i_initlanacl) && PQgetisnull(res, i, i_initrlanacl)) planginfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL; - - if (fout->remoteVersion < 70300) - { - /* - * We need to make a dependency to ensure the function will be - * dumped first. (In 7.3 and later the regular dependency - * mechanism will handle this for us.) - */ - FuncInfo *funcInfo = findFuncByOid(planginfo[i].lanplcallfoid); - - if (funcInfo) - addObjectDependency(&planginfo[i].dobj, - funcInfo->dobj.dumpId); - } } PQclear(res); @@ -7434,25 +6731,13 @@ getCasts(Archive *fout, int *numCasts) "castmethod " "FROM pg_cast ORDER BY 3,4"); } - else if (fout->remoteVersion >= 70300) + else { appendPQExpBufferStr(query, "SELECT tableoid, oid, " "castsource, casttarget, castfunc, castcontext, " "CASE WHEN castfunc = 0 THEN 'b' ELSE 'f' END AS castmethod " "FROM pg_cast ORDER BY 3,4"); } - else - { - appendPQExpBufferStr(query, "SELECT 0 AS tableoid, p.oid, " - "t1.oid AS castsource, t2.oid AS casttarget, " - "p.oid AS castfunc, 'e' AS castcontext, " - "'f' AS castmethod " - "FROM pg_type t1, pg_type t2, pg_proc p " - "WHERE p.pronargs = 1 AND " - "p.proargtypes[0] = t1.oid AND " - "p.prorettype = t2.oid AND p.proname = t2.typname " - "ORDER BY 3,4"); - } res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -7499,22 +6784,6 @@ getCasts(Archive *fout, int *numCasts) sTypeInfo->dobj.name, tTypeInfo->dobj.name); castinfo[i].dobj.name = namebuf.data; - if (fout->remoteVersion < 70300 && - OidIsValid(castinfo[i].castfunc)) - { - /* - * We need to make a dependency to ensure the function will be - * dumped first. (In 7.3 and later the regular dependency - * mechanism handles this for us.) - */ - FuncInfo *funcInfo; - - funcInfo = findFuncByOid(castinfo[i].castfunc); - if (funcInfo) - addObjectDependency(&castinfo[i].dobj, - funcInfo->dobj.dumpId); - } - /* Decide whether we want to dump it */ selectDumpableCast(&(castinfo[i]), fout); @@ -7701,10 +6970,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) /* * we must read the attribute names in attribute number order! because - * we will use the attnum to index into the attnames array later. We - * actually ask to order by "attrelid, attnum" because (at least up to - * 7.3) the planner is not smart enough to realize it needn't re-sort - * the output of an indexscan on pg_attribute_relid_attnum_index. + * we will use the attnum to index into the attnames array later. */ if (g_verbose) write_msg(NULL, "finding the columns and types of table \"%s.%s\"\n", @@ -7736,7 +7002,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) "ON a.atttypid = t.oid " "WHERE a.attrelid = '%u'::pg_catalog.oid " "AND a.attnum > 0::pg_catalog.int2 " - "ORDER BY a.attrelid, a.attnum", + "ORDER BY a.attnum", tbinfo->dobj.catId.oid); } else if (fout->remoteVersion >= 90100) @@ -7760,7 +7026,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) "ON a.atttypid = t.oid " "WHERE a.attrelid = '%u'::pg_catalog.oid " "AND a.attnum > 0::pg_catalog.int2 " - "ORDER BY a.attrelid, a.attnum", + "ORDER BY a.attnum", tbinfo->dobj.catId.oid); } else if (fout->remoteVersion >= 90000) @@ -7778,10 +7044,10 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) "ON a.atttypid = t.oid " "WHERE a.attrelid = '%u'::pg_catalog.oid " "AND a.attnum > 0::pg_catalog.int2 " - "ORDER BY a.attrelid, a.attnum", + "ORDER BY a.attnum", tbinfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 70300) + else { /* need left join here to not fail on dropped columns ... */ appendPQExpBuffer(q, "SELECT a.attnum, a.attname, a.atttypmod, " @@ -7795,50 +7061,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) "ON a.atttypid = t.oid " "WHERE a.attrelid = '%u'::pg_catalog.oid " "AND a.attnum > 0::pg_catalog.int2 " - "ORDER BY a.attrelid, a.attnum", - tbinfo->dobj.catId.oid); - } - else if (fout->remoteVersion >= 70100) - { - /* - * attstattarget doesn't exist in 7.1. It does exist in 7.2, but - * we don't dump it because we can't tell whether it's been - * explicitly set or was just a default. - * - * attislocal doesn't exist before 7.3, either; in older databases - * we assume it's TRUE, else we'd fail to dump non-inherited atts. - */ - appendPQExpBuffer(q, "SELECT a.attnum, a.attname, a.atttypmod, " - "-1 AS attstattarget, a.attstorage, " - "t.typstorage, a.attnotnull, a.atthasdef, " - "false AS attisdropped, a.attlen, " - "a.attalign, true AS attislocal, " - "format_type(t.oid,a.atttypmod) AS atttypname, " - "'' AS attoptions, 0 AS attcollation, " - "NULL AS attfdwoptions " - "FROM pg_attribute a LEFT JOIN pg_type t " - "ON a.atttypid = t.oid " - "WHERE a.attrelid = '%u'::oid " - "AND a.attnum > 0::int2 " - "ORDER BY a.attrelid, a.attnum", - tbinfo->dobj.catId.oid); - } - else - { - /* format_type not available before 7.1 */ - appendPQExpBuffer(q, "SELECT attnum, attname, atttypmod, " - "-1 AS attstattarget, " - "attstorage, attstorage AS typstorage, " - "attnotnull, atthasdef, false AS attisdropped, " - "attlen, attalign, " - "true AS attislocal, " - "(SELECT typname FROM pg_type WHERE oid = atttypid) AS atttypname, " - "'' AS attoptions, 0 AS attcollation, " - "NULL AS attfdwoptions " - "FROM pg_attribute a " - "WHERE attrelid = '%u'::oid " - "AND attnum > 0::int2 " - "ORDER BY attrelid, attnum", + "ORDER BY a.attnum", tbinfo->dobj.catId.oid); } @@ -7924,42 +7147,12 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) tbinfo->dobj.namespace->dobj.name, tbinfo->dobj.name); - resetPQExpBuffer(q); - if (fout->remoteVersion >= 70300) - { - appendPQExpBuffer(q, "SELECT tableoid, oid, adnum, " + printfPQExpBuffer(q, "SELECT tableoid, oid, adnum, " "pg_catalog.pg_get_expr(adbin, adrelid) AS adsrc " - "FROM pg_catalog.pg_attrdef " - "WHERE adrelid = '%u'::pg_catalog.oid", - tbinfo->dobj.catId.oid); - } - else if (fout->remoteVersion >= 70200) - { - /* 7.2 did not have OIDs in pg_attrdef */ - appendPQExpBuffer(q, "SELECT tableoid, 0 AS oid, adnum, " - "pg_get_expr(adbin, adrelid) AS adsrc " - "FROM pg_attrdef " - "WHERE adrelid = '%u'::oid", - tbinfo->dobj.catId.oid); - } - else if (fout->remoteVersion >= 70100) - { - /* no pg_get_expr, so must rely on adsrc */ - appendPQExpBuffer(q, "SELECT tableoid, oid, adnum, adsrc " - "FROM pg_attrdef " - "WHERE adrelid = '%u'::oid", - tbinfo->dobj.catId.oid); - } - else - { - /* no pg_get_expr, no tableoid either */ - appendPQExpBuffer(q, "SELECT " - "(SELECT oid FROM pg_class WHERE relname = 'pg_attrdef') AS tableoid, " - "oid, adnum, adsrc " - "FROM pg_attrdef " - "WHERE adrelid = '%u'::oid", - tbinfo->dobj.catId.oid); - } + "FROM pg_catalog.pg_attrdef " + "WHERE adrelid = '%u'::pg_catalog.oid", + tbinfo->dobj.catId.oid); + res = ExecuteSqlQuery(fout, q->data, PGRES_TUPLES_OK); numDefaults = PQntuples(res); @@ -8005,17 +7198,11 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) if (tbinfo->relkind == RELKIND_VIEW) { attrdefs[j].separate = true; - /* needed in case pre-7.3 DB: */ - addObjectDependency(&attrdefs[j].dobj, - tbinfo->dobj.dumpId); } else if (!shouldPrintColumn(dopt, tbinfo, adnum - 1)) { /* column will be suppressed, print default separately */ attrdefs[j].separate = true; - /* needed in case pre-7.3 DB: */ - addObjectDependency(&attrdefs[j].dobj, - tbinfo->dobj.dumpId); } else { @@ -8077,7 +7264,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) "ORDER BY conname", tbinfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 70400) + else { appendPQExpBuffer(q, "SELECT tableoid, oid, conname, " "pg_catalog.pg_get_constraintdef(oid) AS consrc, " @@ -8088,54 +7275,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) "ORDER BY conname", tbinfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 70300) - { - /* no pg_get_constraintdef, must use consrc */ - appendPQExpBuffer(q, "SELECT tableoid, oid, conname, " - "'CHECK (' || consrc || ')' AS consrc, " - "true AS conislocal, true AS convalidated " - "FROM pg_catalog.pg_constraint " - "WHERE conrelid = '%u'::pg_catalog.oid " - " AND contype = 'c' " - "ORDER BY conname", - tbinfo->dobj.catId.oid); - } - else if (fout->remoteVersion >= 70200) - { - /* 7.2 did not have OIDs in pg_relcheck */ - appendPQExpBuffer(q, "SELECT tableoid, 0 AS oid, " - "rcname AS conname, " - "'CHECK (' || rcsrc || ')' AS consrc, " - "true AS conislocal, true AS convalidated " - "FROM pg_relcheck " - "WHERE rcrelid = '%u'::oid " - "ORDER BY rcname", - tbinfo->dobj.catId.oid); - } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBuffer(q, "SELECT tableoid, oid, " - "rcname AS conname, " - "'CHECK (' || rcsrc || ')' AS consrc, " - "true AS conislocal, true AS convalidated " - "FROM pg_relcheck " - "WHERE rcrelid = '%u'::oid " - "ORDER BY rcname", - tbinfo->dobj.catId.oid); - } - else - { - /* no tableoid in 7.0 */ - appendPQExpBuffer(q, "SELECT " - "(SELECT oid FROM pg_class WHERE relname = 'pg_relcheck') AS tableoid, " - "oid, rcname AS conname, " - "'CHECK (' || rcsrc || ')' AS consrc, " - "true AS conislocal, true AS convalidated " - "FROM pg_relcheck " - "WHERE rcrelid = '%u'::oid " - "ORDER BY rcname", - tbinfo->dobj.catId.oid); - } + res = ExecuteSqlQuery(fout, q->data, PGRES_TUPLES_OK); numConstrs = PQntuples(res); @@ -8303,8 +7443,7 @@ getTSParsers(Archive *fout, int *numTSParsers) prsinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_prsname)); prsinfo[i].dobj.namespace = findNamespace(fout, - atooid(PQgetvalue(res, i, i_prsnamespace)), - prsinfo[i].dobj.catId.oid); + atooid(PQgetvalue(res, i, i_prsnamespace))); prsinfo[i].prsstart = atooid(PQgetvalue(res, i, i_prsstart)); prsinfo[i].prstoken = atooid(PQgetvalue(res, i, i_prstoken)); prsinfo[i].prsend = atooid(PQgetvalue(res, i, i_prsend)); @@ -8390,8 +7529,7 @@ getTSDictionaries(Archive *fout, int *numTSDicts) dictinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_dictname)); dictinfo[i].dobj.namespace = findNamespace(fout, - atooid(PQgetvalue(res, i, i_dictnamespace)), - dictinfo[i].dobj.catId.oid); + atooid(PQgetvalue(res, i, i_dictnamespace))); dictinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); dictinfo[i].dicttemplate = atooid(PQgetvalue(res, i, i_dicttemplate)); if (PQgetisnull(res, i, i_dictinitoption)) @@ -8474,8 +7612,7 @@ getTSTemplates(Archive *fout, int *numTSTemplates) tmplinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_tmplname)); tmplinfo[i].dobj.namespace = findNamespace(fout, - atooid(PQgetvalue(res, i, i_tmplnamespace)), - tmplinfo[i].dobj.catId.oid); + atooid(PQgetvalue(res, i, i_tmplnamespace))); tmplinfo[i].tmplinit = atooid(PQgetvalue(res, i, i_tmplinit)); tmplinfo[i].tmpllexize = atooid(PQgetvalue(res, i, i_tmpllexize)); @@ -8555,8 +7692,7 @@ getTSConfigurations(Archive *fout, int *numTSConfigs) cfginfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_cfgname)); cfginfo[i].dobj.namespace = findNamespace(fout, - atooid(PQgetvalue(res, i, i_cfgnamespace)), - cfginfo[i].dobj.catId.oid); + atooid(PQgetvalue(res, i, i_cfgnamespace))); cfginfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); cfginfo[i].cfgparser = atooid(PQgetvalue(res, i, i_cfgparser)); @@ -8964,8 +8100,7 @@ getDefaultACLs(Archive *fout, int *numDefaultACLs) daclinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_defaclobjtype)); if (nspid != InvalidOid) - daclinfo[i].dobj.namespace = findNamespace(fout, nspid, - daclinfo[i].dobj.catId.oid); + daclinfo[i].dobj.namespace = findNamespace(fout, nspid); else daclinfo[i].dobj.namespace = NULL; @@ -9173,14 +8308,6 @@ findComments(Archive *fout, Oid classoid, Oid objoid, if (ncomments < 0) ncomments = collectComments(fout, &comments); - /* - * Pre-7.2, pg_description does not contain classoid, so collectComments - * just stores a zero. If there's a collision on object OID, well, you - * get duplicate comments. - */ - if (fout->remoteVersion < 70200) - classoid = 0; - /* * Do binary search to find some item matching the object. */ @@ -9268,25 +8395,9 @@ collectComments(Archive *fout, CommentItem **items) query = createPQExpBuffer(); - if (fout->remoteVersion >= 70300) - { - appendPQExpBufferStr(query, "SELECT description, classoid, objoid, objsubid " - "FROM pg_catalog.pg_description " - "ORDER BY classoid, objoid, objsubid"); - } - else if (fout->remoteVersion >= 70200) - { - appendPQExpBufferStr(query, "SELECT description, classoid, objoid, objsubid " - "FROM pg_description " - "ORDER BY classoid, objoid, objsubid"); - } - else - { - /* Note: this will fail to find attribute comments in pre-7.2... */ - appendPQExpBufferStr(query, "SELECT description, 0 AS classoid, objoid, 0 AS objsubid " - "FROM pg_description " - "ORDER BY objoid"); - } + appendPQExpBufferStr(query, "SELECT description, classoid, objoid, objsubid " + "FROM pg_catalog.pg_description " + "ORDER BY classoid, objoid, objsubid"); res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -9468,10 +8579,6 @@ dumpNamespace(Archive *fout, NamespaceInfo *nspinfo) if (!nspinfo->dobj.dump || dopt->dataOnly) return; - /* don't dump dummy namespace from pre-7.3 source */ - if (strlen(nspinfo->dobj.name) == 0) - return; - q = createPQExpBuffer(); delq = createPQExpBuffer(); labelq = createPQExpBuffer(); @@ -10089,71 +9196,35 @@ dumpBaseType(Archive *fout, TypeInfo *tyinfo) "WHERE oid = '%u'::pg_catalog.oid", tyinfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 80400) - { - appendPQExpBuffer(query, "SELECT typlen, " - "typinput, typoutput, typreceive, typsend, " - "typmodin, typmodout, typanalyze, " - "typreceive::pg_catalog.oid AS typreceiveoid, " - "typsend::pg_catalog.oid AS typsendoid, " - "typmodin::pg_catalog.oid AS typmodinoid, " - "typmodout::pg_catalog.oid AS typmodoutoid, " - "typanalyze::pg_catalog.oid AS typanalyzeoid, " - "typcategory, typispreferred, " - "typdelim, typbyval, typalign, typstorage, " - "false AS typcollatable, " - "pg_catalog.pg_get_expr(typdefaultbin, 0) AS typdefaultbin, typdefault " - "FROM pg_catalog.pg_type " - "WHERE oid = '%u'::pg_catalog.oid", - tyinfo->dobj.catId.oid); - } - else if (fout->remoteVersion >= 80300) - { - /* Before 8.4, pg_get_expr does not allow 0 for its second arg */ - appendPQExpBuffer(query, "SELECT typlen, " - "typinput, typoutput, typreceive, typsend, " - "typmodin, typmodout, typanalyze, " - "typreceive::pg_catalog.oid AS typreceiveoid, " - "typsend::pg_catalog.oid AS typsendoid, " - "typmodin::pg_catalog.oid AS typmodinoid, " - "typmodout::pg_catalog.oid AS typmodoutoid, " - "typanalyze::pg_catalog.oid AS typanalyzeoid, " - "'U' AS typcategory, false AS typispreferred, " - "typdelim, typbyval, typalign, typstorage, " - "false AS typcollatable, " - "pg_catalog.pg_get_expr(typdefaultbin, 'pg_catalog.pg_type'::pg_catalog.regclass) AS typdefaultbin, typdefault " - "FROM pg_catalog.pg_type " - "WHERE oid = '%u'::pg_catalog.oid", - tyinfo->dobj.catId.oid); - } - else if (fout->remoteVersion >= 80000) + else if (fout->remoteVersion >= 80400) { appendPQExpBuffer(query, "SELECT typlen, " "typinput, typoutput, typreceive, typsend, " - "'-' AS typmodin, '-' AS typmodout, " - "typanalyze, " + "typmodin, typmodout, typanalyze, " "typreceive::pg_catalog.oid AS typreceiveoid, " "typsend::pg_catalog.oid AS typsendoid, " - "0 AS typmodinoid, 0 AS typmodoutoid, " + "typmodin::pg_catalog.oid AS typmodinoid, " + "typmodout::pg_catalog.oid AS typmodoutoid, " "typanalyze::pg_catalog.oid AS typanalyzeoid, " - "'U' AS typcategory, false AS typispreferred, " + "typcategory, typispreferred, " "typdelim, typbyval, typalign, typstorage, " "false AS typcollatable, " - "pg_catalog.pg_get_expr(typdefaultbin, 'pg_catalog.pg_type'::pg_catalog.regclass) AS typdefaultbin, typdefault " + "pg_catalog.pg_get_expr(typdefaultbin, 0) AS typdefaultbin, typdefault " "FROM pg_catalog.pg_type " "WHERE oid = '%u'::pg_catalog.oid", tyinfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 70400) + else if (fout->remoteVersion >= 80300) { + /* Before 8.4, pg_get_expr does not allow 0 for its second arg */ appendPQExpBuffer(query, "SELECT typlen, " "typinput, typoutput, typreceive, typsend, " - "'-' AS typmodin, '-' AS typmodout, " - "'-' AS typanalyze, " + "typmodin, typmodout, typanalyze, " "typreceive::pg_catalog.oid AS typreceiveoid, " "typsend::pg_catalog.oid AS typsendoid, " - "0 AS typmodinoid, 0 AS typmodoutoid, " - "0 AS typanalyzeoid, " + "typmodin::pg_catalog.oid AS typmodinoid, " + "typmodout::pg_catalog.oid AS typmodoutoid, " + "typanalyze::pg_catalog.oid AS typanalyzeoid, " "'U' AS typcategory, false AS typispreferred, " "typdelim, typbyval, typalign, typstorage, " "false AS typcollatable, " @@ -10162,16 +9233,16 @@ dumpBaseType(Archive *fout, TypeInfo *tyinfo) "WHERE oid = '%u'::pg_catalog.oid", tyinfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 70300) + else { appendPQExpBuffer(query, "SELECT typlen, " - "typinput, typoutput, " - "'-' AS typreceive, '-' AS typsend, " + "typinput, typoutput, typreceive, typsend, " "'-' AS typmodin, '-' AS typmodout, " - "'-' AS typanalyze, " - "0 AS typreceiveoid, 0 AS typsendoid, " + "typanalyze, " + "typreceive::pg_catalog.oid AS typreceiveoid, " + "typsend::pg_catalog.oid AS typsendoid, " "0 AS typmodinoid, 0 AS typmodoutoid, " - "0 AS typanalyzeoid, " + "typanalyze::pg_catalog.oid AS typanalyzeoid, " "'U' AS typcategory, false AS typispreferred, " "typdelim, typbyval, typalign, typstorage, " "false AS typcollatable, " @@ -10180,69 +9251,6 @@ dumpBaseType(Archive *fout, TypeInfo *tyinfo) "WHERE oid = '%u'::pg_catalog.oid", tyinfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 70200) - { - /* - * Note: although pre-7.3 catalogs contain typreceive and typsend, - * ignore them because they are not right. - */ - appendPQExpBuffer(query, "SELECT typlen, " - "typinput, typoutput, " - "'-' AS typreceive, '-' AS typsend, " - "'-' AS typmodin, '-' AS typmodout, " - "'-' AS typanalyze, " - "0 AS typreceiveoid, 0 AS typsendoid, " - "0 AS typmodinoid, 0 AS typmodoutoid, " - "0 AS typanalyzeoid, " - "'U' AS typcategory, false AS typispreferred, " - "typdelim, typbyval, typalign, typstorage, " - "false AS typcollatable, " - "NULL AS typdefaultbin, typdefault " - "FROM pg_type " - "WHERE oid = '%u'::oid", - tyinfo->dobj.catId.oid); - } - else if (fout->remoteVersion >= 70100) - { - /* - * Ignore pre-7.2 typdefault; the field exists but has an unusable - * representation. - */ - appendPQExpBuffer(query, "SELECT typlen, " - "typinput, typoutput, " - "'-' AS typreceive, '-' AS typsend, " - "'-' AS typmodin, '-' AS typmodout, " - "'-' AS typanalyze, " - "0 AS typreceiveoid, 0 AS typsendoid, " - "0 AS typmodinoid, 0 AS typmodoutoid, " - "0 AS typanalyzeoid, " - "'U' AS typcategory, false AS typispreferred, " - "typdelim, typbyval, typalign, typstorage, " - "false AS typcollatable, " - "NULL AS typdefaultbin, NULL AS typdefault " - "FROM pg_type " - "WHERE oid = '%u'::oid", - tyinfo->dobj.catId.oid); - } - else - { - appendPQExpBuffer(query, "SELECT typlen, " - "typinput, typoutput, " - "'-' AS typreceive, '-' AS typsend, " - "'-' AS typmodin, '-' AS typmodout, " - "'-' AS typanalyze, " - "0 AS typreceiveoid, 0 AS typsendoid, " - "0 AS typmodinoid, 0 AS typmodoutoid, " - "0 AS typanalyzeoid, " - "'U' AS typcategory, false AS typispreferred, " - "typdelim, typbyval, typalign, " - "'p'::char AS typstorage, " - "false AS typcollatable, " - "NULL AS typdefaultbin, NULL AS typdefault " - "FROM pg_type " - "WHERE oid = '%u'::oid", - tyinfo->dobj.catId.oid); - } res = ExecuteSqlQueryForSingleRow(fout, query->data); @@ -10300,30 +9308,19 @@ dumpBaseType(Archive *fout, TypeInfo *tyinfo) qtypname, (strcmp(typlen, "-1") == 0) ? "variable" : typlen); - if (fout->remoteVersion >= 70300) - { - /* regproc result is correctly quoted as of 7.3 */ - appendPQExpBuffer(q, ",\n INPUT = %s", typinput); - appendPQExpBuffer(q, ",\n OUTPUT = %s", typoutput); - if (OidIsValid(typreceiveoid)) - appendPQExpBuffer(q, ",\n RECEIVE = %s", typreceive); - if (OidIsValid(typsendoid)) - appendPQExpBuffer(q, ",\n SEND = %s", typsend); - if (OidIsValid(typmodinoid)) - appendPQExpBuffer(q, ",\n TYPMOD_IN = %s", typmodin); - if (OidIsValid(typmodoutoid)) - appendPQExpBuffer(q, ",\n TYPMOD_OUT = %s", typmodout); - if (OidIsValid(typanalyzeoid)) - appendPQExpBuffer(q, ",\n ANALYZE = %s", typanalyze); - } - else - { - /* regproc delivers an unquoted name before 7.3 */ - /* cannot combine these because fmtId uses static result area */ - appendPQExpBuffer(q, ",\n INPUT = %s", fmtId(typinput)); - appendPQExpBuffer(q, ",\n OUTPUT = %s", fmtId(typoutput)); - /* receive/send/typmodin/typmodout/analyze need not be printed */ - } + /* regproc result is sufficiently quoted already */ + appendPQExpBuffer(q, ",\n INPUT = %s", typinput); + appendPQExpBuffer(q, ",\n OUTPUT = %s", typoutput); + if (OidIsValid(typreceiveoid)) + appendPQExpBuffer(q, ",\n RECEIVE = %s", typreceive); + if (OidIsValid(typsendoid)) + appendPQExpBuffer(q, ",\n SEND = %s", typsend); + if (OidIsValid(typmodinoid)) + appendPQExpBuffer(q, ",\n TYPMOD_IN = %s", typmodin); + if (OidIsValid(typmodoutoid)) + appendPQExpBuffer(q, ",\n TYPMOD_OUT = %s", typmodout); + if (OidIsValid(typanalyzeoid)) + appendPQExpBuffer(q, ",\n ANALYZE = %s", typanalyze); if (strcmp(typcollatable, "t") == 0) appendPQExpBufferStr(q, ",\n COLLATABLE = true"); @@ -10468,7 +9465,6 @@ dumpDomain(Archive *fout, TypeInfo *tyinfo) } else { - /* We assume here that remoteVersion must be at least 70300 */ appendPQExpBuffer(query, "SELECT typnotnull, " "pg_catalog.format_type(typbasetype, typtypmod) AS typdefn, " "pg_catalog.pg_get_expr(typdefaultbin, 'pg_catalog.pg_type'::pg_catalog.regclass) AS typdefaultbin, " @@ -10669,9 +9665,8 @@ dumpCompositeType(Archive *fout, TypeInfo *tyinfo) else { /* - * We assume here that remoteVersion must be at least 70300. Since - * ALTER TYPE could not drop columns until 9.1, attisdropped should - * always be false. + * Since ALTER TYPE could not drop columns until 9.1, attisdropped + * should always be false. */ appendPQExpBuffer(query, "SELECT a.attname, " "pg_catalog.format_type(a.atttypid, a.atttypmod) AS atttypdefn, " @@ -10858,7 +9853,6 @@ dumpCompositeTypeColComments(Archive *fout, TypeInfo *tyinfo) query = createPQExpBuffer(); - /* We assume here that remoteVersion must be at least 70300 */ appendPQExpBuffer(query, "SELECT c.tableoid, a.attname, a.attnum " "FROM pg_catalog.pg_class c, pg_catalog.pg_attribute a " @@ -11442,7 +10436,7 @@ dumpFunc(Archive *fout, FuncInfo *finfo) "WHERE oid = '%u'::pg_catalog.oid", finfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 80000) + else { appendPQExpBuffer(query, "SELECT proretset, prosrc, probin, " @@ -11458,58 +10452,6 @@ dumpFunc(Archive *fout, FuncInfo *finfo) "WHERE oid = '%u'::pg_catalog.oid", finfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 70300) - { - appendPQExpBuffer(query, - "SELECT proretset, prosrc, probin, " - "null AS proallargtypes, " - "null AS proargmodes, " - "null AS proargnames, " - "false AS proiswindow, " - "provolatile, proisstrict, prosecdef, " - "false AS proleakproof, " - "null AS proconfig, 0 AS procost, 0 AS prorows, " - "(SELECT lanname FROM pg_catalog.pg_language WHERE oid = prolang) AS lanname " - "FROM pg_catalog.pg_proc " - "WHERE oid = '%u'::pg_catalog.oid", - finfo->dobj.catId.oid); - } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBuffer(query, - "SELECT proretset, prosrc, probin, " - "null AS proallargtypes, " - "null AS proargmodes, " - "null AS proargnames, " - "false AS proiswindow, " - "case when proiscachable then 'i' else 'v' end AS provolatile, " - "proisstrict, " - "false AS prosecdef, " - "false AS proleakproof, " - "null AS proconfig, 0 AS procost, 0 AS prorows, " - "(SELECT lanname FROM pg_language WHERE oid = prolang) AS lanname " - "FROM pg_proc " - "WHERE oid = '%u'::oid", - finfo->dobj.catId.oid); - } - else - { - appendPQExpBuffer(query, - "SELECT proretset, prosrc, probin, " - "null AS proallargtypes, " - "null AS proargmodes, " - "null AS proargnames, " - "false AS proiswindow, " - "CASE WHEN proiscachable THEN 'i' ELSE 'v' END AS provolatile, " - "false AS proisstrict, " - "false AS prosecdef, " - "false AS proleakproof, " - "NULL AS proconfig, 0 AS procost, 0 AS prorows, " - "(SELECT lanname FROM pg_language WHERE oid = prolang) AS lanname " - "FROM pg_proc " - "WHERE oid = '%u'::oid", - finfo->dobj.catId.oid); - } res = ExecuteSqlQueryForSingleRow(fout, query->data); @@ -12082,7 +11024,6 @@ dumpOpr(Archive *fout, OprInfo *oprinfo) PQExpBuffer labelq; PQExpBuffer oprid; PQExpBuffer details; - const char *name; PGresult *res; int i_oprkind; int i_oprcode; @@ -12143,7 +11084,7 @@ dumpOpr(Archive *fout, OprInfo *oprinfo) "WHERE oid = '%u'::pg_catalog.oid", oprinfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 70300) + else { appendPQExpBuffer(query, "SELECT oprkind, " "oprcode::pg_catalog.regprocedure, " @@ -12159,34 +11100,6 @@ dumpOpr(Archive *fout, OprInfo *oprinfo) "WHERE oid = '%u'::pg_catalog.oid", oprinfo->dobj.catId.oid); } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBuffer(query, "SELECT oprkind, oprcode, " - "CASE WHEN oprleft = 0 THEN '-' " - "ELSE format_type(oprleft, NULL) END AS oprleft, " - "CASE WHEN oprright = 0 THEN '-' " - "ELSE format_type(oprright, NULL) END AS oprright, " - "oprcom, oprnegate, oprrest, oprjoin, " - "(oprlsortop != 0) AS oprcanmerge, " - "oprcanhash " - "FROM pg_operator " - "WHERE oid = '%u'::oid", - oprinfo->dobj.catId.oid); - } - else - { - appendPQExpBuffer(query, "SELECT oprkind, oprcode, " - "CASE WHEN oprleft = 0 THEN '-'::name " - "ELSE (SELECT typname FROM pg_type WHERE oid = oprleft) END AS oprleft, " - "CASE WHEN oprright = 0 THEN '-'::name " - "ELSE (SELECT typname FROM pg_type WHERE oid = oprright) END AS oprright, " - "oprcom, oprnegate, oprrest, oprjoin, " - "(oprlsortop != 0) AS oprcanmerge, " - "oprcanhash " - "FROM pg_operator " - "WHERE oid = '%u'::oid", - oprinfo->dobj.catId.oid); - } res = ExecuteSqlQueryForSingleRow(fout, query->data); @@ -12229,12 +11142,8 @@ dumpOpr(Archive *fout, OprInfo *oprinfo) if (strcmp(oprkind, "r") == 0 || strcmp(oprkind, "b") == 0) { - if (fout->remoteVersion >= 70100) - name = oprleft; - else - name = fmtId(oprleft); - appendPQExpBuffer(details, ",\n LEFTARG = %s", name); - appendPQExpBufferStr(oprid, name); + appendPQExpBuffer(details, ",\n LEFTARG = %s", oprleft); + appendPQExpBufferStr(oprid, oprleft); } else appendPQExpBufferStr(oprid, "NONE"); @@ -12242,12 +11151,8 @@ dumpOpr(Archive *fout, OprInfo *oprinfo) if (strcmp(oprkind, "l") == 0 || strcmp(oprkind, "b") == 0) { - if (fout->remoteVersion >= 70100) - name = oprright; - else - name = fmtId(oprright); - appendPQExpBuffer(details, ",\n RIGHTARG = %s", name); - appendPQExpBuffer(oprid, ", %s)", name); + appendPQExpBuffer(details, ",\n RIGHTARG = %s", oprright); + appendPQExpBuffer(oprid, ", %s)", oprright); } else appendPQExpBufferStr(oprid, ", NONE)"); @@ -12334,40 +11239,34 @@ dumpOpr(Archive *fout, OprInfo *oprinfo) * Returns allocated string of what to print, or NULL if function references * is InvalidOid. Returned string is expected to be free'd by the caller. * - * In 7.3 the input is a REGPROCEDURE display; we have to strip the - * argument-types part. In prior versions, the input is a REGPROC display. + * The input is a REGPROCEDURE display; we have to strip the argument-types + * part. */ static char * convertRegProcReference(Archive *fout, const char *proc) { + char *name; + char *paren; + bool inquote; + /* In all cases "-" means a null reference */ if (strcmp(proc, "-") == 0) return NULL; - if (fout->remoteVersion >= 70300) + name = pg_strdup(proc); + /* find non-double-quoted left paren */ + inquote = false; + for (paren = name; *paren; paren++) { - char *name; - char *paren; - bool inquote; - - name = pg_strdup(proc); - /* find non-double-quoted left paren */ - inquote = false; - for (paren = name; *paren; paren++) + if (*paren == '(' && !inquote) { - if (*paren == '(' && !inquote) - { - *paren = '\0'; - break; - } - if (*paren == '"') - inquote = !inquote; + *paren = '\0'; + break; } - return name; + if (*paren == '"') + inquote = !inquote; } - - /* REGPROC before 7.3 does not quote its result */ - return pg_strdup(fmtId(proc)); + return name; } /* @@ -12376,60 +11275,44 @@ convertRegProcReference(Archive *fout, const char *proc) * Returns an allocated string of what to print, or NULL to print nothing. * Caller is responsible for free'ing result string. * - * In 7.3 and up the input is a REGOPERATOR display; we have to strip the - * argument-types part, and add OPERATOR() decoration if the name is - * schema-qualified. In older versions, the input is just a numeric OID, - * which we search our operator list for. + * The input is a REGOPERATOR display; we have to strip the argument-types + * part, and add OPERATOR() decoration if the name is schema-qualified. */ static char * convertOperatorReference(Archive *fout, const char *opr) { - OprInfo *oprInfo; + char *name; + char *oname; + char *ptr; + bool inquote; + bool sawdot; /* In all cases "0" means a null reference */ if (strcmp(opr, "0") == 0) return NULL; - if (fout->remoteVersion >= 70300) - { - char *name; - char *oname; - char *ptr; - bool inquote; - bool sawdot; - - name = pg_strdup(opr); - /* find non-double-quoted left paren, and check for non-quoted dot */ - inquote = false; - sawdot = false; - for (ptr = name; *ptr; ptr++) + name = pg_strdup(opr); + /* find non-double-quoted left paren, and check for non-quoted dot */ + inquote = false; + sawdot = false; + for (ptr = name; *ptr; ptr++) + { + if (*ptr == '"') + inquote = !inquote; + else if (*ptr == '.' && !inquote) + sawdot = true; + else if (*ptr == '(' && !inquote) { - if (*ptr == '"') - inquote = !inquote; - else if (*ptr == '.' && !inquote) - sawdot = true; - else if (*ptr == '(' && !inquote) - { - *ptr = '\0'; - break; - } + *ptr = '\0'; + break; } - /* If not schema-qualified, don't need to add OPERATOR() */ - if (!sawdot) - return name; - oname = psprintf("OPERATOR(%s)", name); - free(name); - return oname; - } - - oprInfo = findOprByOid(atooid(opr)); - if (oprInfo == NULL) - { - write_msg(NULL, "WARNING: could not find operator with OID %s\n", - opr); - return NULL; } - return pg_strdup(oprInfo->dobj.name); + /* If not schema-qualified, don't need to add OPERATOR() */ + if (!sawdot) + return name; + oname = psprintf("OPERATOR(%s)", name); + free(name); + return oname; } /* @@ -12586,14 +11469,6 @@ dumpOpclass(Archive *fout, OpclassInfo *opcinfo) if (!opcinfo->dobj.dump || dopt->dataOnly) return; - /* - * XXX currently we do not implement dumping of operator classes from - * pre-7.3 databases. This could be done but it seems not worth the - * trouble. - */ - if (fout->remoteVersion < 70300) - return; - query = createPQExpBuffer(); q = createPQExpBuffer(); delq = createPQExpBuffer(); @@ -13343,7 +12218,7 @@ dumpConversion(Archive *fout, ConvInfo *convinfo) appendStringLiteralAH(q, conforencoding, fout); appendPQExpBufferStr(q, " TO "); appendStringLiteralAH(q, contoencoding, fout); - /* regproc is automatically quoted in 7.3 and above */ + /* regproc output is already sufficiently quoted */ appendPQExpBuffer(q, " FROM %s;\n", conproc); appendPQExpBuffer(labelq, "CONVERSION %s", fmtId(convinfo->dobj.name)); @@ -13569,7 +12444,7 @@ dumpAgg(Archive *fout, AggInfo *agginfo) "AND p.oid = '%u'::pg_catalog.oid", agginfo->aggfn.dobj.catId.oid); } - else if (fout->remoteVersion >= 70300) + else { appendPQExpBuffer(query, "SELECT aggtransfn, " "aggfinalfn, aggtranstype::pg_catalog.regtype, " @@ -13587,41 +12462,6 @@ dumpAgg(Archive *fout, AggInfo *agginfo) "AND p.oid = '%u'::pg_catalog.oid", agginfo->aggfn.dobj.catId.oid); } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBuffer(query, "SELECT aggtransfn, aggfinalfn, " - "format_type(aggtranstype, NULL) AS aggtranstype, " - "'-' AS aggcombinefn, '-' AS aggserialfn, " - "'-' AS aggdeserialfn, '-' AS aggmtransfn, " - "'-' AS aggminvtransfn, '-' AS aggmfinalfn, " - "0 AS aggmtranstype, false AS aggfinalextra, " - "false AS aggmfinalextra, 0 AS aggsortop, " - "false AS hypothetical, " - "0 AS aggtransspace, agginitval, " - "0 AS aggmtransspace, NULL AS aggminitval, " - "true AS convertok " - "FROM pg_aggregate " - "WHERE oid = '%u'::oid", - agginfo->aggfn.dobj.catId.oid); - } - else - { - appendPQExpBuffer(query, "SELECT aggtransfn1 AS aggtransfn, " - "aggfinalfn, " - "(SELECT typname FROM pg_type WHERE oid = aggtranstype1) AS aggtranstype, " - "'-' AS aggcombinefn, '-' AS aggserialfn, " - "'-' AS aggdeserialfn, '-' AS aggmtransfn, " - "'-' AS aggminvtransfn, '-' AS aggmfinalfn, " - "0 AS aggmtranstype, false AS aggfinalextra, " - "false AS aggmfinalextra, 0 AS aggsortop, " - "false AS hypothetical, " - "0 AS aggtransspace, agginitval1 AS agginitval, " - "0 AS aggmtransspace, NULL AS aggminitval, " - "(aggtransfn2 = 0 and aggtranstype2 = 0 and agginitval2 is null) AS convertok " - "FROM pg_aggregate " - "WHERE oid = '%u'::oid", - agginfo->aggfn.dobj.catId.oid); - } res = ExecuteSqlQueryForSingleRow(fout, query->data); @@ -13701,28 +12541,9 @@ dumpAgg(Archive *fout, AggInfo *agginfo) return; } - if (fout->remoteVersion >= 70300) - { - /* If using 7.3's regproc or regtype, data is already quoted */ - appendPQExpBuffer(details, " SFUNC = %s,\n STYPE = %s", - aggtransfn, - aggtranstype); - } - else if (fout->remoteVersion >= 70100) - { - /* format_type quotes, regproc does not */ - appendPQExpBuffer(details, " SFUNC = %s,\n STYPE = %s", - fmtId(aggtransfn), - aggtranstype); - } - else - { - /* need quotes all around */ - appendPQExpBuffer(details, " SFUNC = %s,\n", - fmtId(aggtransfn)); - appendPQExpBuffer(details, " STYPE = %s", - fmtId(aggtranstype)); - } + /* regproc and regtype output is already sufficiently quoted */ + appendPQExpBuffer(details, " SFUNC = %s,\n STYPE = %s", + aggtransfn, aggtranstype); if (strcmp(aggtransspace, "0") != 0) { @@ -15121,19 +13942,9 @@ createViewAsClause(Archive *fout, TableInfo *tbinfo) int len; /* Fetch the view definition */ - if (fout->remoteVersion >= 70300) - { - /* Beginning in 7.3, viewname is not unique; rely on OID */ - appendPQExpBuffer(query, + appendPQExpBuffer(query, "SELECT pg_catalog.pg_get_viewdef('%u'::pg_catalog.oid) AS viewdef", - tbinfo->dobj.catId.oid); - } - else - { - appendPQExpBufferStr(query, "SELECT definition AS viewdef " - "FROM pg_views WHERE viewname = "); - appendStringLiteralAH(query, tbinfo->dobj.name, fout); - } + tbinfo->dobj.catId.oid); res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -15370,17 +14181,10 @@ dumpTableSchema(Archive *fout, TableInfo *tbinfo) { appendPQExpBufferStr(q, " WITH OPTIONS"); } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBuffer(q, " %s", - tbinfo->atttypnames[j]); - } else { - /* If no format_type, fake it */ appendPQExpBuffer(q, " %s", - myFormatType(tbinfo->atttypnames[j], - tbinfo->atttypmod[j])); + tbinfo->atttypnames[j]); } /* Add collation if not default for the type */ @@ -16283,52 +15087,6 @@ dumpTableConstraintComment(Archive *fout, ConstraintInfo *coninfo) destroyPQExpBuffer(labelq); } -/* - * findLastBuiltInOid - - * find the last built in oid - * - * For 7.1 and 7.2, we do this by retrieving datlastsysoid from the - * pg_database entry for the current database - */ -static Oid -findLastBuiltinOid_V71(Archive *fout, const char *dbname) -{ - PGresult *res; - Oid last_oid; - PQExpBuffer query = createPQExpBuffer(); - - resetPQExpBuffer(query); - appendPQExpBufferStr(query, "SELECT datlastsysoid from pg_database where datname = "); - appendStringLiteralAH(query, dbname, fout); - - res = ExecuteSqlQueryForSingleRow(fout, query->data); - last_oid = atooid(PQgetvalue(res, 0, PQfnumber(res, "datlastsysoid"))); - PQclear(res); - destroyPQExpBuffer(query); - return last_oid; -} - -/* - * findLastBuiltInOid - - * find the last built in oid - * - * For 7.0, we do this by assuming that the last thing that initdb does is to - * create the pg_indexes view. This sucks in general, but seeing that 7.0.x - * initdb won't be changing anymore, it'll do. - */ -static Oid -findLastBuiltinOid_V70(Archive *fout) -{ - PGresult *res; - int last_oid; - - res = ExecuteSqlQueryForSingleRow(fout, - "SELECT oid FROM pg_class WHERE relname = 'pg_indexes'"); - last_oid = atooid(PQgetvalue(res, 0, PQfnumber(res, "oid"))); - PQclear(res); - return last_oid; -} - /* * dumpSequence * write the declaration (not data) of one user-defined sequence @@ -16703,13 +15461,9 @@ dumpTrigger(Archive *fout, TriggerInfo *tginfo) { if (OidIsValid(tginfo->tgconstrrelid)) { - /* If we are using regclass, name is already quoted */ - if (fout->remoteVersion >= 70300) - appendPQExpBuffer(query, " FROM %s\n ", - tginfo->tgconstrrelname); - else - appendPQExpBuffer(query, " FROM %s\n ", - fmtId(tginfo->tgconstrrelname)); + /* regclass output is already quoted */ + appendPQExpBuffer(query, " FROM %s\n ", + tginfo->tgconstrrelname); } if (!tginfo->tgdeferrable) appendPQExpBufferStr(query, "NOT "); @@ -16725,13 +15479,9 @@ dumpTrigger(Archive *fout, TriggerInfo *tginfo) else appendPQExpBufferStr(query, " FOR EACH STATEMENT\n "); - /* In 7.3, result of regproc is already quoted */ - if (fout->remoteVersion >= 70300) - appendPQExpBuffer(query, "EXECUTE PROCEDURE %s(", - tginfo->tgfname); - else - appendPQExpBuffer(query, "EXECUTE PROCEDURE %s(", - fmtId(tginfo->tgfname)); + /* regproc output is already sufficiently quoted */ + appendPQExpBuffer(query, "EXECUTE PROCEDURE %s(", + tginfo->tgfname); tgargs = (char *) PQunescapeBytea((unsigned char *) tginfo->tgargs, &lentgargs); @@ -16923,19 +15673,9 @@ dumpRule(Archive *fout, RuleInfo *rinfo) delcmd = createPQExpBuffer(); labelq = createPQExpBuffer(); - if (fout->remoteVersion >= 70300) - { - appendPQExpBuffer(query, - "SELECT pg_catalog.pg_get_ruledef('%u'::pg_catalog.oid) AS definition", - rinfo->dobj.catId.oid); - } - else - { - /* Rule name was unique before 7.3 ... */ - appendPQExpBuffer(query, - "SELECT pg_get_ruledef('%s') AS definition", - rinfo->dobj.name); - } + appendPQExpBuffer(query, + "SELECT pg_catalog.pg_get_ruledef('%u'::pg_catalog.oid) AS definition", + rinfo->dobj.catId.oid); res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -17325,10 +16065,6 @@ getDependencies(Archive *fout) DumpableObject *dobj, *refdobj; - /* No dependency info available before 7.3 */ - if (fout->remoteVersion < 70300) - return; - if (g_verbose) write_msg(NULL, "reading dependency data\n"); @@ -17680,10 +16416,6 @@ selectSourceSchema(Archive *fout, const char *schemaName) /* This is checked by the callers already */ Assert(schemaName != NULL && *schemaName != '\0'); - /* Not relevant if fetching from pre-7.3 DB */ - if (fout->remoteVersion < 70300) - return; - query = createPQExpBuffer(); appendPQExpBuffer(query, "SET search_path = %s", fmtId(schemaName)); @@ -17699,8 +16431,8 @@ selectSourceSchema(Archive *fout, const char *schemaName) * getFormattedTypeName - retrieve a nicely-formatted type name for the * given type name. * - * NB: in 7.3 and up the result may depend on the currently-selected - * schema; this is why we don't try to cache the names. + * NB: the result may depend on the currently-selected search_path; this is + * why we don't try to cache the names. */ static char * getFormattedTypeName(Archive *fout, Oid oid, OidOptions opts) @@ -17722,36 +16454,13 @@ getFormattedTypeName(Archive *fout, Oid oid, OidOptions opts) } query = createPQExpBuffer(); - if (fout->remoteVersion >= 70300) - { - appendPQExpBuffer(query, "SELECT pg_catalog.format_type('%u'::pg_catalog.oid, NULL)", - oid); - } - else if (fout->remoteVersion >= 70100) - { - appendPQExpBuffer(query, "SELECT format_type('%u'::oid, NULL)", - oid); - } - else - { - appendPQExpBuffer(query, "SELECT typname " - "FROM pg_type " - "WHERE oid = '%u'::oid", - oid); - } + appendPQExpBuffer(query, "SELECT pg_catalog.format_type('%u'::pg_catalog.oid, NULL)", + oid); res = ExecuteSqlQueryForSingleRow(fout, query->data); - if (fout->remoteVersion >= 70100) - { - /* already quoted */ - result = pg_strdup(PQgetvalue(res, 0, 0)); - } - else - { - /* may need to quote it */ - result = pg_strdup(fmtId(PQgetvalue(res, 0, 0))); - } + /* result of format_type is already quoted */ + result = pg_strdup(PQgetvalue(res, 0, 0)); PQclear(res); destroyPQExpBuffer(query); @@ -17759,76 +16468,6 @@ getFormattedTypeName(Archive *fout, Oid oid, OidOptions opts) return result; } -/* - * myFormatType --- local implementation of format_type for use with 7.0. - */ -static char * -myFormatType(const char *typname, int32 typmod) -{ - char *result; - bool isarray = false; - PQExpBuffer buf = createPQExpBuffer(); - - /* Handle array types */ - if (typname[0] == '_') - { - isarray = true; - typname++; - } - - /* Show lengths on bpchar and varchar */ - if (strcmp(typname, "bpchar") == 0) - { - int len = (typmod - VARHDRSZ); - - appendPQExpBufferStr(buf, "character"); - if (len > 1) - appendPQExpBuffer(buf, "(%d)", - typmod - VARHDRSZ); - } - else if (strcmp(typname, "varchar") == 0) - { - appendPQExpBufferStr(buf, "character varying"); - if (typmod != -1) - appendPQExpBuffer(buf, "(%d)", - typmod - VARHDRSZ); - } - else if (strcmp(typname, "numeric") == 0) - { - appendPQExpBufferStr(buf, "numeric"); - if (typmod != -1) - { - int32 tmp_typmod; - int precision; - int scale; - - tmp_typmod = typmod - VARHDRSZ; - precision = (tmp_typmod >> 16) & 0xffff; - scale = tmp_typmod & 0xffff; - appendPQExpBuffer(buf, "(%d,%d)", - precision, scale); - } - } - - /* - * char is an internal single-byte data type; Let's make sure we force it - * through with quotes. - thomas 1998-12-13 - */ - else if (strcmp(typname, "char") == 0) - appendPQExpBufferStr(buf, "\"char\""); - else - appendPQExpBufferStr(buf, fmtId(typname)); - - /* Append array qualifier for array types */ - if (isarray) - appendPQExpBufferStr(buf, "[]"); - - result = pg_strdup(buf->data); - destroyPQExpBuffer(buf); - - return result; -} - /* * Return a column list clause for the given relation. * diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h index 2bfa2d9742..a60cf95733 100644 --- a/src/bin/pg_dump/pg_dump.h +++ b/src/bin/pg_dump/pg_dump.h @@ -605,7 +605,6 @@ extern void parseOidArray(const char *str, Oid *array, int arraysize); extern void sortDumpableObjects(DumpableObject **objs, int numObjs, DumpId preBoundaryId, DumpId postBoundaryId); extern void sortDumpableObjectsByTypeName(DumpableObject **objs, int numObjs); -extern void sortDumpableObjectsByTypeOid(DumpableObject **objs, int numObjs); extern void sortDataAndIndexObjectsBySize(DumpableObject **objs, int numObjs); /* diff --git a/src/bin/pg_dump/pg_dump_sort.c b/src/bin/pg_dump/pg_dump_sort.c index d87f08d356..195b84a0d4 100644 --- a/src/bin/pg_dump/pg_dump_sort.c +++ b/src/bin/pg_dump/pg_dump_sort.c @@ -23,63 +23,7 @@ static const char *modulename = gettext_noop("sorter"); /* - * Sort priority for object types when dumping a pre-7.3 database. - * Objects are sorted by priority levels, and within an equal priority level - * by OID. (This is a relatively crude hack to provide semi-reasonable - * behavior for old databases without full dependency info.) Note: collations, - * extensions, text search, foreign-data, materialized view, event trigger, - * policies, transforms, access methods and default ACL objects can't really - * happen here, so the rather bogus priorities for them don't matter. - * - * NOTE: object-type priorities must match the section assignments made in - * pg_dump.c; that is, PRE_DATA objects must sort before DO_PRE_DATA_BOUNDARY, - * POST_DATA objects must sort after DO_POST_DATA_BOUNDARY, and DATA objects - * must sort between them. - */ -static const int oldObjectTypePriority[] = -{ - 1, /* DO_NAMESPACE */ - 1, /* DO_EXTENSION */ - 2, /* DO_TYPE */ - 2, /* DO_SHELL_TYPE */ - 2, /* DO_FUNC */ - 3, /* DO_AGG */ - 3, /* DO_OPERATOR */ - 3, /* DO_ACCESS_METHOD */ - 4, /* DO_OPCLASS */ - 4, /* DO_OPFAMILY */ - 4, /* DO_COLLATION */ - 5, /* DO_CONVERSION */ - 6, /* DO_TABLE */ - 8, /* DO_ATTRDEF */ - 15, /* DO_INDEX */ - 16, /* DO_RULE */ - 17, /* DO_TRIGGER */ - 14, /* DO_CONSTRAINT */ - 18, /* DO_FK_CONSTRAINT */ - 2, /* DO_PROCLANG */ - 2, /* DO_CAST */ - 11, /* DO_TABLE_DATA */ - 7, /* DO_DUMMY_TYPE */ - 4, /* DO_TSPARSER */ - 4, /* DO_TSDICT */ - 4, /* DO_TSTEMPLATE */ - 4, /* DO_TSCONFIG */ - 4, /* DO_FDW */ - 4, /* DO_FOREIGN_SERVER */ - 19, /* DO_DEFAULT_ACL */ - 4, /* DO_TRANSFORM */ - 9, /* DO_BLOB */ - 12, /* DO_BLOB_DATA */ - 10, /* DO_PRE_DATA_BOUNDARY */ - 13, /* DO_POST_DATA_BOUNDARY */ - 20, /* DO_EVENT_TRIGGER */ - 15, /* DO_REFRESH_MATVIEW */ - 21 /* DO_POLICY */ -}; - -/* - * Sort priority for object types when dumping newer databases. + * Sort priority for database object types. * Objects are sorted by type, and within a type by name. * * NOTE: object-type priorities must match the section assignments made in @@ -87,7 +31,7 @@ static const int oldObjectTypePriority[] = * POST_DATA objects must sort after DO_POST_DATA_BOUNDARY, and DATA objects * must sort between them. */ -static const int newObjectTypePriority[] = +static const int dbObjectTypePriority[] = { 1, /* DO_NAMESPACE */ 4, /* DO_EXTENSION */ @@ -134,7 +78,6 @@ static DumpId postDataBoundId; static int DOTypeNameCompare(const void *p1, const void *p2); -static int DOTypeOidCompare(const void *p1, const void *p2); static bool TopoSort(DumpableObject **objs, int numObjs, DumpableObject **ordering, @@ -266,8 +209,8 @@ DOTypeNameCompare(const void *p1, const void *p2) int cmpval; /* Sort by type */ - cmpval = newObjectTypePriority[obj1->objType] - - newObjectTypePriority[obj2->objType]; + cmpval = dbObjectTypePriority[obj1->objType] - + dbObjectTypePriority[obj2->objType]; if (cmpval != 0) return cmpval; @@ -345,37 +288,6 @@ DOTypeNameCompare(const void *p1, const void *p2) } -/* - * Sort the given objects into a type/OID-based ordering - * - * This is used with pre-7.3 source databases as a crude substitute for the - * lack of dependency information. - */ -void -sortDumpableObjectsByTypeOid(DumpableObject **objs, int numObjs) -{ - if (numObjs > 1) - qsort((void *) objs, numObjs, sizeof(DumpableObject *), - DOTypeOidCompare); -} - -static int -DOTypeOidCompare(const void *p1, const void *p2) -{ - DumpableObject *obj1 = *(DumpableObject *const *) p1; - DumpableObject *obj2 = *(DumpableObject *const *) p2; - int cmpval; - - cmpval = oldObjectTypePriority[obj1->objType] - - oldObjectTypePriority[obj2->objType]; - - if (cmpval != 0) - return cmpval; - - return oidcmp(obj1->catId.oid, obj2->catId.oid); -} - - /* * Sort the given objects into a safe dump order using dependency * information (to the extent we have it available). diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index b5efb46019..82157e5620 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -480,10 +480,7 @@ main(int argc, char *argv[]) dropDBs(conn); if (!roles_only && !no_tablespaces) - { - if (server_version >= 80000) - dropTablespaces(conn); - } + dropTablespaces(conn); if (!tablespaces_only) dropRoles(conn); @@ -505,12 +502,9 @@ main(int argc, char *argv[]) dumpGroups(conn); } + /* Dump tablespaces */ if (!roles_only && !no_tablespaces) - { - /* Dump tablespaces */ - if (server_version >= 80000) - dumpTablespaces(conn); - } + dumpTablespaces(conn); /* Dump CREATE DATABASE commands */ if (binary_upgrade || (!globals_only && !roles_only && !tablespaces_only)) @@ -886,9 +880,8 @@ dumpRoles(PGconn *conn) * We do it this way because config settings for roles could mention the * names of other roles. */ - if (server_version >= 70300) - for (i = 0; i < PQntuples(res); i++) - dumpUserConfig(conn, PQgetvalue(res, i, i_rolname)); + for (i = 0; i < PQntuples(res); i++) + dumpUserConfig(conn, PQgetvalue(res, i, i_rolname)); PQclear(res); @@ -1204,16 +1197,10 @@ dropDBs(PGconn *conn) PGresult *res; int i; - if (server_version >= 70100) - res = executeQuery(conn, - "SELECT datname " - "FROM pg_database d " - "WHERE datallowconn ORDER BY 1"); - else - res = executeQuery(conn, - "SELECT datname " - "FROM pg_database d " - "ORDER BY 1"); + res = executeQuery(conn, + "SELECT datname " + "FROM pg_database d " + "WHERE datallowconn ORDER BY 1"); if (PQntuples(res) > 0) fprintf(OPF, "--\n-- Drop databases\n--\n\n"); @@ -1269,12 +1256,10 @@ dumpCreateDB(PGconn *conn) * We will dump encoding and locale specifications in the CREATE DATABASE * commands for just those databases with values different from defaults. * - * We consider template0's encoding and locale (or, pre-7.1, template1's) - * to define the installation default. Pre-8.4 installations do not have - * per-database locale settings; for them, every database must necessarily - * be using the installation default, so there's no need to do anything - * (which is good, since in very old versions there is no good way to find - * out what the installation locale is anyway...) + * We consider template0's encoding and locale to define the installation + * default. Pre-8.4 installations do not have per-database locale + * settings; for them, every database must necessarily be using the + * installation default, so there's no need to do anything. */ if (server_version >= 80400) res = executeQuery(conn, @@ -1282,18 +1267,12 @@ dumpCreateDB(PGconn *conn) "datcollate, datctype " "FROM pg_database " "WHERE datname = 'template0'"); - else if (server_version >= 70100) - res = executeQuery(conn, - "SELECT pg_encoding_to_char(encoding), " - "null::text AS datcollate, null::text AS datctype " - "FROM pg_database " - "WHERE datname = 'template0'"); else res = executeQuery(conn, "SELECT pg_encoding_to_char(encoding), " "null::text AS datcollate, null::text AS datctype " "FROM pg_database " - "WHERE datname = 'template1'"); + "WHERE datname = 'template0'"); /* If for some reason the template DB isn't there, treat as unknown */ if (PQntuples(res) > 0) @@ -1371,7 +1350,7 @@ dumpCreateDB(PGconn *conn) "(SELECT spcname FROM pg_tablespace t WHERE t.oid = d.dattablespace) AS dattablespace " "FROM pg_database d LEFT JOIN pg_authid u ON (datdba = u.oid) " "WHERE datallowconn ORDER BY 1"); - else if (server_version >= 80000) + else res = executeQuery(conn, "SELECT datname, " "coalesce(usename, (select usename from pg_shadow where usesysid=(select datdba from pg_database where datname='template0'))), " @@ -1382,47 +1361,6 @@ dumpCreateDB(PGconn *conn) "(SELECT spcname FROM pg_tablespace t WHERE t.oid = d.dattablespace) AS dattablespace " "FROM pg_database d LEFT JOIN pg_shadow u ON (datdba = usesysid) " "WHERE datallowconn ORDER BY 1"); - else if (server_version >= 70300) - res = executeQuery(conn, - "SELECT datname, " - "coalesce(usename, (select usename from pg_shadow where usesysid=(select datdba from pg_database where datname='template0'))), " - "pg_encoding_to_char(d.encoding), " - "null::text AS datcollate, null::text AS datctype, datfrozenxid, 0 AS datminmxid, " - "datistemplate, datacl, '' as rdatacl, " - "-1 as datconnlimit, " - "'pg_default' AS dattablespace " - "FROM pg_database d LEFT JOIN pg_shadow u ON (datdba = usesysid) " - "WHERE datallowconn ORDER BY 1"); - else if (server_version >= 70100) - res = executeQuery(conn, - "SELECT datname, " - "coalesce(" - "(select usename from pg_shadow where usesysid=datdba), " - "(select usename from pg_shadow where usesysid=(select datdba from pg_database where datname='template0'))), " - "pg_encoding_to_char(d.encoding), " - "null::text AS datcollate, null::text AS datctype, 0 AS datfrozenxid, 0 AS datminmxid, " - "datistemplate, '' as datacl, '' as rdatacl, " - "-1 as datconnlimit, " - "'pg_default' AS dattablespace " - "FROM pg_database d " - "WHERE datallowconn ORDER BY 1"); - else - { - /* - * Note: 7.0 fails to cope with sub-select in COALESCE, so just deal - * with getting a NULL by not printing any OWNER clause. - */ - res = executeQuery(conn, - "SELECT datname, " - "(select usename from pg_shadow where usesysid=datdba), " - "pg_encoding_to_char(d.encoding), " - "null::text AS datcollate, null::text AS datctype, 0 AS datfrozenxid, 0 AS datminmxid, " - "'f' as datistemplate, " - "'' as datacl, '' as rdatacl, -1 as datconnlimit, " - "'pg_default' AS dattablespace " - "FROM pg_database d " - "ORDER BY 1"); - } for (i = 0; i < PQntuples(res); i++) { @@ -1541,8 +1479,7 @@ dumpCreateDB(PGconn *conn) fprintf(OPF, "%s", buf->data); - if (server_version >= 70300) - dumpDatabaseConfig(conn, dbname); + dumpDatabaseConfig(conn, dbname); free(fdbname); } @@ -1738,10 +1675,7 @@ dumpDatabases(PGconn *conn) PGresult *res; int i; - if (server_version >= 70100) - res = executeQuery(conn, "SELECT datname FROM pg_database WHERE datallowconn ORDER BY 1"); - else - res = executeQuery(conn, "SELECT datname FROM pg_database ORDER BY 1"); + res = executeQuery(conn, "SELECT datname FROM pg_database WHERE datallowconn ORDER BY 1"); for (i = 0; i < PQntuples(res); i++) { @@ -2062,11 +1996,11 @@ connectDatabase(const char *dbname, const char *connection_string, my_version = PG_VERSION_NUM; /* - * We allow the server to be back to 7.0, and up to any minor release of + * We allow the server to be back to 8.0, and up to any minor release of * our own major version. (See also version check in pg_dump.c.) */ if (my_version != server_version - && (server_version < 70000 || + && (server_version < 80000 || (server_version / 100) > (my_version / 100))) { fprintf(stderr, _("server version: %s; %s version: %s\n"), @@ -2076,11 +2010,9 @@ connectDatabase(const char *dbname, const char *connection_string, } /* - * On 7.3 and later, make sure we are not fooled by non-system schemas in - * the search path. + * Make sure we are not fooled by non-system schemas in the search path. */ - if (server_version >= 70300) - executeCommand(conn, "SET search_path = pg_catalog"); + executeCommand(conn, "SET search_path = pg_catalog"); return conn; } -- cgit v1.2.3 From 248776ea06c240ae4605e77369d66bcd7ae4f9e3 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Wed, 12 Oct 2016 17:01:19 -0700 Subject: Remove spurious word. Tatsuo Ishii --- doc/src/sgml/parallel.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/parallel.sgml b/doc/src/sgml/parallel.sgml index c80d42dbef..1e71529eeb 100644 --- a/doc/src/sgml/parallel.sgml +++ b/doc/src/sgml/parallel.sgml @@ -241,7 +241,7 @@ EXPLAIN SELECT * FROM pgbench_accounts WHERE filler LIKE '%x%'; than normal but would produce incorrect results. Instead, the parallel portion of the plan must be what is known internally to the query optimizer as a partial plan; that is, it must constructed - so that each process will which executes the plan will generate only a + so that each process which executes the plan will generate only a subset of the output rows in such a way that each required output row is guaranteed to be generated by exactly one of the cooperating processes. -- cgit v1.2.3 From b1ee762a6138df073d4b2b80c235dd9025a8532c Mon Sep 17 00:00:00 2001 From: Tatsuo Ishii Date: Fri, 14 Oct 2016 07:45:25 +0900 Subject: Fix typo. Confirmed by Tom Lane. --- doc/src/sgml/parallel.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/parallel.sgml b/doc/src/sgml/parallel.sgml index 1e71529eeb..94170b770b 100644 --- a/doc/src/sgml/parallel.sgml +++ b/doc/src/sgml/parallel.sgml @@ -189,7 +189,7 @@ EXPLAIN SELECT * FROM pgbench_accounts WHERE filler LIKE '%x%'; Even when parallel query plan is generated for a particular query, there are several circumstances under which it will be impossible to execute that plan in parallel at execution time. If this occurs, the leader - will execute the portion of the plan between below the Gather + will execute the portion of the plan below the Gather node entirely by itself, almost as if the Gather node were not present. This will happen if any of the following conditions are met: -- cgit v1.2.3 From 13d3180fd14c624bbb274e200e98ddb50e260216 Mon Sep 17 00:00:00 2001 From: Tatsuo Ishii Date: Fri, 14 Oct 2016 09:03:25 +0900 Subject: Fix typo. Confirmed by Michael Paquier. --- doc/src/sgml/parallel.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/parallel.sgml b/doc/src/sgml/parallel.sgml index 94170b770b..d0b438e889 100644 --- a/doc/src/sgml/parallel.sgml +++ b/doc/src/sgml/parallel.sgml @@ -339,7 +339,7 @@ EXPLAIN SELECT * FROM pgbench_accounts WHERE filler LIKE '%x%'; When executing a parallel plan, you can use EXPLAIN (ANALYZE, - VERBOSE) will display per-worker statistics for each plan node. + VERBOSE) to display per-worker statistics for each plan node. This may be useful in determining whether the work is being evenly distributed between all plan nodes and more generally in understanding the performance characteristics of the plan. -- cgit v1.2.3 From 7d3235ba42f8d5fc70c58e242702cc5e2e3549a6 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Mon, 17 Oct 2016 16:31:13 -0400 Subject: By default, set log_line_prefix = '%m [%p] '. This value might not be to everyone's taste; in particular, some people might prefer %t to %m, and others may want %u, %d, or other fields. However, it's a vast improvement on the old default of ''. Christoph Berg --- doc/src/sgml/config.sgml | 14 +++++++++++++- src/backend/utils/misc/guc.c | 2 +- src/backend/utils/misc/postgresql.conf.sample | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index e826c19698..99ff9f5ab5 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -5004,7 +5004,8 @@ local0.* /var/log/postgresql value will pad on the left. Padding can be useful to aid human readability in log files. This parameter can only be set in the postgresql.conf - file or on the server command line. The default is an empty string. + file or on the server command line. The default is + '%m [%p] ' which logs a time stamp and the process ID. @@ -5142,6 +5143,17 @@ FROM pg_stat_activity; include those escapes if you are logging to syslog. + + + + The %q escape is useful when including information that is + only available in session (backend) context like user or database + name. For example: + +log_line_prefix = '%m [%p] %q%u@%d/%a ' + + + diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 622279b058..65660c1bf7 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -3014,7 +3014,7 @@ static struct config_string ConfigureNamesString[] = gettext_noop("If blank, no prefix is used.") }, &Log_line_prefix, - "", + "%m [%p] ", NULL, NULL, NULL }, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 05b1373594..159ada3bc6 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -430,7 +430,7 @@ #log_duration = off #log_error_verbosity = default # terse, default, or verbose messages #log_hostname = off -#log_line_prefix = '' # special values: +#log_line_prefix = '%m [%p] ' # special values: # %a = application name # %u = user name # %d = database name -- cgit v1.2.3 From 0be22457d730da8971f761b9c948f742a12b50b2 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 19 Oct 2016 12:00:00 -0400 Subject: pg_ctl: Add long options for -w and -W From: Vik Fearing --- doc/src/sgml/ref/pg_ctl-ref.sgml | 2 ++ src/bin/pg_ctl/pg_ctl.c | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/pg_ctl-ref.sgml b/doc/src/sgml/ref/pg_ctl-ref.sgml index a00c355f4a..11444e85a8 100644 --- a/doc/src/sgml/ref/pg_ctl-ref.sgml +++ b/doc/src/sgml/ref/pg_ctl-ref.sgml @@ -383,6 +383,7 @@ PostgreSQL documentation + Wait for an operation to complete. This is supported for the @@ -415,6 +416,7 @@ PostgreSQL documentation + Do not wait for an operation to complete. This is the opposite of the diff --git a/src/bin/pg_ctl/pg_ctl.c b/src/bin/pg_ctl/pg_ctl.c index ab10d2f25c..7d97232613 100644 --- a/src/bin/pg_ctl/pg_ctl.c +++ b/src/bin/pg_ctl/pg_ctl.c @@ -1959,8 +1959,8 @@ do_help(void) printf(_(" -s, --silent only print errors, no informational messages\n")); printf(_(" -t, --timeout=SECS seconds to wait when using -w option\n")); printf(_(" -V, --version output version information, then exit\n")); - printf(_(" -w wait until operation completes\n")); - printf(_(" -W do not wait until operation completes\n")); + printf(_(" -w, --wait wait until operation completes\n")); + printf(_(" -W, --no-wait do not wait until operation completes\n")); printf(_(" -?, --help show this help, then exit\n")); printf(_("(The default is to wait for shutdown, but not for start or restart.)\n\n")); printf(_("If the -D option is omitted, the environment variable PGDATA is used.\n")); @@ -2174,6 +2174,8 @@ main(int argc, char **argv) {"silent", no_argument, NULL, 's'}, {"timeout", required_argument, NULL, 't'}, {"core-files", no_argument, NULL, 'c'}, + {"wait", no_argument, NULL, 'w'}, + {"no-wait", no_argument, NULL, 'W'}, {NULL, 0, NULL, 0} }; -- cgit v1.2.3 From c709c6074083a8cc5f1ba431c741ff04e3a8a906 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 19 Oct 2016 12:00:00 -0400 Subject: doc: Consistently use = sign in long options synopses This was already the predominant form in man pages and help output. --- doc/src/sgml/ref/clusterdb.sgml | 2 +- doc/src/sgml/ref/pg_ctl-ref.sgml | 6 +++--- doc/src/sgml/ref/pgupgrade.sgml | 2 +- doc/src/sgml/ref/reindexdb.sgml | 2 +- doc/src/sgml/ref/vacuumdb.sgml | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/clusterdb.sgml b/doc/src/sgml/ref/clusterdb.sgml index c13d74853e..67582fd6e6 100644 --- a/doc/src/sgml/ref/clusterdb.sgml +++ b/doc/src/sgml/ref/clusterdb.sgml @@ -316,7 +316,7 @@ PostgreSQL documentation foo in a database named xyzzy: -$ clusterdb --table foo xyzzy +$ clusterdb --table=foo xyzzy diff --git a/doc/src/sgml/ref/pg_ctl-ref.sgml b/doc/src/sgml/ref/pg_ctl-ref.sgml index 11444e85a8..ea0a6353d8 100644 --- a/doc/src/sgml/ref/pg_ctl-ref.sgml +++ b/doc/src/sgml/ref/pg_ctl-ref.sgml @@ -263,7 +263,7 @@ PostgreSQL documentation - + Specifies the file system location of the database configuration files. If @@ -275,7 +275,7 @@ PostgreSQL documentation - + Append the server log output to @@ -288,7 +288,7 @@ PostgreSQL documentation - + Specifies the shutdown mode. mode diff --git a/doc/src/sgml/ref/pgupgrade.sgml b/doc/src/sgml/ref/pgupgrade.sgml index 96851933cc..d46a730f66 100644 --- a/doc/src/sgml/ref/pgupgrade.sgml +++ b/doc/src/sgml/ref/pgupgrade.sgml @@ -558,7 +558,7 @@ rsync --archive --delete --hard-links --size-only old_pgdata new_pgdata remote_d run using: -psql --username postgres --file script.sql postgres +psql --username=postgres --file=script.sql postgres The scripts can be run in any order and can be deleted once they have diff --git a/doc/src/sgml/ref/reindexdb.sgml b/doc/src/sgml/ref/reindexdb.sgml index 713efc099b..36df949c95 100644 --- a/doc/src/sgml/ref/reindexdb.sgml +++ b/doc/src/sgml/ref/reindexdb.sgml @@ -396,7 +396,7 @@ PostgreSQL documentation To reindex the table foo and the index bar in a database named abcd: -$ reindexdb --table foo --index bar abcd +$ reindexdb --table=foo --index=bar abcd diff --git a/doc/src/sgml/ref/vacuumdb.sgml b/doc/src/sgml/ref/vacuumdb.sgml index 92b8984b7a..4f6fa0d708 100644 --- a/doc/src/sgml/ref/vacuumdb.sgml +++ b/doc/src/sgml/ref/vacuumdb.sgml @@ -430,7 +430,7 @@ PostgreSQL documentation xyzzy, and analyze a single column bar of the table for the optimizer: -$ vacuumdb --analyze --verbose --table 'foo(bar)' xyzzy +$ vacuumdb --analyze --verbose --table='foo(bar)' xyzzy -- cgit v1.2.3 From caf936b09fc7c74844575332b07c667a178cb078 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 19 Oct 2016 12:00:00 -0400 Subject: pg_ctl: Add long option for -o Now all normally used options are covered by long options as well. --- doc/src/sgml/ref/pg_ctl-ref.sgml | 2 ++ src/bin/pg_ctl/pg_ctl.c | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/pg_ctl-ref.sgml b/doc/src/sgml/ref/pg_ctl-ref.sgml index ea0a6353d8..5fb6898699 100644 --- a/doc/src/sgml/ref/pg_ctl-ref.sgml +++ b/doc/src/sgml/ref/pg_ctl-ref.sgml @@ -301,6 +301,7 @@ PostgreSQL documentation + Specifies options to be passed directly to the @@ -316,6 +317,7 @@ PostgreSQL documentation + Specifies options to be passed directly to the diff --git a/src/bin/pg_ctl/pg_ctl.c b/src/bin/pg_ctl/pg_ctl.c index 7d97232613..4b476022c0 100644 --- a/src/bin/pg_ctl/pg_ctl.c +++ b/src/bin/pg_ctl/pg_ctl.c @@ -1972,7 +1972,7 @@ do_help(void) printf(_(" -c, --core-files not applicable on this platform\n")); #endif printf(_(" -l, --log=FILENAME write (or append) server log to FILENAME\n")); - printf(_(" -o OPTIONS command line options to pass to postgres\n" + printf(_(" -o, --options=OPTIONS command line options to pass to postgres\n" " (PostgreSQL server executable) or initdb\n")); printf(_(" -p PATH-TO-POSTGRES normally not necessary\n")); printf(_("\nOptions for stop or restart:\n")); @@ -2171,6 +2171,7 @@ main(int argc, char **argv) {"log", required_argument, NULL, 'l'}, {"mode", required_argument, NULL, 'm'}, {"pgdata", required_argument, NULL, 'D'}, + {"options", required_argument, NULL, 'o'}, {"silent", no_argument, NULL, 's'}, {"timeout", required_argument, NULL, 't'}, {"core-files", no_argument, NULL, 'c'}, -- cgit v1.2.3 From 5d58c07a441414ae29a8e315d2f9868d3d8e20be Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 19 Oct 2016 12:00:00 -0400 Subject: initdb pg_basebackup: Rename --noxxx options to --no-xxx --noclean and --nosync were the only options spelled without a hyphen, so change this for consistency with other options. The options in pg_basebackup have not been in a release, so we just rename them. For initdb, we retain the old variants. Vik Fearing and me --- doc/src/sgml/ref/initdb.sgml | 4 ++-- doc/src/sgml/ref/pg_basebackup.sgml | 4 ++-- src/bin/initdb/initdb.c | 12 +++++++----- src/bin/pg_basebackup/pg_basebackup.c | 8 ++++---- src/bin/pg_basebackup/t/010_pg_basebackup.pl | 2 +- src/test/perl/PostgresNode.pm | 2 +- src/test/regress/pg_regress.c | 2 +- 7 files changed, 18 insertions(+), 16 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/initdb.sgml b/doc/src/sgml/ref/initdb.sgml index 4e339ecce8..31f081ae7a 100644 --- a/doc/src/sgml/ref/initdb.sgml +++ b/doc/src/sgml/ref/initdb.sgml @@ -235,7 +235,7 @@ PostgreSQL documentation - + By default, initdb will wait for all files to be @@ -355,7 +355,7 @@ PostgreSQL documentation - + By default, when initdb diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml index 55e913f70d..7cb690dded 100644 --- a/doc/src/sgml/ref/pg_basebackup.sgml +++ b/doc/src/sgml/ref/pg_basebackup.sgml @@ -400,7 +400,7 @@ PostgreSQL documentation - + By default, when pg_basebackup aborts with an @@ -440,7 +440,7 @@ PostgreSQL documentation - + By default, pg_basebackup will wait for all files diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index e52e67df61..9e23f64130 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -2402,8 +2402,8 @@ usage(const char *progname) printf(_(" -d, --debug generate lots of debugging output\n")); printf(_(" -k, --data-checksums use data page checksums\n")); printf(_(" -L DIRECTORY where to find the input files\n")); - printf(_(" -n, --noclean do not clean up after errors\n")); - printf(_(" -N, --nosync do not wait for changes to be written safely to disk\n")); + printf(_(" -n, --no-clean do not clean up after errors\n")); + printf(_(" -N, --no-sync do not wait for changes to be written safely to disk\n")); printf(_(" -s, --show show internal settings\n")); printf(_(" -S, --sync-only only sync data directory\n")); printf(_("\nOther options:\n")); @@ -3078,8 +3078,10 @@ main(int argc, char *argv[]) {"version", no_argument, NULL, 'V'}, {"debug", no_argument, NULL, 'd'}, {"show", no_argument, NULL, 's'}, - {"noclean", no_argument, NULL, 'n'}, - {"nosync", no_argument, NULL, 'N'}, + {"noclean", no_argument, NULL, 'n'}, /* for backwards compatibility */ + {"no-clean", no_argument, NULL, 'n'}, + {"nosync", no_argument, NULL, 'N'}, /* for backwards compatibility */ + {"no-sync", no_argument, NULL, 'N'}, {"sync-only", no_argument, NULL, 'S'}, {"xlogdir", required_argument, NULL, 'X'}, {"data-checksums", no_argument, NULL, 'k'}, @@ -3165,7 +3167,7 @@ main(int argc, char *argv[]) break; case 'n': noclean = true; - printf(_("Running in noclean mode. Mistakes will not be cleaned up.\n")); + printf(_("Running in no-clean mode. Mistakes will not be cleaned up.\n")); break; case 'N': do_sync = false; diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index 0f5d9d6a87..76e8f449fe 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -329,8 +329,8 @@ usage(void) printf(_(" -c, --checkpoint=fast|spread\n" " set fast or spread checkpointing\n")); printf(_(" -l, --label=LABEL set backup label\n")); - printf(_(" -n, --noclean do not clean up after errors\n")); - printf(_(" -N, --nosync do not wait for changes to be written safely to disk\n")); + printf(_(" -n, --no-clean do not clean up after errors\n")); + printf(_(" -N, --no-sync do not wait for changes to be written safely to disk\n")); printf(_(" -P, --progress show progress information\n")); printf(_(" -v, --verbose output verbose messages\n")); printf(_(" -V, --version output version information, then exit\n")); @@ -2006,8 +2006,8 @@ main(int argc, char **argv) {"gzip", no_argument, NULL, 'z'}, {"compress", required_argument, NULL, 'Z'}, {"label", required_argument, NULL, 'l'}, - {"noclean", no_argument, NULL, 'n'}, - {"nosync", no_argument, NULL, 'N'}, + {"no-clean", no_argument, NULL, 'n'}, + {"no-sync", no_argument, NULL, 'N'}, {"dbname", required_argument, NULL, 'd'}, {"host", required_argument, NULL, 'h'}, {"port", required_argument, NULL, 'p'}, diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/010_pg_basebackup.pl index a52bd4e124..fcedfed2b2 100644 --- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl +++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl @@ -44,7 +44,7 @@ ok(! -d "$tempdir/backup", 'backup directory was cleaned up'); $node->command_fails( [ 'pg_basebackup', '-D', "$tempdir/backup", '-n' ], - 'failing run with noclean option'); + 'failing run with no-clean option'); ok(-d "$tempdir/backup", 'backup directory was created and left behind'); diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm index 535d6c0e7c..6e5a75a050 100644 --- a/src/test/perl/PostgresNode.pm +++ b/src/test/perl/PostgresNode.pm @@ -484,7 +484,7 @@ sub backup print "# Taking pg_basebackup $backup_name from node \"$name\"\n"; TestLib::system_or_bail('pg_basebackup', '-D', $backup_path, '-p', $port, - '-x', '--nosync'); + '-x', '--no-sync'); print "# Backup finished\n"; } diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c index 762adb8443..f2dedbbc8a 100644 --- a/src/test/regress/pg_regress.c +++ b/src/test/regress/pg_regress.c @@ -2239,7 +2239,7 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc /* initdb */ header(_("initializing database system")); snprintf(buf, sizeof(buf), - "\"%s%sinitdb\" -D \"%s/data\" --noclean --nosync%s%s > \"%s/log/initdb.log\" 2>&1", + "\"%s%sinitdb\" -D \"%s/data\" --no-clean --no-sync%s%s > \"%s/log/initdb.log\" 2>&1", bindir ? bindir : "", bindir ? "/" : "", temp_instance, -- cgit v1.2.3 From 9ffe4a8b4cbb713bf8137f8414f02d97b6b2eb08 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 19 Oct 2016 12:00:00 -0400 Subject: Make getrusage() output a little more readable Reviewed-by: Robert Haas Reviewed-by: Peter Geoghegan --- doc/src/sgml/ref/vacuum.sgml | 12 ++++++------ src/backend/tcop/postgres.c | 10 +++++----- src/backend/utils/misc/pg_rusage.c | 6 +++--- 3 files changed, 14 insertions(+), 14 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/vacuum.sgml b/doc/src/sgml/ref/vacuum.sgml index dee1c5bad3..f18180a2fa 100644 --- a/doc/src/sgml/ref/vacuum.sgml +++ b/doc/src/sgml/ref/vacuum.sgml @@ -253,27 +253,27 @@ INFO: vacuuming "public.onek" INFO: index "onek_unique1" now contains 1000 tuples in 14 pages DETAIL: 3000 index tuples were removed. 0 index pages have been deleted, 0 are currently reusable. -CPU 0.01s/0.08u sec elapsed 0.18 sec. +CPU: user: 0.08 s, system: 0.01 s, elapsed: 0.18 s. INFO: index "onek_unique2" now contains 1000 tuples in 16 pages DETAIL: 3000 index tuples were removed. 0 index pages have been deleted, 0 are currently reusable. -CPU 0.00s/0.07u sec elapsed 0.23 sec. +CPU: user: 0.07 s, system: 0.00 s, elapsed: 0.23 s. INFO: index "onek_hundred" now contains 1000 tuples in 13 pages DETAIL: 3000 index tuples were removed. 0 index pages have been deleted, 0 are currently reusable. -CPU 0.01s/0.08u sec elapsed 0.17 sec. +CPU: user: 0.08 s, system: 0.01 s, elapsed: 0.17 s. INFO: index "onek_stringu1" now contains 1000 tuples in 48 pages DETAIL: 3000 index tuples were removed. 0 index pages have been deleted, 0 are currently reusable. -CPU 0.01s/0.09u sec elapsed 0.59 sec. +CPU: user: 0.09 s, system: 0.01 s, elapsed: 0.59 s. INFO: "onek": removed 3000 tuples in 108 pages -DETAIL: CPU 0.01s/0.06u sec elapsed 0.07 sec. +DETAIL: CPU: user: 0.06 s, system: 0.01 s, elapsed: 0.07 s. INFO: "onek": found 3000 removable, 1000 nonremovable tuples in 143 pages DETAIL: 0 dead tuples cannot be removed yet. There were 0 unused item pointers. Skipped 0 pages due to buffer pins. 0 pages are entirely empty. -CPU 0.07s/0.39u sec elapsed 1.56 sec. +CPU: user: 0.39 s, system: 0.07 s, elapsed: 1.56 s. INFO: analyzing "public.onek" INFO: "onek": 36 pages, 1000 rows sampled, 1000 estimated total rows VACUUM diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index 2347f1bcdc..599874e743 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -4422,15 +4422,15 @@ ShowUsage(const char *title) appendStringInfoString(&str, "! system usage stats:\n"); appendStringInfo(&str, - "!\t%ld.%06ld elapsed %ld.%06ld user %ld.%06ld system sec\n", - (long) (elapse_t.tv_sec - Save_t.tv_sec), - (long) (elapse_t.tv_usec - Save_t.tv_usec), + "!\t%ld.%06ld s user, %ld.%06ld s system, %ld.%06ld s elapsed\n", (long) (r.ru_utime.tv_sec - Save_r.ru_utime.tv_sec), (long) (r.ru_utime.tv_usec - Save_r.ru_utime.tv_usec), (long) (r.ru_stime.tv_sec - Save_r.ru_stime.tv_sec), - (long) (r.ru_stime.tv_usec - Save_r.ru_stime.tv_usec)); + (long) (r.ru_stime.tv_usec - Save_r.ru_stime.tv_usec), + (long) (elapse_t.tv_sec - Save_t.tv_sec), + (long) (elapse_t.tv_usec - Save_t.tv_usec)); appendStringInfo(&str, - "!\t[%ld.%06ld user %ld.%06ld sys total]\n", + "!\t[%ld.%06ld s user, %ld.%06ld s system total]\n", (long) user.tv_sec, (long) user.tv_usec, (long) sys.tv_sec, diff --git a/src/backend/utils/misc/pg_rusage.c b/src/backend/utils/misc/pg_rusage.c index 8781a383c0..6602663e42 100644 --- a/src/backend/utils/misc/pg_rusage.c +++ b/src/backend/utils/misc/pg_rusage.c @@ -61,11 +61,11 @@ pg_rusage_show(const PGRUsage *ru0) } snprintf(result, sizeof(result), - "CPU %d.%02ds/%d.%02du sec elapsed %d.%02d sec", - (int) (ru1.ru.ru_stime.tv_sec - ru0->ru.ru_stime.tv_sec), - (int) (ru1.ru.ru_stime.tv_usec - ru0->ru.ru_stime.tv_usec) / 10000, + "CPU: user: %d.%02d s, system: %d.%02d s, elapsed: %d.%02d s", (int) (ru1.ru.ru_utime.tv_sec - ru0->ru.ru_utime.tv_sec), (int) (ru1.ru.ru_utime.tv_usec - ru0->ru.ru_utime.tv_usec) / 10000, + (int) (ru1.ru.ru_stime.tv_sec - ru0->ru.ru_stime.tv_sec), + (int) (ru1.ru.ru_stime.tv_usec - ru0->ru.ru_stime.tv_usec) / 10000, (int) (ru1.tv.tv_sec - ru0->tv.tv_sec), (int) (ru1.tv.tv_usec - ru0->tv.tv_usec) / 10000); -- cgit v1.2.3 From f82ec32ac30ae7e3ec7c84067192535b2ff8ec0e Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Thu, 20 Oct 2016 11:24:37 -0400 Subject: Rename "pg_xlog" directory to "pg_wal". "xlog" is not a particularly clear abbreviation for "write-ahead log", and it sometimes confuses users into believe that the contents of the "pg_xlog" directory are not critical data, leading to unpleasant consequences. So, rename the directory to "pg_wal". This patch modifies pg_upgrade and pg_basebackup to understand both the old and new directory layouts; the former is necessary given the purpose of the tool, while the latter merely avoids an unnecessary backward-compatibility break. We may wish to consider renaming other programs, switches, and functions which still use the old "xlog" naming to also refer to "wal". However, that's still under discussion, so let's do just this much for now. Discussion: CAB7nPqTeC-8+zux8_-4ZD46V7YPwooeFxgndfsq5Rg8ibLVm1A@mail.gmail.com Michael Paquier --- doc/src/sgml/backup.sgml | 28 ++++---- doc/src/sgml/config.sgml | 6 +- doc/src/sgml/func.sgml | 2 +- doc/src/sgml/high-availability.sgml | 14 ++-- doc/src/sgml/perform.sgml | 2 +- doc/src/sgml/protocol.sgml | 6 +- doc/src/sgml/ref/pg_resetxlog.sgml | 10 +-- doc/src/sgml/ref/pg_rewind.sgml | 4 +- doc/src/sgml/ref/pg_xlogdump.sgml | 2 +- doc/src/sgml/ref/pgtestfsync.sgml | 4 +- doc/src/sgml/ref/pgupgrade.sgml | 4 +- doc/src/sgml/storage.sgml | 2 +- doc/src/sgml/wal.sgml | 10 +-- src/backend/access/transam/timeline.c | 4 +- src/backend/access/transam/xlog.c | 100 +++++++++++++-------------- src/backend/access/transam/xlogarchive.c | 2 +- src/backend/access/transam/xlogfuncs.c | 2 +- src/backend/replication/README | 2 +- src/backend/replication/basebackup.c | 20 +++--- src/backend/replication/walreceiver.c | 6 +- src/backend/replication/walsender.c | 4 +- src/backend/storage/file/fd.c | 16 ++--- src/bin/initdb/initdb.c | 12 ++-- src/bin/pg_basebackup/pg_basebackup.c | 80 +++++++++++++-------- src/bin/pg_basebackup/t/010_pg_basebackup.pl | 8 +-- src/bin/pg_resetxlog/pg_resetxlog.c | 2 +- src/bin/pg_rewind/copy_fetch.c | 4 +- src/bin/pg_rewind/filemap.c | 8 +-- src/bin/pg_rewind/parsexlog.c | 2 +- src/bin/pg_rewind/t/004_pg_xlog_symlink.pl | 10 +-- src/bin/pg_upgrade/exec.c | 79 +++++++++++++-------- src/bin/pg_xlogdump/pg_xlogdump.c | 2 +- src/common/file_utils.c | 39 +++++++---- src/include/access/xlog_internal.h | 2 +- src/include/catalog/catversion.h | 2 +- src/include/common/file_utils.h | 3 +- src/include/postmaster/pgarch.h | 2 +- 37 files changed, 279 insertions(+), 226 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml index 95d0ff3149..6eaed1efbe 100644 --- a/doc/src/sgml/backup.sgml +++ b/doc/src/sgml/backup.sgml @@ -472,7 +472,7 @@ tar -cf backup.tar /usr/local/pgsql/data At all times, PostgreSQL maintains a - write ahead log (WAL) in the pg_xlog/ + write ahead log (WAL) in the pg_wal/ subdirectory of the cluster's data directory. The log records every change made to the database's data files. This log exists primarily for crash-safety purposes: if the system crashes, the @@ -616,7 +616,7 @@ archive_command = 'copy "%p" "C:\\server\\archivedir\\%f"' # Windows %p and %f parameters have been replaced, the actual command executed might look like this: -test ! -f /mnt/server/archivedir/00000001000000A900000065 && cp pg_xlog/00000001000000A900000065 /mnt/server/archivedir/00000001000000A900000065 +test ! -f /mnt/server/archivedir/00000001000000A900000065 && cp pg_wal/00000001000000A900000065 /mnt/server/archivedir/00000001000000A900000065 A similar command will be generated for each new file to be archived. @@ -668,9 +668,9 @@ test ! -f /mnt/server/archivedir/00000001000000A900000065 && cp pg_xlog/ fills, nothing further can be archived until the tape is swapped. You should ensure that any error condition or request to a human operator is reported appropriately so that the situation can be - resolved reasonably quickly. The pg_xlog/ directory will + resolved reasonably quickly. The pg_wal/ directory will continue to fill with WAL segment files until the situation is resolved. - (If the file system containing pg_xlog/ fills up, + (If the file system containing pg_wal/ fills up, PostgreSQL will do a PANIC shutdown. No committed transactions will be lost, but the database will remain offline until you free some space.) @@ -682,7 +682,7 @@ test ! -f /mnt/server/archivedir/00000001000000A900000065 && cp pg_xlog/ operation continues even if the archiving process falls a little behind. If archiving falls significantly behind, this will increase the amount of data that would be lost in the event of a disaster. It will also mean that - the pg_xlog/ directory will contain large numbers of + the pg_wal/ directory will contain large numbers of not-yet-archived segment files, which could eventually exceed available disk space. You are advised to monitor the archiving process to ensure that it is working as you intend. @@ -743,7 +743,7 @@ test ! -f /mnt/server/archivedir/00000001000000A900000065 && cp pg_xlog/ configuration file reload. If you wish to temporarily stop archiving, one way to do it is to set archive_command to the empty string (''). - This will cause WAL files to accumulate in pg_xlog/ until a + This will cause WAL files to accumulate in pg_wal/ until a working archive_command is re-established. @@ -1062,10 +1062,10 @@ SELECT pg_stop_backup(); You should, however, omit from the backup the files within the - cluster's pg_xlog/ subdirectory. This + cluster's pg_wal/ subdirectory. This slight adjustment is worthwhile because it reduces the risk of mistakes when restoring. This is easy to arrange if - pg_xlog/ is a symbolic link pointing to someplace outside + pg_wal/ is a symbolic link pointing to someplace outside the cluster directory, which is a common setup anyway for performance reasons. You might also want to exclude postmaster.pid and postmaster.opts, which record information @@ -1149,7 +1149,7 @@ SELECT pg_stop_backup(); location in case you need them later. Note that this precaution will require that you have enough free space on your system to hold two copies of your existing database. If you do not have enough space, - you should at least save the contents of the cluster's pg_xlog + you should at least save the contents of the cluster's pg_wal subdirectory, as it might contain logs which were not archived before the system went down. @@ -1172,9 +1172,9 @@ SELECT pg_stop_backup(); - Remove any files present in pg_xlog/; these came from the + Remove any files present in pg_wal/; these came from the file system backup and are therefore probably obsolete rather than current. - If you didn't archive pg_xlog/ at all, then recreate + If you didn't archive pg_wal/ at all, then recreate it with proper permissions, being careful to ensure that you re-establish it as a symbolic link if you had it set up that way before. @@ -1183,7 +1183,7 @@ SELECT pg_stop_backup(); If you have unarchived WAL segment files that you saved in step 2, - copy them into pg_xlog/. (It is best to copy them, + copy them into pg_wal/. (It is best to copy them, not move them, so you still have the unmodified files if a problem occurs and you have to start over.) @@ -1265,9 +1265,9 @@ restore_command = 'cp /mnt/server/archivedir/%f %p' WAL segments that cannot be found in the archive will be sought in - pg_xlog/; this allows use of recent un-archived segments. + pg_wal/; this allows use of recent un-archived segments. However, segments that are available from the archive will be used in - preference to files in pg_xlog/. + preference to files in pg_wal/. diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 99ff9f5ab5..adab2f8378 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -2932,7 +2932,7 @@ include_dir 'conf.d' Specifies the minimum number of past log file segments kept in the - pg_xlog + pg_wal directory, in case a standby server needs to fetch them for streaming replication. Each segment is normally 16 megabytes. If a standby server connected to the sending server falls behind by more than @@ -2946,7 +2946,7 @@ include_dir 'conf.d' This sets only the minimum number of segments retained in - pg_xlog; the system might need to retain more segments + pg_wal; the system might need to retain more segments for WAL archival or to recover from a checkpoint. If wal_keep_segments is zero (the default), the system doesn't keep any extra segments for standby purposes, so the number @@ -3322,7 +3322,7 @@ include_dir 'conf.d' Specify how long the standby server should wait when WAL data is not available from any sources (streaming replication, - local pg_xlog or WAL archive) before retrying to + local pg_wal or WAL archive) before retrying to retrieve WAL data. This parameter can only be set in the postgresql.conf file or on the server command line. The default value is 5 seconds. Units are milliseconds if not specified. diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index a58835082b..2e64cc430c 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -15327,7 +15327,7 @@ SELECT * FROM pg_ls_dir('.') WITH ORDINALITY AS t(ls,n); pg_snapshots | 13 pg_multixact | 14 PG_VERSION | 15 - pg_xlog | 16 + pg_wal | 16 pg_hba.conf | 17 pg_stat_tmp | 18 pg_subtrans | 19 diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml index 06f49dba5d..5bedaf27a2 100644 --- a/doc/src/sgml/high-availability.sgml +++ b/doc/src/sgml/high-availability.sgml @@ -594,24 +594,24 @@ protocol to make nodes agree on a serializable transactional order. (see ) or directly from the master over a TCP connection (streaming replication). The standby server will also attempt to restore any WAL found in the standby cluster's - pg_xlog directory. That typically happens after a server + pg_wal directory. That typically happens after a server restart, when the standby replays again WAL that was streamed from the master before the restart, but you can also manually copy files to - pg_xlog at any time to have them replayed. + pg_wal at any time to have them replayed. At startup, the standby begins by restoring all WAL available in the archive location, calling restore_command. Once it reaches the end of WAL available there and restore_command - fails, it tries to restore any WAL available in the pg_xlog directory. + fails, it tries to restore any WAL available in the pg_wal directory. If that fails, and streaming replication has been configured, the standby tries to connect to the primary server and start streaming WAL - from the last valid record found in archive or pg_xlog. If that fails + from the last valid record found in archive or pg_wal. If that fails or streaming replication is not configured, or if the connection is later disconnected, the standby goes back to step 1 and tries to restore the file from the archive again. This loop of retries from the - archive, pg_xlog, and via streaming replication goes on until the server + archive, pg_wal, and via streaming replication goes on until the server is stopped or failover is triggered by a trigger file. @@ -619,7 +619,7 @@ protocol to make nodes agree on a serializable transactional order. Standby mode is exited and the server switches to normal operation when pg_ctl promote is run or a trigger file is found (trigger_file). Before failover, - any WAL immediately available in the archive or in pg_xlog will be + any WAL immediately available in the archive or in pg_wal will be restored, but no attempt is made to connect to the master. @@ -895,7 +895,7 @@ primary_conninfo = 'host=192.168.1.50 port=5432 user=foo password=foopass' However, these methods often result in retaining more WAL segments than required, whereas replication slots retain only the number of segments known to be needed. An advantage of these methods is that they bound - the space requirement for pg_xlog; there is currently no way + the space requirement for pg_wal; there is currently no way to do this using replication slots. diff --git a/doc/src/sgml/perform.sgml b/doc/src/sgml/perform.sgml index 7bcbfa7611..8d30fd1384 100644 --- a/doc/src/sgml/perform.sgml +++ b/doc/src/sgml/perform.sgml @@ -1612,7 +1612,7 @@ SELECT * FROM x, y, a, b, c WHERE something AND somethingelse; Increase and ; this reduces the frequency of checkpoints, but increases the storage requirements of - /pg_xlog. + /pg_wal. diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index 3384e73448..50cf527427 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -1947,7 +1947,7 @@ The commands accepted in walsender mode are: Include the necessary WAL segments in the backup. This will include all the files between start and stop backup in the - pg_xlog directory of the base directory tar + pg_wal directory of the base directory tar file. @@ -2076,8 +2076,8 @@ The commands accepted in walsender mode are: - pg_xlog, including subdirectories. If the backup is run - with WAL files included, a synthesized version of pg_xlog will be + pg_wal, including subdirectories. If the backup is run + with WAL files included, a synthesized version of pg_wal will be included, but it will only contain the files necessary for the backup to work, not the rest of the contents. diff --git a/doc/src/sgml/ref/pg_resetxlog.sgml b/doc/src/sgml/ref/pg_resetxlog.sgml index fd9d0be6f4..c949c5e849 100644 --- a/doc/src/sgml/ref/pg_resetxlog.sgml +++ b/doc/src/sgml/ref/pg_resetxlog.sgml @@ -173,22 +173,22 @@ PostgreSQL documentation The WAL starting address should be larger than any WAL segment file name currently existing in - the directory pg_xlog under the data directory. + the directory pg_wal under the data directory. These names are also in hexadecimal and have three parts. The first part is the timeline ID and should usually be kept the same. For example, if 00000001000000320000004A is the - largest entry in pg_xlog, use -l 00000001000000320000004B or higher. + largest entry in pg_wal, use -l 00000001000000320000004B or higher. pg_resetxlog itself looks at the files in - pg_xlog and chooses a default diff --git a/doc/src/sgml/ref/pg_rewind.sgml b/doc/src/sgml/ref/pg_rewind.sgml index 42ebfbfdef..371c4a475f 100644 --- a/doc/src/sgml/ref/pg_rewind.sgml +++ b/doc/src/sgml/ref/pg_rewind.sgml @@ -61,14 +61,14 @@ PostgreSQL documentation pg_rewind examines the timeline histories of the source and target clusters to determine the point where they diverged, and - expects to find WAL in the target cluster's pg_xlog directory + expects to find WAL in the target cluster's pg_wal directory reaching all the way back to the point of divergence. The point of divergence can be found either on the target timeline, the source timeline, or their common ancestor. In the typical failover scenario where the target cluster was shut down soon after the divergence, this is not a problem, but if the target cluster ran for a long time after the divergence, the old WAL files might no longer be present. In that case, they can be manually - copied from the WAL archive to the pg_xlog directory, or + copied from the WAL archive to the pg_wal directory, or fetched on startup by configuring recovery.conf. The use of pg_rewind is not limited to failover, e.g. a standby server can be promoted, run some write transactions, and then rewinded diff --git a/doc/src/sgml/ref/pg_xlogdump.sgml b/doc/src/sgml/ref/pg_xlogdump.sgml index 177caab00d..cfb6d87259 100644 --- a/doc/src/sgml/ref/pg_xlogdump.sgml +++ b/doc/src/sgml/ref/pg_xlogdump.sgml @@ -118,7 +118,7 @@ PostgreSQL documentation Directory in which to find log segment files. The default is to search - for them in the pg_xlog subdirectory of the current + for them in the pg_wal subdirectory of the current directory. diff --git a/doc/src/sgml/ref/pgtestfsync.sgml b/doc/src/sgml/ref/pgtestfsync.sgml index 6e134c75df..5856356b42 100644 --- a/doc/src/sgml/ref/pgtestfsync.sgml +++ b/doc/src/sgml/ref/pgtestfsync.sgml @@ -57,8 +57,8 @@ Specifies the file name to write test data in. This file should be in the same file system that the - pg_xlog directory is or will be placed in. - (pg_xlog contains the WAL files.) + pg_wal directory is or will be placed in. + (pg_wal contains the WAL files.) The default is pg_test_fsync.out in the current directory. diff --git a/doc/src/sgml/ref/pgupgrade.sgml b/doc/src/sgml/ref/pgupgrade.sgml index d46a730f66..ad28526296 100644 --- a/doc/src/sgml/ref/pgupgrade.sgml +++ b/doc/src/sgml/ref/pgupgrade.sgml @@ -345,7 +345,7 @@ NET STOP postgresql-9.0 your old cluster once you start the new cluster after the upgrade. Link mode also requires that the old and new cluster data directories be in the - same file system. (Tablespaces and pg_xlog can be on + same file system. (Tablespaces and pg_wal can be on different file systems.) See pg_upgrade --help for a full list of options. @@ -508,7 +508,7 @@ rsync --archive --delete --hard-links --size-only old_pgdata new_pgdata remote_d If you have tablespaces, you will need to run a similar rsync command for each tablespace directory. If you - have relocated pg_xlog outside the data directories, + have relocated pg_wal outside the data directories, rsync must be run on those directories too. diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml index 1b812bd0a9..fddb69bad3 100644 --- a/doc/src/sgml/storage.sgml +++ b/doc/src/sgml/storage.sgml @@ -141,7 +141,7 @@ Item - pg_xlog + pg_wal Subdirectory containing WAL (Write Ahead Log) files diff --git a/doc/src/sgml/wal.sgml b/doc/src/sgml/wal.sgml index fe3b588c72..346aa769a8 100644 --- a/doc/src/sgml/wal.sgml +++ b/doc/src/sgml/wal.sgml @@ -557,7 +557,7 @@ - The number of WAL segment files in pg_xlog directory depends on + The number of WAL segment files in pg_wal directory depends on min_wal_size, max_wal_size and the amount of WAL generated in previous checkpoint cycles. When old log segment files are no longer needed, they are removed or recycled (that is, @@ -582,7 +582,7 @@ kept at all times. Also, if WAL archiving is used, old segments can not be removed or recycled until they are archived. If WAL archiving cannot keep up with the pace that WAL is generated, or if archive_command - fails repeatedly, old WAL files will accumulate in pg_xlog + fails repeatedly, old WAL files will accumulate in pg_wal until the situation is resolved. A slow or failed standby server that uses a replication slot will have the same effect (see ). @@ -594,7 +594,7 @@ which are similar to checkpoints in normal operation: the server forces all its state to disk, updates the pg_control file to indicate that the already-processed WAL data need not be scanned again, - and then recycles any old log segment files in the pg_xlog + and then recycles any old log segment files in the pg_wal directory. Restartpoints can't be performed more frequently than checkpoints in the master because restartpoints can only be performed at checkpoint records. @@ -750,7 +750,7 @@ WAL logs are stored in the directory - pg_xlog under the data directory, as a set of + pg_wal under the data directory, as a set of segment files, normally each 16 MB in size (but the size can be changed by altering the + + The transaction log files are written to a separate file + named pg_wal.tar (if the server is a version + earlier than 10, the file will be named pg_xlog.tar). + @@ -353,7 +363,8 @@ PostgreSQL documentation Enables gzip compression of tar file output, with the default compression level. Compression is only available when using - the tar format. + the tar format, and the suffix .gz will + automatically be added to all tar filenames. @@ -366,7 +377,8 @@ PostgreSQL documentation Enables gzip compression of tar file output, and specifies the compression level (0 through 9, 0 being no compression and 9 being best compression). Compression is only available when using the tar - format. + format, and the suffix .gz will + automatically be added to all tar filenames. diff --git a/src/bin/pg_basebackup/Makefile b/src/bin/pg_basebackup/Makefile index fa1ce8b24d..52ac9e9fb8 100644 --- a/src/bin/pg_basebackup/Makefile +++ b/src/bin/pg_basebackup/Makefile @@ -19,7 +19,7 @@ include $(top_builddir)/src/Makefile.global override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS) LDFLAGS += -L$(top_builddir)/src/fe_utils -lpgfeutils -lpq -OBJS=receivelog.o streamutil.o $(WIN32RES) +OBJS=receivelog.o streamutil.o walmethods.o $(WIN32RES) all: pg_basebackup pg_receivexlog pg_recvlogical diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index b82b8e1b26..16cab978d0 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -449,7 +449,7 @@ typedef struct { PGconn *bgconn; XLogRecPtr startptr; - char xlogdir[MAXPGPATH]; + char xlog[MAXPGPATH]; /* directory or tarfile depending on mode */ char *sysidentifier; int timeline; } logstreamer_param; @@ -470,9 +470,13 @@ LogStreamerMain(logstreamer_param *param) stream.synchronous = false; stream.do_sync = do_sync; stream.mark_done = true; - stream.basedir = param->xlogdir; stream.partial_suffix = NULL; + if (format == 'p') + stream.walmethod = CreateWalDirectoryMethod(param->xlog, do_sync); + else + stream.walmethod = CreateWalTarMethod(param->xlog, compresslevel, do_sync); + if (!ReceiveXlogStream(param->bgconn, &stream)) /* @@ -482,6 +486,14 @@ LogStreamerMain(logstreamer_param *param) */ return 1; + if (!stream.walmethod->finish()) + { + fprintf(stderr, + _("%s: could not finish writing WAL files: %s\n"), + progname, strerror(errno)); + return 1; + } + PQfinish(param->bgconn); return 0; } @@ -533,28 +545,32 @@ StartLogStreamer(char *startpos, uint32 timeline, char *sysidentifier) exit(1); /* In post-10 cluster, pg_xlog has been renamed to pg_wal */ - snprintf(param->xlogdir, sizeof(param->xlogdir), "%s/%s", + snprintf(param->xlog, sizeof(param->xlog), "%s/%s", basedir, PQserverVersion(conn) < MINIMUM_VERSION_FOR_PG_WAL ? "pg_xlog" : "pg_wal"); - /* - * Create pg_wal/archive_status or pg_xlog/archive_status (and thus - * pg_wal or pg_xlog) depending on the target server so we can write to - * basedir/pg_wal or basedir/pg_xlog as the directory entry in the tar - * file may arrive later. - */ - snprintf(statusdir, sizeof(statusdir), "%s/%s/archive_status", - basedir, - PQserverVersion(conn) < MINIMUM_VERSION_FOR_PG_WAL ? - "pg_xlog" : "pg_wal"); - if (pg_mkdir_p(statusdir, S_IRWXU) != 0 && errno != EEXIST) + if (format == 'p') { - fprintf(stderr, - _("%s: could not create directory \"%s\": %s\n"), - progname, statusdir, strerror(errno)); - disconnect_and_exit(1); + /* + * Create pg_wal/archive_status or pg_xlog/archive_status (and thus + * pg_wal or pg_xlog) depending on the target server so we can write to + * basedir/pg_wal or basedir/pg_xlog as the directory entry in the tar + * file may arrive later. + */ + snprintf(statusdir, sizeof(statusdir), "%s/%s/archive_status", + basedir, + PQserverVersion(conn) < MINIMUM_VERSION_FOR_PG_WAL ? + "pg_xlog" : "pg_wal"); + + if (pg_mkdir_p(statusdir, S_IRWXU) != 0 && errno != EEXIST) + { + fprintf(stderr, + _("%s: could not create directory \"%s\": %s\n"), + progname, statusdir, strerror(errno)); + disconnect_and_exit(1); + } } /* @@ -2245,16 +2261,6 @@ main(int argc, char **argv) exit(1); } - if (format != 'p' && streamwal) - { - fprintf(stderr, - _("%s: WAL streaming can only be used in plain mode\n"), - progname); - fprintf(stderr, _("Try \"%s --help\" for more information.\n"), - progname); - exit(1); - } - if (replication_slot && !streamwal) { fprintf(stderr, diff --git a/src/bin/pg_basebackup/pg_receivexlog.c b/src/bin/pg_basebackup/pg_receivexlog.c index a58a251a59..bbdf96edfd 100644 --- a/src/bin/pg_basebackup/pg_receivexlog.c +++ b/src/bin/pg_basebackup/pg_receivexlog.c @@ -338,11 +338,19 @@ StreamLog(void) stream.synchronous = synchronous; stream.do_sync = true; stream.mark_done = false; - stream.basedir = basedir; + stream.walmethod = CreateWalDirectoryMethod(basedir, stream.do_sync); stream.partial_suffix = ".partial"; ReceiveXlogStream(conn, &stream); + if (!stream.walmethod->finish()) + { + fprintf(stderr, + _("%s: could not finish writing WAL files: %s\n"), + progname, strerror(errno)); + return; + } + PQfinish(conn); conn = NULL; } diff --git a/src/bin/pg_basebackup/receivelog.c b/src/bin/pg_basebackup/receivelog.c index b0fa916b44..fcd0269473 100644 --- a/src/bin/pg_basebackup/receivelog.c +++ b/src/bin/pg_basebackup/receivelog.c @@ -30,7 +30,7 @@ /* fd and filename for currently open WAL file */ -static int walfile = -1; +static Walfile *walfile = NULL; static char current_walfile_name[MAXPGPATH] = ""; static bool reportFlushPosition = false; static XLogRecPtr lastFlushPosition = InvalidXLogRecPtr; @@ -56,29 +56,23 @@ static bool ReadEndOfStreamingResult(PGresult *res, XLogRecPtr *startpos, uint32 *timeline); static bool -mark_file_as_archived(const char *basedir, const char *fname, bool do_sync) +mark_file_as_archived(StreamCtl *stream, const char *fname) { - int fd; + Walfile *f; static char tmppath[MAXPGPATH]; - snprintf(tmppath, sizeof(tmppath), "%s/archive_status/%s.done", - basedir, fname); + snprintf(tmppath, sizeof(tmppath), "archive_status/%s.done", + fname); - fd = open(tmppath, O_WRONLY | O_CREAT | PG_BINARY, S_IRUSR | S_IWUSR); - if (fd < 0) + f = stream->walmethod->open_for_write(tmppath, NULL, 0); + if (f == NULL) { fprintf(stderr, _("%s: could not create archive status file \"%s\": %s\n"), - progname, tmppath, strerror(errno)); + progname, tmppath, stream->walmethod->getlasterror()); return false; } - close(fd); - - if (do_sync && fsync_fname(tmppath, false, progname) != 0) - return false; - - if (do_sync && fsync_parent_path(tmppath, progname) != 0) - return false; + stream->walmethod->close(f, CLOSE_NORMAL); return true; } @@ -95,121 +89,82 @@ mark_file_as_archived(const char *basedir, const char *fname, bool do_sync) static bool open_walfile(StreamCtl *stream, XLogRecPtr startpoint) { - int f; + Walfile *f; char fn[MAXPGPATH]; - struct stat statbuf; - char *zerobuf; - int bytes; + ssize_t size; XLogSegNo segno; XLByteToSeg(startpoint, segno); XLogFileName(current_walfile_name, stream->timeline, segno); - snprintf(fn, sizeof(fn), "%s/%s%s", stream->basedir, current_walfile_name, + snprintf(fn, sizeof(fn), "%s%s", current_walfile_name, stream->partial_suffix ? stream->partial_suffix : ""); - f = open(fn, O_WRONLY | O_CREAT | PG_BINARY, S_IRUSR | S_IWUSR); - if (f == -1) - { - fprintf(stderr, - _("%s: could not open transaction log file \"%s\": %s\n"), - progname, fn, strerror(errno)); - return false; - } /* - * Verify that the file is either empty (just created), or a complete - * XLogSegSize segment. Anything in between indicates a corrupt file. + * When streaming to files, if an existing file exists we verify that it's + * either empty (just created), or a complete XLogSegSize segment (in + * which case it has been created and padded). Anything else indicates a + * corrupt file. + * + * When streaming to tar, no file with this name will exist before, so we + * never have to verify a size. */ - if (fstat(f, &statbuf) != 0) + if (stream->walmethod->existsfile(fn)) { - fprintf(stderr, - _("%s: could not stat transaction log file \"%s\": %s\n"), - progname, fn, strerror(errno)); - close(f); - return false; - } - if (statbuf.st_size == XLogSegSize) - { - /* - * fsync, in case of a previous crash between padding and fsyncing the - * file. - */ - if (stream->do_sync) + size = stream->walmethod->get_file_size(fn); + if (size < 0) { - if (fsync_fname(fn, false, progname) != 0 || - fsync_parent_path(fn, progname) != 0) + fprintf(stderr, + _("%s: could not get size of transaction log file \"%s\": %s\n"), + progname, fn, stream->walmethod->getlasterror()); + return false; + } + if (size == XLogSegSize) + { + /* Already padded file. Open it for use */ + f = stream->walmethod->open_for_write(current_walfile_name, stream->partial_suffix, 0); + if (f == NULL) { - /* error already printed */ - close(f); + fprintf(stderr, + _("%s: could not open existing transaction log file \"%s\": %s\n"), + progname, fn, stream->walmethod->getlasterror()); return false; } - } - /* File is open and ready to use */ - walfile = f; - return true; - } - if (statbuf.st_size != 0) - { - fprintf(stderr, - _("%s: transaction log file \"%s\" has %d bytes, should be 0 or %d\n"), - progname, fn, (int) statbuf.st_size, XLogSegSize); - close(f); - return false; - } + /* fsync file in case of a previous crash */ + if (!stream->walmethod->fsync(f)) + { + stream->walmethod->close(f, CLOSE_UNLINK); + return false; + } - /* - * New, empty, file. So pad it to 16Mb with zeroes. If we fail partway - * through padding, we should attempt to unlink the file on failure, so as - * not to leave behind a partially-filled file. - */ - zerobuf = pg_malloc0(XLOG_BLCKSZ); - for (bytes = 0; bytes < XLogSegSize; bytes += XLOG_BLCKSZ) - { - errno = 0; - if (write(f, zerobuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) + walfile = f; + return true; + } + if (size != 0) { /* if write didn't set errno, assume problem is no disk space */ if (errno == 0) errno = ENOSPC; fprintf(stderr, - _("%s: could not pad transaction log file \"%s\": %s\n"), - progname, fn, strerror(errno)); - free(zerobuf); - close(f); - unlink(fn); + _("%s: transaction log file \"%s\" has %d bytes, should be 0 or %d\n"), + progname, fn, (int) size, XLogSegSize); return false; } + /* File existed and was empty, so fall through and open */ } - free(zerobuf); - /* - * fsync WAL file and containing directory, to ensure the file is - * persistently created and zeroed. That's particularly important when - * using synchronous mode, where the file is modified and fsynced - * in-place, without a directory fsync. - */ - if (stream->do_sync) - { - if (fsync_fname(fn, false, progname) != 0 || - fsync_parent_path(fn, progname) != 0) - { - /* error already printed */ - close(f); - return false; - } - } + /* No file existed, so create one */ - if (lseek(f, SEEK_SET, 0) != 0) + f = stream->walmethod->open_for_write(current_walfile_name, stream->partial_suffix, XLogSegSize); + if (f == NULL) { fprintf(stderr, - _("%s: could not seek to beginning of transaction log file \"%s\": %s\n"), - progname, fn, strerror(errno)); - close(f); + _("%s: could not open transaction log file \"%s\": %s\n"), + progname, fn, stream->walmethod->getlasterror()); return false; } - /* File is open and ready to use */ walfile = f; return true; } @@ -223,59 +178,46 @@ static bool close_walfile(StreamCtl *stream, XLogRecPtr pos) { off_t currpos; + int r; - if (walfile == -1) + if (walfile == NULL) return true; - currpos = lseek(walfile, 0, SEEK_CUR); + currpos = stream->walmethod->get_current_pos(walfile); if (currpos == -1) { fprintf(stderr, _("%s: could not determine seek position in file \"%s\": %s\n"), - progname, current_walfile_name, strerror(errno)); - close(walfile); - walfile = -1; + progname, current_walfile_name, stream->walmethod->getlasterror()); + stream->walmethod->close(walfile, CLOSE_UNLINK); + walfile = NULL; + return false; } - if (stream->do_sync && fsync(walfile) != 0) + if (stream->partial_suffix) { - fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"), - progname, current_walfile_name, strerror(errno)); - close(walfile); - walfile = -1; - return false; + if (currpos == XLOG_SEG_SIZE) + r = stream->walmethod->close(walfile, CLOSE_NORMAL); + else + { + fprintf(stderr, + _("%s: not renaming \"%s%s\", segment is not complete\n"), + progname, current_walfile_name, stream->partial_suffix); + r = stream->walmethod->close(walfile, CLOSE_NO_RENAME); + } } + else + r = stream->walmethod->close(walfile, CLOSE_NORMAL); - if (close(walfile) != 0) + walfile = NULL; + + if (r != 0) { fprintf(stderr, _("%s: could not close file \"%s\": %s\n"), - progname, current_walfile_name, strerror(errno)); - walfile = -1; + progname, current_walfile_name, stream->walmethod->getlasterror()); return false; } - walfile = -1; - - /* - * If we finished writing a .partial file, rename it into place. - */ - if (currpos == XLOG_SEG_SIZE && stream->partial_suffix) - { - char oldfn[MAXPGPATH]; - char newfn[MAXPGPATH]; - - snprintf(oldfn, sizeof(oldfn), "%s/%s%s", stream->basedir, current_walfile_name, stream->partial_suffix); - snprintf(newfn, sizeof(newfn), "%s/%s", stream->basedir, current_walfile_name); - if (durable_rename(oldfn, newfn, progname) != 0) - { - /* durable_rename produced a log entry */ - return false; - } - } - else if (stream->partial_suffix) - fprintf(stderr, - _("%s: not renaming \"%s%s\", segment is not complete\n"), - progname, current_walfile_name, stream->partial_suffix); /* * Mark file as archived if requested by the caller - pg_basebackup needs @@ -286,8 +228,7 @@ close_walfile(StreamCtl *stream, XLogRecPtr pos) if (currpos == XLOG_SEG_SIZE && stream->mark_done) { /* writes error message if failed */ - if (!mark_file_as_archived(stream->basedir, current_walfile_name, - stream->do_sync)) + if (!mark_file_as_archived(stream, current_walfile_name)) return false; } @@ -302,9 +243,7 @@ close_walfile(StreamCtl *stream, XLogRecPtr pos) static bool existsTimeLineHistoryFile(StreamCtl *stream) { - char path[MAXPGPATH]; char histfname[MAXFNAMELEN]; - int fd; /* * Timeline 1 never has a history file. We treat that as if it existed, @@ -315,31 +254,15 @@ existsTimeLineHistoryFile(StreamCtl *stream) TLHistoryFileName(histfname, stream->timeline); - snprintf(path, sizeof(path), "%s/%s", stream->basedir, histfname); - - fd = open(path, O_RDONLY | PG_BINARY, 0); - if (fd < 0) - { - if (errno != ENOENT) - fprintf(stderr, _("%s: could not open timeline history file \"%s\": %s\n"), - progname, path, strerror(errno)); - return false; - } - else - { - close(fd); - return true; - } + return stream->walmethod->existsfile(histfname); } static bool writeTimeLineHistoryFile(StreamCtl *stream, char *filename, char *content) { int size = strlen(content); - char path[MAXPGPATH]; - char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; - int fd; + Walfile *f; /* * Check that the server's idea of how timeline history files should be @@ -353,53 +276,31 @@ writeTimeLineHistoryFile(StreamCtl *stream, char *filename, char *content) return false; } - snprintf(path, sizeof(path), "%s/%s", stream->basedir, histfname); - - /* - * Write into a temp file name. - */ - snprintf(tmppath, MAXPGPATH, "%s.tmp", path); - - unlink(tmppath); - - fd = open(tmppath, O_WRONLY | O_CREAT | PG_BINARY, S_IRUSR | S_IWUSR); - if (fd < 0) + f = stream->walmethod->open_for_write(histfname, ".tmp", 0); + if (f == NULL) { fprintf(stderr, _("%s: could not create timeline history file \"%s\": %s\n"), - progname, tmppath, strerror(errno)); + progname, histfname, stream->walmethod->getlasterror()); return false; } - errno = 0; - if ((int) write(fd, content, size) != size) + if ((int) stream->walmethod->write(f, content, size) != size) { - int save_errno = errno; + fprintf(stderr, _("%s: could not write timeline history file \"%s\": %s\n"), + progname, histfname, stream->walmethod->getlasterror()); /* * If we fail to make the file, delete it to release disk space */ - close(fd); - unlink(tmppath); - errno = save_errno; + stream->walmethod->close(f, CLOSE_UNLINK); - fprintf(stderr, _("%s: could not write timeline history file \"%s\": %s\n"), - progname, tmppath, strerror(errno)); return false; } - if (close(fd) != 0) + if (stream->walmethod->close(f, CLOSE_NORMAL) != 0) { fprintf(stderr, _("%s: could not close file \"%s\": %s\n"), - progname, tmppath, strerror(errno)); - return false; - } - - /* - * Now move the completed history file into place with its final name. - */ - if (durable_rename(tmppath, path, progname) < 0) - { - /* durable_rename produced a log entry */ + progname, histfname, stream->walmethod->getlasterror()); return false; } @@ -407,8 +308,7 @@ writeTimeLineHistoryFile(StreamCtl *stream, char *filename, char *content) if (stream->mark_done) { /* writes error message if failed */ - if (!mark_file_as_archived(stream->basedir, histfname, - stream->do_sync)) + if (!mark_file_as_archived(stream, histfname)) return false; } @@ -618,7 +518,9 @@ ReceiveXlogStream(PGconn *conn, StreamCtl *stream) { /* * Fetch the timeline history file for this timeline, if we don't have - * it already. + * it already. When streaming log to tar, this will always return + * false, as we are never streaming into an existing file and + * therefore there can be no pre-existing timeline history file. */ if (!existsTimeLineHistoryFile(stream)) { @@ -777,10 +679,10 @@ ReceiveXlogStream(PGconn *conn, StreamCtl *stream) } error: - if (walfile != -1 && close(walfile) != 0) + if (walfile != NULL && stream->walmethod->close(walfile, CLOSE_NORMAL) != 0) fprintf(stderr, _("%s: could not close file \"%s\": %s\n"), - progname, current_walfile_name, strerror(errno)); - walfile = -1; + progname, current_walfile_name, stream->walmethod->getlasterror()); + walfile = NULL; return false; } @@ -864,12 +766,12 @@ HandleCopyStream(PGconn *conn, StreamCtl *stream, * If synchronous option is true, issue sync command as soon as there * are WAL data which has not been flushed yet. */ - if (stream->synchronous && lastFlushPosition < blockpos && walfile != -1) + if (stream->synchronous && lastFlushPosition < blockpos && walfile != NULL) { - if (stream->do_sync && fsync(walfile) != 0) + if (stream->walmethod->fsync(walfile) != 0) { fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"), - progname, current_walfile_name, strerror(errno)); + progname, current_walfile_name, stream->walmethod->getlasterror()); goto error; } lastFlushPosition = blockpos; @@ -1100,7 +1002,7 @@ ProcessKeepaliveMsg(PGconn *conn, StreamCtl *stream, char *copybuf, int len, if (replyRequested && still_sending) { if (reportFlushPosition && lastFlushPosition < blockpos && - walfile != -1) + walfile != NULL) { /* * If a valid flush location needs to be reported, flush the @@ -1109,10 +1011,10 @@ ProcessKeepaliveMsg(PGconn *conn, StreamCtl *stream, char *copybuf, int len, * data has been successfully replicated or not, at the normal * shutdown of the server. */ - if (stream->do_sync && fsync(walfile) != 0) + if (stream->walmethod->fsync(walfile) != 0) { fprintf(stderr, _("%s: could not fsync file \"%s\": %s\n"), - progname, current_walfile_name, strerror(errno)); + progname, current_walfile_name, stream->walmethod->getlasterror()); return false; } lastFlushPosition = blockpos; @@ -1170,7 +1072,7 @@ ProcessXLogDataMsg(PGconn *conn, StreamCtl *stream, char *copybuf, int len, * Verify that the initial location in the stream matches where we think * we are. */ - if (walfile == -1) + if (walfile == NULL) { /* No file open yet */ if (xlogoff != 0) @@ -1184,12 +1086,11 @@ ProcessXLogDataMsg(PGconn *conn, StreamCtl *stream, char *copybuf, int len, else { /* More data in existing segment */ - /* XXX: store seek value don't reseek all the time */ - if (lseek(walfile, 0, SEEK_CUR) != xlogoff) + if (stream->walmethod->get_current_pos(walfile) != xlogoff) { fprintf(stderr, _("%s: got WAL data offset %08x, expected %08x\n"), - progname, xlogoff, (int) lseek(walfile, 0, SEEK_CUR)); + progname, xlogoff, (int) stream->walmethod->get_current_pos(walfile)); return false; } } @@ -1210,7 +1111,7 @@ ProcessXLogDataMsg(PGconn *conn, StreamCtl *stream, char *copybuf, int len, else bytes_to_write = bytes_left; - if (walfile == -1) + if (walfile == NULL) { if (!open_walfile(stream, *blockpos)) { @@ -1219,14 +1120,13 @@ ProcessXLogDataMsg(PGconn *conn, StreamCtl *stream, char *copybuf, int len, } } - if (write(walfile, - copybuf + hdr_len + bytes_written, - bytes_to_write) != bytes_to_write) + if (stream->walmethod->write(walfile, copybuf + hdr_len + bytes_written, + bytes_to_write) != bytes_to_write) { fprintf(stderr, _("%s: could not write %u bytes to WAL file \"%s\": %s\n"), progname, bytes_to_write, current_walfile_name, - strerror(errno)); + stream->walmethod->getlasterror()); return false; } diff --git a/src/bin/pg_basebackup/receivelog.h b/src/bin/pg_basebackup/receivelog.h index 7a3bbc5080..b5913ea995 100644 --- a/src/bin/pg_basebackup/receivelog.h +++ b/src/bin/pg_basebackup/receivelog.h @@ -13,6 +13,7 @@ #define RECEIVELOG_H #include "libpq-fe.h" +#include "walmethods.h" #include "access/xlogdefs.h" @@ -41,7 +42,7 @@ typedef struct StreamCtl stream_stop_callback stream_stop; /* Stop streaming when returns true */ - char *basedir; /* Received segments written to this dir */ + WalWriteMethod *walmethod; /* How to write the WAL */ char *partial_suffix; /* Suffix appended to partially received files */ } StreamCtl; diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/010_pg_basebackup.pl index 579d7a15fb..91eb84e238 100644 --- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl +++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl @@ -4,7 +4,7 @@ use Cwd; use Config; use PostgresNode; use TestLib; -use Test::More tests => 67; +use Test::More tests => 69; program_help_ok('pg_basebackup'); program_version_ok('pg_basebackup'); @@ -237,6 +237,10 @@ $node->command_ok( 'pg_basebackup -X stream runs'); ok(grep(/^[0-9A-F]{24}$/, slurp_dir("$tempdir/backupxf/pg_wal")), 'WAL files copied'); +$node->command_ok( + [ 'pg_basebackup', '-D', "$tempdir/backupxst", '-X', 'stream', '-Ft' ], + 'pg_basebackup -X stream runs in tar mode'); +ok(-f "$tempdir/backupxst/pg_wal.tar", "tar file was created"); $node->command_fails( [ 'pg_basebackup', '-D', "$tempdir/fail", '-S', 'slot1' ], diff --git a/src/bin/pg_basebackup/walmethods.c b/src/bin/pg_basebackup/walmethods.c new file mode 100644 index 0000000000..e0ec752bbd --- /dev/null +++ b/src/bin/pg_basebackup/walmethods.c @@ -0,0 +1,886 @@ +/*------------------------------------------------------------------------- + * + * walmethods.c - implementations of different ways to write received wal + * + * NOTE! The caller must ensure that only one method is instantiated in + * any given program, and that it's only instantiated once! + * + * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/bin/pg_basebackup/walmethods.c + *------------------------------------------------------------------------- + */ + +#include "postgres_fe.h" + +#include +#include +#include +#ifdef HAVE_LIBZ +#include +#endif + +#include "pgtar.h" +#include "common/file_utils.h" + +#include "receivelog.h" +#include "streamutil.h" + +/* Size of zlib buffer for .tar.gz */ +#define ZLIB_OUT_SIZE 4096 + +/*------------------------------------------------------------------------- + * WalDirectoryMethod - write wal to a directory looking like pg_xlog + *------------------------------------------------------------------------- + */ + +/* + * Global static data for this method + */ +typedef struct DirectoryMethodData +{ + char *basedir; + bool sync; +} DirectoryMethodData; +static DirectoryMethodData *dir_data = NULL; + +/* + * Local file handle + */ +typedef struct DirectoryMethodFile +{ + int fd; + off_t currpos; + char *pathname; + char *fullpath; + char *temp_suffix; +} DirectoryMethodFile; + +static char * +dir_getlasterror(void) +{ + /* Directory method always sets errno, so just use strerror */ + return strerror(errno); +} + +static Walfile +dir_open_for_write(const char *pathname, const char *temp_suffix, size_t pad_to_size) +{ + static char tmppath[MAXPGPATH]; + int fd; + DirectoryMethodFile *f; + + snprintf(tmppath, sizeof(tmppath), "%s/%s%s", + dir_data->basedir, pathname, temp_suffix ? temp_suffix : ""); + + fd = open(tmppath, O_WRONLY | O_CREAT | PG_BINARY, S_IRUSR | S_IWUSR); + if (fd < 0) + return NULL; + + if (pad_to_size) + { + /* Always pre-pad on regular files */ + char *zerobuf; + int bytes; + + zerobuf = pg_malloc0(XLOG_BLCKSZ); + for (bytes = 0; bytes < pad_to_size; bytes += XLOG_BLCKSZ) + { + if (write(fd, zerobuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) + { + int save_errno = errno; + + pg_free(zerobuf); + close(fd); + errno = save_errno; + return NULL; + } + } + pg_free(zerobuf); + + if (lseek(fd, 0, SEEK_SET) != 0) + { + int save_errno = errno; + + close(fd); + errno = save_errno; + return NULL; + } + } + + /* + * fsync WAL file and containing directory, to ensure the file is + * persistently created and zeroed (if padded). That's particularly + * important when using synchronous mode, where the file is modified and + * fsynced in-place, without a directory fsync. + */ + if (dir_data->sync) + { + if (fsync_fname(tmppath, false, progname) != 0 || + fsync_parent_path(tmppath, progname) != 0) + { + close(fd); + return NULL; + } + } + + f = pg_malloc0(sizeof(DirectoryMethodFile)); + f->fd = fd; + f->currpos = 0; + f->pathname = pg_strdup(pathname); + f->fullpath = pg_strdup(tmppath); + if (temp_suffix) + f->temp_suffix = pg_strdup(temp_suffix); + + return f; +} + +static ssize_t +dir_write(Walfile f, const void *buf, size_t count) +{ + ssize_t r; + DirectoryMethodFile *df = (DirectoryMethodFile *) f; + + Assert(f != NULL); + + r = write(df->fd, buf, count); + if (r > 0) + df->currpos += r; + return r; +} + +static off_t +dir_get_current_pos(Walfile f) +{ + Assert(f != NULL); + + /* Use a cached value to prevent lots of reseeks */ + return ((DirectoryMethodFile *) f)->currpos; +} + +static int +dir_close(Walfile f, WalCloseMethod method) +{ + int r; + DirectoryMethodFile *df = (DirectoryMethodFile *) f; + static char tmppath[MAXPGPATH]; + static char tmppath2[MAXPGPATH]; + + Assert(f != NULL); + + r = close(df->fd); + + if (r == 0) + { + /* Build path to the current version of the file */ + if (method == CLOSE_NORMAL && df->temp_suffix) + { + /* + * If we have a temp prefix, normal operation is to rename the + * file. + */ + snprintf(tmppath, sizeof(tmppath), "%s/%s%s", + dir_data->basedir, df->pathname, df->temp_suffix); + snprintf(tmppath2, sizeof(tmppath2), "%s/%s", + dir_data->basedir, df->pathname); + r = durable_rename(tmppath, tmppath2, progname); + } + else if (method == CLOSE_UNLINK) + { + /* Unlink the file once it's closed */ + snprintf(tmppath, sizeof(tmppath), "%s/%s%s", + dir_data->basedir, df->pathname, df->temp_suffix ? df->temp_suffix : ""); + r = unlink(tmppath); + } + else + { + /* + * Else either CLOSE_NORMAL and no temp suffix, or + * CLOSE_NO_RENAME. In this case, fsync the file and containing + * directory if sync mode is requested. + */ + if (dir_data->sync) + { + r = fsync_fname(df->fullpath, false, progname); + if (r == 0) + r = fsync_parent_path(df->fullpath, progname); + } + } + } + + pg_free(df->pathname); + pg_free(df->fullpath); + if (df->temp_suffix) + pg_free(df->temp_suffix); + pg_free(df); + + return r; +} + +static int +dir_fsync(Walfile f) +{ + Assert(f != NULL); + + if (!dir_data->sync) + return 0; + + return fsync(((DirectoryMethodFile *) f)->fd); +} + +static ssize_t +dir_get_file_size(const char *pathname) +{ + struct stat statbuf; + static char tmppath[MAXPGPATH]; + + snprintf(tmppath, sizeof(tmppath), "%s/%s", + dir_data->basedir, pathname); + + if (stat(tmppath, &statbuf) != 0) + return -1; + + return statbuf.st_size; +} + +static bool +dir_existsfile(const char *pathname) +{ + static char tmppath[MAXPGPATH]; + int fd; + + snprintf(tmppath, sizeof(tmppath), "%s/%s", + dir_data->basedir, pathname); + + fd = open(tmppath, O_RDONLY | PG_BINARY, 0); + if (fd < 0) + return false; + close(fd); + return true; +} + +static bool +dir_finish(void) +{ + if (dir_data->sync) + { + /* + * Files are fsynced when they are closed, but we need to fsync the + * directory entry here as well. + */ + if (fsync_fname(dir_data->basedir, true, progname) != 0) + return false; + } + return true; +} + + +WalWriteMethod * +CreateWalDirectoryMethod(const char *basedir, bool sync) +{ + WalWriteMethod *method; + + method = pg_malloc0(sizeof(WalWriteMethod)); + method->open_for_write = dir_open_for_write; + method->write = dir_write; + method->get_current_pos = dir_get_current_pos; + method->get_file_size = dir_get_file_size; + method->close = dir_close; + method->fsync = dir_fsync; + method->existsfile = dir_existsfile; + method->finish = dir_finish; + method->getlasterror = dir_getlasterror; + + dir_data = pg_malloc0(sizeof(DirectoryMethodData)); + dir_data->basedir = pg_strdup(basedir); + dir_data->sync = sync; + + return method; +} + + +/*------------------------------------------------------------------------- + * WalTarMethod - write wal to a tar file containing pg_xlog contents + *------------------------------------------------------------------------- + */ + +typedef struct TarMethodFile +{ + off_t ofs_start; /* Where does the *header* for this file start */ + off_t currpos; + char header[512]; + char *pathname; + size_t pad_to_size; +} TarMethodFile; + +typedef struct TarMethodData +{ + char *tarfilename; + int fd; + int compression; + bool sync; + TarMethodFile *currentfile; + char lasterror[1024]; +#ifdef HAVE_LIBZ + z_streamp zp; + void *zlibOut; +#endif +} TarMethodData; +static TarMethodData *tar_data = NULL; + +#define tar_clear_error() tar_data->lasterror[0] = '\0' +#define tar_set_error(msg) strlcpy(tar_data->lasterror, msg, sizeof(tar_data->lasterror)) + +static char * +tar_getlasterror(void) +{ + /* + * If a custom error is set, return that one. Otherwise, assume errno is + * set and return that one. + */ + if (tar_data->lasterror[0]) + return tar_data->lasterror; + return strerror(errno); +} + +#ifdef HAVE_LIBZ +static bool +tar_write_compressed_data(void *buf, size_t count, bool flush) +{ + tar_data->zp->next_in = buf; + tar_data->zp->avail_in = count; + + while (tar_data->zp->avail_in || flush) + { + int r; + + r = deflate(tar_data->zp, flush ? Z_FINISH : Z_NO_FLUSH); + if (r == Z_STREAM_ERROR) + { + tar_set_error("deflate failed"); + return false; + } + + if (tar_data->zp->avail_out < ZLIB_OUT_SIZE) + { + size_t len = ZLIB_OUT_SIZE - tar_data->zp->avail_out; + + if (write(tar_data->fd, tar_data->zlibOut, len) != len) + return false; + + tar_data->zp->next_out = tar_data->zlibOut; + tar_data->zp->avail_out = ZLIB_OUT_SIZE; + } + + if (r == Z_STREAM_END) + break; + } + + if (flush) + { + /* Reset the stream for writing */ + if (deflateReset(tar_data->zp) != Z_OK) + { + tar_set_error("deflateReset failed"); + return false; + } + } + + return true; +} +#endif + +static ssize_t +tar_write(Walfile f, const void *buf, size_t count) +{ + ssize_t r; + + Assert(f != NULL); + tar_clear_error(); + + /* Tarfile will always be positioned at the end */ + if (!tar_data->compression) + { + r = write(tar_data->fd, buf, count); + if (r > 0) + ((TarMethodFile *) f)->currpos += r; + return r; + } +#ifdef HAVE_LIBZ + else + { + if (!tar_write_compressed_data((void *) buf, count, false)) + return -1; + ((TarMethodFile *) f)->currpos += count; + return count; + } +#endif +} + +static bool +tar_write_padding_data(TarMethodFile * f, size_t bytes) +{ + char *zerobuf = pg_malloc0(XLOG_BLCKSZ); + size_t bytesleft = bytes; + + while (bytesleft) + { + size_t bytestowrite = bytesleft > XLOG_BLCKSZ ? XLOG_BLCKSZ : bytesleft; + + size_t r = tar_write(f, zerobuf, bytestowrite); + + if (r < 0) + return false; + bytesleft -= r; + } + return true; +} + +static Walfile +tar_open_for_write(const char *pathname, const char *temp_suffix, size_t pad_to_size) +{ + int save_errno; + static char tmppath[MAXPGPATH]; + + tar_clear_error(); + + if (tar_data->fd < 0) + { + /* + * We open the tar file only when we first try to write to it. + */ + tar_data->fd = open(tar_data->tarfilename, + O_WRONLY | O_CREAT | PG_BINARY, S_IRUSR | S_IWUSR); + if (tar_data->fd < 0) + return NULL; + +#ifdef HAVE_LIBZ + if (tar_data->compression) + { + tar_data->zp = (z_streamp) pg_malloc(sizeof(z_stream)); + tar_data->zp->zalloc = Z_NULL; + tar_data->zp->zfree = Z_NULL; + tar_data->zp->opaque = Z_NULL; + tar_data->zp->next_out = tar_data->zlibOut; + tar_data->zp->avail_out = ZLIB_OUT_SIZE; + + /* + * Initialize deflation library. Adding the magic value 16 to the + * default 15 for the windowBits parameter makes the output be + * gzip instead of zlib. + */ + if (deflateInit2(tar_data->zp, tar_data->compression, Z_DEFLATED, 15 + 16, 8, Z_DEFAULT_STRATEGY) != Z_OK) + { + pg_free(tar_data->zp); + tar_data->zp = NULL; + tar_set_error("deflateInit2 failed"); + return NULL; + } + } +#endif + + /* There's no tar header itself, the file starts with regular files */ + } + + Assert(tar_data->currentfile == NULL); + if (tar_data->currentfile != NULL) + { + tar_set_error("implementation error: tar files can't have more than one open file\n"); + return NULL; + } + + tar_data->currentfile = pg_malloc0(sizeof(TarMethodFile)); + + snprintf(tmppath, sizeof(tmppath), "%s%s", + pathname, temp_suffix ? temp_suffix : ""); + + /* Create a header with size set to 0 - we will fill out the size on close */ + if (tarCreateHeader(tar_data->currentfile->header, tmppath, NULL, 0, S_IRUSR | S_IWUSR, 0, 0, time(NULL)) != TAR_OK) + { + pg_free(tar_data->currentfile); + tar_data->currentfile = NULL; + tar_set_error("could not create tar header"); + return NULL; + } + +#ifdef HAVE_LIBZ + if (tar_data->compression) + { + /* Flush existing data */ + if (!tar_write_compressed_data(NULL, 0, true)) + return NULL; + + /* Turn off compression for header */ + if (deflateParams(tar_data->zp, 0, 0) != Z_OK) + { + tar_set_error("deflateParams failed"); + return NULL; + } + } +#endif + + tar_data->currentfile->ofs_start = lseek(tar_data->fd, 0, SEEK_CUR); + if (tar_data->currentfile->ofs_start == -1) + { + save_errno = errno; + pg_free(tar_data->currentfile); + tar_data->currentfile = NULL; + errno = save_errno; + return NULL; + } + tar_data->currentfile->currpos = 0; + + if (!tar_data->compression) + { + if (write(tar_data->fd, tar_data->currentfile->header, 512) != 512) + { + save_errno = errno; + pg_free(tar_data->currentfile); + tar_data->currentfile = NULL; + errno = save_errno; + return NULL; + } + } +#ifdef HAVE_LIBZ + else + { + /* Write header through the zlib APIs but with no compression */ + if (!tar_write_compressed_data(tar_data->currentfile->header, 512, true)) + return NULL; + + /* Re-enable compression for the rest of the file */ + if (deflateParams(tar_data->zp, tar_data->compression, 0) != Z_OK) + { + tar_set_error("deflateParams failed"); + return NULL; + } + } +#endif + + tar_data->currentfile->pathname = pg_strdup(pathname); + + /* + * Uncompressed files are padded on creation, but for compression we can't + * do that + */ + if (pad_to_size) + { + tar_data->currentfile->pad_to_size = pad_to_size; + if (!tar_data->compression) + { + /* Uncompressed, so pad now */ + tar_write_padding_data(tar_data->currentfile, pad_to_size); + /* Seek back to start */ + if (lseek(tar_data->fd, tar_data->currentfile->ofs_start + 512, SEEK_SET) != tar_data->currentfile->ofs_start + 512) + return NULL; + + tar_data->currentfile->currpos = 0; + } + } + + return tar_data->currentfile; +} + +static ssize_t +tar_get_file_size(const char *pathname) +{ + tar_clear_error(); + + /* Currently not used, so not supported */ + errno = ENOSYS; + return -1; +} + +static off_t +tar_get_current_pos(Walfile f) +{ + Assert(f != NULL); + tar_clear_error(); + + return ((TarMethodFile *) f)->currpos; +} + +static int +tar_fsync(Walfile f) +{ + Assert(f != NULL); + tar_clear_error(); + + /* + * Always sync the whole tarfile, because that's all we can do. This makes + * no sense on compressed files, so just ignore those. + */ + if (tar_data->compression) + return 0; + + return fsync(tar_data->fd); +} + +static int +tar_close(Walfile f, WalCloseMethod method) +{ + ssize_t filesize; + int padding; + TarMethodFile *tf = (TarMethodFile *) f; + + Assert(f != NULL); + tar_clear_error(); + + if (method == CLOSE_UNLINK) + { + if (tar_data->compression) + { + tar_set_error("unlink not supported with compression"); + return -1; + } + + /* + * Unlink the file that we just wrote to the tar. We do this by + * truncating it to the start of the header. This is safe as we only + * allow writing of the very last file. + */ + if (ftruncate(tar_data->fd, tf->ofs_start) != 0) + return -1; + + pg_free(tf->pathname); + pg_free(tf); + tar_data->currentfile = NULL; + + return 0; + } + + /* + * Pad the file itself with zeroes if necessary. Note that this is + * different from the tar format padding -- this is the padding we asked + * for when the file was opened. + */ + if (tf->pad_to_size) + { + if (tar_data->compression) + { + /* + * A compressed tarfile is padded on close since we cannot know + * the size of the compressed output until the end. + */ + size_t sizeleft = tf->pad_to_size - tf->currpos; + + if (sizeleft) + { + if (!tar_write_padding_data(tf, sizeleft)) + return -1; + } + } + else + { + /* + * An uncompressed tarfile was padded on creation, so just adjust + * the current position as if we seeked to the end. + */ + tf->currpos = tf->pad_to_size; + } + } + + /* + * Get the size of the file, and pad the current data up to the nearest + * 512 byte boundary. + */ + filesize = tar_get_current_pos(f); + padding = ((filesize + 511) & ~511) - filesize; + if (padding) + { + char zerobuf[512]; + + MemSet(zerobuf, 0, padding); + if (tar_write(f, zerobuf, padding) != padding) + return -1; + } + + +#ifdef HAVE_LIBZ + if (tar_data->compression) + { + /* Flush the current buffer */ + if (!tar_write_compressed_data(NULL, 0, true)) + { + errno = EINVAL; + return -1; + } + } +#endif + + /* + * Now go back and update the header with the correct filesize and + * possibly also renaming the file. We overwrite the entire current header + * when done, including the checksum. + */ + print_tar_number(&(tf->header[124]), 12, filesize); + + if (method == CLOSE_NORMAL) + + /* + * We overwrite it with what it was before if we have no tempname, + * since we're going to write the buffer anyway. + */ + strlcpy(&(tf->header[0]), tf->pathname, 100); + + print_tar_number(&(tf->header[148]), 8, tarChecksum(((TarMethodFile *) f)->header)); + if (lseek(tar_data->fd, tf->ofs_start, SEEK_SET) != ((TarMethodFile *) f)->ofs_start) + return -1; + if (!tar_data->compression) + { + if (write(tar_data->fd, tf->header, 512) != 512) + return -1; + } +#ifdef HAVE_LIBZ + else + { + /* Turn off compression */ + if (deflateParams(tar_data->zp, 0, 0) != Z_OK) + { + tar_set_error("deflateParams failed"); + return -1; + } + + /* Overwrite the header, assuming the size will be the same */ + if (!tar_write_compressed_data(tar_data->currentfile->header, 512, true)) + return -1; + + /* Turn compression back on */ + if (deflateParams(tar_data->zp, tar_data->compression, 0) != Z_OK) + { + tar_set_error("deflateParams failed"); + return -1; + } + } +#endif + + /* Move file pointer back down to end, so we can write the next file */ + if (lseek(tar_data->fd, 0, SEEK_END) < 0) + return -1; + + /* Always fsync on close, so the padding gets fsynced */ + tar_fsync(f); + + /* Clean up and done */ + pg_free(tf->pathname); + pg_free(tf); + tar_data->currentfile = NULL; + + return 0; +} + +static bool +tar_existsfile(const char *pathname) +{ + tar_clear_error(); + /* We only deal with new tarfiles, so nothing externally created exists */ + return false; +} + +static bool +tar_finish(void) +{ + char zerobuf[1024]; + + tar_clear_error(); + + if (tar_data->currentfile) + { + if (tar_close(tar_data->currentfile, CLOSE_NORMAL) != 0) + return false; + } + + /* A tarfile always ends with two empty blocks */ + MemSet(zerobuf, 0, sizeof(zerobuf)); + if (!tar_data->compression) + { + if (write(tar_data->fd, zerobuf, sizeof(zerobuf)) != sizeof(zerobuf)) + return false; + } +#ifdef HAVE_LIBZ + else + { + if (!tar_write_compressed_data(zerobuf, sizeof(zerobuf), false)) + return false; + + /* Also flush all data to make sure the gzip stream is finished */ + tar_data->zp->next_in = NULL; + tar_data->zp->avail_in = 0; + while (true) + { + int r; + + r = deflate(tar_data->zp, Z_FINISH); + + if (r == Z_STREAM_ERROR) + { + tar_set_error("deflate failed"); + return false; + } + if (tar_data->zp->avail_out < ZLIB_OUT_SIZE) + { + size_t len = ZLIB_OUT_SIZE - tar_data->zp->avail_out; + + if (write(tar_data->fd, tar_data->zlibOut, len) != len) + return false; + } + if (r == Z_STREAM_END) + break; + } + + if (deflateEnd(tar_data->zp) != Z_OK) + { + tar_set_error("deflateEnd failed"); + return false; + } + } +#endif + + /* sync the empty blocks as well, since they're after the last file */ + fsync(tar_data->fd); + + if (close(tar_data->fd) != 0) + return false; + + tar_data->fd = -1; + + if (tar_data->sync) + { + if (fsync_fname(tar_data->tarfilename, false, progname) != 0) + return false; + if (fsync_parent_path(tar_data->tarfilename, progname) != 0) + return false; + } + + return true; +} + +WalWriteMethod * +CreateWalTarMethod(const char *tarbase, int compression, bool sync) +{ + WalWriteMethod *method; + const char *suffix = (compression != 0) ? ".tar.gz" : ".tar"; + + method = pg_malloc0(sizeof(WalWriteMethod)); + method->open_for_write = tar_open_for_write; + method->write = tar_write; + method->get_current_pos = tar_get_current_pos; + method->get_file_size = tar_get_file_size; + method->close = tar_close; + method->fsync = tar_fsync; + method->existsfile = tar_existsfile; + method->finish = tar_finish; + method->getlasterror = tar_getlasterror; + + tar_data = pg_malloc0(sizeof(TarMethodData)); + tar_data->tarfilename = pg_malloc0(strlen(tarbase) + strlen(suffix) + 1); + sprintf(tar_data->tarfilename, "%s%s", tarbase, suffix); + tar_data->fd = -1; + tar_data->compression = compression; + tar_data->sync = sync; + if (compression) + tar_data->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1); + + return method; +} diff --git a/src/bin/pg_basebackup/walmethods.h b/src/bin/pg_basebackup/walmethods.h new file mode 100644 index 0000000000..fa58f812f6 --- /dev/null +++ b/src/bin/pg_basebackup/walmethods.h @@ -0,0 +1,45 @@ +/*------------------------------------------------------------------------- + * + * walmethods.h + * + * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/bin/pg_basebackup/walmethods.h + *------------------------------------------------------------------------- + */ + + +typedef void *Walfile; + +typedef enum +{ + CLOSE_NORMAL, + CLOSE_UNLINK, + CLOSE_NO_RENAME, +} WalCloseMethod; + +typedef struct WalWriteMethod WalWriteMethod; +struct WalWriteMethod +{ + Walfile(*open_for_write) (const char *pathname, const char *temp_suffix, size_t pad_to_size); + int (*close) (Walfile f, WalCloseMethod method); + bool (*existsfile) (const char *pathname); + ssize_t (*get_file_size) (const char *pathname); + + ssize_t (*write) (Walfile f, const void *buf, size_t count); + off_t (*get_current_pos) (Walfile f); + int (*fsync) (Walfile f); + bool (*finish) (void); + char *(*getlasterror) (void); +}; + +/* + * Available WAL methods: + * - WalDirectoryMethod - write WAL to regular files in a standard pg_xlog + * - TarDirectoryMethod - write WAL to a tarfile corresponding to pg_xlog + * (only implements the methods required for pg_basebackup, + * not all those required for pg_receivexlog) + */ +WalWriteMethod *CreateWalDirectoryMethod(const char *basedir, bool sync); +WalWriteMethod *CreateWalTarMethod(const char *tarbase, int compression, bool sync); diff --git a/src/include/pgtar.h b/src/include/pgtar.h index 45ca400f98..1d179f0df1 100644 --- a/src/include/pgtar.h +++ b/src/include/pgtar.h @@ -22,4 +22,5 @@ enum tarError extern enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime); extern uint64 read_tar_number(const char *s, int len); +extern void print_tar_number(char *s, int len, uint64 val); extern int tarChecksum(char *header); diff --git a/src/port/tar.c b/src/port/tar.c index 52a2113a47..f1da959dac 100644 --- a/src/port/tar.c +++ b/src/port/tar.c @@ -16,7 +16,7 @@ * support only non-negative numbers, so we don't worry about the GNU rules * for handling negative numbers.) */ -static void +void print_tar_number(char *s, int len, uint64 val) { if (val < (((uint64) 1) << ((len - 1) * 3))) -- cgit v1.2.3 From 7d80417d3dfc88b0c03b5c08a18b29f9d430e217 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 23 Oct 2016 22:13:28 -0400 Subject: Release notes for 9.6.1, 9.5.5, 9.4.10, 9.3.15, 9.2.19, 9.1.24. --- doc/src/sgml/release-9.1.sgml | 210 +++++++++++ doc/src/sgml/release-9.2.sgml | 266 ++++++++++++++ doc/src/sgml/release-9.3.sgml | 329 +++++++++++++++++ doc/src/sgml/release-9.4.sgml | 466 ++++++++++++++++++++++++ doc/src/sgml/release-9.5.sgml | 812 ++++++++++++++++++++++++++++++++++++++++++ doc/src/sgml/release-9.6.sgml | 586 ++---------------------------- 6 files changed, 2103 insertions(+), 566 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/release-9.1.sgml b/doc/src/sgml/release-9.1.sgml index 26b709056f..edacfbf355 100644 --- a/doc/src/sgml/release-9.1.sgml +++ b/doc/src/sgml/release-9.1.sgml @@ -1,6 +1,216 @@ + + Release 9.1.24 + + + Release Date + 2016-10-27 + + + + This release contains a variety of fixes from 9.1.23. + For information about new features in the 9.1 major release, see + . + + + + This is expected to be the last PostgreSQL release + in the 9.1.X series. Users are encouraged to update to a newer + release branch soon. + + + + Migration to Version 9.1.24 + + + A dump/restore is not required for those running 9.1.X. + + + + However, if you are upgrading from a version earlier than 9.1.16, + see . + + + + + + Changes + + + + + + Fix EvalPlanQual rechecks involving CTE scans (Tom Lane) + + + + The recheck would always see the CTE as returning no rows, typically + leading to failure to update rows that were recently updated. + + + + + + Fix improper repetition of previous results from hashed aggregation in + a subquery (Andrew Gierth) + + + + The test to see if we can reuse a previously-computed hash table of + the aggregate state values neglected the possibility of an outer query + reference appearing in an aggregate argument expression. A change in + the value of such a reference should lead to recalculating the hash + table, but did not. + + + + + + Fix timeout length when VACUUM is waiting for exclusive + table lock so that it can truncate the table (Simon Riggs) + + + + The timeout was meant to be 50 milliseconds, but it was actually only + 50 microseconds, causing VACUUM to give up on truncation + much more easily than intended. Set it to the intended value. + + + + + + Remove artificial restrictions on the values accepted + by numeric_in() and numeric_recv() + (Tom Lane) + + + + We allow numeric values up to the limit of the storage format (more + than 1e100000), so it seems fairly pointless + that numeric_in() rejected scientific-notation exponents + above 1000. Likewise, it was silly for numeric_recv() to + reject more than 1000 digits in an input value. + + + + + + Avoid very-low-probability data corruption due to testing tuple + visibility without holding buffer lock (Thomas Munro, Peter Geoghegan, + Tom Lane) + + + + + + Fix file descriptor leakage when truncating a temporary relation of + more than 1GB (Andres Freund) + + + + + + Disallow starting a standalone backend with standby_mode + turned on (Michael Paquier) + + + + This can't do anything useful, since there will be no WAL receiver + process to fetch more WAL data; and it could result in misbehavior + in code that wasn't designed with this situation in mind. + + + + + + Don't try to share SSL contexts across multiple connections + in libpq (Heikki Linnakangas) + + + + This led to assorted corner-case bugs, particularly when trying to use + different SSL parameters for different connections. + + + + + + Avoid corner-case memory leak in libpq (Tom Lane) + + + + The reported problem involved leaking an error report + during PQreset(), but there might be related cases. + + + + + + Make ecpg's + + + + + Fix contrib/intarray/bench/bench.pl to print the results + of the EXPLAIN it does when given the + + + + + Prevent failure of obsolete dynamic time zone abbreviations (Tom Lane) + + + + If a dynamic time zone abbreviation does not match any entry in the + referenced time zone, treat it as equivalent to the time zone name. + This avoids unexpected failures when IANA removes abbreviations from + their time zone database, as they did in tzdata + release 2016f and seem likely to do again in the future. The + consequences were not limited to not recognizing the individual + abbreviation; any mismatch caused + the pg_timezone_abbrevs view to fail altogether. + + + + + + Update time zone data files to tzdata release 2016h + for DST law changes in Palestine and Turkey, plus historical + corrections for Turkey and some regions of Russia. + Switch to numeric abbreviations for some time zones in Antarctica, + the former Soviet Union, and Sri Lanka. + + + + The IANA time zone database previously provided textual abbreviations + for all time zones, sometimes making up abbreviations that have little + or no currency among the local population. They are in process of + reversing that policy in favor of using numeric UTC offsets in zones + where there is no evidence of real-world use of an English + abbreviation. At least for the time being, PostgreSQL + will continue to accept such removed abbreviations for timestamp input. + But they will not be shown in the pg_timezone_names + view nor used for output. + + + + In this update, AMT is no longer shown as being in use to + mean Armenia Time. Therefore, we have changed the Default + abbreviation set to interpret it as Amazon Time, thus UTC-4 not UTC+4. + + + + + + + + Release 9.1.23 diff --git a/doc/src/sgml/release-9.2.sgml b/doc/src/sgml/release-9.2.sgml index 0f6e3d127f..49430389d9 100644 --- a/doc/src/sgml/release-9.2.sgml +++ b/doc/src/sgml/release-9.2.sgml @@ -1,6 +1,272 @@ + + Release 9.2.19 + + + Release Date + 2016-10-27 + + + + This release contains a variety of fixes from 9.2.18. + For information about new features in the 9.2 major release, see + . + + + + Migration to Version 9.2.19 + + + A dump/restore is not required for those running 9.2.X. + + + + However, if you are upgrading from a version earlier than 9.2.11, + see . + + + + + + Changes + + + + + + Fix EvalPlanQual rechecks involving CTE scans (Tom Lane) + + + + The recheck would always see the CTE as returning no rows, typically + leading to failure to update rows that were recently updated. + + + + + + Fix improper repetition of previous results from hashed aggregation in + a subquery (Andrew Gierth) + + + + The test to see if we can reuse a previously-computed hash table of + the aggregate state values neglected the possibility of an outer query + reference appearing in an aggregate argument expression. A change in + the value of such a reference should lead to recalculating the hash + table, but did not. + + + + + + Fix EXPLAIN to emit valid XML when + is on (Markus Winand) + + + + Previously the XML output-format option produced syntactically invalid + tags such as <I/O-Read-Time>. That is now + rendered as <I-O-Read-Time>. + + + + + + Suppress printing of zeroes for unmeasured times + in EXPLAIN (Maksim Milyutin) + + + + Certain option combinations resulted in printing zero values for times + that actually aren't ever measured in that combination. Our general + policy in EXPLAIN is not to print such fields at all, so + do that consistently in all cases. + + + + + + Fix timeout length when VACUUM is waiting for exclusive + table lock so that it can truncate the table (Simon Riggs) + + + + The timeout was meant to be 50 milliseconds, but it was actually only + 50 microseconds, causing VACUUM to give up on truncation + much more easily than intended. Set it to the intended value. + + + + + + Fix bugs in merging inherited CHECK constraints while + creating or altering a table (Tom Lane, Amit Langote) + + + + Allow identical CHECK constraints to be added to a parent + and child table in either order. Prevent merging of a valid + constraint from the parent table with a NOT VALID + constraint on the child. Likewise, prevent merging of a NO + INHERIT child constraint with an inherited constraint. + + + + + + Remove artificial restrictions on the values accepted + by numeric_in() and numeric_recv() + (Tom Lane) + + + + We allow numeric values up to the limit of the storage format (more + than 1e100000), so it seems fairly pointless + that numeric_in() rejected scientific-notation exponents + above 1000. Likewise, it was silly for numeric_recv() to + reject more than 1000 digits in an input value. + + + + + + Avoid very-low-probability data corruption due to testing tuple + visibility without holding buffer lock (Thomas Munro, Peter Geoghegan, + Tom Lane) + + + + + + Fix file descriptor leakage when truncating a temporary relation of + more than 1GB (Andres Freund) + + + + + + Disallow starting a standalone backend with standby_mode + turned on (Michael Paquier) + + + + This can't do anything useful, since there will be no WAL receiver + process to fetch more WAL data; and it could result in misbehavior + in code that wasn't designed with this situation in mind. + + + + + + Don't try to share SSL contexts across multiple connections + in libpq (Heikki Linnakangas) + + + + This led to assorted corner-case bugs, particularly when trying to use + different SSL parameters for different connections. + + + + + + Avoid corner-case memory leak in libpq (Tom Lane) + + + + The reported problem involved leaking an error report + during PQreset(), but there might be related cases. + + + + + + Make ecpg's + + + + + In pg_dump, never dump range constructor functions + (Tom Lane) + + + + This oversight led to pg_upgrade failures with + extensions containing range types, due to duplicate creation of the + constructor functions. + + + + + + Fix contrib/intarray/bench/bench.pl to print the results + of the EXPLAIN it does when given the + + + + + Update Windows time zone mapping to recognize some time zone names + added in recent Windows versions (Michael Paquier) + + + + + + Prevent failure of obsolete dynamic time zone abbreviations (Tom Lane) + + + + If a dynamic time zone abbreviation does not match any entry in the + referenced time zone, treat it as equivalent to the time zone name. + This avoids unexpected failures when IANA removes abbreviations from + their time zone database, as they did in tzdata + release 2016f and seem likely to do again in the future. The + consequences were not limited to not recognizing the individual + abbreviation; any mismatch caused + the pg_timezone_abbrevs view to fail altogether. + + + + + + Update time zone data files to tzdata release 2016h + for DST law changes in Palestine and Turkey, plus historical + corrections for Turkey and some regions of Russia. + Switch to numeric abbreviations for some time zones in Antarctica, + the former Soviet Union, and Sri Lanka. + + + + The IANA time zone database previously provided textual abbreviations + for all time zones, sometimes making up abbreviations that have little + or no currency among the local population. They are in process of + reversing that policy in favor of using numeric UTC offsets in zones + where there is no evidence of real-world use of an English + abbreviation. At least for the time being, PostgreSQL + will continue to accept such removed abbreviations for timestamp input. + But they will not be shown in the pg_timezone_names + view nor used for output. + + + + In this update, AMT is no longer shown as being in use to + mean Armenia Time. Therefore, we have changed the Default + abbreviation set to interpret it as Amazon Time, thus UTC-4 not UTC+4. + + + + + + + + Release 9.2.18 diff --git a/doc/src/sgml/release-9.3.sgml b/doc/src/sgml/release-9.3.sgml index e321f4b31c..81205a40c7 100644 --- a/doc/src/sgml/release-9.3.sgml +++ b/doc/src/sgml/release-9.3.sgml @@ -1,6 +1,335 @@ + + Release 9.3.15 + + + Release Date + 2016-10-27 + + + + This release contains a variety of fixes from 9.3.14. + For information about new features in the 9.3 major release, see + . + + + + Migration to Version 9.3.15 + + + A dump/restore is not required for those running 9.3.X. + + + + However, if your installation has been affected by the bug described in + the first changelog entry below, then after updating you may need + to take action to repair corrupted free space maps. + + + + Also, if you are upgrading from a version earlier than 9.3.9, + see . + + + + + + Changes + + + + + + Fix WAL-logging of truncation of relation free space maps and + visibility maps (Pavan Deolasee, Heikki Linnakangas) + + + + It was possible for these files to not be correctly restored during + crash recovery, or to be written incorrectly on a standby server. + Bogus entries in a free space map could lead to attempts to access + pages that have been truncated away from the relation itself, typically + producing errors like could not read block XXX: + read only 0 of 8192 bytes. Checksum failures in the + visibility map are also possible, if checksumming is enabled. + + + + Procedures for determining whether there is a problem and repairing it + if so are discussed at + . + + + + + + Fix SELECT FOR UPDATE/SHARE to correctly lock tuples that + have been updated by a subsequently-aborted transaction + (Álvaro Herrera) + + + + In 9.5 and later, the SELECT would sometimes fail to + return such tuples at all. A failure has not been proven to occur in + earlier releases, but might be possible with concurrent updates. + + + + + + Fix EvalPlanQual rechecks involving CTE scans (Tom Lane) + + + + The recheck would always see the CTE as returning no rows, typically + leading to failure to update rows that were recently updated. + + + + + + Fix improper repetition of previous results from hashed aggregation in + a subquery (Andrew Gierth) + + + + The test to see if we can reuse a previously-computed hash table of + the aggregate state values neglected the possibility of an outer query + reference appearing in an aggregate argument expression. A change in + the value of such a reference should lead to recalculating the hash + table, but did not. + + + + + + Fix EXPLAIN to emit valid XML when + is on (Markus Winand) + + + + Previously the XML output-format option produced syntactically invalid + tags such as <I/O-Read-Time>. That is now + rendered as <I-O-Read-Time>. + + + + + + Suppress printing of zeroes for unmeasured times + in EXPLAIN (Maksim Milyutin) + + + + Certain option combinations resulted in printing zero values for times + that actually aren't ever measured in that combination. Our general + policy in EXPLAIN is not to print such fields at all, so + do that consistently in all cases. + + + + + + Fix timeout length when VACUUM is waiting for exclusive + table lock so that it can truncate the table (Simon Riggs) + + + + The timeout was meant to be 50 milliseconds, but it was actually only + 50 microseconds, causing VACUUM to give up on truncation + much more easily than intended. Set it to the intended value. + + + + + + Fix bugs in merging inherited CHECK constraints while + creating or altering a table (Tom Lane, Amit Langote) + + + + Allow identical CHECK constraints to be added to a parent + and child table in either order. Prevent merging of a valid + constraint from the parent table with a NOT VALID + constraint on the child. Likewise, prevent merging of a NO + INHERIT child constraint with an inherited constraint. + + + + + + Remove artificial restrictions on the values accepted + by numeric_in() and numeric_recv() + (Tom Lane) + + + + We allow numeric values up to the limit of the storage format (more + than 1e100000), so it seems fairly pointless + that numeric_in() rejected scientific-notation exponents + above 1000. Likewise, it was silly for numeric_recv() to + reject more than 1000 digits in an input value. + + + + + + Avoid very-low-probability data corruption due to testing tuple + visibility without holding buffer lock (Thomas Munro, Peter Geoghegan, + Tom Lane) + + + + + + Fix file descriptor leakage when truncating a temporary relation of + more than 1GB (Andres Freund) + + + + + + Disallow starting a standalone backend with standby_mode + turned on (Michael Paquier) + + + + This can't do anything useful, since there will be no WAL receiver + process to fetch more WAL data; and it could result in misbehavior + in code that wasn't designed with this situation in mind. + + + + + + Don't try to share SSL contexts across multiple connections + in libpq (Heikki Linnakangas) + + + + This led to assorted corner-case bugs, particularly when trying to use + different SSL parameters for different connections. + + + + + + Avoid corner-case memory leak in libpq (Tom Lane) + + + + The reported problem involved leaking an error report + during PQreset(), but there might be related cases. + + + + + + Make ecpg's + + + + + In pg_dump, never dump range constructor functions + (Tom Lane) + + + + This oversight led to pg_upgrade failures with + extensions containing range types, due to duplicate creation of the + constructor functions. + + + + + + In pg_xlogdump, retry opening new WAL segments when + using + + + This allows for a possible delay in the server's creation of the next + segment. + + + + + + Fix pg_xlogdump to cope with a WAL file that begins + with a continuation record spanning more than one page (Pavan + Deolasee) + + + + + + Fix contrib/intarray/bench/bench.pl to print the results + of the EXPLAIN it does when given the + + + + + Update Windows time zone mapping to recognize some time zone names + added in recent Windows versions (Michael Paquier) + + + + + + Prevent failure of obsolete dynamic time zone abbreviations (Tom Lane) + + + + If a dynamic time zone abbreviation does not match any entry in the + referenced time zone, treat it as equivalent to the time zone name. + This avoids unexpected failures when IANA removes abbreviations from + their time zone database, as they did in tzdata + release 2016f and seem likely to do again in the future. The + consequences were not limited to not recognizing the individual + abbreviation; any mismatch caused + the pg_timezone_abbrevs view to fail altogether. + + + + + + Update time zone data files to tzdata release 2016h + for DST law changes in Palestine and Turkey, plus historical + corrections for Turkey and some regions of Russia. + Switch to numeric abbreviations for some time zones in Antarctica, + the former Soviet Union, and Sri Lanka. + + + + The IANA time zone database previously provided textual abbreviations + for all time zones, sometimes making up abbreviations that have little + or no currency among the local population. They are in process of + reversing that policy in favor of using numeric UTC offsets in zones + where there is no evidence of real-world use of an English + abbreviation. At least for the time being, PostgreSQL + will continue to accept such removed abbreviations for timestamp input. + But they will not be shown in the pg_timezone_names + view nor used for output. + + + + In this update, AMT is no longer shown as being in use to + mean Armenia Time. Therefore, we have changed the Default + abbreviation set to interpret it as Amazon Time, thus UTC-4 not UTC+4. + + + + + + + + Release 9.3.14 diff --git a/doc/src/sgml/release-9.4.sgml b/doc/src/sgml/release-9.4.sgml index 51896924c9..94b028a065 100644 --- a/doc/src/sgml/release-9.4.sgml +++ b/doc/src/sgml/release-9.4.sgml @@ -1,6 +1,472 @@ + + Release 9.4.10 + + + Release Date + 2016-10-27 + + + + This release contains a variety of fixes from 9.4.9. + For information about new features in the 9.4 major release, see + . + + + + Migration to Version 9.4.10 + + + A dump/restore is not required for those running 9.4.X. + + + + However, if your installation has been affected by the bug described in + the first changelog entry below, then after updating you may need + to take action to repair corrupted free space maps. + + + + Also, if you are upgrading from a version earlier than 9.4.6, + see . + + + + + Changes + + + + + + Fix WAL-logging of truncation of relation free space maps and + visibility maps (Pavan Deolasee, Heikki Linnakangas) + + + + It was possible for these files to not be correctly restored during + crash recovery, or to be written incorrectly on a standby server. + Bogus entries in a free space map could lead to attempts to access + pages that have been truncated away from the relation itself, typically + producing errors like could not read block XXX: + read only 0 of 8192 bytes. Checksum failures in the + visibility map are also possible, if checksumming is enabled. + + + + Procedures for determining whether there is a problem and repairing it + if so are discussed at + . + + + + + + Fix incorrect creation of GIN index WAL records on big-endian machines + (Tom Lane) + + + + The typical symptom was unexpected GIN leaf action errors + during WAL replay. + + + + + + Fix SELECT FOR UPDATE/SHARE to correctly lock tuples that + have been updated by a subsequently-aborted transaction + (Álvaro Herrera) + + + + In 9.5 and later, the SELECT would sometimes fail to + return such tuples at all. A failure has not been proven to occur in + earlier releases, but might be possible with concurrent updates. + + + + + + Fix EvalPlanQual rechecks involving CTE scans (Tom Lane) + + + + The recheck would always see the CTE as returning no rows, typically + leading to failure to update rows that were recently updated. + + + + + + Fix improper repetition of previous results from hashed aggregation in + a subquery (Andrew Gierth) + + + + The test to see if we can reuse a previously-computed hash table of + the aggregate state values neglected the possibility of an outer query + reference appearing in an aggregate argument expression. A change in + the value of such a reference should lead to recalculating the hash + table, but did not. + + + + + + Fix query-lifespan memory leak in a bulk UPDATE on a table + with a PRIMARY KEY or REPLICA IDENTITY index + (Tom Lane) + + + + + + Fix EXPLAIN to emit valid XML when + is on (Markus Winand) + + + + Previously the XML output-format option produced syntactically invalid + tags such as <I/O-Read-Time>. That is now + rendered as <I-O-Read-Time>. + + + + + + Suppress printing of zeroes for unmeasured times + in EXPLAIN (Maksim Milyutin) + + + + Certain option combinations resulted in printing zero values for times + that actually aren't ever measured in that combination. Our general + policy in EXPLAIN is not to print such fields at all, so + do that consistently in all cases. + + + + + + Fix timeout length when VACUUM is waiting for exclusive + table lock so that it can truncate the table (Simon Riggs) + + + + The timeout was meant to be 50 milliseconds, but it was actually only + 50 microseconds, causing VACUUM to give up on truncation + much more easily than intended. Set it to the intended value. + + + + + + Fix bugs in merging inherited CHECK constraints while + creating or altering a table (Tom Lane, Amit Langote) + + + + Allow identical CHECK constraints to be added to a parent + and child table in either order. Prevent merging of a valid + constraint from the parent table with a NOT VALID + constraint on the child. Likewise, prevent merging of a NO + INHERIT child constraint with an inherited constraint. + + + + + + Remove artificial restrictions on the values accepted + by numeric_in() and numeric_recv() + (Tom Lane) + + + + We allow numeric values up to the limit of the storage format (more + than 1e100000), so it seems fairly pointless + that numeric_in() rejected scientific-notation exponents + above 1000. Likewise, it was silly for numeric_recv() to + reject more than 1000 digits in an input value. + + + + + + Avoid very-low-probability data corruption due to testing tuple + visibility without holding buffer lock (Thomas Munro, Peter Geoghegan, + Tom Lane) + + + + + + Fix logical WAL decoding to work properly when a subtransaction's WAL + output is large enough to spill to disk (Andres Freund) + + + + + + + Fix buffer overread in logical WAL decoding (Tom Lane) + + + + Logical decoding of a tuple update record read 23 bytes too many, + which was usually harmless but with very bad luck could result in a + crash. + + + + + + Fix file descriptor leakage when truncating a temporary relation of + more than 1GB (Andres Freund) + + + + + + Disallow starting a standalone backend with standby_mode + turned on (Michael Paquier) + + + + This can't do anything useful, since there will be no WAL receiver + process to fetch more WAL data; and it could result in misbehavior + in code that wasn't designed with this situation in mind. + + + + + + Properly initialize replication slot state when recycling a + previously-used slot (Michael Paquier) + + + + This failure to reset all of the fields of the slot could + prevent VACUUM from removing dead tuples. + + + + + + Round shared-memory allocation request to a multiple of the actual + huge page size when attempting to use huge pages on Linux (Tom Lane) + + + + This avoids possible failures during munmap() on systems + with atypical default huge page sizes. Except in crash-recovery + cases, there were no ill effects other than a log message. + + + + + + Use a more random value for the dynamic shared memory control + segment's ID (Robert Haas, Tom Lane) + + + + Previously, the same value would be chosen every time, because it was + derived from random() but srandom() had not + yet been called. While relatively harmless, this was not the intended + behavior. + + + + + + On Windows, retry creation of the dynamic shared memory control + segment after an access-denied error (Kyotaro Horiguchi, Amit Kapila) + + + + Windows sometimes returns ERROR_ACCESS_DENIED rather + than ERROR_ALREADY_EXISTS when there is an existing + segment. This led to postmaster startup failure due to believing that + the former was an unrecoverable error. + + + + + + Don't try to share SSL contexts across multiple connections + in libpq (Heikki Linnakangas) + + + + This led to assorted corner-case bugs, particularly when trying to use + different SSL parameters for different connections. + + + + + + Avoid corner-case memory leak in libpq (Tom Lane) + + + + The reported problem involved leaking an error report + during PQreset(), but there might be related cases. + + + + + + Make ecpg's + + + + + Fix pgbench's calculation of average latency + (Fabien Coelho) + + + + The calculation was incorrect when there were \sleep + commands in the script, or when the test duration was specified in + number of transactions rather than total time. + + + + + + In pg_dump, never dump range constructor functions + (Tom Lane) + + + + This oversight led to pg_upgrade failures with + extensions containing range types, due to duplicate creation of the + constructor functions. + + + + + + In pg_xlogdump, retry opening new WAL segments when + using + + + This allows for a possible delay in the server's creation of the next + segment. + + + + + + Fix pg_xlogdump to cope with a WAL file that begins + with a continuation record spanning more than one page (Pavan + Deolasee) + + + + + + Fix contrib/pg_buffercache to work + when shared_buffers exceeds 256GB (KaiGai Kohei) + + + + + + Fix contrib/intarray/bench/bench.pl to print the results + of the EXPLAIN it does when given the + + + + + Install TAP test infrastructure so that it's available for extension + testing (Craig Ringer) + + + + When PostgreSQL has been configured + with + + + + + In MSVC builds, include pg_recvlogical in a + client-only installation (MauMau) + + + + + + Update Windows time zone mapping to recognize some time zone names + added in recent Windows versions (Michael Paquier) + + + + + + Prevent failure of obsolete dynamic time zone abbreviations (Tom Lane) + + + + If a dynamic time zone abbreviation does not match any entry in the + referenced time zone, treat it as equivalent to the time zone name. + This avoids unexpected failures when IANA removes abbreviations from + their time zone database, as they did in tzdata + release 2016f and seem likely to do again in the future. The + consequences were not limited to not recognizing the individual + abbreviation; any mismatch caused + the pg_timezone_abbrevs view to fail altogether. + + + + + + Update time zone data files to tzdata release 2016h + for DST law changes in Palestine and Turkey, plus historical + corrections for Turkey and some regions of Russia. + Switch to numeric abbreviations for some time zones in Antarctica, + the former Soviet Union, and Sri Lanka. + + + + The IANA time zone database previously provided textual abbreviations + for all time zones, sometimes making up abbreviations that have little + or no currency among the local population. They are in process of + reversing that policy in favor of using numeric UTC offsets in zones + where there is no evidence of real-world use of an English + abbreviation. At least for the time being, PostgreSQL + will continue to accept such removed abbreviations for timestamp input. + But they will not be shown in the pg_timezone_names + view nor used for output. + + + + In this update, AMT is no longer shown as being in use to + mean Armenia Time. Therefore, we have changed the Default + abbreviation set to interpret it as Amazon Time, thus UTC-4 not UTC+4. + + + + + + + + Release 9.4.9 diff --git a/doc/src/sgml/release-9.5.sgml b/doc/src/sgml/release-9.5.sgml index c3f0f7051e..2102b1dd29 100644 --- a/doc/src/sgml/release-9.5.sgml +++ b/doc/src/sgml/release-9.5.sgml @@ -1,6 +1,818 @@ + + Release 9.5.5 + + + Release Date + 2016-10-27 + + + + This release contains a variety of fixes from 9.5.4. + For information about new features in the 9.5 major release, see + . + + + + Migration to Version 9.5.5 + + + A dump/restore is not required for those running 9.5.X. + + + + However, if your installation has been affected by the bug described in + the first changelog entry below, then after updating you may need + to take action to repair corrupted free space maps. + + + + Also, if you are upgrading from a version earlier than 9.5.2, + see . + + + + + Changes + + + + + + Fix WAL-logging of truncation of relation free space maps and + visibility maps (Pavan Deolasee, Heikki Linnakangas) + + + + It was possible for these files to not be correctly restored during + crash recovery, or to be written incorrectly on a standby server. + Bogus entries in a free space map could lead to attempts to access + pages that have been truncated away from the relation itself, typically + producing errors like could not read block XXX: + read only 0 of 8192 bytes. Checksum failures in the + visibility map are also possible, if checksumming is enabled. + + + + Procedures for determining whether there is a problem and repairing it + if so are discussed at + . + + + + + + + Fix incorrect creation of GIN index WAL records on big-endian machines + (Tom Lane) + + + + The typical symptom was unexpected GIN leaf action errors + during WAL replay. + + + + + + + Fix SELECT FOR UPDATE/SHARE to correctly lock tuples that + have been updated by a subsequently-aborted transaction + (Álvaro Herrera) + + + + In 9.5 and later, the SELECT would sometimes fail to + return such tuples at all. A failure has not been proven to occur in + earlier releases, but might be possible with concurrent updates. + + + + + + + Fix EvalPlanQual rechecks involving CTE scans (Tom Lane) + + + + The recheck would always see the CTE as returning no rows, typically + leading to failure to update rows that were recently updated. + + + + + + + Fix deletion of speculatively inserted TOAST tuples when backing out + of INSERT ... ON CONFLICT (Oskari Saarenmaa) + + + + In the race condition where two transactions try to insert conflicting + tuples at about the same time, the loser would fail with + an attempted to delete invisible tuple error if its + insertion included any TOAST'ed fields. + + + + + + Don't throw serialization errors for self-conflicting insertions + in INSERT ... ON CONFLICT (Thomas Munro, Peter Geoghegan) + + + + + + + Fix improper repetition of previous results from hashed aggregation in + a subquery (Andrew Gierth) + + + + The test to see if we can reuse a previously-computed hash table of + the aggregate state values neglected the possibility of an outer query + reference appearing in an aggregate argument expression. A change in + the value of such a reference should lead to recalculating the hash + table, but did not. + + + + + + + Fix query-lifespan memory leak in a bulk UPDATE on a table + with a PRIMARY KEY or REPLICA IDENTITY index + (Tom Lane) + + + + + + Fix COPY with a column name list from a table that has + row-level security enabled (Adam Brightwell) + + + + + + Fix EXPLAIN to emit valid XML when + is on (Markus Winand) + + + + Previously the XML output-format option produced syntactically invalid + tags such as <I/O-Read-Time>. That is now + rendered as <I-O-Read-Time>. + + + + + + + Suppress printing of zeroes for unmeasured times + in EXPLAIN (Maksim Milyutin) + + + + Certain option combinations resulted in printing zero values for times + that actually aren't ever measured in that combination. Our general + policy in EXPLAIN is not to print such fields at all, so + do that consistently in all cases. + + + + + + Fix statistics update for TRUNCATE in a prepared + transaction (Stas Kelvich) + + + + + + + Fix timeout length when VACUUM is waiting for exclusive + table lock so that it can truncate the table (Simon Riggs) + + + + The timeout was meant to be 50 milliseconds, but it was actually only + 50 microseconds, causing VACUUM to give up on truncation + much more easily than intended. Set it to the intended value. + + + + + + Fix bugs in merging inherited CHECK constraints while + creating or altering a table (Tom Lane, Amit Langote) + + + + Allow identical CHECK constraints to be added to a parent + and child table in either order. Prevent merging of a valid + constraint from the parent table with a NOT VALID + constraint on the child. Likewise, prevent merging of a NO + INHERIT child constraint with an inherited constraint. + + + + + + Show a sensible value + in pg_settings.unit + for min_wal_size and max_wal_size (Tom Lane) + + + + + + + Remove artificial restrictions on the values accepted + by numeric_in() and numeric_recv() + (Tom Lane) + + + + We allow numeric values up to the limit of the storage format (more + than 1e100000), so it seems fairly pointless + that numeric_in() rejected scientific-notation exponents + above 1000. Likewise, it was silly for numeric_recv() to + reject more than 1000 digits in an input value. + + + + + + Avoid very-low-probability data corruption due to testing tuple + visibility without holding buffer lock (Thomas Munro, Peter Geoghegan, + Tom Lane) + + + + + + Fix logical WAL decoding to work properly when a subtransaction's WAL + output is large enough to spill to disk (Andres Freund) + + + + + + + Fix possible sorting error when aborting use of abbreviated keys + (Peter Geoghegan) + + + + In the worst case, this could result in a corrupt btree index, which + would need to be rebuilt using REINDEX. However, the + situation is believed to be rare. + + + + + + + Fix file descriptor leakage when truncating a temporary relation of + more than 1GB (Andres Freund) + + + + + + + Disallow starting a standalone backend with standby_mode + turned on (Michael Paquier) + + + + This can't do anything useful, since there will be no WAL receiver + process to fetch more WAL data; and it could result in misbehavior + in code that wasn't designed with this situation in mind. + + + + + + + Properly initialize replication slot state when recycling a + previously-used slot (Michael Paquier) + + + + This failure to reset all of the fields of the slot could + prevent VACUUM from removing dead tuples. + + + + + + Round shared-memory allocation request to a multiple of the actual + huge page size when attempting to use huge pages on Linux (Tom Lane) + + + + This avoids possible failures during munmap() on systems + with atypical default huge page sizes. Except in crash-recovery + cases, there were no ill effects other than a log message. + + + + + + + Use a more random value for the dynamic shared memory control + segment's ID (Robert Haas, Tom Lane) + + + + Previously, the same value would be chosen every time, because it was + derived from random() but srandom() had not + yet been called. While relatively harmless, this was not the intended + behavior. + + + + + + + On Windows, retry creation of the dynamic shared memory control + segment after an access-denied error (Kyotaro Horiguchi, Amit Kapila) + + + + Windows sometimes returns ERROR_ACCESS_DENIED rather + than ERROR_ALREADY_EXISTS when there is an existing + segment. This led to postmaster startup failure due to believing that + the former was an unrecoverable error. + + + + + + + Fix PL/pgSQL to not misbehave with parameters and + local variables of type int2vector or oidvector + (Tom Lane) + + + + + + Don't try to share SSL contexts across multiple connections + in libpq (Heikki Linnakangas) + + + + This led to assorted corner-case bugs, particularly when trying to use + different SSL parameters for different connections. + + + + + + Avoid corner-case memory leak in libpq (Tom Lane) + + + + The reported problem involved leaking an error report + during PQreset(), but there might be related cases. + + + + + + + Make ecpg's + + + + + + Fix pgbench's calculation of average latency + (Fabien Coelho) + + + + The calculation was incorrect when there were \sleep + commands in the script, or when the test duration was specified in + number of transactions rather than total time. + + + + + + In pg_upgrade, check library loadability in name order + (Tom Lane) + + + + This is a workaround to deal with cross-extension dependencies from + language transform modules to their base language and data type + modules. + + + + + + + In pg_dump, never dump range constructor functions + (Tom Lane) + + + + This oversight led to pg_upgrade failures with + extensions containing range types, due to duplicate creation of the + constructor functions. + + + + + + + In pg_dump with + + + + + + Make pg_receivexlog work correctly + with + + + + + Disallow specifying both + + + + + Make pg_rewind turn off synchronous_commit + in its session on the source server (Michael Banck, Michael Paquier) + + + + This allows pg_rewind to work even when the source + server is using synchronous replication that is not working for some + reason. + + + + + + In pg_xlogdump, retry opening new WAL segments when + using + + + This allows for a possible delay in the server's creation of the next + segment. + + + + + + + Fix pg_xlogdump to cope with a WAL file that begins + with a continuation record spanning more than one page (Pavan + Deolasee) + + + + + + + Fix contrib/pg_buffercache to work + when shared_buffers exceeds 256GB (KaiGai Kohei) + + + + + + + Fix contrib/intarray/bench/bench.pl to print the results + of the EXPLAIN it does when given the + + + + + + Support OpenSSL 1.1.0 (Heikki Linnakangas) + + + + + + + Install TAP test infrastructure so that it's available for extension + testing (Craig Ringer) + + + + When PostgreSQL has been configured + with + + + + + + In MSVC builds, include pg_recvlogical in a + client-only installation (MauMau) + + + + + + + Update Windows time zone mapping to recognize some time zone names + added in recent Windows versions (Michael Paquier) + + + + + + + Prevent failure of obsolete dynamic time zone abbreviations (Tom Lane) + + + + If a dynamic time zone abbreviation does not match any entry in the + referenced time zone, treat it as equivalent to the time zone name. + This avoids unexpected failures when IANA removes abbreviations from + their time zone database, as they did in tzdata + release 2016f and seem likely to do again in the future. The + consequences were not limited to not recognizing the individual + abbreviation; any mismatch caused + the pg_timezone_abbrevs view to fail altogether. + + + + + + Update time zone data files to tzdata release 2016h + for DST law changes in Palestine and Turkey, plus historical + corrections for Turkey and some regions of Russia. + Switch to numeric abbreviations for some time zones in Antarctica, + the former Soviet Union, and Sri Lanka. + + + + The IANA time zone database previously provided textual abbreviations + for all time zones, sometimes making up abbreviations that have little + or no currency among the local population. They are in process of + reversing that policy in favor of using numeric UTC offsets in zones + where there is no evidence of real-world use of an English + abbreviation. At least for the time being, PostgreSQL + will continue to accept such removed abbreviations for timestamp input. + But they will not be shown in the pg_timezone_names + view nor used for output. + + + + In this update, AMT is no longer shown as being in use to + mean Armenia Time. Therefore, we have changed the Default + abbreviation set to interpret it as Amazon Time, thus UTC-4 not UTC+4. + + + + + + + + Release 9.5.4 diff --git a/doc/src/sgml/release-9.6.sgml b/doc/src/sgml/release-9.6.sgml index ebdeda4445..8b3f51428d 100644 --- a/doc/src/sgml/release-9.6.sgml +++ b/doc/src/sgml/release-9.6.sgml @@ -62,7 +62,7 @@ Branch: REL9_3_STABLE [1c02ee314] 2016-10-19 15:00:34 +0300 Procedures for determining whether there is a problem and repairing it if so are discussed at - . + . @@ -96,89 +96,20 @@ Branch: REL9_6_STABLE [b6d906073] 2016-09-30 20:39:06 -0400 with contrib/pg_visibility's pg_truncate_visibility_map() function. For more information see - . - - - - - - - Fix incorrect creation of GIN index WAL records on big-endian machines - (Tom Lane) - - - - The typical symptom was unexpected GIN leaf action errors - during WAL replay. - - - - - - - Fix SELECT FOR UPDATE/SHARE to correctly lock tuples that - have been updated by a subsequently-aborted transaction - (Álvaro Herrera) - - - - In 9.5 and later, the SELECT would sometimes fail to - return such tuples at all. A failure has not been proven to occur in - earlier releases, but might be possible with concurrent updates. + . - Fix EvalPlanQual rechecks involving CTE scans (Tom Lane) - - - - The recheck would always see the CTE as returning no rows, typically - leading to failure to update rows that were recently updated. - - - - - - - Fix deletion of speculatively inserted TOAST tuples when backing out - of INSERT ... ON CONFLICT (Oskari Saarenmaa) - - - - In the race condition where two transactions try to insert conflicting - tuples at about the same time, the loser would fail with - an attempted to delete invisible tuple error if its - insertion included any TOAST'ed fields. + Don't throw serialization errors for self-conflicting insertions + in INSERT ... ON CONFLICT (Thomas Munro, Peter Geoghegan) @@ -219,46 +150,6 @@ Branch: REL9_6_STABLE [dca25c256] 2016-10-09 12:49:37 -0400 - - Fix improper repetition of previous results from hashed aggregation in - a subquery (Andrew Gierth) - - - - The test to see if we can reuse a previously-computed hash table of - the aggregate state values neglected the possibility of an outer query - reference appearing in an aggregate argument expression. A change in - the value of such a reference should lead to recalculating the hash - table, but did not. - - - - - - - Fix query-lifespan memory leak in a bulk UPDATE on a table - with a PRIMARY KEY or REPLICA IDENTITY index - (Tom Lane) - - - - - - - Suppress printing of zeroes for unmeasured times - in EXPLAIN (Maksim Milyutin) - - - - Certain option combinations resulted in printing zero values for times - that actually aren't ever measured in that combination. Our general - policy in EXPLAIN is not to print such fields at all, so - do that consistently in all cases. - - - - - - - Fix timeout length when VACUUM is waiting for exclusive - table lock so that it can truncate the table (Simon Riggs) - - - - The timeout was meant to be 50 milliseconds, but it was actually only - 50 microseconds, causing VACUUM to give up on truncation - much more easily than intended. Set it to the intended value. - - - - - - Remove artificial restrictions on the values accepted - by numeric_in() and numeric_recv() - (Tom Lane) - - - - We allow numeric values up to the limit of the storage format (more - than 1e100000), so it seems fairly pointless - that numeric_in() rejected scientific-notation exponents - above 1000. Likewise, it was silly for numeric_recv() to - reject more than 1000 digits in an input value. + Avoid very-low-probability data corruption due to testing tuple + visibility without holding buffer lock (Thomas Munro, Peter Geoghegan, + Tom Lane) @@ -464,100 +306,6 @@ Branch: REL9_6_STABLE [32841fa32] 2016-09-28 11:22:39 -0400 - - Fix buffer overread in logical WAL decoding (Tom Lane) - - - - Logical decoding of a tuple update record read 23 bytes too many, - which was usually harmless but with very bad luck could result in a - crash. - - - - - - - Fix possible sorting error when aborting use of abbreviated keys - (Peter Geoghegan) - - - - In the worst case, this could result in a corrupt btree index, which - would need to be rebuilt using REINDEX. However, the - situation is believed to be rare. - - - - - - - Fix file descriptor leakage when truncating a temporary relation of - more than 1GB (Andres Freund) - - - - - - - Disallow starting a standalone backend with standby_mode - turned on (Michael Paquier) - - - - This can't do anything useful, since there will be no WAL receiver - process to fetch more WAL data; and it could result in misbehavior - in code that wasn't designed with this situation in mind. - - - - - - - Properly initialize replication slot state when recycling a - previously-used slot (Michael Paquier) - - - - This failure to reset all of the fields of the slot could - prevent VACUUM from removing dead tuples. - - - - - - - Use a more random value for the dynamic shared memory control - segment's ID (Robert Haas, Tom Lane) - - - - Previously, the same value would be chosen every time, because it was - derived from random() but srandom() had not - yet been called. While relatively harmless, this was not the intended - behavior. - - - - - - - On Windows, retry creation of the dynamic shared memory control - segment after an access-denied error (Kyotaro Horiguchi, Amit Kapila) - - - - Windows sometimes returns ERROR_ACCESS_DENIED rather - than ERROR_ALREADY_EXISTS when there is an existing - segment. This led to postmaster startup failure due to believing that - the former was an unrecoverable error. - - - - - - - Fix PL/pgSQL to not misbehave with parameters and - local variables of type int2vector or oidvector - (Tom Lane) - - - - - - - Make ecpg's - - - - - - Fix pgbench's calculation of average latency - (Fabien Coelho) - - - - The calculation was incorrect when there were \sleep - commands in the script, or when the test duration was specified in - number of transactions rather than total time. - - - - - - - In pg_dump, never dump range constructor functions - (Tom Lane) - - - - This oversight led to pg_upgrade failures with - extensions containing range types, due to duplicate creation of the - constructor functions. - - - - - - - In pg_dump with - - - - @@ -827,20 +437,6 @@ Branch: REL9_6_STABLE [1749332ec] 2016-10-07 09:51:28 -0400 - - Make pg_receivexlog work correctly - with - - - - - - Fix pg_xlogdump to cope with a WAL file that begins - with a continuation record spanning more than one page (Pavan - Deolasee) - - - - - - - Fix contrib/pg_buffercache to work - when shared_buffers exceeds 256GB (KaiGai Kohei) - - - - - - - Fix contrib/intarray/bench/bench.pl to print the results - of the EXPLAIN it does when given the - - - - - - Support OpenSSL 1.1.0 (Heikki Linnakangas) - - - - - - - Install TAP test infrastructure so that it's available for extension - testing (Craig Ringer) - - - - When PostgreSQL has been configured - with - - - - - - In MSVC builds, include pg_recvlogical in a - client-only installation (MauMau) - - - - - - - Update Windows time zone mapping to recognize some time zone names - added in recent Windows versions (Michael Paquier) - - - - - - - Prevent failure of obsolete dynamic time zone abbreviations (Tom Lane) - - - - If a dynamic time zone abbreviation does not match any entry in the - referenced time zone, treat it as equivalent to the time zone name. - This avoids unexpected failures when IANA removes abbreviations from - their time zone database, as they did in tzdata - release 2016f and seem likely to do again in the future. The - consequences were not limited to not recognizing the individual - abbreviation; any mismatch caused - the pg_timezone_abbrevs view to fail altogether. - - - - - + + Preserve commit timestamps across server restart + (Julien Rouhaud, Craig Ringer) + + + + With turned on, old + commit timestamps became inaccessible after a clean server restart. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +

+ + + + + + + + + + + + +

+ + + + + + +
+

+ + + +

+
+ + + + + + + +
+
+ + + + + + +
+

+ + + +

+
+ + + + + + + +
+
+ + + + + + + + + + + +
+ + + + + + + + + + +
+ + + + + + +

+ +

+ +
+ + + + + + + +
+
+ + + + + + + + + + + + + + | + + + + + + + + + + -- cgit v1.2.3 From 59fa9d2d9da46097dd4da5c5f1f07e22a288fccf Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 14 Oct 2016 12:00:00 -0400 Subject: pg_test_timing: Add NLS Also straighten out use of time unit abbreviations a bit. Reviewed-by: Michael Paquier --- doc/src/sgml/ref/pgtesttiming.sgml | 16 ++++++++-------- src/bin/pg_test_timing/nls.mk | 4 ++++ src/bin/pg_test_timing/pg_test_timing.c | 28 ++++++++++++++++------------ 3 files changed, 28 insertions(+), 20 deletions(-) create mode 100644 src/bin/pg_test_timing/nls.mk (limited to 'doc/src') diff --git a/doc/src/sgml/ref/pgtesttiming.sgml b/doc/src/sgml/ref/pgtesttiming.sgml index f07a0600ff..e3539cf764 100644 --- a/doc/src/sgml/ref/pgtesttiming.sgml +++ b/doc/src/sgml/ref/pgtesttiming.sgml @@ -96,9 +96,9 @@ Testing timing overhead for 3 seconds. -Per loop time including overhead: 35.96 nsec +Per loop time including overhead: 35.96 ns Histogram of timing durations: -< usec % of total count + < us % of total count 1 96.40465 80435604 2 3.59518 2999652 4 0.00015 126 @@ -109,9 +109,9 @@ Histogram of timing durations: Note that different units are used for the per loop time than the - histogram. The loop can have resolution within a few nanoseconds (nsec), + histogram. The loop can have resolution within a few nanoseconds (ns), while the individual timing calls can only resolve down to one microsecond - (usec). + (us). @@ -157,9 +157,9 @@ EXPLAIN ANALYZE SELECT COUNT(*) FROM t; tsc hpet acpi_pm # echo acpi_pm > /sys/devices/system/clocksource/clocksource0/current_clocksource # pg_test_timing -Per loop time including overhead: 722.92 nsec +Per loop time including overhead: 722.92 ns Histogram of timing durations: -< usec % of total count + < us % of total count 1 27.84870 1155682 2 72.05956 2990371 4 0.07810 3241 @@ -170,7 +170,7 @@ Histogram of timing durations: In this configuration, the sample EXPLAIN ANALYZE above - takes 115.9 ms. That's 1061 nsec of timing overhead, again a small multiple + takes 115.9 ms. That's 1061 ns of timing overhead, again a small multiple of what's measured directly by this utility. That much timing overhead means the actual query itself is only taking a tiny fraction of the accounted for time, most of it is being consumed in overhead instead. In @@ -211,7 +211,7 @@ $ pg_test_timing Testing timing overhead for 3 seconds. Per timing duration including loop overhead: 97.75 ns Histogram of timing durations: -< usec % of total count + < us % of total count 1 90.23734 27694571 2 9.75277 2993204 4 0.00981 3010 diff --git a/src/bin/pg_test_timing/nls.mk b/src/bin/pg_test_timing/nls.mk new file mode 100644 index 0000000000..e12ea5cfdb --- /dev/null +++ b/src/bin/pg_test_timing/nls.mk @@ -0,0 +1,4 @@ +# src/bin/pg_test_timing/nls.mk +CATALOG_NAME = pg_test_timing +AVAIL_LANGUAGES = +GETTEXT_FILES = pg_test_timing.c diff --git a/src/bin/pg_test_timing/pg_test_timing.c b/src/bin/pg_test_timing/pg_test_timing.c index e5c11de6bb..2f1ab7cd60 100644 --- a/src/bin/pg_test_timing/pg_test_timing.c +++ b/src/bin/pg_test_timing/pg_test_timing.c @@ -25,6 +25,7 @@ main(int argc, char *argv[]) { uint64 loop_count; + set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_test_timing")); progname = get_progname(argv[0]); handle_args(argc, argv); @@ -51,7 +52,7 @@ handle_args(int argc, char *argv[]) { if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0) { - printf("Usage: %s [-d DURATION]\n", progname); + printf(_("Usage: %s [-d DURATION]\n"), progname); exit(0); } if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0) @@ -71,7 +72,7 @@ handle_args(int argc, char *argv[]) break; default: - fprintf(stderr, "Try \"%s --help\" for more information.\n", + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); exit(1); break; @@ -81,23 +82,26 @@ handle_args(int argc, char *argv[]) if (argc > optind) { fprintf(stderr, - "%s: too many command-line arguments (first is \"%s\")\n", + _("%s: too many command-line arguments (first is \"%s\")\n"), progname, argv[optind]); - fprintf(stderr, "Try \"%s --help\" for more information.\n", + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); exit(1); } if (test_duration > 0) { - printf("Testing timing overhead for %d seconds.\n", test_duration); + printf(ngettext("Testing timing overhead for %d second.\n", + "Testing timing overhead for %d seconds.\n", + test_duration), + test_duration); } else { fprintf(stderr, - "%s: duration must be a positive integer (duration is \"%d\")\n", + _("%s: duration must be a positive integer (duration is \"%d\")\n"), progname, test_duration); - fprintf(stderr, "Try \"%s --help\" for more information.\n", + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); exit(1); } @@ -133,8 +137,8 @@ test_timing(int32 duration) /* Did time go backwards? */ if (diff < 0) { - printf("Detected clock going backwards in time.\n"); - printf("Time warp: %d microseconds\n", diff); + fprintf(stderr, _("Detected clock going backwards in time.\n")); + fprintf(stderr, _("Time warp: %d ms\n"), diff); exit(1); } @@ -157,7 +161,7 @@ test_timing(int32 duration) INSTR_TIME_SUBTRACT(end_time, start_time); - printf("Per loop time including overhead: %0.2f nsec\n", + printf(_("Per loop time including overhead: %0.2f ns\n"), INSTR_TIME_GET_DOUBLE(end_time) * 1e9 / loop_count); return loop_count; @@ -173,8 +177,8 @@ output(uint64 loop_count) while (max_bit > 0 && histogram[max_bit] == 0) max_bit--; - printf("Histogram of timing durations:\n"); - printf("%6s %10s %10s\n", "< usec", "% of total", "count"); + printf(_("Histogram of timing durations:\n")); + printf("%6s %10s %10s\n", _("< us"), _("% of total"), _("count")); for (i = 0; i <= max_bit; i++) { -- cgit v1.2.3 From 8c48375e5f43ebd832f93c9166d1fe0e639ff806 Mon Sep 17 00:00:00 2001 From: Kevin Grittner Date: Fri, 4 Nov 2016 10:49:50 -0500 Subject: Implement syntax for transition tables in AFTER triggers. This is infrastructure for the complete SQL standard feature. No support is included at this point for execution nodes or PLs. The intent is to add that soon. As this patch leaves things, standard syntax can create tuplestores to contain old and/or new versions of rows affected by a statement. References to these tuplestores are in the TriggerData structure. C triggers can access the tuplestores directly, so they are usable, but they cannot yet be referenced within a SQL statement. --- doc/src/sgml/catalogs.sgml | 16 ++ doc/src/sgml/ref/create_trigger.sgml | 94 ++++++++-- src/backend/commands/tablecmds.c | 5 +- src/backend/commands/trigger.c | 327 +++++++++++++++++++++++++++++++++-- src/backend/nodes/copyfuncs.c | 16 ++ src/backend/nodes/equalfuncs.c | 14 ++ src/backend/nodes/outfuncs.c | 13 ++ src/backend/parser/gram.y | 70 +++++++- src/backend/utils/adt/ruleutils.c | 23 +++ src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_trigger.h | 13 +- src/include/commands/trigger.h | 2 + src/include/nodes/nodes.h | 1 + src/include/nodes/parsenodes.h | 17 ++ src/include/parser/kwlist.h | 3 + src/include/utils/reltrigger.h | 7 + 16 files changed, 580 insertions(+), 43 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 29738b07cb..bac169a19e 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -6231,6 +6231,22 @@ representation) for the trigger's WHEN condition, or null if none + + + tgoldtable + name + + REFERENCING clause name for OLD TABLE, + or null if none + + + + tgnewtable + name + + REFERENCING clause name for NEW TABLE, + or null if none +
diff --git a/doc/src/sgml/ref/create_trigger.sgml b/doc/src/sgml/ref/create_trigger.sgml index 4bde815012..8590e226e3 100644 --- a/doc/src/sgml/ref/create_trigger.sgml +++ b/doc/src/sgml/ref/create_trigger.sgml @@ -25,6 +25,7 @@ CREATE [ CONSTRAINT ] TRIGGER name ON table_name [ FROM referenced_table_name ] [ NOT DEFERRABLE | [ DEFERRABLE ] [ INITIALLY IMMEDIATE | INITIALLY DEFERRED ] ] + [ REFERENCING { { OLD | NEW } TABLE [ AS ] transition_relation_name } [ ... ] ] [ FOR [ EACH ] { ROW | STATEMENT } ] [ WHEN ( condition ) ] EXECUTE PROCEDURE function_name ( arguments ) @@ -177,6 +178,15 @@ CREATE [ CONSTRAINT ] TRIGGER name when the constraints they implement are violated. + + The REFERENCING option is only allowed for an AFTER + trigger which is not a constraint trigger. OLD TABLE may only + be specified once, and only on a trigger which can fire on + UPDATE or DELETE. NEW TABLE may only + be specified once, and only on a trigger which can fire on + UPDATE or INSERT. + + SELECT does not modify any rows so you cannot create SELECT triggers. Rules and views are more @@ -281,6 +291,40 @@ UPDATE OF column_name1 [, column_name2 + + REFERENCING + + + This immediately preceeds the declaration of one or two relations which + can be used to read the before and/or after images of all rows directly + affected by the triggering statement. An AFTER EACH ROW + trigger is allowed to use both these transition relation names and the + row names (OLD and NEW) which reference each + individual row for which the trigger fires. + + + + + + OLD TABLE + NEW TABLE + + + This specifies whether the named relation contains the before or after + images for rows affected by the statement which fired the trigger. + + + + + + transition_relation_name + + + The (unqualified) name to be used within the trigger for this relation. + + + + FOR EACH ROW FOR EACH STATEMENT @@ -474,6 +518,30 @@ CREATE TRIGGER view_insert FOR EACH ROW EXECUTE PROCEDURE view_insert_row(); + + Execute the function check_transfer_balances_to_zero for each + statement to confirm that the transfer rows offset to a net of + zero: + + +CREATE TRIGGER transfer_insert + AFTER INSERT ON transfer + FOR EACH STATEMENT + REFERENCING NEW TABLE AS inserted + EXECUTE PROCEDURE check_transfer_balances_to_zero(); + + + Execute the function check_matching_pairs for each row to + confirm that changes are made to matching pairs at the same time (by the + same statement): + + +CREATE TRIGGER paired_items_update + AFTER UPDATE ON paired_items + FOR EACH ROW + REFERENCING NEW TABLE AS newtab OLD TABLE AS oldtab + EXECUTE PROCEDURE check_matching_pairs(); + @@ -502,24 +570,14 @@ CREATE TRIGGER view_insert - SQL allows you to define aliases for the old - and new rows or tables for use in the definition - of the triggered action (e.g., CREATE TRIGGER ... ON - tablename REFERENCING OLD ROW AS somename NEW ROW AS othername - ...). Since PostgreSQL - allows trigger procedures to be written in any number of - user-defined languages, access to the data is handled in a - language-specific way. - - - - - - PostgreSQL does not allow the old and new - tables to be referenced in statement-level triggers, i.e., the tables - that contain all the old and/or new rows, which are referred to by the - OLD TABLE and NEW TABLE clauses in - the SQL standard. + While transition tables for AFTER triggers are specified + using the REFERENCING clause in the standard way, the row + variables used in FOR EACH ROW triggers may not be + specified in REFERENCING clause. They are available in a + manner which is dependent on the language in which the trigger function + is written. Some languages effectively behave as though there is a + REFERENCING clause containing OLD ROW AS OLD NEW + ROW AS NEW. diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 2137372c23..f97bee5b0e 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -7430,7 +7430,7 @@ validateForeignKeyConstraint(char *conname, trig.tgconstraint = constraintOid; trig.tgdeferrable = FALSE; trig.tginitdeferred = FALSE; - /* we needn't fill in tgargs or tgqual */ + /* we needn't fill in remaining fields */ /* * See if we can do it with a single LEFT JOIN query. A FALSE result @@ -7514,6 +7514,7 @@ CreateFKCheckTrigger(Oid myRelOid, Oid refRelOid, Constraint *fkconstraint, } fk_trigger->columns = NIL; + fk_trigger->transitionRels = NIL; fk_trigger->whenClause = NULL; fk_trigger->isconstraint = true; fk_trigger->deferrable = fkconstraint->deferrable; @@ -7557,6 +7558,7 @@ createForeignKeyTriggers(Relation rel, Oid refRelOid, Constraint *fkconstraint, fk_trigger->timing = TRIGGER_TYPE_AFTER; fk_trigger->events = TRIGGER_TYPE_DELETE; fk_trigger->columns = NIL; + fk_trigger->transitionRels = NIL; fk_trigger->whenClause = NULL; fk_trigger->isconstraint = true; fk_trigger->constrrel = NULL; @@ -7611,6 +7613,7 @@ createForeignKeyTriggers(Relation rel, Oid refRelOid, Constraint *fkconstraint, fk_trigger->timing = TRIGGER_TYPE_AFTER; fk_trigger->events = TRIGGER_TYPE_UPDATE; fk_trigger->columns = NIL; + fk_trigger->transitionRels = NIL; fk_trigger->whenClause = NULL; fk_trigger->isconstraint = true; fk_trigger->constrrel = NULL; diff --git a/src/backend/commands/trigger.c b/src/backend/commands/trigger.c index 9de22a13d7..1c264b7736 100644 --- a/src/backend/commands/trigger.c +++ b/src/backend/commands/trigger.c @@ -164,6 +164,8 @@ CreateTrigger(CreateTrigStmt *stmt, const char *queryString, Oid constrrelid = InvalidOid; ObjectAddress myself, referenced; + char *oldtablename = NULL; + char *newtablename = NULL; if (OidIsValid(relOid)) rel = heap_open(relOid, ShareRowExclusiveLock); @@ -309,6 +311,87 @@ CreateTrigger(CreateTrigStmt *stmt, const char *queryString, errmsg("INSTEAD OF triggers cannot have column lists"))); } + /* + * We don't yet support naming ROW transition variables, but the parser + * recognizes the syntax so we can give a nicer message here. + * + * Per standard, REFERENCING TABLE names are only allowed on AFTER + * triggers. Per standard, REFERENCING ROW names are not allowed with FOR + * EACH STATEMENT. Per standard, each OLD/NEW, ROW/TABLE permutation is + * only allowed once. Per standard, OLD may not be specified when + * creating a trigger only for INSERT, and NEW may not be specified when + * creating a trigger only for DELETE. + * + * Notice that the standard allows an AFTER ... FOR EACH ROW trigger to + * reference both ROW and TABLE transition data. + */ + if (stmt->transitionRels != NIL) + { + List *varList = stmt->transitionRels; + ListCell *lc; + + foreach(lc, varList) + { + TriggerTransition *tt = (TriggerTransition *) lfirst(lc); + + Assert(IsA(tt, TriggerTransition)); + + if (!(tt->isTable)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("ROW variable naming in the REFERENCING clause is not supported"), + errhint("Use OLD TABLE or NEW TABLE for naming transition tables."))); + + /* + * Because of the above test, we omit further ROW-related testing + * below. If we later allow naming OLD and NEW ROW variables, + * adjustments will be needed below. + */ + + if (stmt->timing != TRIGGER_TYPE_AFTER) + ereport(ERROR, + (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), + errmsg("transition table name can only be specified for an AFTER trigger"))); + + if (tt->isNew) + { + if (!(TRIGGER_FOR_INSERT(tgtype) || + TRIGGER_FOR_UPDATE(tgtype))) + ereport(ERROR, + (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), + errmsg("NEW TABLE can only be specified for an INSERT or UPDATE trigger"))); + + if (newtablename != NULL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), + errmsg("NEW TABLE cannot be specified multiple times"))); + + newtablename = tt->name; + } + else + { + if (!(TRIGGER_FOR_DELETE(tgtype) || + TRIGGER_FOR_UPDATE(tgtype))) + ereport(ERROR, + (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), + errmsg("OLD TABLE can only be specified for a DELETE or UPDATE trigger"))); + + if (oldtablename != NULL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), + errmsg("OLD TABLE cannot be specified multiple times"))); + + oldtablename = tt->name; + } + } + + if (newtablename != NULL && oldtablename != NULL && + strcmp(newtablename, oldtablename) == 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), + errmsg("OLD TABLE name and NEW TABLE name cannot be the same"))); + } + /* * Parse the WHEN clause, if any */ @@ -664,6 +747,17 @@ CreateTrigger(CreateTrigStmt *stmt, const char *queryString, else nulls[Anum_pg_trigger_tgqual - 1] = true; + if (oldtablename) + values[Anum_pg_trigger_tgoldtable - 1] = DirectFunctionCall1(namein, + CStringGetDatum(oldtablename)); + else + nulls[Anum_pg_trigger_tgoldtable - 1] = true; + if (newtablename) + values[Anum_pg_trigger_tgnewtable - 1] = DirectFunctionCall1(namein, + CStringGetDatum(newtablename)); + else + nulls[Anum_pg_trigger_tgnewtable - 1] = true; + tuple = heap_form_tuple(tgrel->rd_att, values, nulls); /* force tuple to have the desired OID */ @@ -682,6 +776,10 @@ CreateTrigger(CreateTrigStmt *stmt, const char *queryString, pfree(DatumGetPointer(values[Anum_pg_trigger_tgname - 1])); pfree(DatumGetPointer(values[Anum_pg_trigger_tgargs - 1])); pfree(DatumGetPointer(values[Anum_pg_trigger_tgattr - 1])); + if (oldtablename) + pfree(DatumGetPointer(values[Anum_pg_trigger_tgoldtable - 1])); + if (newtablename) + pfree(DatumGetPointer(values[Anum_pg_trigger_tgnewtable - 1])); /* * Update relation's pg_class entry. Crucial side-effect: other backends @@ -1584,6 +1682,23 @@ RelationBuildTriggers(Relation relation) } else build->tgargs = NULL; + + datum = fastgetattr(htup, Anum_pg_trigger_tgoldtable, + tgrel->rd_att, &isnull); + if (!isnull) + build->tgoldtable = + DatumGetCString(DirectFunctionCall1(nameout, datum)); + else + build->tgoldtable = NULL; + + datum = fastgetattr(htup, Anum_pg_trigger_tgnewtable, + tgrel->rd_att, &isnull); + if (!isnull) + build->tgnewtable = + DatumGetCString(DirectFunctionCall1(nameout, datum)); + else + build->tgnewtable = NULL; + datum = fastgetattr(htup, Anum_pg_trigger_tgqual, tgrel->rd_att, &isnull); if (!isnull) @@ -1680,6 +1795,19 @@ SetTriggerFlags(TriggerDesc *trigdesc, Trigger *trigger) trigdesc->trig_truncate_after_statement |= TRIGGER_TYPE_MATCHES(tgtype, TRIGGER_TYPE_STATEMENT, TRIGGER_TYPE_AFTER, TRIGGER_TYPE_TRUNCATE); + + trigdesc->trig_insert_new_table |= + (TRIGGER_FOR_INSERT(tgtype) && + TRIGGER_USES_TRANSITION_TABLE(trigger->tgnewtable)); + trigdesc->trig_update_old_table |= + (TRIGGER_FOR_UPDATE(tgtype) && + TRIGGER_USES_TRANSITION_TABLE(trigger->tgoldtable)); + trigdesc->trig_update_new_table |= + (TRIGGER_FOR_UPDATE(tgtype) && + TRIGGER_USES_TRANSITION_TABLE(trigger->tgnewtable)); + trigdesc->trig_delete_old_table |= + (TRIGGER_FOR_DELETE(tgtype) && + TRIGGER_USES_TRANSITION_TABLE(trigger->tgoldtable)); } /* @@ -1729,6 +1857,10 @@ CopyTriggerDesc(TriggerDesc *trigdesc) } if (trigger->tgqual) trigger->tgqual = pstrdup(trigger->tgqual); + if (trigger->tgoldtable) + trigger->tgoldtable = pstrdup(trigger->tgoldtable); + if (trigger->tgnewtable) + trigger->tgnewtable = pstrdup(trigger->tgnewtable); trigger++; } @@ -1761,6 +1893,10 @@ FreeTriggerDesc(TriggerDesc *trigdesc) } if (trigger->tgqual) pfree(trigger->tgqual); + if (trigger->tgoldtable) + pfree(trigger->tgoldtable); + if (trigger->tgnewtable) + pfree(trigger->tgnewtable); trigger++; } pfree(trigdesc->triggers); @@ -1839,6 +1975,18 @@ equalTriggerDescs(TriggerDesc *trigdesc1, TriggerDesc *trigdesc2) return false; else if (strcmp(trig1->tgqual, trig2->tgqual) != 0) return false; + if (trig1->tgoldtable == NULL && trig2->tgoldtable == NULL) + /* ok */ ; + else if (trig1->tgoldtable == NULL || trig2->tgoldtable == NULL) + return false; + else if (strcmp(trig1->tgoldtable, trig2->tgoldtable) != 0) + return false; + if (trig1->tgnewtable == NULL && trig2->tgnewtable == NULL) + /* ok */ ; + else if (trig1->tgnewtable == NULL || trig2->tgnewtable == NULL) + return false; + else if (strcmp(trig1->tgnewtable, trig2->tgnewtable) != 0) + return false; } } else if (trigdesc2 != NULL) @@ -1870,6 +2018,18 @@ ExecCallTriggerFunc(TriggerData *trigdata, Datum result; MemoryContext oldContext; + /* + * Protect against code paths that may fail to initialize transition table + * info. + */ + Assert(((TRIGGER_FIRED_BY_INSERT(trigdata->tg_event) || + TRIGGER_FIRED_BY_UPDATE(trigdata->tg_event) || + TRIGGER_FIRED_BY_DELETE(trigdata->tg_event)) && + TRIGGER_FIRED_AFTER(trigdata->tg_event) && + !(trigdata->tg_event & AFTER_TRIGGER_DEFERRABLE) && + !(trigdata->tg_event & AFTER_TRIGGER_INITDEFERRED)) || + (trigdata->tg_oldtable == NULL && trigdata->tg_newtable == NULL)); + finfo += tgindx; /* @@ -1960,6 +2120,8 @@ ExecBSInsertTriggers(EState *estate, ResultRelInfo *relinfo) LocTriggerData.tg_relation = relinfo->ri_RelationDesc; LocTriggerData.tg_trigtuple = NULL; LocTriggerData.tg_newtuple = NULL; + LocTriggerData.tg_oldtable = NULL; + LocTriggerData.tg_newtable = NULL; LocTriggerData.tg_trigtuplebuf = InvalidBuffer; LocTriggerData.tg_newtuplebuf = InvalidBuffer; for (i = 0; i < trigdesc->numtriggers; i++) @@ -2017,6 +2179,8 @@ ExecBRInsertTriggers(EState *estate, ResultRelInfo *relinfo, TRIGGER_EVENT_BEFORE; LocTriggerData.tg_relation = relinfo->ri_RelationDesc; LocTriggerData.tg_newtuple = NULL; + LocTriggerData.tg_oldtable = NULL; + LocTriggerData.tg_newtable = NULL; LocTriggerData.tg_newtuplebuf = InvalidBuffer; for (i = 0; i < trigdesc->numtriggers; i++) { @@ -2070,7 +2234,8 @@ ExecARInsertTriggers(EState *estate, ResultRelInfo *relinfo, { TriggerDesc *trigdesc = relinfo->ri_TrigDesc; - if (trigdesc && trigdesc->trig_insert_after_row) + if (trigdesc && + (trigdesc->trig_insert_after_row || trigdesc->trig_insert_new_table)) AfterTriggerSaveEvent(estate, relinfo, TRIGGER_EVENT_INSERT, true, NULL, trigtuple, recheckIndexes, NULL); } @@ -2092,6 +2257,8 @@ ExecIRInsertTriggers(EState *estate, ResultRelInfo *relinfo, TRIGGER_EVENT_INSTEAD; LocTriggerData.tg_relation = relinfo->ri_RelationDesc; LocTriggerData.tg_newtuple = NULL; + LocTriggerData.tg_oldtable = NULL; + LocTriggerData.tg_newtable = NULL; LocTriggerData.tg_newtuplebuf = InvalidBuffer; for (i = 0; i < trigdesc->numtriggers; i++) { @@ -2159,6 +2326,8 @@ ExecBSDeleteTriggers(EState *estate, ResultRelInfo *relinfo) LocTriggerData.tg_relation = relinfo->ri_RelationDesc; LocTriggerData.tg_trigtuple = NULL; LocTriggerData.tg_newtuple = NULL; + LocTriggerData.tg_oldtable = NULL; + LocTriggerData.tg_newtable = NULL; LocTriggerData.tg_trigtuplebuf = InvalidBuffer; LocTriggerData.tg_newtuplebuf = InvalidBuffer; for (i = 0; i < trigdesc->numtriggers; i++) @@ -2230,6 +2399,8 @@ ExecBRDeleteTriggers(EState *estate, EPQState *epqstate, TRIGGER_EVENT_BEFORE; LocTriggerData.tg_relation = relinfo->ri_RelationDesc; LocTriggerData.tg_newtuple = NULL; + LocTriggerData.tg_oldtable = NULL; + LocTriggerData.tg_newtable = NULL; LocTriggerData.tg_newtuplebuf = InvalidBuffer; for (i = 0; i < trigdesc->numtriggers; i++) { @@ -2273,7 +2444,8 @@ ExecARDeleteTriggers(EState *estate, ResultRelInfo *relinfo, { TriggerDesc *trigdesc = relinfo->ri_TrigDesc; - if (trigdesc && trigdesc->trig_delete_after_row) + if (trigdesc && + (trigdesc->trig_delete_after_row || trigdesc->trig_delete_old_table)) { HeapTuple trigtuple; @@ -2310,6 +2482,8 @@ ExecIRDeleteTriggers(EState *estate, ResultRelInfo *relinfo, TRIGGER_EVENT_INSTEAD; LocTriggerData.tg_relation = relinfo->ri_RelationDesc; LocTriggerData.tg_newtuple = NULL; + LocTriggerData.tg_oldtable = NULL; + LocTriggerData.tg_newtable = NULL; LocTriggerData.tg_newtuplebuf = InvalidBuffer; for (i = 0; i < trigdesc->numtriggers; i++) { @@ -2363,6 +2537,8 @@ ExecBSUpdateTriggers(EState *estate, ResultRelInfo *relinfo) LocTriggerData.tg_relation = relinfo->ri_RelationDesc; LocTriggerData.tg_trigtuple = NULL; LocTriggerData.tg_newtuple = NULL; + LocTriggerData.tg_oldtable = NULL; + LocTriggerData.tg_newtable = NULL; LocTriggerData.tg_trigtuplebuf = InvalidBuffer; LocTriggerData.tg_newtuplebuf = InvalidBuffer; for (i = 0; i < trigdesc->numtriggers; i++) @@ -2464,6 +2640,8 @@ ExecBRUpdateTriggers(EState *estate, EPQState *epqstate, TRIGGER_EVENT_ROW | TRIGGER_EVENT_BEFORE; LocTriggerData.tg_relation = relinfo->ri_RelationDesc; + LocTriggerData.tg_oldtable = NULL; + LocTriggerData.tg_newtable = NULL; updatedCols = GetUpdatedColumns(relinfo, estate); for (i = 0; i < trigdesc->numtriggers; i++) { @@ -2528,7 +2706,8 @@ ExecARUpdateTriggers(EState *estate, ResultRelInfo *relinfo, { TriggerDesc *trigdesc = relinfo->ri_TrigDesc; - if (trigdesc && trigdesc->trig_update_after_row) + if (trigdesc && (trigdesc->trig_update_after_row || + trigdesc->trig_update_old_table || trigdesc->trig_update_new_table)) { HeapTuple trigtuple; @@ -2567,6 +2746,8 @@ ExecIRUpdateTriggers(EState *estate, ResultRelInfo *relinfo, TRIGGER_EVENT_ROW | TRIGGER_EVENT_INSTEAD; LocTriggerData.tg_relation = relinfo->ri_RelationDesc; + LocTriggerData.tg_oldtable = NULL; + LocTriggerData.tg_newtable = NULL; for (i = 0; i < trigdesc->numtriggers; i++) { Trigger *trigger = &trigdesc->triggers[i]; @@ -2635,6 +2816,8 @@ ExecBSTruncateTriggers(EState *estate, ResultRelInfo *relinfo) LocTriggerData.tg_relation = relinfo->ri_RelationDesc; LocTriggerData.tg_trigtuple = NULL; LocTriggerData.tg_newtuple = NULL; + LocTriggerData.tg_oldtable = NULL; + LocTriggerData.tg_newtable = NULL; LocTriggerData.tg_trigtuplebuf = InvalidBuffer; LocTriggerData.tg_newtuplebuf = InvalidBuffer; for (i = 0; i < trigdesc->numtriggers; i++) @@ -3163,8 +3346,11 @@ typedef struct AfterTriggerEventList * fdw_tuplestores[query_depth] is a tuplestore containing the foreign tuples * needed for the current query. * - * maxquerydepth is just the allocated length of query_stack and - * fdw_tuplestores. + * old_tuplestores[query_depth] and new_tuplestores[query_depth] hold the + * transition relations for the current query. + * + * maxquerydepth is just the allocated length of query_stack and the + * tuplestores. * * state_stack is a stack of pointers to saved copies of the SET CONSTRAINTS * state data; each subtransaction level that modifies that state first @@ -3193,7 +3379,9 @@ typedef struct AfterTriggersData AfterTriggerEventList events; /* deferred-event list */ int query_depth; /* current query list index */ AfterTriggerEventList *query_stack; /* events pending from each query */ - Tuplestorestate **fdw_tuplestores; /* foreign tuples from each query */ + Tuplestorestate **fdw_tuplestores; /* foreign tuples for one row from each query */ + Tuplestorestate **old_tuplestores; /* all old tuples from each query */ + Tuplestorestate **new_tuplestores; /* all new tuples from each query */ int maxquerydepth; /* allocated len of above array */ MemoryContext event_cxt; /* memory context for events, if any */ @@ -3222,14 +3410,16 @@ static SetConstraintState SetConstraintStateAddItem(SetConstraintState state, /* - * Gets the current query fdw tuplestore and initializes it if necessary + * Gets a current query transition tuplestore and initializes it if necessary. + * This can be holding a single transition row tuple (in the case of an FDW) + * or a transition table (for an AFTER trigger). */ static Tuplestorestate * -GetCurrentFDWTuplestore(void) +GetTriggerTransitionTuplestore(Tuplestorestate **tss) { Tuplestorestate *ret; - ret = afterTriggers.fdw_tuplestores[afterTriggers.query_depth]; + ret = tss[afterTriggers.query_depth]; if (ret == NULL) { MemoryContext oldcxt; @@ -3256,7 +3446,7 @@ GetCurrentFDWTuplestore(void) CurrentResourceOwner = saveResourceOwner; MemoryContextSwitchTo(oldcxt); - afterTriggers.fdw_tuplestores[afterTriggers.query_depth] = ret; + tss[afterTriggers.query_depth] = ret; } return ret; @@ -3554,7 +3744,9 @@ AfterTriggerExecute(AfterTriggerEvent event, { case AFTER_TRIGGER_FDW_FETCH: { - Tuplestorestate *fdw_tuplestore = GetCurrentFDWTuplestore(); + Tuplestorestate *fdw_tuplestore = + GetTriggerTransitionTuplestore + (afterTriggers.fdw_tuplestores); if (!tuplestore_gettupleslot(fdw_tuplestore, true, false, trig_tuple_slot1)) @@ -3623,6 +3815,20 @@ AfterTriggerExecute(AfterTriggerEvent event, } } + /* + * Set up the tuplestore information. + */ + if (LocTriggerData.tg_trigger->tgoldtable) + LocTriggerData.tg_oldtable = + GetTriggerTransitionTuplestore(afterTriggers.old_tuplestores); + else + LocTriggerData.tg_oldtable = NULL; + if (LocTriggerData.tg_trigger->tgnewtable) + LocTriggerData.tg_newtable = + GetTriggerTransitionTuplestore(afterTriggers.new_tuplestores); + else + LocTriggerData.tg_newtable = NULL; + /* * Setup the remaining trigger information */ @@ -3912,6 +4118,8 @@ AfterTriggerBeginXact(void) Assert(afterTriggers.state == NULL); Assert(afterTriggers.query_stack == NULL); Assert(afterTriggers.fdw_tuplestores == NULL); + Assert(afterTriggers.old_tuplestores == NULL); + Assert(afterTriggers.new_tuplestores == NULL); Assert(afterTriggers.maxquerydepth == 0); Assert(afterTriggers.event_cxt == NULL); Assert(afterTriggers.events.head == NULL); @@ -3956,6 +4164,8 @@ AfterTriggerEndQuery(EState *estate) { AfterTriggerEventList *events; Tuplestorestate *fdw_tuplestore; + Tuplestorestate *old_tuplestore; + Tuplestorestate *new_tuplestore; /* Must be inside a query, too */ Assert(afterTriggers.query_depth >= 0); @@ -4014,6 +4224,18 @@ AfterTriggerEndQuery(EState *estate) tuplestore_end(fdw_tuplestore); afterTriggers.fdw_tuplestores[afterTriggers.query_depth] = NULL; } + old_tuplestore = afterTriggers.old_tuplestores[afterTriggers.query_depth]; + if (old_tuplestore) + { + tuplestore_end(old_tuplestore); + afterTriggers.old_tuplestores[afterTriggers.query_depth] = NULL; + } + new_tuplestore = afterTriggers.new_tuplestores[afterTriggers.query_depth]; + if (new_tuplestore) + { + tuplestore_end(new_tuplestore); + afterTriggers.new_tuplestores[afterTriggers.query_depth] = NULL; + } afterTriggerFreeEventList(&afterTriggers.query_stack[afterTriggers.query_depth]); afterTriggers.query_depth--; @@ -4127,6 +4349,8 @@ AfterTriggerEndXact(bool isCommit) */ afterTriggers.query_stack = NULL; afterTriggers.fdw_tuplestores = NULL; + afterTriggers.old_tuplestores = NULL; + afterTriggers.new_tuplestores = NULL; afterTriggers.maxquerydepth = 0; afterTriggers.state = NULL; @@ -4259,6 +4483,18 @@ AfterTriggerEndSubXact(bool isCommit) tuplestore_end(ts); afterTriggers.fdw_tuplestores[afterTriggers.query_depth] = NULL; } + ts = afterTriggers.old_tuplestores[afterTriggers.query_depth]; + if (ts) + { + tuplestore_end(ts); + afterTriggers.old_tuplestores[afterTriggers.query_depth] = NULL; + } + ts = afterTriggers.new_tuplestores[afterTriggers.query_depth]; + if (ts) + { + tuplestore_end(ts); + afterTriggers.new_tuplestores[afterTriggers.query_depth] = NULL; + } afterTriggerFreeEventList(&afterTriggers.query_stack[afterTriggers.query_depth]); } @@ -4338,6 +4574,12 @@ AfterTriggerEnlargeQueryState(void) afterTriggers.fdw_tuplestores = (Tuplestorestate **) MemoryContextAllocZero(TopTransactionContext, new_alloc * sizeof(Tuplestorestate *)); + afterTriggers.old_tuplestores = (Tuplestorestate **) + MemoryContextAllocZero(TopTransactionContext, + new_alloc * sizeof(Tuplestorestate *)); + afterTriggers.new_tuplestores = (Tuplestorestate **) + MemoryContextAllocZero(TopTransactionContext, + new_alloc * sizeof(Tuplestorestate *)); afterTriggers.maxquerydepth = new_alloc; } else @@ -4353,9 +4595,19 @@ AfterTriggerEnlargeQueryState(void) afterTriggers.fdw_tuplestores = (Tuplestorestate **) repalloc(afterTriggers.fdw_tuplestores, new_alloc * sizeof(Tuplestorestate *)); + afterTriggers.old_tuplestores = (Tuplestorestate **) + repalloc(afterTriggers.old_tuplestores, + new_alloc * sizeof(Tuplestorestate *)); + afterTriggers.new_tuplestores = (Tuplestorestate **) + repalloc(afterTriggers.new_tuplestores, + new_alloc * sizeof(Tuplestorestate *)); /* Clear newly-allocated slots for subsequent lazy initialization. */ memset(afterTriggers.fdw_tuplestores + old_alloc, 0, (new_alloc - old_alloc) * sizeof(Tuplestorestate *)); + memset(afterTriggers.old_tuplestores + old_alloc, + 0, (new_alloc - old_alloc) * sizeof(Tuplestorestate *)); + memset(afterTriggers.new_tuplestores + old_alloc, + 0, (new_alloc - old_alloc) * sizeof(Tuplestorestate *)); afterTriggers.maxquerydepth = new_alloc; } @@ -4800,7 +5052,14 @@ AfterTriggerPendingOnRel(Oid relid) * * NOTE: this is called whenever there are any triggers associated with * the event (even if they are disabled). This function decides which - * triggers actually need to be queued. + * triggers actually need to be queued. It is also called after each row, + * even if there are no triggers for that event, if there are any AFTER + * STATEMENT triggers for the statement which use transition tables, so that + * the transition tuplestores can be built. + * + * Transition tuplestores are built now, rather than when events are pulled + * off of the queue because AFTER ROW triggers are allowed to select from the + * transition tables for the statement. * ---------- */ static void @@ -4831,6 +5090,46 @@ AfterTriggerSaveEvent(EState *estate, ResultRelInfo *relinfo, if (afterTriggers.query_depth >= afterTriggers.maxquerydepth) AfterTriggerEnlargeQueryState(); + /* + * If the relation has AFTER ... FOR EACH ROW triggers, capture rows into + * transition tuplestores for this depth. + */ + if (row_trigger) + { + if ((event == TRIGGER_EVENT_DELETE && + trigdesc->trig_delete_old_table) || + (event == TRIGGER_EVENT_UPDATE && + trigdesc->trig_update_old_table)) + { + Tuplestorestate *old_tuplestore; + + Assert(oldtup != NULL); + old_tuplestore = + GetTriggerTransitionTuplestore + (afterTriggers.old_tuplestores); + tuplestore_puttuple(old_tuplestore, oldtup); + } + if ((event == TRIGGER_EVENT_INSERT && + trigdesc->trig_insert_new_table) || + (event == TRIGGER_EVENT_UPDATE && + trigdesc->trig_update_new_table)) + { + Tuplestorestate *new_tuplestore; + + Assert(newtup != NULL); + new_tuplestore = + GetTriggerTransitionTuplestore + (afterTriggers.new_tuplestores); + tuplestore_puttuple(new_tuplestore, newtup); + } + + /* If transition tables are the only reason we're here, return. */ + if ((event == TRIGGER_EVENT_DELETE && !trigdesc->trig_delete_after_row) || + (event == TRIGGER_EVENT_INSERT && !trigdesc->trig_insert_after_row) || + (event == TRIGGER_EVENT_UPDATE && !trigdesc->trig_update_after_row)) + return; + } + /* * Validate the event code and collect the associated tuple CTIDs. * @@ -4928,7 +5227,9 @@ AfterTriggerSaveEvent(EState *estate, ResultRelInfo *relinfo, { if (fdw_tuplestore == NULL) { - fdw_tuplestore = GetCurrentFDWTuplestore(); + fdw_tuplestore = + GetTriggerTransitionTuplestore + (afterTriggers.fdw_tuplestores); new_event.ate_flags = AFTER_TRIGGER_FDW_FETCH; } else diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index 71714bc1d6..04e49b7795 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -2718,6 +2718,18 @@ _copyRoleSpec(const RoleSpec *from) return newnode; } +static TriggerTransition * +_copyTriggerTransition(const TriggerTransition *from) +{ + TriggerTransition *newnode = makeNode(TriggerTransition); + + COPY_STRING_FIELD(name); + COPY_SCALAR_FIELD(isNew); + COPY_SCALAR_FIELD(isTable); + + return newnode; +} + static Query * _copyQuery(const Query *from) { @@ -3893,6 +3905,7 @@ _copyCreateTrigStmt(const CreateTrigStmt *from) COPY_NODE_FIELD(columns); COPY_NODE_FIELD(whenClause); COPY_SCALAR_FIELD(isconstraint); + COPY_NODE_FIELD(transitionRels); COPY_SCALAR_FIELD(deferrable); COPY_SCALAR_FIELD(initdeferred); COPY_NODE_FIELD(constrrel); @@ -5088,6 +5101,9 @@ copyObject(const void *from) case T_RoleSpec: retval = _copyRoleSpec(from); break; + case T_TriggerTransition: + retval = _copyTriggerTransition(from); + break; /* * MISCELLANEOUS NODES diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index 29a090fc48..2eaf41c37f 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -1905,6 +1905,7 @@ _equalCreateTrigStmt(const CreateTrigStmt *a, const CreateTrigStmt *b) COMPARE_NODE_FIELD(columns); COMPARE_NODE_FIELD(whenClause); COMPARE_SCALAR_FIELD(isconstraint); + COMPARE_NODE_FIELD(transitionRels); COMPARE_SCALAR_FIELD(deferrable); COMPARE_SCALAR_FIELD(initdeferred); COMPARE_NODE_FIELD(constrrel); @@ -2634,6 +2635,16 @@ _equalRoleSpec(const RoleSpec *a, const RoleSpec *b) return true; } +static bool +_equalTriggerTransition(const TriggerTransition *a, const TriggerTransition *b) +{ + COMPARE_STRING_FIELD(name); + COMPARE_SCALAR_FIELD(isNew); + COMPARE_SCALAR_FIELD(isTable); + + return true; +} + /* * Stuff from pg_list.h */ @@ -3387,6 +3398,9 @@ equal(const void *a, const void *b) case T_RoleSpec: retval = _equalRoleSpec(a, b); break; + case T_TriggerTransition: + retval = _equalTriggerTransition(a, b); + break; default: elog(ERROR, "unrecognized node type: %d", diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index ae869547f3..748b687929 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -2561,6 +2561,16 @@ _outXmlSerialize(StringInfo str, const XmlSerialize *node) WRITE_LOCATION_FIELD(location); } +static void +_outTriggerTransition(StringInfo str, const TriggerTransition *node) +{ + WRITE_NODE_TYPE("TRIGGERTRANSITION"); + + WRITE_STRING_FIELD(name); + WRITE_BOOL_FIELD(isNew); + WRITE_BOOL_FIELD(isTable); +} + static void _outColumnDef(StringInfo str, const ColumnDef *node) { @@ -3852,6 +3862,9 @@ outNode(StringInfo str, const void *obj) case T_ForeignKeyCacheInfo: _outForeignKeyCacheInfo(str, obj); break; + case T_TriggerTransition: + _outTriggerTransition(str, obj); + break; default: diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 5547fc8658..0ec1cd345b 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -310,6 +310,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); %type TriggerEvents TriggerOneEvent %type TriggerFuncArg %type TriggerWhen +%type TransitionRelName +%type TransitionRowOrTable TransitionOldOrNew +%type TriggerTransition %type event_trigger_when_list event_trigger_value_list %type event_trigger_when_item @@ -374,6 +377,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); create_generic_options alter_generic_options relation_expr_list dostmt_opt_list transform_element_list transform_type_list + TriggerTransitions TriggerReferencing %type group_by_list %type group_by_item empty_grouping_set rollup_clause cube_clause @@ -610,11 +614,11 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); MAPPING MATCH MATERIALIZED MAXVALUE METHOD MINUTE_P MINVALUE MODE MONTH_P MOVE - NAME_P NAMES NATIONAL NATURAL NCHAR NEXT NO NONE + NAME_P NAMES NATIONAL NATURAL NCHAR NEW NEXT NO NONE NOT NOTHING NOTIFY NOTNULL NOWAIT NULL_P NULLIF NULLS_P NUMERIC - OBJECT_P OF OFF OFFSET OIDS ON ONLY OPERATOR OPTION OPTIONS OR + OBJECT_P OF OFF OFFSET OIDS OLD ON ONLY OPERATOR OPTION OPTIONS OR ORDER ORDINALITY OUT_P OUTER_P OVER OVERLAPS OVERLAY OWNED OWNER PARALLEL PARSER PARTIAL PARTITION PASSING PASSWORD PLACING PLANS POLICY @@ -623,8 +627,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); QUOTE - RANGE READ REAL REASSIGN RECHECK RECURSIVE REF REFERENCES REFRESH REINDEX - RELATIVE_P RELEASE RENAME REPEATABLE REPLACE REPLICA + RANGE READ REAL REASSIGN RECHECK RECURSIVE REF REFERENCES REFERENCING + REFRESH REINDEX RELATIVE_P RELEASE RENAME REPEATABLE REPLACE REPLICA RESET RESTART RESTRICT RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP ROW ROWS RULE @@ -4748,19 +4752,20 @@ CreateAmStmt: CREATE ACCESS METHOD name TYPE_P INDEX HANDLER handler_name CreateTrigStmt: CREATE TRIGGER name TriggerActionTime TriggerEvents ON - qualified_name TriggerForSpec TriggerWhen + qualified_name TriggerReferencing TriggerForSpec TriggerWhen EXECUTE PROCEDURE func_name '(' TriggerFuncArgs ')' { CreateTrigStmt *n = makeNode(CreateTrigStmt); n->trigname = $3; n->relation = $7; - n->funcname = $12; - n->args = $14; - n->row = $8; + n->funcname = $13; + n->args = $15; + n->row = $9; n->timing = $4; n->events = intVal(linitial($5)); n->columns = (List *) lsecond($5); - n->whenClause = $9; + n->whenClause = $10; + n->transitionRels = $8; n->isconstraint = FALSE; n->deferrable = FALSE; n->initdeferred = FALSE; @@ -4782,6 +4787,7 @@ CreateTrigStmt: n->events = intVal(linitial($6)); n->columns = (List *) lsecond($6); n->whenClause = $14; + n->transitionRels = NIL; n->isconstraint = TRUE; processCASbits($10, @10, "TRIGGER", &n->deferrable, &n->initdeferred, NULL, @@ -4834,6 +4840,49 @@ TriggerOneEvent: { $$ = list_make2(makeInteger(TRIGGER_TYPE_TRUNCATE), NIL); } ; +TriggerReferencing: + REFERENCING TriggerTransitions { $$ = $2; } + | /*EMPTY*/ { $$ = NIL; } + ; + +TriggerTransitions: + TriggerTransition { $$ = list_make1($1); } + | TriggerTransitions TriggerTransition { $$ = lappend($1, $2); } + ; + +TriggerTransition: + TransitionOldOrNew TransitionRowOrTable opt_as TransitionRelName + { + TriggerTransition *n = makeNode(TriggerTransition); + n->name = $4; + n->isNew = $1; + n->isTable = $2; + $$ = (Node *)n; + } + ; + +TransitionOldOrNew: + NEW { $$ = TRUE; } + | OLD { $$ = FALSE; } + ; + +TransitionRowOrTable: + TABLE { $$ = TRUE; } + /* + * According to the standard, lack of a keyword here implies ROW. + * Support for that would require prohibiting ROW entirely here, + * reserving the keyword ROW, and/or requiring AS (instead of + * allowing it to be optional, as the standard specifies) as the + * next token. Requiring ROW seems cleanest and easiest to + * explain. + */ + | ROW { $$ = FALSE; } + ; + +TransitionRelName: + ColId { $$ = $1; } + ; + TriggerForSpec: FOR TriggerForOptEach TriggerForType { @@ -13810,6 +13859,7 @@ unreserved_keyword: | MOVE | NAME_P | NAMES + | NEW | NEXT | NO | NOTHING @@ -13820,6 +13870,7 @@ unreserved_keyword: | OF | OFF | OIDS + | OLD | OPERATOR | OPTION | OPTIONS @@ -13851,6 +13902,7 @@ unreserved_keyword: | RECHECK | RECURSIVE | REF + | REFERENCING | REFRESH | REINDEX | RELATIVE_P diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c index 8a81d7a078..a3a4174abf 100644 --- a/src/backend/utils/adt/ruleutils.c +++ b/src/backend/utils/adt/ruleutils.c @@ -813,6 +813,8 @@ pg_get_triggerdef_worker(Oid trigid, bool pretty) SysScanDesc tgscan; int findx = 0; char *tgname; + char *tgoldtable; + char *tgnewtable; Oid argtypes[1]; /* dummy */ Datum value; bool isnull; @@ -924,6 +926,27 @@ pg_get_triggerdef_worker(Oid trigid, bool pretty) appendStringInfoString(&buf, "IMMEDIATE "); } + value = fastgetattr(ht_trig, Anum_pg_trigger_tgoldtable, + tgrel->rd_att, &isnull); + if (!isnull) + tgoldtable = NameStr(*((NameData *) DatumGetPointer(value))); + else + tgoldtable = NULL; + value = fastgetattr(ht_trig, Anum_pg_trigger_tgnewtable, + tgrel->rd_att, &isnull); + if (!isnull) + tgnewtable = NameStr(*((NameData *) DatumGetPointer(value))); + else + tgnewtable = NULL; + if (tgoldtable != NULL || tgnewtable != NULL) + { + appendStringInfoString(&buf, "REFERENCING "); + if (tgoldtable != NULL) + appendStringInfo(&buf, "OLD TABLE AS %s ", tgoldtable); + if (tgnewtable != NULL) + appendStringInfo(&buf, "NEW TABLE AS %s ", tgnewtable); + } + if (TRIGGER_FOR_ROW(trigrec->tgtype)) appendStringInfoString(&buf, "FOR EACH ROW "); else diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index cd3048db86..880559650a 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201610201 +#define CATALOG_VERSION_NO 201611041 #endif diff --git a/src/include/catalog/pg_trigger.h b/src/include/catalog/pg_trigger.h index eb39c50e63..da6a7f3a2e 100644 --- a/src/include/catalog/pg_trigger.h +++ b/src/include/catalog/pg_trigger.h @@ -59,6 +59,8 @@ CATALOG(pg_trigger,2620) #ifdef CATALOG_VARLEN bytea tgargs BKI_FORCE_NOT_NULL; /* first\000second\000tgnargs\000 */ pg_node_tree tgqual; /* WHEN expression, or NULL if none */ + NameData tgoldtable; /* old transition table, or NULL if none */ + NameData tgnewtable; /* new transition table, or NULL if none */ #endif } FormData_pg_trigger; @@ -73,7 +75,7 @@ typedef FormData_pg_trigger *Form_pg_trigger; * compiler constants for pg_trigger * ---------------- */ -#define Natts_pg_trigger 15 +#define Natts_pg_trigger 17 #define Anum_pg_trigger_tgrelid 1 #define Anum_pg_trigger_tgname 2 #define Anum_pg_trigger_tgfoid 3 @@ -89,6 +91,8 @@ typedef FormData_pg_trigger *Form_pg_trigger; #define Anum_pg_trigger_tgattr 13 #define Anum_pg_trigger_tgargs 14 #define Anum_pg_trigger_tgqual 15 +#define Anum_pg_trigger_tgoldtable 16 +#define Anum_pg_trigger_tgnewtable 17 /* Bits within tgtype */ #define TRIGGER_TYPE_ROW (1 << 0) @@ -142,4 +146,11 @@ typedef FormData_pg_trigger *Form_pg_trigger; #define TRIGGER_TYPE_MATCHES(type, level, timing, event) \ (((type) & (TRIGGER_TYPE_LEVEL_MASK | TRIGGER_TYPE_TIMING_MASK | (event))) == ((level) | (timing) | (event))) +/* + * Macro to determine whether tgnewtable or tgoldtable has been specified for + * a trigger. + */ +#define TRIGGER_USES_TRANSITION_TABLE(namepointer) \ + ((namepointer) != (char *) NULL) + #endif /* PG_TRIGGER_H */ diff --git a/src/include/commands/trigger.h b/src/include/commands/trigger.h index 0ed7c86eb2..c6e3e2c234 100644 --- a/src/include/commands/trigger.h +++ b/src/include/commands/trigger.h @@ -37,6 +37,8 @@ typedef struct TriggerData Trigger *tg_trigger; Buffer tg_trigtuplebuf; Buffer tg_newtuplebuf; + Tuplestorestate *tg_oldtable; + Tuplestorestate *tg_newtable; } TriggerData; /* diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index 88297bbe80..cb9307cd00 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -453,6 +453,7 @@ typedef enum NodeTag T_OnConflictClause, T_CommonTableExpr, T_RoleSpec, + T_TriggerTransition, /* * TAGS FOR REPLICATION GRAMMAR PARSE NODES (replnodes.h) diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 6de2cab6b2..9b600a5f76 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -1204,6 +1204,21 @@ typedef struct CommonTableExpr ((Query *) (cte)->ctequery)->targetList : \ ((Query *) (cte)->ctequery)->returningList) +/* + * TriggerTransition - + * representation of transition row or table naming clause + * + * Only transition tables are initially supported in the syntax, and only for + * AFTER triggers, but other permutations are accepted by the parser so we can + * give a meaningful message from C code. + */ +typedef struct TriggerTransition +{ + NodeTag type; + char *name; + bool isNew; + bool isTable; +} TriggerTransition; /***************************************************************************** * Optimizable Statements @@ -2105,6 +2120,8 @@ typedef struct CreateTrigStmt List *columns; /* column names, or NIL for all columns */ Node *whenClause; /* qual expression, or NULL if none */ bool isconstraint; /* This is a constraint trigger */ + /* explicitly named transition data */ + List *transitionRels; /* TriggerTransition nodes, or NIL if none */ /* The remaining fields are only used for constraint triggers */ bool deferrable; /* [NOT] DEFERRABLE */ bool initdeferred; /* INITIALLY {DEFERRED|IMMEDIATE} */ diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h index 17ffef53a7..77d873beca 100644 --- a/src/include/parser/kwlist.h +++ b/src/include/parser/kwlist.h @@ -251,6 +251,7 @@ PG_KEYWORD("names", NAMES, UNRESERVED_KEYWORD) PG_KEYWORD("national", NATIONAL, COL_NAME_KEYWORD) PG_KEYWORD("natural", NATURAL, TYPE_FUNC_NAME_KEYWORD) PG_KEYWORD("nchar", NCHAR, COL_NAME_KEYWORD) +PG_KEYWORD("new", NEW, UNRESERVED_KEYWORD) PG_KEYWORD("next", NEXT, UNRESERVED_KEYWORD) PG_KEYWORD("no", NO, UNRESERVED_KEYWORD) PG_KEYWORD("none", NONE, COL_NAME_KEYWORD) @@ -268,6 +269,7 @@ PG_KEYWORD("of", OF, UNRESERVED_KEYWORD) PG_KEYWORD("off", OFF, UNRESERVED_KEYWORD) PG_KEYWORD("offset", OFFSET, RESERVED_KEYWORD) PG_KEYWORD("oids", OIDS, UNRESERVED_KEYWORD) +PG_KEYWORD("old", OLD, UNRESERVED_KEYWORD) PG_KEYWORD("on", ON, RESERVED_KEYWORD) PG_KEYWORD("only", ONLY, RESERVED_KEYWORD) PG_KEYWORD("operator", OPERATOR, UNRESERVED_KEYWORD) @@ -313,6 +315,7 @@ PG_KEYWORD("recheck", RECHECK, UNRESERVED_KEYWORD) PG_KEYWORD("recursive", RECURSIVE, UNRESERVED_KEYWORD) PG_KEYWORD("ref", REF, UNRESERVED_KEYWORD) PG_KEYWORD("references", REFERENCES, RESERVED_KEYWORD) +PG_KEYWORD("referencing", REFERENCING, UNRESERVED_KEYWORD) PG_KEYWORD("refresh", REFRESH, UNRESERVED_KEYWORD) PG_KEYWORD("reindex", REINDEX, UNRESERVED_KEYWORD) PG_KEYWORD("relative", RELATIVE_P, UNRESERVED_KEYWORD) diff --git a/src/include/utils/reltrigger.h b/src/include/utils/reltrigger.h index e87f2283ec..756b417128 100644 --- a/src/include/utils/reltrigger.h +++ b/src/include/utils/reltrigger.h @@ -39,6 +39,8 @@ typedef struct Trigger int16 *tgattr; char **tgargs; char *tgqual; + char *tgoldtable; + char *tgnewtable; } Trigger; typedef struct TriggerDesc @@ -68,6 +70,11 @@ typedef struct TriggerDesc /* there are no row-level truncate triggers */ bool trig_truncate_before_statement; bool trig_truncate_after_statement; + /* Is there at least one trigger specifying each transition relation? */ + bool trig_insert_new_table; + bool trig_update_old_table; + bool trig_update_new_table; + bool trig_delete_old_table; } TriggerDesc; #endif /* RELTRIGGER_H */ -- cgit v1.2.3 From 20540710e83f2873707c284a0c0693f0b57156c4 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 4 Nov 2016 18:29:53 -0400 Subject: Delete contrib/xml2's legacy implementation of xml_is_well_formed(). This function is unreferenced in modern usage; it was superseded in 9.1 by a core function of the same name. It has been left in place in the C code only so that pre-9.1 SQL definitions of the contrib/xml2 functions would continue to work. Six years seems like enough time for people to have updated to the extension-style version of the xml2 module, so let's drop this. The key reason for not keeping it any longer is that we want to stick an explicit PGDLLEXPORT into PG_FUNCTION_INFO_V1(), and the similarity of name to the core function creates a conflict that compilers will complain about. Extracted from a larger patch for that purpose. I'm committing this change separately to give it more visibility in the commit logs. While at it, remove the documentation entry that claimed that xml_is_well_formed() is a function provided by contrib/xml2, and instead mention the even more ancient alias xml_valid(). Laurenz Albe, doc change by me Patch: --- contrib/xml2/xpath.c | 45 --------------------------------------------- doc/src/sgml/xml2.sgml | 10 +++++----- 2 files changed, 5 insertions(+), 50 deletions(-) (limited to 'doc/src') diff --git a/contrib/xml2/xpath.c b/contrib/xml2/xpath.c index ac28996867..28445be369 100644 --- a/contrib/xml2/xpath.c +++ b/contrib/xml2/xpath.c @@ -81,51 +81,6 @@ pgxml_parser_init(PgXmlStrictness strictness) } -/* - * Returns true if document is well-formed - * - * Note: this has been superseded by a core function. We still have to - * have it in the contrib module so that existing SQL-level references - * to the function won't fail; but in normal usage with up-to-date SQL - * definitions for the contrib module, this won't be called. - */ - -PG_FUNCTION_INFO_V1(xml_is_well_formed); - -Datum -xml_is_well_formed(PG_FUNCTION_ARGS) -{ - text *t = PG_GETARG_TEXT_P(0); /* document buffer */ - bool result = false; - int32 docsize = VARSIZE(t) - VARHDRSZ; - xmlDocPtr doctree; - PgXmlErrorContext *xmlerrcxt; - - xmlerrcxt = pgxml_parser_init(PG_XML_STRICTNESS_LEGACY); - - PG_TRY(); - { - doctree = xmlParseMemory((char *) VARDATA(t), docsize); - - result = (doctree != NULL); - - if (doctree != NULL) - xmlFreeDoc(doctree); - } - PG_CATCH(); - { - pg_xml_done(xmlerrcxt, true); - - PG_RE_THROW(); - } - PG_END_TRY(); - - pg_xml_done(xmlerrcxt, false); - - PG_RETURN_BOOL(result); -} - - /* Encodes special characters (<, >, &, " and \r) as XML entities */ PG_FUNCTION_INFO_V1(xml_encode_special_chars); diff --git a/doc/src/sgml/xml2.sgml b/doc/src/sgml/xml2.sgml index a40172c36d..9bbc9e75d7 100644 --- a/doc/src/sgml/xml2.sgml +++ b/doc/src/sgml/xml2.sgml @@ -53,7 +53,7 @@ - xml_is_well_formed(document) + xml_valid(document) @@ -62,10 +62,10 @@ This parses the document text in its parameter and returns true if the - document is well-formed XML. (Note: before PostgreSQL 8.2, this - function was called xml_valid(). That is the wrong name - since validity and well-formedness have different meanings in XML. - The old name is still available, but is deprecated.) + document is well-formed XML. (Note: this is an alias for the standard + PostgreSQL function xml_is_well_formed(). The + name xml_valid() is technically incorrect since validity + and well-formedness have different meanings in XML.) -- cgit v1.2.3 From c8ead2a3974d3eada145a0e18940150039493cc9 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 4 Nov 2016 19:04:56 -0400 Subject: Provide DLLEXPORT markers for C functions via PG_FUNCTION_INFO_V1 macro. Second try at the change originally made in commit 8518583cd; this time with contrib updates so that manual extern declarations are also marked with PGDLLEXPORT. The release notes should point this out as a significant source-code change for extension authors, since they'll have to make similar additions to avoid trouble on Windows. Laurenz Albe, doc change by me Patch: --- contrib/hstore/hstore.h | 2 +- contrib/ltree/ltree.h | 40 ++++++++++++++++++++-------------------- doc/src/sgml/xfunc.sgml | 17 +++++++++++++++++ src/include/fmgr.h | 7 +++---- 4 files changed, 41 insertions(+), 25 deletions(-) (limited to 'doc/src') diff --git a/contrib/hstore/hstore.h b/contrib/hstore/hstore.h index 6bab08b7de..6303fa4061 100644 --- a/contrib/hstore/hstore.h +++ b/contrib/hstore/hstore.h @@ -194,7 +194,7 @@ extern Pairs *hstoreArrayToPairs(ArrayType *a, int *npairs); #if HSTORE_POLLUTE_NAMESPACE #define HSTORE_POLLUTE(newname_,oldname_) \ PG_FUNCTION_INFO_V1(oldname_); \ - Datum newname_(PG_FUNCTION_ARGS); \ + extern PGDLLEXPORT Datum newname_(PG_FUNCTION_ARGS); \ Datum oldname_(PG_FUNCTION_ARGS) { return newname_(fcinfo); } \ extern int no_such_variable #else diff --git a/contrib/ltree/ltree.h b/contrib/ltree/ltree.h index c604357dbf..c7aa7f8818 100644 --- a/contrib/ltree/ltree.h +++ b/contrib/ltree/ltree.h @@ -130,30 +130,30 @@ typedef struct /* use in array iterator */ -Datum ltree_isparent(PG_FUNCTION_ARGS); -Datum ltree_risparent(PG_FUNCTION_ARGS); -Datum ltq_regex(PG_FUNCTION_ARGS); -Datum ltq_rregex(PG_FUNCTION_ARGS); -Datum lt_q_regex(PG_FUNCTION_ARGS); -Datum lt_q_rregex(PG_FUNCTION_ARGS); -Datum ltxtq_exec(PG_FUNCTION_ARGS); -Datum ltxtq_rexec(PG_FUNCTION_ARGS); -Datum _ltq_regex(PG_FUNCTION_ARGS); -Datum _ltq_rregex(PG_FUNCTION_ARGS); -Datum _lt_q_regex(PG_FUNCTION_ARGS); -Datum _lt_q_rregex(PG_FUNCTION_ARGS); -Datum _ltxtq_exec(PG_FUNCTION_ARGS); -Datum _ltxtq_rexec(PG_FUNCTION_ARGS); -Datum _ltree_isparent(PG_FUNCTION_ARGS); -Datum _ltree_risparent(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum ltree_isparent(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum ltree_risparent(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum ltq_regex(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum ltq_rregex(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum lt_q_regex(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum lt_q_rregex(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum ltxtq_exec(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum ltxtq_rexec(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum _ltq_regex(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum _ltq_rregex(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum _lt_q_regex(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum _lt_q_rregex(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum _ltxtq_exec(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum _ltxtq_rexec(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum _ltree_isparent(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum _ltree_risparent(PG_FUNCTION_ARGS); /* Concatenation functions */ -Datum ltree_addltree(PG_FUNCTION_ARGS); -Datum ltree_addtext(PG_FUNCTION_ARGS); -Datum ltree_textadd(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum ltree_addltree(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum ltree_addtext(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum ltree_textadd(PG_FUNCTION_ARGS); /* Util function */ -Datum ltree_in(PG_FUNCTION_ARGS); +extern PGDLLEXPORT Datum ltree_in(PG_FUNCTION_ARGS); bool ltree_execute(ITEM *curitem, void *checkval, bool calcnot, bool (*chkcond) (void *checkval, ITEM *val)); diff --git a/doc/src/sgml/xfunc.sgml b/doc/src/sgml/xfunc.sgml index de6a466efc..6060e61857 100644 --- a/doc/src/sgml/xfunc.sgml +++ b/doc/src/sgml/xfunc.sgml @@ -2577,6 +2577,23 @@ concat_text(PG_FUNCTION_ARGS) error messages to this effect. + + + + To work correctly on Windows, C-language functions need + to be marked with PGDLLEXPORT, unless you use a build + process that marks all global functions that way. In simple cases + this detail will be handled transparently by + the PG_FUNCTION_INFO_V1 macro. However, if you write + explicit external declarations (perhaps in header files), be sure + to write them like this: + +extern PGDLLEXPORT Datum funcname(PG_FUNCTION_ARGS); + + or you'll get compiler complaints when building on Windows. (On + other platforms, the PGDLLEXPORT macro does nothing.) + + diff --git a/src/include/fmgr.h b/src/include/fmgr.h index 0878418516..3668ac3f6e 100644 --- a/src/include/fmgr.h +++ b/src/include/fmgr.h @@ -350,12 +350,11 @@ typedef const Pg_finfo_record *(*PGFInfoFunction) (void); * * On Windows, the function and info function must be exported. Our normal * build processes take care of that via .DEF files or --export-all-symbols. - * Module authors using a different build process might need to manually - * declare the function PGDLLEXPORT. We do that automatically here for the - * info function, since authors shouldn't need to be explicitly aware of it. + * Module authors using a different build process might do it differently, + * so we declare these functions PGDLLEXPORT for their convenience. */ #define PG_FUNCTION_INFO_V1(funcname) \ -extern Datum funcname(PG_FUNCTION_ARGS); \ +extern PGDLLEXPORT Datum funcname(PG_FUNCTION_ARGS); \ extern PGDLLEXPORT const Pg_finfo_record * CppConcat(pg_finfo_,funcname)(void); \ const Pg_finfo_record * \ CppConcat(pg_finfo_,funcname) (void) \ -- cgit v1.2.3 From 6feb69f6cef8b1bd2829700e25e402f22e86f3bd Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 4 Nov 2016 12:00:00 -0400 Subject: doc: Port page header customizations to XSLT --- doc/src/sgml/stylesheet.xsl | 138 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) (limited to 'doc/src') diff --git a/doc/src/sgml/stylesheet.xsl b/doc/src/sgml/stylesheet.xsl index 39c9df28ad..42e8cce368 100644 --- a/doc/src/sgml/stylesheet.xsl +++ b/doc/src/sgml/stylesheet.xsl @@ -246,4 +246,142 @@ set toc,title + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + +
+ +
+ + + + + + + + + + + + + + + +   + + + + + + + + + + + + + + + + + +   + + + + + + +   + + + + + + + + + + + + + + + + + + +  |  + + +   + + +   + + + + + + + + + + + + + + + +
+ + +
+ +
+ + + -- cgit v1.2.3 From d49cc588ca589cd378b5862fa5704eaade4a1380 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 4 Nov 2016 12:00:00 -0400 Subject: doc: Don't reformat .fo files before processing by fop This messes up the whitespace in the output PDF document in some places. --- doc/src/sgml/Makefile | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/Makefile b/doc/src/sgml/Makefile index 5df2f04dd6..84c94e8ae0 100644 --- a/doc/src/sgml/Makefile +++ b/doc/src/sgml/Makefile @@ -270,20 +270,16 @@ htmlhelp: stylesheet-hh.xsl postgres.xml $(XMLLINT) --noout --valid postgres.xml $(XSLTPROC) $(XSLTPROCFLAGS) $^ -%-A4.fo.tmp: stylesheet-fo.xsl %.xml +%-A4.fo: stylesheet-fo.xsl %.xml $(XMLLINT) --noout --valid $*.xml $(XSLTPROC) $(XSLTPROCFLAGS) --stringparam paper.type A4 -o $@ $^ -%-US.fo.tmp: stylesheet-fo.xsl %.xml +%-US.fo: stylesheet-fo.xsl %.xml $(XMLLINT) --noout --valid $*.xml $(XSLTPROC) $(XSLTPROCFLAGS) --stringparam paper.type USletter -o $@ $^ FOP = fop -# reformat FO output so that locations of errors are easier to find -%.fo: %.fo.tmp - $(XMLLINT) --format --output $@ $^ - .SECONDARY: postgres-A4.fo postgres-US.fo %-fop.pdf: %.fo @@ -404,7 +400,7 @@ clean: # index rm -f HTML.index $(GENERATED_SGML) # XSLT - rm -f postgres.xml postgres.xmltmp htmlhelp.hhp toc.hhc index.hhk *.fo *.fo.tmp + rm -f postgres.xml postgres.xmltmp htmlhelp.hhp toc.hhc index.hhk *.fo # EPUB rm -f postgres.epub # Texinfo -- cgit v1.2.3 From fd2664dcb71102a5d66d2453182c010fb219496c Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 6 Nov 2016 14:43:13 -0500 Subject: Rationalize and document pltcl's handling of magic ".tupno" array element. For a very long time, pltcl's spi_exec and spi_execp commands have had a behavior of storing the current row number as an element of output arrays, but this was never documented. Fix that. For an equally long time, pltcl_trigger_handler had a behavior of silently ignoring ".tupno" as an output column name, evidently so that the result of spi_exec could be used directly as a trigger result tuple. Not sure how useful that really is, but in any case it's bad that it would break attempts to use ".tupno" as an actual column name. We can fix it by not checking for ".tupno" until after we check for a column name match. This comports with the effective behavior of spi_exec[p] that ".tupno" is only magic when you don't have an actual column named that. In passing, wordsmith the description of returning modified tuples from a pltcl trigger. Noted while working on Jim Nasby's patch to support composite results from pltcl. The inability to return trigger tuples using ".tupno" as a column name is a bug, so back-patch to all supported branches. --- doc/src/sgml/pltcl.sgml | 54 ++++++++++++++++++++++++++++++++----------------- src/pl/tcl/pltcl.c | 23 +++++++++++++-------- 2 files changed, 50 insertions(+), 27 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/pltcl.sgml b/doc/src/sgml/pltcl.sgml index 805cc89dc9..52fc44940c 100644 --- a/doc/src/sgml/pltcl.sgml +++ b/doc/src/sgml/pltcl.sgml @@ -296,20 +296,22 @@ $$ LANGUAGE pltcl; If the command is a SELECT statement, the values of the result columns are placed into Tcl variables named after the columns. If the -array option is given, the column values are - instead stored into the named associative array, with the - column names used as array indexes. + instead stored into elements of the named associative array, with the + column names used as array indexes. In addition, the current row + number within the result (counting from zero) is stored into the array + element named .tupno, unless that name is + in use as a column name in the result. If the command is a SELECT statement and no loop-body script is given, then only the first row of results are stored into - Tcl variables; remaining rows, if any, are ignored. No storing occurs - if the - query returns no rows. (This case can be detected by checking the - result of spi_exec.) For example: + Tcl variables or array elements; remaining rows, if any, are ignored. + No storing occurs if the query returns no rows. (This case can be + detected by checking the result of spi_exec.) + For example: spi_exec "SELECT count(*) AS cnt FROM pg_proc" - will set the Tcl variable $cnt to the number of rows in the pg_proc system catalog. @@ -317,15 +319,15 @@ spi_exec "SELECT count(*) AS cnt FROM pg_proc" If the optional loop-body argument is given, it is a piece of Tcl script that is executed once for each row in the query result. (loop-body is ignored if the given - command is not a SELECT.) The values of the current row's columns - are stored into Tcl variables before each iteration. For example: - + command is not a SELECT.) + The values of the current row's columns + are stored into Tcl variables or array elements before each iteration. + For example: spi_exec -array C "SELECT * FROM pg_class" { elog DEBUG "have table $C(relname)" } - will print a log message for every row of pg_class. This feature works similarly to other Tcl looping constructs; in particular continue and break work in the @@ -667,21 +669,35 @@ SELECT 'doesn''t' AS ret The return value from a trigger procedure can be one of the strings - OK or SKIP, or a list as returned by the - array get Tcl command. If the return value is OK, - the operation (INSERT/UPDATE/DELETE) that fired the trigger will proceed + OK or SKIP, or a list of column name/value pairs. + If the return value is OK, + the operation (INSERT/UPDATE/DELETE) + that fired the trigger will proceed normally. SKIP tells the trigger manager to silently suppress the operation for this row. If a list is returned, it tells PL/Tcl to - return a modified row to the trigger manager. This is only meaningful + return a modified row to the trigger manager; the contents of the + modified row are specified by the column names and values in the list. + Any columns not mentioned in the list are set to null. + Returning a modified row is only meaningful for row-level BEFORE INSERT or UPDATE - triggers for which the modified row will be inserted instead of the one + triggers, for which the modified row will be inserted instead of the one given in $NEW; or for row-level INSTEAD OF INSERT or UPDATE triggers where the returned row - is used to support INSERT RETURNING and - UPDATE RETURNING commands. The return value is ignored for - other types of triggers. + is used as the source data for INSERT RETURNING or + UPDATE RETURNING clauses. + In row-level BEFORE DELETE or INSTEAD + OF DELETE triggers, returning a modified row has the same + effect as returning OK, that is the operation proceeds. + The trigger return value is ignored for all other types of triggers. + + + The result list can be made from an array representation of the + modified tuple with the array get Tcl command. + + + Here's a little example trigger procedure that forces an integer value in a table to keep track of the number of updates that are performed on the diff --git a/src/pl/tcl/pltcl.c b/src/pl/tcl/pltcl.c index 9d72f47f59..44fcf68054 100644 --- a/src/pl/tcl/pltcl.c +++ b/src/pl/tcl/pltcl.c @@ -1118,21 +1118,23 @@ pltcl_trigger_handler(PG_FUNCTION_ARGS, bool pltrusted) Oid typioparam; FmgrInfo finfo; - /************************************************************ - * Ignore ".tupno" pseudo elements (see pltcl_set_tuple_values) - ************************************************************/ - if (strcmp(ret_name, ".tupno") == 0) - continue; - /************************************************************ * Get the attribute number + * + * We silently ignore ".tupno", if it's present but doesn't match + * any actual output column. This allows direct use of a row + * returned by pltcl_set_tuple_values(). ************************************************************/ attnum = SPI_fnumber(tupdesc, ret_name); if (attnum == SPI_ERROR_NOATTRIBUTE) + { + if (strcmp(ret_name, ".tupno") == 0) + continue; ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("unrecognized attribute \"%s\"", ret_name))); + } if (attnum <= 0) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -2703,8 +2705,7 @@ pltcl_set_tuple_values(Tcl_Interp *interp, const char *arrayname, const char *nullname = NULL; /************************************************************ - * Prepare pointers for Tcl_SetVar2() below and in array - * mode set the .tupno element + * Prepare pointers for Tcl_SetVar2() below ************************************************************/ if (arrayname == NULL) { @@ -2715,6 +2716,12 @@ pltcl_set_tuple_values(Tcl_Interp *interp, const char *arrayname, { arrptr = &arrayname; nameptr = &attname; + + /* + * When outputting to an array, fill the ".tupno" element with the + * current tuple number. This will be overridden below if ".tupno" is + * in use as an actual field name in the rowtype. + */ Tcl_SetVar2Ex(interp, arrayname, ".tupno", Tcl_NewWideIntObj(tupno), 0); } -- cgit v1.2.3 From 26abb50c490dee191df21282bc940b94118550aa Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 6 Nov 2016 17:56:05 -0500 Subject: Support PL/Tcl functions that return composite types and/or sets. Jim Nasby, rather heavily editorialized by me Patch: --- doc/src/sgml/pltcl.sgml | 73 ++++-- src/pl/tcl/expected/pltcl_queries.out | 61 +++++ src/pl/tcl/expected/pltcl_setup.out | 13 + src/pl/tcl/pltcl.c | 430 +++++++++++++++++++++++++++++----- src/pl/tcl/sql/pltcl_queries.sql | 33 +++ src/pl/tcl/sql/pltcl_setup.sql | 16 ++ 6 files changed, 545 insertions(+), 81 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/pltcl.sgml b/doc/src/sgml/pltcl.sgml index 52fc44940c..8afaf4ad36 100644 --- a/doc/src/sgml/pltcl.sgml +++ b/doc/src/sgml/pltcl.sgml @@ -94,11 +94,11 @@ $$ LANGUAGE pltcl; The body of the function is simply a piece of Tcl script. - When the function is called, the argument values are passed as - variables $1 ... $n to the - Tcl script. The result is returned - from the Tcl code in the usual way, with a return - statement. + When the function is called, the argument values are passed to the + Tcl script as variables named 1 + ... n. The result is + returned from the Tcl code in the usual way, with + a return statement. @@ -173,17 +173,57 @@ $$ LANGUAGE pltcl; - There is currently no support for returning a composite-type - result value, nor for returning sets. + PL/Tcl functions can return composite-type results, too. To do this, + the Tcl code must return a list of column name/value pairs matching + the expected result type. Any column names omitted from the list + are returned as nulls, and an error is raised if there are unexpected + column names. Here is an example: + + +CREATE FUNCTION square_cube(in int, out squared int, out cubed int) AS $$ + return [list squared [expr {$1 * $1}] cubed [expr {$1 * $1 * $1}]] +$$ LANGUAGE pltcl; + + + + The result list can be made from an array representation of the + desired tuple with the array get Tcl command. For example: + + +CREATE FUNCTION raise_pay(employee, delta int) RETURNS employee AS $$ + set 1(salary) [expr {$1(salary) + $2}] + return [array get 1] +$$ LANGUAGE pltcl; + + + + - PL/Tcl does not currently have full support for - domain types: it treats a domain the same as the underlying scalar - type. This means that constraints associated with the domain will - not be enforced. This is not an issue for function arguments, but - it is a hazard if you declare a PL/Tcl function - as returning a domain type. + PL/Tcl functions can return sets. To do this, the Tcl code should + call return_next once per row to be returned, + passing either the appropriate value when returning a scalar type, + or a list of column name/value pairs when returning a composite type. + Here is an example returning a scalar type: + + +CREATE FUNCTION sequence(int, int) RETURNS SETOF int AS $$ + for {set i $1} {$i < $2} {incr i} { + return_next $i + } +$$ LANGUAGE pltcl; + + + and here is one returning a composite type: + + +CREATE FUNCTION table_of_squares(int, int) RETURNS TABLE (x int, x2 int) AS $$ + for {set i $1} {$i < $2} {incr i} { + return_next [list x $i x2 [expr {$i * $i}]] + } +$$ LANGUAGE pltcl; + @@ -195,10 +235,9 @@ $$ LANGUAGE pltcl; The argument values supplied to a PL/Tcl function's code are simply the input arguments converted to text form (just as if they had been displayed by a SELECT statement). Conversely, the - return - command will accept any string that is acceptable input format for - the function's declared return type. So, within the PL/Tcl function, - all values are just text strings. + return and return_next commands will accept + any string that is acceptable input format for the function's declared + result type, or for the specified column of a composite result type. diff --git a/src/pl/tcl/expected/pltcl_queries.out b/src/pl/tcl/expected/pltcl_queries.out index 6cb1fdbb61..3a9fef3447 100644 --- a/src/pl/tcl/expected/pltcl_queries.out +++ b/src/pl/tcl/expected/pltcl_queries.out @@ -303,3 +303,64 @@ select tcl_lastoid('t2') > 0; t (1 row) +-- test some error cases +CREATE FUNCTION tcl_error(OUT a int, OUT b int) AS $$return {$$ LANGUAGE pltcl; +SELECT tcl_error(); +ERROR: missing close-brace +CREATE FUNCTION bad_record(OUT a text, OUT b text) AS $$return [list a]$$ LANGUAGE pltcl; +SELECT bad_record(); +ERROR: column name/value list must have even number of elements +CREATE FUNCTION bad_field(OUT a text, OUT b text) AS $$return [list a 1 b 2 cow 3]$$ LANGUAGE pltcl; +SELECT bad_field(); +ERROR: column name/value list contains nonexistent column name "cow" +-- test compound return +select * from tcl_test_cube_squared(5); + squared | cubed +---------+------- + 25 | 125 +(1 row) + +-- test SRF +select * from tcl_test_squared_rows(0,5); + x | y +---+---- + 0 | 0 + 1 | 1 + 2 | 4 + 3 | 9 + 4 | 16 +(5 rows) + +select * from tcl_test_sequence(0,5) as a; + a +--- + 0 + 1 + 2 + 3 + 4 +(5 rows) + +select 1, tcl_test_sequence(0,5); + ?column? | tcl_test_sequence +----------+------------------- + 1 | 0 + 1 | 1 + 1 | 2 + 1 | 3 + 1 | 4 +(5 rows) + +CREATE FUNCTION non_srf() RETURNS int AS $$return_next 1$$ LANGUAGE pltcl; +select non_srf(); +ERROR: return_next cannot be used in non-set-returning functions +CREATE FUNCTION bad_record_srf(OUT a text, OUT b text) RETURNS SETOF record AS $$ +return_next [list a] +$$ LANGUAGE pltcl; +SELECT bad_record_srf(); +ERROR: column name/value list must have even number of elements +CREATE FUNCTION bad_field_srf(OUT a text, OUT b text) RETURNS SETOF record AS $$ +return_next [list a 1 b 2 cow 3] +$$ LANGUAGE pltcl; +SELECT bad_field_srf(); +ERROR: column name/value list contains nonexistent column name "cow" diff --git a/src/pl/tcl/expected/pltcl_setup.out b/src/pl/tcl/expected/pltcl_setup.out index e65e9e3ff7..ed99d9b492 100644 --- a/src/pl/tcl/expected/pltcl_setup.out +++ b/src/pl/tcl/expected/pltcl_setup.out @@ -555,6 +555,19 @@ NOTICE: tclsnitch: ddl_command_start DROP TABLE NOTICE: tclsnitch: ddl_command_end DROP TABLE drop event trigger tcl_a_snitch; drop event trigger tcl_b_snitch; +CREATE FUNCTION tcl_test_cube_squared(in int, out squared int, out cubed int) AS $$ + return [list squared [expr {$1 * $1}] cubed [expr {$1 * $1 * $1}]] +$$ language pltcl; +CREATE FUNCTION tcl_test_squared_rows(int,int) RETURNS TABLE (x int, y int) AS $$ + for {set i $1} {$i < $2} {incr i} { + return_next [list y [expr {$i * $i}] x $i] + } +$$ language pltcl; +CREATE FUNCTION tcl_test_sequence(int,int) RETURNS SETOF int AS $$ + for {set i $1} {$i < $2} {incr i} { + return_next $i + } +$$ language pltcl; -- test use of errorCode in error handling create function tcl_error_handling_test() returns text as $$ global errorCode diff --git a/src/pl/tcl/pltcl.c b/src/pl/tcl/pltcl.c index 97d1f7ef7d..3d529c2e7d 100644 --- a/src/pl/tcl/pltcl.c +++ b/src/pl/tcl/pltcl.c @@ -21,6 +21,7 @@ #include "commands/trigger.h" #include "executor/spi.h" #include "fmgr.h" +#include "funcapi.h" #include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/makefuncs.h" @@ -123,6 +124,9 @@ typedef struct pltcl_interp_desc * problem to manage its memory when we replace a proc definition. We do * not clean up pltcl_proc_descs when a pg_proc row is deleted, only when * it is updated, and the same policy applies to Tcl's copy as well.) + * + * Note that the data in this struct is shared across all active calls; + * nothing except the fn_refcount should be changed by a call instance. **********************************************************************/ typedef struct pltcl_proc_desc { @@ -137,6 +141,8 @@ typedef struct pltcl_proc_desc pltcl_interp_desc *interp_desc; /* interpreter to use */ FmgrInfo result_in_func; /* input function for fn's result type */ Oid result_typioparam; /* param to pass to same */ + bool fn_retisset; /* true if function returns a set */ + bool fn_retistuple; /* true if function returns composite */ int nargs; /* number of arguments */ /* these arrays have nargs entries: */ FmgrInfo *arg_out_func; /* output fns for arg types */ @@ -188,6 +194,32 @@ typedef struct pltcl_proc_ptr } pltcl_proc_ptr; +/********************************************************************** + * Per-call state + **********************************************************************/ +typedef struct pltcl_call_state +{ + /* Call info struct, or NULL in a trigger */ + FunctionCallInfo fcinfo; + + /* Function we're executing (NULL if not yet identified) */ + pltcl_proc_desc *prodesc; + + /* + * Information for SRFs and functions returning composite types. + * ret_tupdesc and attinmeta are set up if either fn_retistuple or + * fn_retisset, since even a scalar-returning SRF needs a tuplestore. + */ + TupleDesc ret_tupdesc; /* return rowtype, if retistuple or retisset */ + AttInMetadata *attinmeta; /* metadata for building tuples of that type */ + + ReturnSetInfo *rsi; /* passed-in ReturnSetInfo, if any */ + Tuplestorestate *tuple_store; /* SRFs accumulate result here */ + MemoryContext tuple_store_cxt; /* context and resowner for tuplestore */ + ResourceOwner tuple_store_owner; +} pltcl_call_state; + + /********************************************************************** * Global data **********************************************************************/ @@ -196,9 +228,8 @@ static Tcl_Interp *pltcl_hold_interp = NULL; static HTAB *pltcl_interp_htab = NULL; static HTAB *pltcl_proc_htab = NULL; -/* these are saved and restored by pltcl_handler */ -static FunctionCallInfo pltcl_current_fcinfo = NULL; -static pltcl_proc_desc *pltcl_current_prodesc = NULL; +/* this is saved and restored by pltcl_handler */ +static pltcl_call_state *pltcl_current_call_state = NULL; /********************************************************************** * Lookup table for SQLSTATE condition names @@ -225,10 +256,12 @@ static void pltcl_init_load_unknown(Tcl_Interp *interp); static Datum pltcl_handler(PG_FUNCTION_ARGS, bool pltrusted); -static Datum pltcl_func_handler(PG_FUNCTION_ARGS, bool pltrusted); - -static HeapTuple pltcl_trigger_handler(PG_FUNCTION_ARGS, bool pltrusted); -static void pltcl_event_trigger_handler(PG_FUNCTION_ARGS, bool pltrusted); +static Datum pltcl_func_handler(PG_FUNCTION_ARGS, pltcl_call_state *call_state, + bool pltrusted); +static HeapTuple pltcl_trigger_handler(PG_FUNCTION_ARGS, pltcl_call_state *call_state, + bool pltrusted); +static void pltcl_event_trigger_handler(PG_FUNCTION_ARGS, pltcl_call_state *call_state, + bool pltrusted); static void throw_tcl_error(Tcl_Interp *interp, const char *proname); @@ -246,7 +279,8 @@ static int pltcl_argisnull(ClientData cdata, Tcl_Interp *interp, int objc, Tcl_Obj *const objv[]); static int pltcl_returnnull(ClientData cdata, Tcl_Interp *interp, int objc, Tcl_Obj *const objv[]); - +static int pltcl_returnnext(ClientData cdata, Tcl_Interp *interp, + int objc, Tcl_Obj *const objv[]); static int pltcl_SPI_execute(ClientData cdata, Tcl_Interp *interp, int objc, Tcl_Obj *const objv[]); static int pltcl_process_SPI_result(Tcl_Interp *interp, @@ -265,6 +299,10 @@ static int pltcl_SPI_lastoid(ClientData cdata, Tcl_Interp *interp, static void pltcl_set_tuple_values(Tcl_Interp *interp, const char *arrayname, uint64 tupno, HeapTuple tuple, TupleDesc tupdesc); static Tcl_Obj *pltcl_build_tuple_argument(HeapTuple tuple, TupleDesc tupdesc); +static HeapTuple pltcl_build_tuple_result(Tcl_Interp *interp, + Tcl_Obj **kvObjv, int kvObjc, + pltcl_call_state *call_state); +static void pltcl_init_tuple_store(pltcl_call_state *call_state); /* @@ -432,7 +470,8 @@ pltcl_init_interp(pltcl_interp_desc *interp_desc, bool pltrusted) pltcl_argisnull, NULL, NULL); Tcl_CreateObjCommand(interp, "return_null", pltcl_returnnull, NULL, NULL); - + Tcl_CreateObjCommand(interp, "return_next", + pltcl_returnnext, NULL, NULL); Tcl_CreateObjCommand(interp, "spi_exec", pltcl_SPI_execute, NULL, NULL); Tcl_CreateObjCommand(interp, "spi_prepare", @@ -625,29 +664,33 @@ pltclu_call_handler(PG_FUNCTION_ARGS) } +/********************************************************************** + * pltcl_handler() - Handler for function and trigger calls, for + * both trusted and untrusted interpreters. + **********************************************************************/ static Datum pltcl_handler(PG_FUNCTION_ARGS, bool pltrusted) { Datum retval; - FunctionCallInfo save_fcinfo; - pltcl_proc_desc *save_prodesc; - pltcl_proc_desc *this_prodesc; + pltcl_call_state current_call_state; + pltcl_call_state *save_call_state; /* - * Ensure that static pointers are saved/restored properly + * Initialize current_call_state to nulls/zeroes; in particular, set its + * prodesc pointer to null. Anything that sets it non-null should + * increase the prodesc's fn_refcount at the same time. We'll decrease + * the refcount, and then delete the prodesc if it's no longer referenced, + * on the way out of this function. This ensures that prodescs live as + * long as needed even if somebody replaces the originating pg_proc row + * while they're executing. */ - save_fcinfo = pltcl_current_fcinfo; - save_prodesc = pltcl_current_prodesc; + memset(¤t_call_state, 0, sizeof(current_call_state)); /* - * Reset pltcl_current_prodesc to null. Anything that sets it non-null - * should increase the prodesc's fn_refcount at the same time. We'll - * decrease the refcount, and then delete the prodesc if it's no longer - * referenced, on the way out of this function. This ensures that - * prodescs live as long as needed even if somebody replaces the - * originating pg_proc row while they're executing. + * Ensure that static pointer is saved/restored properly */ - pltcl_current_prodesc = NULL; + save_call_state = pltcl_current_call_state; + pltcl_current_call_state = ¤t_call_state; PG_TRY(); { @@ -657,47 +700,46 @@ pltcl_handler(PG_FUNCTION_ARGS, bool pltrusted) */ if (CALLED_AS_TRIGGER(fcinfo)) { - pltcl_current_fcinfo = NULL; - retval = PointerGetDatum(pltcl_trigger_handler(fcinfo, pltrusted)); + /* invoke the trigger handler */ + retval = PointerGetDatum(pltcl_trigger_handler(fcinfo, + ¤t_call_state, + pltrusted)); } else if (CALLED_AS_EVENT_TRIGGER(fcinfo)) { - pltcl_current_fcinfo = NULL; - pltcl_event_trigger_handler(fcinfo, pltrusted); + /* invoke the event trigger handler */ + pltcl_event_trigger_handler(fcinfo, ¤t_call_state, pltrusted); retval = (Datum) 0; } else { - pltcl_current_fcinfo = fcinfo; - retval = pltcl_func_handler(fcinfo, pltrusted); + /* invoke the regular function handler */ + current_call_state.fcinfo = fcinfo; + retval = pltcl_func_handler(fcinfo, ¤t_call_state, pltrusted); } } PG_CATCH(); { - /* Restore globals, then clean up the prodesc refcount if any */ - this_prodesc = pltcl_current_prodesc; - pltcl_current_fcinfo = save_fcinfo; - pltcl_current_prodesc = save_prodesc; - if (this_prodesc != NULL) + /* Restore static pointer, then clean up the prodesc refcount if any */ + pltcl_current_call_state = save_call_state; + if (current_call_state.prodesc != NULL) { - Assert(this_prodesc->fn_refcount > 0); - if (--this_prodesc->fn_refcount == 0) - MemoryContextDelete(this_prodesc->fn_cxt); + Assert(current_call_state.prodesc->fn_refcount > 0); + if (--current_call_state.prodesc->fn_refcount == 0) + MemoryContextDelete(current_call_state.prodesc->fn_cxt); } PG_RE_THROW(); } PG_END_TRY(); - /* Restore globals, then clean up the prodesc refcount if any */ + /* Restore static pointer, then clean up the prodesc refcount if any */ /* (We're being paranoid in case an error is thrown in context deletion) */ - this_prodesc = pltcl_current_prodesc; - pltcl_current_fcinfo = save_fcinfo; - pltcl_current_prodesc = save_prodesc; - if (this_prodesc != NULL) + pltcl_current_call_state = save_call_state; + if (current_call_state.prodesc != NULL) { - Assert(this_prodesc->fn_refcount > 0); - if (--this_prodesc->fn_refcount == 0) - MemoryContextDelete(this_prodesc->fn_cxt); + Assert(current_call_state.prodesc->fn_refcount > 0); + if (--current_call_state.prodesc->fn_refcount == 0) + MemoryContextDelete(current_call_state.prodesc->fn_cxt); } return retval; @@ -708,7 +750,8 @@ pltcl_handler(PG_FUNCTION_ARGS, bool pltrusted) * pltcl_func_handler() - Handler for regular function calls **********************************************************************/ static Datum -pltcl_func_handler(PG_FUNCTION_ARGS, bool pltrusted) +pltcl_func_handler(PG_FUNCTION_ARGS, pltcl_call_state *call_state, + bool pltrusted) { pltcl_proc_desc *prodesc; Tcl_Interp *volatile interp; @@ -725,11 +768,32 @@ pltcl_func_handler(PG_FUNCTION_ARGS, bool pltrusted) prodesc = compile_pltcl_function(fcinfo->flinfo->fn_oid, InvalidOid, false, pltrusted); - pltcl_current_prodesc = prodesc; + call_state->prodesc = prodesc; prodesc->fn_refcount++; interp = prodesc->interp_desc->interp; + /* + * If we're a SRF, check caller can handle materialize mode, and save + * relevant info into call_state. We must ensure that the returned + * tuplestore is owned by the caller's context, even if we first create it + * inside a subtransaction. + */ + if (prodesc->fn_retisset) + { + ReturnSetInfo *rsi = (ReturnSetInfo *) fcinfo->resultinfo; + + if (!rsi || !IsA(rsi, ReturnSetInfo) || + (rsi->allowedModes & SFRM_Materialize) == 0) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("set-valued function called in context that cannot accept a set"))); + + call_state->rsi = rsi; + call_state->tuple_store_cxt = rsi->econtext->ecxt_per_query_memory; + call_state->tuple_store_owner = CurrentResourceOwner; + } + /************************************************************ * Create the tcl command to call the internal * proc in the Tcl interpreter @@ -838,11 +902,72 @@ pltcl_func_handler(PG_FUNCTION_ARGS, bool pltrusted) if (SPI_finish() != SPI_OK_FINISH) elog(ERROR, "SPI_finish() failed"); - if (fcinfo->isnull) + if (prodesc->fn_retisset) + { + ReturnSetInfo *rsi = call_state->rsi; + + /* We already checked this is OK */ + rsi->returnMode = SFRM_Materialize; + + /* If we produced any tuples, send back the result */ + if (call_state->tuple_store) + { + rsi->setResult = call_state->tuple_store; + if (call_state->ret_tupdesc) + { + MemoryContext oldcxt; + + oldcxt = MemoryContextSwitchTo(call_state->tuple_store_cxt); + rsi->setDesc = CreateTupleDescCopy(call_state->ret_tupdesc); + MemoryContextSwitchTo(oldcxt); + } + } + retval = (Datum) 0; + fcinfo->isnull = true; + } + else if (fcinfo->isnull) + { retval = InputFunctionCall(&prodesc->result_in_func, NULL, prodesc->result_typioparam, -1); + } + else if (prodesc->fn_retistuple) + { + TupleDesc td; + HeapTuple tup; + Tcl_Obj *resultObj; + Tcl_Obj **resultObjv; + int resultObjc; + + /* + * Set up data about result type. XXX it's tempting to consider + * caching this in the prodesc, in the common case where the rowtype + * is determined by the function not the calling query. But we'd have + * to be able to deal with ADD/DROP/ALTER COLUMN events when the + * result type is a named composite type, so it's not exactly trivial. + * Maybe worth improving someday. + */ + if (get_call_result_type(fcinfo, NULL, &td) != TYPEFUNC_COMPOSITE) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("function returning record called in context " + "that cannot accept type record"))); + + Assert(!call_state->ret_tupdesc); + Assert(!call_state->attinmeta); + call_state->ret_tupdesc = td; + call_state->attinmeta = TupleDescGetAttInMetadata(td); + + /* Convert function result to tuple */ + resultObj = Tcl_GetObjResult(interp); + if (Tcl_ListObjGetElements(interp, resultObj, &resultObjc, &resultObjv) == TCL_ERROR) + throw_tcl_error(interp, prodesc->user_proname); + + tup = pltcl_build_tuple_result(interp, resultObjv, resultObjc, + call_state); + retval = HeapTupleGetDatum(tup); + } else retval = InputFunctionCall(&prodesc->result_in_func, utf_u2e(Tcl_GetStringResult(interp)), @@ -857,7 +982,8 @@ pltcl_func_handler(PG_FUNCTION_ARGS, bool pltrusted) * pltcl_trigger_handler() - Handler for trigger calls **********************************************************************/ static HeapTuple -pltcl_trigger_handler(PG_FUNCTION_ARGS, bool pltrusted) +pltcl_trigger_handler(PG_FUNCTION_ARGS, pltcl_call_state *call_state, + bool pltrusted) { pltcl_proc_desc *prodesc; Tcl_Interp *volatile interp; @@ -886,7 +1012,7 @@ pltcl_trigger_handler(PG_FUNCTION_ARGS, bool pltrusted) false, /* not an event trigger */ pltrusted); - pltcl_current_prodesc = prodesc; + call_state->prodesc = prodesc; prodesc->fn_refcount++; interp = prodesc->interp_desc->interp; @@ -1169,7 +1295,8 @@ pltcl_trigger_handler(PG_FUNCTION_ARGS, bool pltrusted) * pltcl_event_trigger_handler() - Handler for event trigger calls **********************************************************************/ static void -pltcl_event_trigger_handler(PG_FUNCTION_ARGS, bool pltrusted) +pltcl_event_trigger_handler(PG_FUNCTION_ARGS, pltcl_call_state *call_state, + bool pltrusted) { pltcl_proc_desc *prodesc; Tcl_Interp *volatile interp; @@ -1185,7 +1312,7 @@ pltcl_event_trigger_handler(PG_FUNCTION_ARGS, bool pltrusted) prodesc = compile_pltcl_function(fcinfo->flinfo->fn_oid, InvalidOid, true, pltrusted); - pltcl_current_prodesc = prodesc; + call_state->prodesc = prodesc; prodesc->fn_refcount++; interp = prodesc->interp_desc->interp; @@ -1389,10 +1516,11 @@ compile_pltcl_function(Oid fn_oid, Oid tgreloid, procStruct->prorettype); typeStruct = (Form_pg_type) GETSTRUCT(typeTup); - /* Disallow pseudotype result, except VOID */ + /* Disallow pseudotype result, except VOID and RECORD */ if (typeStruct->typtype == TYPTYPE_PSEUDO) { - if (procStruct->prorettype == VOIDOID) + if (procStruct->prorettype == VOIDOID || + procStruct->prorettype == RECORDOID) /* okay */ ; else if (procStruct->prorettype == TRIGGEROID || procStruct->prorettype == EVTTRIGGEROID) @@ -1406,16 +1534,15 @@ compile_pltcl_function(Oid fn_oid, Oid tgreloid, format_type_be(procStruct->prorettype)))); } - if (typeStruct->typtype == TYPTYPE_COMPOSITE) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("PL/Tcl functions cannot return composite types"))); - fmgr_info_cxt(typeStruct->typinput, &(prodesc->result_in_func), proc_cxt); prodesc->result_typioparam = getTypeIOParam(typeTup); + prodesc->fn_retisset = procStruct->proretset; + prodesc->fn_retistuple = (procStruct->prorettype == RECORDOID || + typeStruct->typtype == TYPTYPE_COMPOSITE); + ReleaseSysCache(typeTup); } @@ -1914,7 +2041,7 @@ pltcl_argisnull(ClientData cdata, Tcl_Interp *interp, int objc, Tcl_Obj *const objv[]) { int argno; - FunctionCallInfo fcinfo = pltcl_current_fcinfo; + FunctionCallInfo fcinfo = pltcl_current_call_state->fcinfo; /************************************************************ * Check call syntax @@ -1967,7 +2094,7 @@ static int pltcl_returnnull(ClientData cdata, Tcl_Interp *interp, int objc, Tcl_Obj *const objv[]) { - FunctionCallInfo fcinfo = pltcl_current_fcinfo; + FunctionCallInfo fcinfo = pltcl_current_call_state->fcinfo; /************************************************************ * Check call syntax @@ -1998,6 +2125,95 @@ pltcl_returnnull(ClientData cdata, Tcl_Interp *interp, } +/********************************************************************** + * pltcl_returnnext() - Add a row to the result tuplestore in a SRF. + **********************************************************************/ +static int +pltcl_returnnext(ClientData cdata, Tcl_Interp *interp, + int objc, Tcl_Obj *const objv[]) +{ + pltcl_call_state *call_state = pltcl_current_call_state; + FunctionCallInfo fcinfo = call_state->fcinfo; + pltcl_proc_desc *prodesc = call_state->prodesc; + int result = TCL_OK; + MemoryContext tmpcxt; + MemoryContext oldcxt; + + /* + * Check that we're called as a set-returning function + */ + if (fcinfo == NULL) + { + Tcl_SetObjResult(interp, + Tcl_NewStringObj("return_next cannot be used in triggers", -1)); + return TCL_ERROR; + } + + if (!prodesc->fn_retisset) + { + Tcl_SetObjResult(interp, + Tcl_NewStringObj("return_next cannot be used in non-set-returning functions", -1)); + return TCL_ERROR; + } + + /* + * Check call syntax + */ + if (objc != 2) + { + Tcl_WrongNumArgs(interp, 1, objv, "result"); + return TCL_ERROR; + } + + /* Set up tuple store if first output row */ + if (call_state->tuple_store == NULL) + pltcl_init_tuple_store(call_state); + + /* Make short-lived context to run input functions in */ + tmpcxt = AllocSetContextCreate(CurrentMemoryContext, + "pltcl_returnnext", + ALLOCSET_SMALL_SIZES); + oldcxt = MemoryContextSwitchTo(tmpcxt); + + if (prodesc->fn_retistuple) + { + Tcl_Obj **rowObjv; + int rowObjc; + + /* result should be a list, so break it down */ + if (Tcl_ListObjGetElements(interp, objv[1], &rowObjc, &rowObjv) == TCL_ERROR) + result = TCL_ERROR; + else + { + HeapTuple tuple; + + SPI_push(); + tuple = pltcl_build_tuple_result(interp, rowObjv, rowObjc, + call_state); + tuplestore_puttuple(call_state->tuple_store, tuple); + SPI_pop(); + } + } + else + { + Datum retval; + bool isNull = false; + + retval = InputFunctionCall(&prodesc->result_in_func, + utf_u2e((char *) Tcl_GetString(objv[1])), + prodesc->result_typioparam, + -1); + tuplestore_putvalues(call_state->tuple_store, call_state->ret_tupdesc, + &retval, &isNull); + } + + MemoryContextSwitchTo(oldcxt); + MemoryContextDelete(tmpcxt); + + return result; +} + + /*---------- * Support for running SPI operations inside subtransactions * @@ -2164,7 +2380,7 @@ pltcl_SPI_execute(ClientData cdata, Tcl_Interp *interp, { UTF_BEGIN; spi_rc = SPI_execute(UTF_U2E(Tcl_GetString(objv[query_idx])), - pltcl_current_prodesc->fn_readonly, count); + pltcl_current_call_state->prodesc->fn_readonly, count); UTF_END; my_rc = pltcl_process_SPI_result(interp, @@ -2414,7 +2630,7 @@ pltcl_SPI_prepare(ClientData cdata, Tcl_Interp *interp, * Insert a hashtable entry for the plan and return * the key to the caller ************************************************************/ - query_hash = &pltcl_current_prodesc->interp_desc->query_hash; + query_hash = &pltcl_current_call_state->prodesc->interp_desc->query_hash; hashent = Tcl_CreateHashEntry(query_hash, qdesc->qname, &hashnew); Tcl_SetHashValue(hashent, (ClientData) qdesc); @@ -2503,7 +2719,7 @@ pltcl_SPI_execute_plan(ClientData cdata, Tcl_Interp *interp, return TCL_ERROR; } - query_hash = &pltcl_current_prodesc->interp_desc->query_hash; + query_hash = &pltcl_current_call_state->prodesc->interp_desc->query_hash; hashent = Tcl_FindHashEntry(query_hash, Tcl_GetString(objv[i])); if (hashent == NULL) @@ -2618,7 +2834,8 @@ pltcl_SPI_execute_plan(ClientData cdata, Tcl_Interp *interp, * Execute the plan ************************************************************/ spi_rc = SPI_execute_plan(qdesc->plan, argvalues, nulls, - pltcl_current_prodesc->fn_readonly, count); + pltcl_current_call_state->prodesc->fn_readonly, + count); my_rc = pltcl_process_SPI_result(interp, arrayname, @@ -2808,3 +3025,88 @@ pltcl_build_tuple_argument(HeapTuple tuple, TupleDesc tupdesc) return retobj; } + +/********************************************************************** + * pltcl_build_tuple_result() - Build a tuple of function's result rowtype + * from a Tcl list of column names and values + * + * Note: this function leaks memory. Even if we made it clean up its own + * mess, there's no way to prevent the datatype input functions it calls + * from leaking. Run it in a short-lived context, unless we're about to + * exit the procedure anyway. + * + * Also, caller is responsible for doing SPI_push/SPI_pop if calling from + * inside SPI environment. + **********************************************************************/ +static HeapTuple +pltcl_build_tuple_result(Tcl_Interp *interp, Tcl_Obj **kvObjv, int kvObjc, + pltcl_call_state *call_state) +{ + char **values; + int i; + + if (kvObjc % 2 != 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("column name/value list must have even number of elements"))); + + values = (char **) palloc0(call_state->ret_tupdesc->natts * sizeof(char *)); + + for (i = 0; i < kvObjc; i += 2) + { + char *fieldName = utf_e2u(Tcl_GetString(kvObjv[i])); + int attn = SPI_fnumber(call_state->ret_tupdesc, fieldName); + + if (attn <= 0 || call_state->ret_tupdesc->attrs[attn - 1]->attisdropped) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_COLUMN), + errmsg("column name/value list contains nonexistent column name \"%s\"", + fieldName))); + + values[attn - 1] = utf_e2u(Tcl_GetString(kvObjv[i + 1])); + } + + return BuildTupleFromCStrings(call_state->attinmeta, values); +} + +/********************************************************************** + * pltcl_init_tuple_store() - Initialize the result tuplestore for a SRF + **********************************************************************/ +static void +pltcl_init_tuple_store(pltcl_call_state *call_state) +{ + ReturnSetInfo *rsi = call_state->rsi; + MemoryContext oldcxt; + ResourceOwner oldowner; + + /* Should be in a SRF */ + Assert(rsi); + /* Should be first time through */ + Assert(!call_state->tuple_store); + Assert(!call_state->attinmeta); + + /* We expect caller to provide an appropriate result tupdesc */ + Assert(rsi->expectedDesc); + call_state->ret_tupdesc = rsi->expectedDesc; + + /* + * Switch to the right memory context and resource owner for storing the + * tuplestore. If we're within a subtransaction opened for an exception + * block, for example, we must still create the tuplestore in the resource + * owner that was active when this function was entered, and not in the + * subtransaction's resource owner. + */ + oldcxt = MemoryContextSwitchTo(call_state->tuple_store_cxt); + oldowner = CurrentResourceOwner; + CurrentResourceOwner = call_state->tuple_store_owner; + + call_state->tuple_store = + tuplestore_begin_heap(rsi->allowedModes & SFRM_Materialize_Random, + false, work_mem); + + /* Build attinmeta in this context, too */ + call_state->attinmeta = TupleDescGetAttInMetadata(call_state->ret_tupdesc); + + CurrentResourceOwner = oldowner; + MemoryContextSwitchTo(oldcxt); +} diff --git a/src/pl/tcl/sql/pltcl_queries.sql b/src/pl/tcl/sql/pltcl_queries.sql index a0a9619a9b..0ebfe65340 100644 --- a/src/pl/tcl/sql/pltcl_queries.sql +++ b/src/pl/tcl/sql/pltcl_queries.sql @@ -97,3 +97,36 @@ create temp table t1 (f1 int); select tcl_lastoid('t1'); create temp table t2 (f1 int) with oids; select tcl_lastoid('t2') > 0; + +-- test some error cases +CREATE FUNCTION tcl_error(OUT a int, OUT b int) AS $$return {$$ LANGUAGE pltcl; +SELECT tcl_error(); + +CREATE FUNCTION bad_record(OUT a text, OUT b text) AS $$return [list a]$$ LANGUAGE pltcl; +SELECT bad_record(); + +CREATE FUNCTION bad_field(OUT a text, OUT b text) AS $$return [list a 1 b 2 cow 3]$$ LANGUAGE pltcl; +SELECT bad_field(); + +-- test compound return +select * from tcl_test_cube_squared(5); + +-- test SRF +select * from tcl_test_squared_rows(0,5); + +select * from tcl_test_sequence(0,5) as a; + +select 1, tcl_test_sequence(0,5); + +CREATE FUNCTION non_srf() RETURNS int AS $$return_next 1$$ LANGUAGE pltcl; +select non_srf(); + +CREATE FUNCTION bad_record_srf(OUT a text, OUT b text) RETURNS SETOF record AS $$ +return_next [list a] +$$ LANGUAGE pltcl; +SELECT bad_record_srf(); + +CREATE FUNCTION bad_field_srf(OUT a text, OUT b text) RETURNS SETOF record AS $$ +return_next [list a 1 b 2 cow 3] +$$ LANGUAGE pltcl; +SELECT bad_field_srf(); diff --git a/src/pl/tcl/sql/pltcl_setup.sql b/src/pl/tcl/sql/pltcl_setup.sql index 8df65a5816..58f38d53aa 100644 --- a/src/pl/tcl/sql/pltcl_setup.sql +++ b/src/pl/tcl/sql/pltcl_setup.sql @@ -596,6 +596,22 @@ drop table foo; drop event trigger tcl_a_snitch; drop event trigger tcl_b_snitch; +CREATE FUNCTION tcl_test_cube_squared(in int, out squared int, out cubed int) AS $$ + return [list squared [expr {$1 * $1}] cubed [expr {$1 * $1 * $1}]] +$$ language pltcl; + +CREATE FUNCTION tcl_test_squared_rows(int,int) RETURNS TABLE (x int, y int) AS $$ + for {set i $1} {$i < $2} {incr i} { + return_next [list y [expr {$i * $i}] x $i] + } +$$ language pltcl; + +CREATE FUNCTION tcl_test_sequence(int,int) RETURNS SETOF int AS $$ + for {set i $1} {$i < $2} {incr i} { + return_next $i + } +$$ language pltcl; + -- test use of errorCode in error handling create function tcl_error_handling_test() returns text as $$ -- cgit v1.2.3 From 33cb96ba1a84c612491fb5794674a649d1a6a4d6 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 7 Nov 2016 10:19:22 -0500 Subject: Revert "Provide DLLEXPORT markers for C functions via PG_FUNCTION_INFO_V1 macro." This reverts commit c8ead2a3974d3eada145a0e18940150039493cc9. Seems there is no way to do this that doesn't cause MSVC to give warnings, so let's just go back to the way we've been doing it. Discussion: <11843.1478358206@sss.pgh.pa.us> --- contrib/hstore/hstore.h | 2 +- contrib/ltree/ltree.h | 40 ++++++++++++++++++++-------------------- doc/src/sgml/xfunc.sgml | 17 ----------------- src/include/fmgr.h | 7 ++++--- 4 files changed, 25 insertions(+), 41 deletions(-) (limited to 'doc/src') diff --git a/contrib/hstore/hstore.h b/contrib/hstore/hstore.h index 6303fa4061..6bab08b7de 100644 --- a/contrib/hstore/hstore.h +++ b/contrib/hstore/hstore.h @@ -194,7 +194,7 @@ extern Pairs *hstoreArrayToPairs(ArrayType *a, int *npairs); #if HSTORE_POLLUTE_NAMESPACE #define HSTORE_POLLUTE(newname_,oldname_) \ PG_FUNCTION_INFO_V1(oldname_); \ - extern PGDLLEXPORT Datum newname_(PG_FUNCTION_ARGS); \ + Datum newname_(PG_FUNCTION_ARGS); \ Datum oldname_(PG_FUNCTION_ARGS) { return newname_(fcinfo); } \ extern int no_such_variable #else diff --git a/contrib/ltree/ltree.h b/contrib/ltree/ltree.h index c7aa7f8818..c604357dbf 100644 --- a/contrib/ltree/ltree.h +++ b/contrib/ltree/ltree.h @@ -130,30 +130,30 @@ typedef struct /* use in array iterator */ -extern PGDLLEXPORT Datum ltree_isparent(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum ltree_risparent(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum ltq_regex(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum ltq_rregex(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum lt_q_regex(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum lt_q_rregex(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum ltxtq_exec(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum ltxtq_rexec(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum _ltq_regex(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum _ltq_rregex(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum _lt_q_regex(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum _lt_q_rregex(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum _ltxtq_exec(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum _ltxtq_rexec(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum _ltree_isparent(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum _ltree_risparent(PG_FUNCTION_ARGS); +Datum ltree_isparent(PG_FUNCTION_ARGS); +Datum ltree_risparent(PG_FUNCTION_ARGS); +Datum ltq_regex(PG_FUNCTION_ARGS); +Datum ltq_rregex(PG_FUNCTION_ARGS); +Datum lt_q_regex(PG_FUNCTION_ARGS); +Datum lt_q_rregex(PG_FUNCTION_ARGS); +Datum ltxtq_exec(PG_FUNCTION_ARGS); +Datum ltxtq_rexec(PG_FUNCTION_ARGS); +Datum _ltq_regex(PG_FUNCTION_ARGS); +Datum _ltq_rregex(PG_FUNCTION_ARGS); +Datum _lt_q_regex(PG_FUNCTION_ARGS); +Datum _lt_q_rregex(PG_FUNCTION_ARGS); +Datum _ltxtq_exec(PG_FUNCTION_ARGS); +Datum _ltxtq_rexec(PG_FUNCTION_ARGS); +Datum _ltree_isparent(PG_FUNCTION_ARGS); +Datum _ltree_risparent(PG_FUNCTION_ARGS); /* Concatenation functions */ -extern PGDLLEXPORT Datum ltree_addltree(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum ltree_addtext(PG_FUNCTION_ARGS); -extern PGDLLEXPORT Datum ltree_textadd(PG_FUNCTION_ARGS); +Datum ltree_addltree(PG_FUNCTION_ARGS); +Datum ltree_addtext(PG_FUNCTION_ARGS); +Datum ltree_textadd(PG_FUNCTION_ARGS); /* Util function */ -extern PGDLLEXPORT Datum ltree_in(PG_FUNCTION_ARGS); +Datum ltree_in(PG_FUNCTION_ARGS); bool ltree_execute(ITEM *curitem, void *checkval, bool calcnot, bool (*chkcond) (void *checkval, ITEM *val)); diff --git a/doc/src/sgml/xfunc.sgml b/doc/src/sgml/xfunc.sgml index 6060e61857..de6a466efc 100644 --- a/doc/src/sgml/xfunc.sgml +++ b/doc/src/sgml/xfunc.sgml @@ -2577,23 +2577,6 @@ concat_text(PG_FUNCTION_ARGS) error messages to this effect. - - - - To work correctly on Windows, C-language functions need - to be marked with PGDLLEXPORT, unless you use a build - process that marks all global functions that way. In simple cases - this detail will be handled transparently by - the PG_FUNCTION_INFO_V1 macro. However, if you write - explicit external declarations (perhaps in header files), be sure - to write them like this: - -extern PGDLLEXPORT Datum funcname(PG_FUNCTION_ARGS); - - or you'll get compiler complaints when building on Windows. (On - other platforms, the PGDLLEXPORT macro does nothing.) - - diff --git a/src/include/fmgr.h b/src/include/fmgr.h index 3668ac3f6e..0878418516 100644 --- a/src/include/fmgr.h +++ b/src/include/fmgr.h @@ -350,11 +350,12 @@ typedef const Pg_finfo_record *(*PGFInfoFunction) (void); * * On Windows, the function and info function must be exported. Our normal * build processes take care of that via .DEF files or --export-all-symbols. - * Module authors using a different build process might do it differently, - * so we declare these functions PGDLLEXPORT for their convenience. + * Module authors using a different build process might need to manually + * declare the function PGDLLEXPORT. We do that automatically here for the + * info function, since authors shouldn't need to be explicitly aware of it. */ #define PG_FUNCTION_INFO_V1(funcname) \ -extern PGDLLEXPORT Datum funcname(PG_FUNCTION_ARGS); \ +extern Datum funcname(PG_FUNCTION_ARGS); \ extern PGDLLEXPORT const Pg_finfo_record * CppConcat(pg_finfo_,funcname)(void); \ const Pg_finfo_record * \ CppConcat(pg_finfo_,funcname) (void) \ -- cgit v1.2.3 From 36ac6d0e793087153a452df6502d0ef32a780db6 Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Tue, 8 Nov 2016 18:34:59 +0100 Subject: Fix typo --- doc/src/sgml/ref/pg_basebackup.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml index e66a7ae8ee..1f15a17d0e 100644 --- a/doc/src/sgml/ref/pg_basebackup.sgml +++ b/doc/src/sgml/ref/pg_basebackup.sgml @@ -88,7 +88,7 @@ PostgreSQL documentation There is no guarantee that all WAL files required for the backup are archived at the end of backup. If you are planning to use the backup for an archive recovery and want to ensure that all required files are available at that moment, - you need to include them into the backup by using -x option. + you need to include them into the backup by using the -x option. -- cgit v1.2.3 From 6d30fb1f75a57d80f80e27770d39d88f8aa32d28 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 8 Nov 2016 13:11:15 -0500 Subject: Make SPI_fnumber() reject dropped columns. There's basically no scenario where it's sensible for this to match dropped columns, so put a test for dropped-ness into SPI_fnumber() itself, and excise the test from the small number of callers that were paying attention to the case. (Most weren't :-(.) In passing, normalize tests at call sites: always reject attnum <= 0 if we're disallowing system columns. Previously there was a mixture of "< 0" and "<= 0" tests. This makes no practical difference since SPI_fnumber() never returns 0, but I'm feeling pedantic today. Also, in the places that are actually live user-facing code and not legacy cruft, distinguish "column not found" from "can't handle system column". Per discussion with Jim Nasby; thi supersedes his original patch that just changed the behavior at one call site. Discussion: --- contrib/spi/autoinc.c | 2 +- contrib/spi/insert_username.c | 2 +- contrib/spi/moddatetime.c | 4 ++-- contrib/spi/refint.c | 5 +++-- contrib/spi/timetravel.c | 4 ++-- doc/src/sgml/spi.sgml | 2 +- src/backend/executor/spi.c | 3 ++- src/backend/utils/adt/tsvector_op.c | 1 + src/pl/plperl/plperl.c | 7 ++++++- src/pl/tcl/pltcl.c | 11 +---------- src/test/regress/regress.c | 9 +++++---- 11 files changed, 25 insertions(+), 25 deletions(-) (limited to 'doc/src') diff --git a/contrib/spi/autoinc.c b/contrib/spi/autoinc.c index 41eae4fdc4..fc657a7c06 100644 --- a/contrib/spi/autoinc.c +++ b/contrib/spi/autoinc.c @@ -71,7 +71,7 @@ autoinc(PG_FUNCTION_ARGS) int32 val; Datum seqname; - if (attnum < 0) + if (attnum <= 0) ereport(ERROR, (errcode(ERRCODE_TRIGGERED_ACTION_EXCEPTION), errmsg("\"%s\" has no attribute \"%s\"", diff --git a/contrib/spi/insert_username.c b/contrib/spi/insert_username.c index 3812525c4c..617c60a81c 100644 --- a/contrib/spi/insert_username.c +++ b/contrib/spi/insert_username.c @@ -67,7 +67,7 @@ insert_username(PG_FUNCTION_ARGS) attnum = SPI_fnumber(tupdesc, args[0]); - if (attnum < 0) + if (attnum <= 0) ereport(ERROR, (errcode(ERRCODE_TRIGGERED_ACTION_EXCEPTION), errmsg("\"%s\" has no attribute \"%s\"", relname, args[0]))); diff --git a/contrib/spi/moddatetime.c b/contrib/spi/moddatetime.c index c6d33b7355..cd700fe6d1 100644 --- a/contrib/spi/moddatetime.c +++ b/contrib/spi/moddatetime.c @@ -84,9 +84,9 @@ moddatetime(PG_FUNCTION_ARGS) /* * This is where we check to see if the field we are supposed to update - * even exists. The above function must return -1 if name not found? + * even exists. */ - if (attnum < 0) + if (attnum <= 0) ereport(ERROR, (errcode(ERRCODE_TRIGGERED_ACTION_EXCEPTION), errmsg("\"%s\" has no attribute \"%s\"", diff --git a/contrib/spi/refint.c b/contrib/spi/refint.c index 01dd717522..78cfedf219 100644 --- a/contrib/spi/refint.c +++ b/contrib/spi/refint.c @@ -135,7 +135,7 @@ check_primary_key(PG_FUNCTION_ARGS) int fnumber = SPI_fnumber(tupdesc, args[i]); /* Bad guys may give us un-existing column in CREATE TRIGGER */ - if (fnumber < 0) + if (fnumber <= 0) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("there is no attribute \"%s\" in relation \"%s\"", @@ -362,7 +362,7 @@ check_foreign_key(PG_FUNCTION_ARGS) int fnumber = SPI_fnumber(tupdesc, args[i]); /* Bad guys may give us un-existing column in CREATE TRIGGER */ - if (fnumber < 0) + if (fnumber <= 0) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("there is no attribute \"%s\" in relation \"%s\"", @@ -469,6 +469,7 @@ check_foreign_key(PG_FUNCTION_ARGS) char *type; fn = SPI_fnumber(tupdesc, args_temp[k - 1]); + Assert(fn > 0); /* already checked above */ nv = SPI_getvalue(newtuple, tupdesc, fn); type = SPI_gettype(tupdesc, fn); diff --git a/contrib/spi/timetravel.c b/contrib/spi/timetravel.c index 5a345841c6..30dcfd4d3e 100644 --- a/contrib/spi/timetravel.c +++ b/contrib/spi/timetravel.c @@ -157,7 +157,7 @@ timetravel(PG_FUNCTION_ARGS) for (i = 0; i < MinAttrNum; i++) { attnum[i] = SPI_fnumber(tupdesc, args[i]); - if (attnum[i] < 0) + if (attnum[i] <= 0) elog(ERROR, "timetravel (%s): there is no attribute %s", relname, args[i]); if (SPI_gettypeid(tupdesc, attnum[i]) != ABSTIMEOID) elog(ERROR, "timetravel (%s): attribute %s must be of abstime type", @@ -166,7 +166,7 @@ timetravel(PG_FUNCTION_ARGS) for (; i < argc; i++) { attnum[i] = SPI_fnumber(tupdesc, args[i]); - if (attnum[i] < 0) + if (attnum[i] <= 0) elog(ERROR, "timetravel (%s): there is no attribute %s", relname, args[i]); if (SPI_gettypeid(tupdesc, attnum[i]) != TEXTOID) elog(ERROR, "timetravel (%s): attribute %s must be of text type", diff --git a/doc/src/sgml/spi.sgml b/doc/src/sgml/spi.sgml index 9ae7126ae7..817a5d0120 100644 --- a/doc/src/sgml/spi.sgml +++ b/doc/src/sgml/spi.sgml @@ -2891,7 +2891,7 @@ int SPI_fnumber(TupleDesc rowdesc, const char * Return Value - Column number (count starts at 1), or + Column number (count starts at 1 for user-defined columns), or SPI_ERROR_NOATTRIBUTE if the named column was not found. diff --git a/src/backend/executor/spi.c b/src/backend/executor/spi.c index 38767ae4ce..8e650bc412 100644 --- a/src/backend/executor/spi.c +++ b/src/backend/executor/spi.c @@ -824,7 +824,8 @@ SPI_fnumber(TupleDesc tupdesc, const char *fname) for (res = 0; res < tupdesc->natts; res++) { - if (namestrcmp(&tupdesc->attrs[res]->attname, fname) == 0) + if (namestrcmp(&tupdesc->attrs[res]->attname, fname) == 0 && + !tupdesc->attrs[res]->attisdropped) return res + 1; } diff --git a/src/backend/utils/adt/tsvector_op.c b/src/backend/utils/adt/tsvector_op.c index ad5a254c57..0e9ae5ff9c 100644 --- a/src/backend/utils/adt/tsvector_op.c +++ b/src/backend/utils/adt/tsvector_op.c @@ -2242,6 +2242,7 @@ tsvector_update_trigger(PG_FUNCTION_ARGS, bool config_column) (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("tsvector column \"%s\" does not exist", trigger->tgargs[0]))); + /* This will effectively reject system columns, so no separate test: */ if (!IsBinaryCoercible(SPI_gettypeid(rel->rd_att, tsvector_attr_num), TSVECTOROID)) ereport(ERROR, diff --git a/src/pl/plperl/plperl.c b/src/pl/plperl/plperl.c index 4d993e7371..461986cda3 100644 --- a/src/pl/plperl/plperl.c +++ b/src/pl/plperl/plperl.c @@ -1062,11 +1062,16 @@ plperl_build_tuple_result(HV *perlhash, TupleDesc td) char *key = hek2cstr(he); int attn = SPI_fnumber(td, key); - if (attn <= 0 || td->attrs[attn - 1]->attisdropped) + if (attn == SPI_ERROR_NOATTRIBUTE) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("Perl hash contains nonexistent column \"%s\"", key))); + if (attn <= 0) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot set system attribute \"%s\"", + key))); values[attn - 1] = plperl_sv_to_datum(val, td->attrs[attn - 1]->atttypid, diff --git a/src/pl/tcl/pltcl.c b/src/pl/tcl/pltcl.c index 3e52113ee2..20809102ef 100644 --- a/src/pl/tcl/pltcl.c +++ b/src/pl/tcl/pltcl.c @@ -603,6 +603,7 @@ pltcl_init_load_unknown(Tcl_Interp *interp) * leave this code as DString - it's only executed once per session ************************************************************/ fno = SPI_fnumber(SPI_tuptable->tupdesc, "modsrc"); + Assert(fno > 0); Tcl_DStringInit(&unknown_src); @@ -1259,12 +1260,6 @@ pltcl_trigger_handler(PG_FUNCTION_ARGS, pltcl_call_state *call_state, errmsg("cannot set system attribute \"%s\"", ret_name))); - /************************************************************ - * Ignore dropped columns - ************************************************************/ - if (tupdesc->attrs[attnum - 1]->attisdropped) - continue; - /************************************************************ * Lookup the attribute type's input function ************************************************************/ @@ -3077,10 +3072,6 @@ pltcl_build_tuple_result(Tcl_Interp *interp, Tcl_Obj **kvObjv, int kvObjc, errmsg("cannot set system attribute \"%s\"", fieldName))); - /* Ignore dropped attributes */ - if (call_state->ret_tupdesc->attrs[attn - 1]->attisdropped) - continue; - values[attn - 1] = utf_e2u(Tcl_GetString(kvObjv[i + 1])); } diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index e7826a4513..119a59ab07 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -523,11 +523,12 @@ ttdummy(PG_FUNCTION_ARGS) for (i = 0; i < 2; i++) { attnum[i] = SPI_fnumber(tupdesc, args[i]); - if (attnum[i] < 0) - elog(ERROR, "ttdummy (%s): there is no attribute %s", relname, args[i]); + if (attnum[i] <= 0) + elog(ERROR, "ttdummy (%s): there is no attribute %s", + relname, args[i]); if (SPI_gettypeid(tupdesc, attnum[i]) != INT4OID) - elog(ERROR, "ttdummy (%s): attributes %s and %s must be of abstime type", - relname, args[0], args[1]); + elog(ERROR, "ttdummy (%s): attribute %s must be of integer type", + relname, args[i]); } oldon = SPI_getbinval(trigtuple, tupdesc, attnum[0], &isnull); -- cgit v1.2.3 From 9257f0787257022e31c61cd77449127adfccf37f Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 8 Nov 2016 15:36:36 -0500 Subject: Replace uses of SPI_modifytuple that intend to allocate in current context. Invent a new function heap_modify_tuple_by_cols() that is functionally equivalent to SPI_modifytuple except that it always allocates its result by simple palloc. I chose however to make the API details a bit more like heap_modify_tuple: pass a tupdesc rather than a Relation, and use bool convention for the isnull array. Use this function in place of SPI_modifytuple at all call sites where the intended behavior is to allocate in current context. (There actually are only two call sites left that depend on the old behavior, which makes me wonder if we should just drop this function rather than keep it.) This new function is easier to use than heap_modify_tuple() for purposes of replacing a single column (or, really, any fixed number of columns). There are a number of places where it would simplify the code to change over, but I resisted that temptation for the moment ... everywhere except in plpgsql's exec_assign_value(); changing that might offer some small performance benefit, so I did it. This is on the way to removing SPI_push/SPI_pop, but it seems like good code cleanup in its own right. Discussion: <9633.1478552022@sss.pgh.pa.us> --- contrib/spi/autoinc.c | 13 ++++--- contrib/spi/insert_username.c | 12 +++---- contrib/spi/moddatetime.c | 21 ++++------- contrib/spi/timetravel.c | 25 ++++++------- doc/src/sgml/spi.sgml | 5 +-- src/backend/access/common/heaptuple.c | 66 +++++++++++++++++++++++++++++++++++ src/backend/utils/adt/tsvector_op.c | 16 ++++----- src/include/access/htup_details.h | 6 ++++ src/pl/plpgsql/src/pl_exec.c | 57 ++++++++++-------------------- src/test/regress/regress.c | 11 ++---- 10 files changed, 137 insertions(+), 95 deletions(-) (limited to 'doc/src') diff --git a/contrib/spi/autoinc.c b/contrib/spi/autoinc.c index fc657a7c06..54f85a3709 100644 --- a/contrib/spi/autoinc.c +++ b/contrib/spi/autoinc.c @@ -3,6 +3,7 @@ */ #include "postgres.h" +#include "access/htup_details.h" #include "catalog/pg_type.h" #include "commands/sequence.h" #include "commands/trigger.h" @@ -23,6 +24,7 @@ autoinc(PG_FUNCTION_ARGS) int *chattrs; /* attnums of attributes to change */ int chnattrs = 0; /* # of above */ Datum *newvals; /* vals of above */ + bool *newnulls; /* null flags for above */ char **args; /* arguments */ char *relname; /* triggered relation name */ Relation rel; /* triggered relation */ @@ -64,6 +66,7 @@ autoinc(PG_FUNCTION_ARGS) chattrs = (int *) palloc(nargs / 2 * sizeof(int)); newvals = (Datum *) palloc(nargs / 2 * sizeof(Datum)); + newnulls = (bool *) palloc(nargs / 2 * sizeof(bool)); for (i = 0; i < nargs;) { @@ -102,6 +105,7 @@ autoinc(PG_FUNCTION_ARGS) newvals[chnattrs] = DirectFunctionCall1(nextval, seqname); newvals[chnattrs] = Int32GetDatum((int32) DatumGetInt64(newvals[chnattrs])); } + newnulls[chnattrs] = false; pfree(DatumGetTextP(seqname)); chnattrs++; i++; @@ -109,16 +113,15 @@ autoinc(PG_FUNCTION_ARGS) if (chnattrs > 0) { - rettuple = SPI_modifytuple(rel, rettuple, chnattrs, chattrs, newvals, NULL); - if (rettuple == NULL) - /* internal error */ - elog(ERROR, "autoinc (%s): %d returned by SPI_modifytuple", - relname, SPI_result); + rettuple = heap_modify_tuple_by_cols(rettuple, tupdesc, + chnattrs, chattrs, + newvals, newnulls); } pfree(relname); pfree(chattrs); pfree(newvals); + pfree(newnulls); return PointerGetDatum(rettuple); } diff --git a/contrib/spi/insert_username.c b/contrib/spi/insert_username.c index 617c60a81c..a2e1747ff7 100644 --- a/contrib/spi/insert_username.c +++ b/contrib/spi/insert_username.c @@ -1,6 +1,4 @@ /* - * insert_username.c - * $Modified: Thu Oct 16 08:13:42 1997 by brook $ * contrib/spi/insert_username.c * * insert user name in response to a trigger @@ -8,6 +6,7 @@ */ #include "postgres.h" +#include "access/htup_details.h" #include "catalog/pg_type.h" #include "commands/trigger.h" #include "executor/spi.h" @@ -26,6 +25,7 @@ insert_username(PG_FUNCTION_ARGS) Trigger *trigger; /* to get trigger name */ int nargs; /* # of arguments */ Datum newval; /* new value of column */ + bool newnull; /* null flag */ char **args; /* arguments */ char *relname; /* triggered relation name */ Relation rel; /* triggered relation */ @@ -80,13 +80,11 @@ insert_username(PG_FUNCTION_ARGS) /* create fields containing name */ newval = CStringGetTextDatum(GetUserNameFromId(GetUserId(), false)); + newnull = false; /* construct new tuple */ - rettuple = SPI_modifytuple(rel, rettuple, 1, &attnum, &newval, NULL); - if (rettuple == NULL) - /* internal error */ - elog(ERROR, "insert_username (\"%s\"): %d returned by SPI_modifytuple", - relname, SPI_result); + rettuple = heap_modify_tuple_by_cols(rettuple, tupdesc, + 1, &attnum, &newval, &newnull); pfree(relname); diff --git a/contrib/spi/moddatetime.c b/contrib/spi/moddatetime.c index cd700fe6d1..2d1f22c4e1 100644 --- a/contrib/spi/moddatetime.c +++ b/contrib/spi/moddatetime.c @@ -15,6 +15,7 @@ OH, me, I'm Terry Mackintosh */ #include "postgres.h" +#include "access/htup_details.h" #include "catalog/pg_type.h" #include "executor/spi.h" #include "commands/trigger.h" @@ -34,6 +35,7 @@ moddatetime(PG_FUNCTION_ARGS) int attnum; /* positional number of field to change */ Oid atttypid; /* type OID of field to change */ Datum newdt; /* The current datetime. */ + bool newdtnull; /* null flag for it */ char **args; /* arguments */ char *relname; /* triggered relation name */ Relation rel; /* triggered relation */ @@ -115,22 +117,13 @@ moddatetime(PG_FUNCTION_ARGS) args[0], relname))); newdt = (Datum) 0; /* keep compiler quiet */ } + newdtnull = false; -/* 1 is the number of items in the arrays attnum and newdt. - attnum is the positional number of the field to be updated. - newdt is the new datetime stamp. - NOTE that attnum and newdt are not arrays, but then a 1 element array - is not an array any more then they are. Thus, they can be considered a - one element array. -*/ - rettuple = SPI_modifytuple(rel, rettuple, 1, &attnum, &newdt, NULL); - - if (rettuple == NULL) - /* internal error */ - elog(ERROR, "moddatetime (%s): %d returned by SPI_modifytuple", - relname, SPI_result); + /* Replace the attnum'th column with newdt */ + rettuple = heap_modify_tuple_by_cols(rettuple, tupdesc, + 1, &attnum, &newdt, &newdtnull); -/* Clean up */ + /* Clean up */ pfree(relname); return PointerGetDatum(rettuple); diff --git a/contrib/spi/timetravel.c b/contrib/spi/timetravel.c index 30dcfd4d3e..2733aa231e 100644 --- a/contrib/spi/timetravel.c +++ b/contrib/spi/timetravel.c @@ -11,6 +11,7 @@ #include +#include "access/htup_details.h" #include "catalog/pg_type.h" #include "commands/trigger.h" #include "executor/spi.h" @@ -183,13 +184,13 @@ timetravel(PG_FUNCTION_ARGS) int chnattrs = 0; int chattrs[MaxAttrNum]; Datum newvals[MaxAttrNum]; - char newnulls[MaxAttrNum]; + bool newnulls[MaxAttrNum]; oldtimeon = SPI_getbinval(trigtuple, tupdesc, attnum[a_time_on], &isnull); if (isnull) { newvals[chnattrs] = GetCurrentAbsoluteTime(); - newnulls[chnattrs] = ' '; + newnulls[chnattrs] = false; chattrs[chnattrs] = attnum[a_time_on]; chnattrs++; } @@ -201,7 +202,7 @@ timetravel(PG_FUNCTION_ARGS) (chnattrs > 0 && DatumGetInt32(newvals[a_time_on]) >= NOEND_ABSTIME)) elog(ERROR, "timetravel (%s): %s is infinity", relname, args[a_time_on]); newvals[chnattrs] = NOEND_ABSTIME; - newnulls[chnattrs] = ' '; + newnulls[chnattrs] = false; chattrs[chnattrs] = attnum[a_time_off]; chnattrs++; } @@ -220,21 +221,23 @@ timetravel(PG_FUNCTION_ARGS) { /* clear update_user value */ newvals[chnattrs] = nulltext; - newnulls[chnattrs] = 'n'; + newnulls[chnattrs] = true; chattrs[chnattrs] = attnum[a_upd_user]; chnattrs++; /* clear delete_user value */ newvals[chnattrs] = nulltext; - newnulls[chnattrs] = 'n'; + newnulls[chnattrs] = true; chattrs[chnattrs] = attnum[a_del_user]; chnattrs++; /* set insert_user value */ newvals[chnattrs] = newuser; - newnulls[chnattrs] = ' '; + newnulls[chnattrs] = false; chattrs[chnattrs] = attnum[a_ins_user]; chnattrs++; } - rettuple = SPI_modifytuple(rel, trigtuple, chnattrs, chattrs, newvals, newnulls); + rettuple = heap_modify_tuple_by_cols(trigtuple, tupdesc, + chnattrs, chattrs, + newvals, newnulls); return PointerGetDatum(rettuple); /* end of INSERT */ } @@ -395,13 +398,11 @@ timetravel(PG_FUNCTION_ARGS) chnattrs++; } - rettuple = SPI_modifytuple(rel, newtuple, chnattrs, chattrs, newvals, newnulls); - /* - * SPI_copytuple allocates tmptuple in upper executor context - have - * to free allocation using SPI_pfree + * Use SPI_modifytuple() here because we are inside SPI environment + * but rettuple must be allocated in caller's context. */ - /* SPI_pfree(tmptuple); */ + rettuple = SPI_modifytuple(rel, newtuple, chnattrs, chattrs, newvals, newnulls); } else /* DELETE case */ diff --git a/doc/src/sgml/spi.sgml b/doc/src/sgml/spi.sgml index 817a5d0120..39133c9038 100644 --- a/doc/src/sgml/spi.sgml +++ b/doc/src/sgml/spi.sgml @@ -3382,8 +3382,9 @@ char * SPI_getnspname(Relation rel) repalloc, or SPI utility functions (except for SPI_copytuple, SPI_returntuple, - SPI_modifytuple, and - SPI_palloc) are made in this context. When a + SPI_modifytuple, + SPI_palloc, and + SPI_datumTransfer) are made in this context. When a procedure disconnects from the SPI manager (via SPI_finish) the current context is restored to the upper executor context, and all allocations made in the diff --git a/src/backend/access/common/heaptuple.c b/src/backend/access/common/heaptuple.c index 6d0f3f3767..e27ec78b71 100644 --- a/src/backend/access/common/heaptuple.c +++ b/src/backend/access/common/heaptuple.c @@ -846,6 +846,72 @@ heap_modify_tuple(HeapTuple tuple, return newTuple; } +/* + * heap_modify_tuple_by_cols + * form a new tuple from an old tuple and a set of replacement values. + * + * This is like heap_modify_tuple, except that instead of specifying which + * column(s) to replace by a boolean map, an array of target column numbers + * is used. This is often more convenient when a fixed number of columns + * are to be replaced. The replCols, replValues, and replIsnull arrays must + * be of length nCols. Target column numbers are indexed from 1. + * + * The result is allocated in the current memory context. + */ +HeapTuple +heap_modify_tuple_by_cols(HeapTuple tuple, + TupleDesc tupleDesc, + int nCols, + int *replCols, + Datum *replValues, + bool *replIsnull) +{ + int numberOfAttributes = tupleDesc->natts; + Datum *values; + bool *isnull; + HeapTuple newTuple; + int i; + + /* + * allocate and fill values and isnull arrays from the tuple, then replace + * selected columns from the input arrays. + */ + values = (Datum *) palloc(numberOfAttributes * sizeof(Datum)); + isnull = (bool *) palloc(numberOfAttributes * sizeof(bool)); + + heap_deform_tuple(tuple, tupleDesc, values, isnull); + + for (i = 0; i < nCols; i++) + { + int attnum = replCols[i]; + + if (attnum <= 0 || attnum > numberOfAttributes) + elog(ERROR, "invalid column number %d", attnum); + values[attnum - 1] = replValues[i]; + isnull[attnum - 1] = replIsnull[i]; + } + + /* + * create a new tuple from the values and isnull arrays + */ + newTuple = heap_form_tuple(tupleDesc, values, isnull); + + pfree(values); + pfree(isnull); + + /* + * copy the identification info of the old tuple: t_ctid, t_self, and OID + * (if any) + */ + newTuple->t_data->t_ctid = tuple->t_data->t_ctid; + newTuple->t_self = tuple->t_self; + newTuple->t_tableOid = tuple->t_tableOid; + if (tupleDesc->tdhasoid) + HeapTupleSetOid(newTuple, HeapTupleGetOid(tuple)); + + return newTuple; +} + /* * heap_deform_tuple * Given a tuple, extract data into values/isnull arrays; this is diff --git a/src/backend/utils/adt/tsvector_op.c b/src/backend/utils/adt/tsvector_op.c index 0e9ae5ff9c..c9d5060f2c 100644 --- a/src/backend/utils/adt/tsvector_op.c +++ b/src/backend/utils/adt/tsvector_op.c @@ -2329,8 +2329,10 @@ tsvector_update_trigger(PG_FUNCTION_ARGS, bool config_column) if (prs.curwords) { datum = PointerGetDatum(make_tsvector(&prs)); - rettuple = SPI_modifytuple(rel, rettuple, 1, &tsvector_attr_num, - &datum, NULL); + isnull = false; + rettuple = heap_modify_tuple_by_cols(rettuple, rel->rd_att, + 1, &tsvector_attr_num, + &datum, &isnull); pfree(DatumGetPointer(datum)); } else @@ -2340,14 +2342,12 @@ tsvector_update_trigger(PG_FUNCTION_ARGS, bool config_column) SET_VARSIZE(out, CALCDATASIZE(0, 0)); out->size = 0; datum = PointerGetDatum(out); - rettuple = SPI_modifytuple(rel, rettuple, 1, &tsvector_attr_num, - &datum, NULL); + isnull = false; + rettuple = heap_modify_tuple_by_cols(rettuple, rel->rd_att, + 1, &tsvector_attr_num, + &datum, &isnull); pfree(prs.words); } - if (rettuple == NULL) /* internal error */ - elog(ERROR, "tsvector_update_trigger: %d returned by SPI_modifytuple", - SPI_result); - return PointerGetDatum(rettuple); } diff --git a/src/include/access/htup_details.h b/src/include/access/htup_details.h index d7e5fad11e..8fb1f6ddea 100644 --- a/src/include/access/htup_details.h +++ b/src/include/access/htup_details.h @@ -805,6 +805,12 @@ extern HeapTuple heap_modify_tuple(HeapTuple tuple, Datum *replValues, bool *replIsnull, bool *doReplace); +extern HeapTuple heap_modify_tuple_by_cols(HeapTuple tuple, + TupleDesc tupleDesc, + int nCols, + int *replCols, + Datum *replValues, + bool *replIsnull); extern void heap_deform_tuple(HeapTuple tuple, TupleDesc tupleDesc, Datum *values, bool *isnull); extern void heap_freetuple(HeapTuple htup); diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c index 042b31fd77..91e1f8dd3f 100644 --- a/src/pl/plpgsql/src/pl_exec.c +++ b/src/pl/plpgsql/src/pl_exec.c @@ -4562,10 +4562,9 @@ exec_assign_value(PLpgSQL_execstate *estate, PLpgSQL_rec *rec; int fno; HeapTuple newtup; - int natts; - Datum *values; - bool *nulls; - bool *replaces; + int colnums[1]; + Datum values[1]; + bool nulls[1]; Oid atttype; int32 atttypmod; @@ -4584,9 +4583,8 @@ exec_assign_value(PLpgSQL_execstate *estate, errdetail("The tuple structure of a not-yet-assigned record is indeterminate."))); /* - * Get the number of the records field to change and the - * number of attributes in the tuple. Note: disallow system - * column names because the code below won't cope. + * Get the number of the record field to change. Disallow + * system columns because the code below won't cope. */ fno = SPI_fnumber(rec->tupdesc, recfield->fieldname); if (fno <= 0) @@ -4594,42 +4592,25 @@ exec_assign_value(PLpgSQL_execstate *estate, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("record \"%s\" has no field \"%s\"", rec->refname, recfield->fieldname))); - fno--; - natts = rec->tupdesc->natts; - - /* - * Set up values/control arrays for heap_modify_tuple. For all - * the attributes except the one we want to replace, use the - * value that's in the old tuple. - */ - values = eval_mcontext_alloc(estate, sizeof(Datum) * natts); - nulls = eval_mcontext_alloc(estate, sizeof(bool) * natts); - replaces = eval_mcontext_alloc(estate, sizeof(bool) * natts); - - memset(replaces, false, sizeof(bool) * natts); - replaces[fno] = true; + colnums[0] = fno; /* * Now insert the new value, being careful to cast it to the * right type. */ - atttype = rec->tupdesc->attrs[fno]->atttypid; - atttypmod = rec->tupdesc->attrs[fno]->atttypmod; - values[fno] = exec_cast_value(estate, - value, - &isNull, - valtype, - valtypmod, - atttype, - atttypmod); - nulls[fno] = isNull; - - /* - * Now call heap_modify_tuple() to create a new tuple that - * replaces the old one in the record. - */ - newtup = heap_modify_tuple(rec->tup, rec->tupdesc, - values, nulls, replaces); + atttype = rec->tupdesc->attrs[fno - 1]->atttypid; + atttypmod = rec->tupdesc->attrs[fno - 1]->atttypmod; + values[0] = exec_cast_value(estate, + value, + &isNull, + valtype, + valtypmod, + atttype, + atttypmod); + nulls[0] = isNull; + + newtup = heap_modify_tuple_by_cols(rec->tup, rec->tupdesc, + 1, colnums, values, nulls); if (rec->freetup) heap_freetuple(rec->tup); diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 119a59ab07..32703fcdcf 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -639,15 +639,8 @@ ttdummy(PG_FUNCTION_ARGS) /* Tuple to return to upper Executor ... */ if (newtuple) /* UPDATE */ - { - HeapTuple tmptuple; - - tmptuple = SPI_copytuple(trigtuple); - rettuple = SPI_modifytuple(rel, tmptuple, 1, &(attnum[1]), &newoff, NULL); - SPI_freetuple(tmptuple); - } - else - /* DELETE */ + rettuple = SPI_modifytuple(rel, trigtuple, 1, &(attnum[1]), &newoff, NULL); + else /* DELETE */ rettuple = trigtuple; SPI_finish(); /* don't forget say Bye to SPI mgr */ -- cgit v1.2.3 From 1833f1a1c3b0e12b3ea40d49bf11898eedae5248 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 8 Nov 2016 17:39:45 -0500 Subject: Simplify code by getting rid of SPI_push, SPI_pop, SPI_restore_connection. The idea behind SPI_push was to allow transitioning back into an "unconnected" state when a SPI-using procedure calls unrelated code that might or might not invoke SPI. That sounds good, but in practice the only thing it does for us is to catch cases where a called SPI-using function forgets to call SPI_connect --- which is a highly improbable failure mode, since it would be exposed immediately by direct testing of said function. As against that, we've had multiple bugs induced by forgetting to call SPI_push/SPI_pop around code that might invoke SPI-using functions; these are much harder to catch and indeed have gone undetected for years in some cases. And we've had to band-aid around some problems of this ilk by introducing conditional push/pop pairs in some places, which really kind of defeats the purpose altogether; if we can't draw bright lines between connected and unconnected code, what's the point? Hence, get rid of SPI_push[_conditional], SPI_pop[_conditional], and the underlying state variable _SPI_curid. It turns out SPI_restore_connection can go away too, which is a nice side benefit since it was never more than a kluge. Provide no-op macros for the deleted functions so as to avoid an API break for external modules. A side effect of this removal is that SPI_palloc and allied functions no longer permit being called when unconnected; they'll throw an error instead. The apparent usefulness of the previous behavior was a mirage as well, because it was depended on by only a few places (which I fixed in preceding commits), and it posed a risk of allocations being unexpectedly long-lived if someone forgot a SPI_push call. Discussion: <20808.1478481403@sss.pgh.pa.us> --- doc/src/sgml/spi.sgml | 180 ++++++++----------------------- src/backend/executor/spi.c | 200 +++++++++-------------------------- src/backend/utils/adt/xml.c | 6 -- src/backend/utils/cache/plancache.c | 13 --- src/backend/utils/fmgr/fmgr.c | 48 +-------- src/include/executor/spi.h | 12 ++- src/pl/plperl/plperl.c | 78 -------------- src/pl/plpgsql/src/pl_exec.c | 21 ---- src/pl/plpython/plpy_exec.c | 2 - src/pl/plpython/plpy_spi.c | 13 --- src/pl/plpython/plpy_subxactobject.c | 7 -- src/pl/tcl/pltcl.c | 18 ---- 12 files changed, 105 insertions(+), 493 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/spi.sgml b/doc/src/sgml/spi.sgml index 39133c9038..836ce0822f 100644 --- a/doc/src/sgml/spi.sgml +++ b/doc/src/sgml/spi.sgml @@ -90,21 +90,6 @@ int SPI_connect(void) function if you want to execute commands through SPI. Some utility SPI functions can be called from unconnected procedures. - - - If your procedure is already connected, - SPI_connect will return the error code - SPI_ERROR_CONNECT. This could happen if - a procedure that has called SPI_connect - directly calls another procedure that calls - SPI_connect. While recursive calls to the - SPI manager are permitted when an SQL command - called through SPI invokes another function that uses - SPI, directly nested calls to - SPI_connect and - SPI_finish are forbidden. - (But see SPI_push and SPI_pop.) - @@ -164,13 +149,6 @@ int SPI_finish(void) abort the transaction via elog(ERROR). In that case SPI will clean itself up automatically. - - - If SPI_finish is called without having a valid - connection, it will return SPI_ERROR_UNCONNECTED. - There is no fundamental problem with this; it means that the SPI - manager has nothing to do. - @@ -200,86 +178,6 @@ int SPI_finish(void) - - SPI_push - - - SPI_push - 3 - - - - SPI_push - push SPI stack to allow recursive SPI usage - - - - -void SPI_push(void) - - - - - Description - - - SPI_push should be called before executing another - procedure that might itself wish to use SPI. - After SPI_push, SPI is no longer in a - connected state, and SPI function calls will be rejected unless - a fresh SPI_connect is done. This ensures a clean - separation between your procedure's SPI state and that of another procedure - you call. After the other procedure returns, call - SPI_pop to restore access to your own SPI state. - - - - Note that SPI_execute and related functions - automatically do the equivalent of SPI_push before - passing control back to the SQL execution engine, so it is not necessary - for you to worry about this when using those functions. - Only when you are directly calling arbitrary code that might contain - SPI_connect calls do you need to issue - SPI_push and SPI_pop. - - - - - - - - - SPI_pop - - - SPI_pop - 3 - - - - SPI_pop - pop SPI stack to return from recursive SPI usage - - - - -void SPI_pop(void) - - - - - Description - - - SPI_pop pops the previous environment from the - SPI call stack. See SPI_push. - - - - - - - SPI_execute @@ -3361,17 +3259,8 @@ char * SPI_getnspname(Relation rel) upper executor context, that is, the memory context that was current when SPI_connect was called, which is precisely the right context for a value returned from your - procedure. - - - - If SPI_palloc is called while the procedure is - not connected to SPI, then it acts the same as a normal - palloc. Before a procedure connects to the - SPI manager, the current memory context is the upper executor - context, so all allocations made by the procedure via - palloc or by SPI utility functions are made in - this context. + procedure. Several of the other utility procedures described in + this section also return objects created in the upper executor context. @@ -3379,25 +3268,14 @@ char * SPI_getnspname(Relation rel) context of the procedure, which is created by SPI_connect, is made the current context. All allocations made by palloc, - repalloc, or SPI utility functions (except for - SPI_copytuple, - SPI_returntuple, - SPI_modifytuple, - SPI_palloc, and - SPI_datumTransfer) are made in this context. When a + repalloc, or SPI utility functions (except as + described in this section) are made in this context. When a procedure disconnects from the SPI manager (via SPI_finish) the current context is restored to the upper executor context, and all allocations made in the procedure memory context are freed and cannot be used any more. - - All functions described in this section can be used by both - connected and unconnected procedures. In an unconnected procedure, - they act the same as the underlying ordinary server functions - (palloc, etc.). - - @@ -3426,6 +3304,11 @@ void * SPI_palloc(Size size) SPI_palloc allocates memory in the upper executor context. + + + This function can only be used while connected to SPI. + Otherwise, it throws an error. + @@ -3605,6 +3488,12 @@ HeapTuple SPI_copytuple(HeapTuple row) row from a trigger. In a function declared to return a composite type, use SPI_returntuple instead. + + + This function can only be used while connected to SPI. + Otherwise, it returns NULL and sets SPI_result to + SPI_ERROR_UNCONNECTED. + @@ -3626,8 +3515,8 @@ HeapTuple SPI_copytuple(HeapTuple row) Return Value - the copied row; NULL only if - tuple is NULL + the copied row, or NULL on error + (see SPI_result for an error indication) @@ -3663,6 +3552,12 @@ HeapTupleHeader SPI_returntuple(HeapTuple row, TupleDesc before returning. + + This function can only be used while connected to SPI. + Otherwise, it returns NULL and sets SPI_result to + SPI_ERROR_UNCONNECTED. + + Note that this should be used for functions that are declared to return composite types. It is not used for triggers; use @@ -3699,10 +3594,9 @@ HeapTupleHeader SPI_returntuple(HeapTuple row, TupleDesc Return Value - HeapTupleHeader pointing to copied row; - NULL only if - row or rowdesc is - NULL + HeapTupleHeader pointing to copied row, + or NULL on error + (see SPI_result for an error indication) @@ -3736,6 +3630,13 @@ HeapTuple SPI_modifytuple(Relation rel, HeapTuple SPI_modifytuple creates a new row by substituting new values for selected columns, copying the original row's columns at other positions. The input row is not modified. + The new row is returned in the upper executor context. + + + + This function can only be used while connected to SPI. + Otherwise, it returns NULL and sets SPI_result to + SPI_ERROR_UNCONNECTED. @@ -3821,8 +3722,8 @@ HeapTuple SPI_modifytuple(Relation rel, HeapTuple new row with modifications, allocated in the upper executor - context; NULL only if row - is NULL + context, or NULL on error + (see SPI_result for an error indication) @@ -3845,11 +3746,20 @@ HeapTuple SPI_modifytuple(Relation rel, HeapTuple if colnum contains an invalid column number (less - than or equal to 0 or greater than the number of column in + than or equal to 0 or greater than the number of columns in row) + + + SPI_ERROR_UNCONNECTED + + + if SPI is not active + + + diff --git a/src/backend/executor/spi.c b/src/backend/executor/spi.c index 8e650bc412..80fc4c4725 100644 --- a/src/backend/executor/spi.c +++ b/src/backend/executor/spi.c @@ -44,8 +44,7 @@ int SPI_result; static _SPI_connection *_SPI_stack = NULL; static _SPI_connection *_SPI_current = NULL; static int _SPI_stack_depth = 0; /* allocated size of _SPI_stack */ -static int _SPI_connected = -1; -static int _SPI_curid = -1; +static int _SPI_connected = -1; /* current stack index */ static Portal SPI_cursor_open_internal(const char *name, SPIPlanPtr plan, ParamListInfo paramLI, bool read_only); @@ -86,13 +85,7 @@ SPI_connect(void) { int newdepth; - /* - * When procedure called by Executor _SPI_curid expected to be equal to - * _SPI_connected - */ - if (_SPI_curid != _SPI_connected) - return SPI_ERROR_CONNECT; - + /* Enlarge stack if necessary */ if (_SPI_stack == NULL) { if (_SPI_connected != -1 || _SPI_stack_depth != 0) @@ -117,9 +110,7 @@ SPI_connect(void) } } - /* - * We're entering procedure where _SPI_curid == _SPI_connected - 1 - */ + /* Enter new stack level */ _SPI_connected++; Assert(_SPI_connected >= 0 && _SPI_connected < _SPI_stack_depth); @@ -178,14 +169,9 @@ SPI_finish(void) SPI_lastoid = InvalidOid; SPI_tuptable = NULL; - /* - * After _SPI_begin_call _SPI_connected == _SPI_curid. Now we are closing - * connection to SPI and returning to upper Executor and so _SPI_connected - * must be equal to _SPI_curid. - */ + /* Exit stack level */ _SPI_connected--; - _SPI_curid--; - if (_SPI_connected == -1) + if (_SPI_connected < 0) _SPI_current = NULL; else _SPI_current = &(_SPI_stack[_SPI_connected]); @@ -212,7 +198,7 @@ AtEOXact_SPI(bool isCommit) _SPI_current = _SPI_stack = NULL; _SPI_stack_depth = 0; - _SPI_connected = _SPI_curid = -1; + _SPI_connected = -1; SPI_processed = 0; SPI_lastoid = InvalidOid; SPI_tuptable = NULL; @@ -258,8 +244,7 @@ AtEOSubXact_SPI(bool isCommit, SubTransactionId mySubid) * be already gone. */ _SPI_connected--; - _SPI_curid = _SPI_connected; - if (_SPI_connected == -1) + if (_SPI_connected < 0) _SPI_current = NULL; else _SPI_current = &(_SPI_stack[_SPI_connected]); @@ -313,53 +298,6 @@ AtEOSubXact_SPI(bool isCommit, SubTransactionId mySubid) } -/* Pushes SPI stack to allow recursive SPI calls */ -void -SPI_push(void) -{ - _SPI_curid++; -} - -/* Pops SPI stack to allow recursive SPI calls */ -void -SPI_pop(void) -{ - _SPI_curid--; -} - -/* Conditional push: push only if we're inside a SPI procedure */ -bool -SPI_push_conditional(void) -{ - bool pushed = (_SPI_curid != _SPI_connected); - - if (pushed) - { - _SPI_curid++; - /* We should now be in a state where SPI_connect would succeed */ - Assert(_SPI_curid == _SPI_connected); - } - return pushed; -} - -/* Conditional pop: pop only if SPI_push_conditional pushed */ -void -SPI_pop_conditional(bool pushed) -{ - /* We should be in a state where SPI_connect would succeed */ - Assert(_SPI_curid == _SPI_connected); - if (pushed) - _SPI_curid--; -} - -/* Restore state of SPI stack after aborting a subtransaction */ -void -SPI_restore_connection(void) -{ - Assert(_SPI_connected >= 0); - _SPI_curid = _SPI_connected - 1; -} - /* Parse, plan, and execute a query string */ int SPI_execute(const char *src, bool read_only, long tcount) @@ -691,7 +629,7 @@ SPI_freeplan(SPIPlanPtr plan) HeapTuple SPI_copytuple(HeapTuple tuple) { - MemoryContext oldcxt = NULL; + MemoryContext oldcxt; HeapTuple ctuple; if (tuple == NULL) @@ -700,17 +638,17 @@ SPI_copytuple(HeapTuple tuple) return NULL; } - if (_SPI_curid + 1 == _SPI_connected) /* connected */ + if (_SPI_current == NULL) { - if (_SPI_current != &(_SPI_stack[_SPI_curid + 1])) - elog(ERROR, "SPI stack corrupted"); - oldcxt = MemoryContextSwitchTo(_SPI_current->savedcxt); + SPI_result = SPI_ERROR_UNCONNECTED; + return NULL; } + oldcxt = MemoryContextSwitchTo(_SPI_current->savedcxt); + ctuple = heap_copytuple(tuple); - if (oldcxt) - MemoryContextSwitchTo(oldcxt); + MemoryContextSwitchTo(oldcxt); return ctuple; } @@ -718,7 +656,7 @@ SPI_copytuple(HeapTuple tuple) HeapTupleHeader SPI_returntuple(HeapTuple tuple, TupleDesc tupdesc) { - MemoryContext oldcxt = NULL; + MemoryContext oldcxt; HeapTupleHeader dtup; if (tuple == NULL || tupdesc == NULL) @@ -727,22 +665,22 @@ SPI_returntuple(HeapTuple tuple, TupleDesc tupdesc) return NULL; } + if (_SPI_current == NULL) + { + SPI_result = SPI_ERROR_UNCONNECTED; + return NULL; + } + /* For RECORD results, make sure a typmod has been assigned */ if (tupdesc->tdtypeid == RECORDOID && tupdesc->tdtypmod < 0) assign_record_type_typmod(tupdesc); - if (_SPI_curid + 1 == _SPI_connected) /* connected */ - { - if (_SPI_current != &(_SPI_stack[_SPI_curid + 1])) - elog(ERROR, "SPI stack corrupted"); - oldcxt = MemoryContextSwitchTo(_SPI_current->savedcxt); - } + oldcxt = MemoryContextSwitchTo(_SPI_current->savedcxt); dtup = DatumGetHeapTupleHeader(heap_copy_tuple_as_datum(tuple, tupdesc)); - if (oldcxt) - MemoryContextSwitchTo(oldcxt); + MemoryContextSwitchTo(oldcxt); return dtup; } @@ -751,7 +689,7 @@ HeapTuple SPI_modifytuple(Relation rel, HeapTuple tuple, int natts, int *attnum, Datum *Values, const char *Nulls) { - MemoryContext oldcxt = NULL; + MemoryContext oldcxt; HeapTuple mtuple; int numberOfAttributes; Datum *v; @@ -764,13 +702,16 @@ SPI_modifytuple(Relation rel, HeapTuple tuple, int natts, int *attnum, return NULL; } - if (_SPI_curid + 1 == _SPI_connected) /* connected */ + if (_SPI_current == NULL) { - if (_SPI_current != &(_SPI_stack[_SPI_curid + 1])) - elog(ERROR, "SPI stack corrupted"); - oldcxt = MemoryContextSwitchTo(_SPI_current->savedcxt); + SPI_result = SPI_ERROR_UNCONNECTED; + return NULL; } + + oldcxt = MemoryContextSwitchTo(_SPI_current->savedcxt); + SPI_result = 0; + numberOfAttributes = rel->rd_att->natts; v = (Datum *) palloc(numberOfAttributes * sizeof(Datum)); n = (bool *) palloc(numberOfAttributes * sizeof(bool)); @@ -810,8 +751,7 @@ SPI_modifytuple(Relation rel, HeapTuple tuple, int natts, int *attnum, pfree(v); pfree(n); - if (oldcxt) - MemoryContextSwitchTo(oldcxt); + MemoryContextSwitchTo(oldcxt); return mtuple; } @@ -980,22 +920,10 @@ SPI_getnspname(Relation rel) void * SPI_palloc(Size size) { - MemoryContext oldcxt = NULL; - void *pointer; - - if (_SPI_curid + 1 == _SPI_connected) /* connected */ - { - if (_SPI_current != &(_SPI_stack[_SPI_curid + 1])) - elog(ERROR, "SPI stack corrupted"); - oldcxt = MemoryContextSwitchTo(_SPI_current->savedcxt); - } - - pointer = palloc(size); - - if (oldcxt) - MemoryContextSwitchTo(oldcxt); + if (_SPI_current == NULL) + elog(ERROR, "SPI_palloc called while not connected to SPI"); - return pointer; + return MemoryContextAlloc(_SPI_current->savedcxt, size); } void * @@ -1015,20 +943,17 @@ SPI_pfree(void *pointer) Datum SPI_datumTransfer(Datum value, bool typByVal, int typLen) { - MemoryContext oldcxt = NULL; + MemoryContext oldcxt; Datum result; - if (_SPI_curid + 1 == _SPI_connected) /* connected */ - { - if (_SPI_current != &(_SPI_stack[_SPI_curid + 1])) - elog(ERROR, "SPI stack corrupted"); - oldcxt = MemoryContextSwitchTo(_SPI_current->savedcxt); - } + if (_SPI_current == NULL) + elog(ERROR, "SPI_datumTransfer called while not connected to SPI"); + + oldcxt = MemoryContextSwitchTo(_SPI_current->savedcxt); result = datumTransfer(value, typByVal, typLen); - if (oldcxt) - MemoryContextSwitchTo(oldcxt); + MemoryContextSwitchTo(oldcxt); return result; } @@ -1050,17 +975,12 @@ SPI_freetuptable(SPITupleTable *tuptable) return; /* - * Since this function might be called during error recovery, it seems - * best not to insist that the caller be actively connected. We just - * search the topmost SPI context, connected or not. + * Search only the topmost SPI context for a matching tuple table. */ - if (_SPI_connected >= 0) + if (_SPI_current != NULL) { slist_mutable_iter siter; - if (_SPI_current != &(_SPI_stack[_SPI_connected])) - elog(ERROR, "SPI stack corrupted"); - /* find tuptable in active list, then remove it */ slist_foreach_modify(siter, &_SPI_current->tuptables) { @@ -1168,13 +1088,9 @@ SPI_cursor_open_with_args(const char *name, /* We needn't copy the plan; SPI_cursor_open_internal will do so */ - /* Adjust stack so that SPI_cursor_open_internal doesn't complain */ - _SPI_curid--; - result = SPI_cursor_open_internal(name, &plan, paramLI, read_only); /* And clean up */ - _SPI_curid++; _SPI_end_call(true); return result; @@ -1723,14 +1639,8 @@ spi_dest_startup(DestReceiver *self, int operation, TupleDesc typeinfo) MemoryContext oldcxt; MemoryContext tuptabcxt; - /* - * When called by Executor _SPI_curid expected to be equal to - * _SPI_connected - */ - if (_SPI_curid != _SPI_connected || _SPI_connected < 0) - elog(ERROR, "improper call to spi_dest_startup"); - if (_SPI_current != &(_SPI_stack[_SPI_curid])) - elog(ERROR, "SPI stack corrupted"); + if (_SPI_current == NULL) + elog(ERROR, "spi_dest_startup called while not connected to SPI"); if (_SPI_current->tuptable != NULL) elog(ERROR, "improper call to spi_dest_startup"); @@ -1775,14 +1685,8 @@ spi_printtup(TupleTableSlot *slot, DestReceiver *self) SPITupleTable *tuptable; MemoryContext oldcxt; - /* - * When called by Executor _SPI_curid expected to be equal to - * _SPI_connected - */ - if (_SPI_curid != _SPI_connected || _SPI_connected < 0) - elog(ERROR, "improper call to spi_printtup"); - if (_SPI_current != &(_SPI_stack[_SPI_curid])) - elog(ERROR, "SPI stack corrupted"); + if (_SPI_current == NULL) + elog(ERROR, "spi_printtup called while not connected to SPI"); tuptable = _SPI_current->tuptable; if (tuptable == NULL) @@ -2534,11 +2438,8 @@ _SPI_procmem(void) static int _SPI_begin_call(bool execmem) { - if (_SPI_curid + 1 != _SPI_connected) + if (_SPI_current == NULL) return SPI_ERROR_UNCONNECTED; - _SPI_curid++; - if (_SPI_current != &(_SPI_stack[_SPI_curid])) - elog(ERROR, "SPI stack corrupted"); if (execmem) /* switch to the Executor memory context */ _SPI_execmem(); @@ -2554,11 +2455,6 @@ _SPI_begin_call(bool execmem) static int _SPI_end_call(bool procmem) { - /* - * We're returning to procedure where _SPI_curid == _SPI_connected - 1 - */ - _SPI_curid--; - if (procmem) /* switch to the procedure memory context */ { _SPI_procmem(); diff --git a/src/backend/utils/adt/xml.c b/src/backend/utils/adt/xml.c index b144920ec6..057c3bfd7c 100644 --- a/src/backend/utils/adt/xml.c +++ b/src/backend/utils/adt/xml.c @@ -2644,8 +2644,6 @@ schema_to_xml_internal(Oid nspid, const char *xmlschema, bool nulls, relid_list = schema_get_xml_visible_tables(nspid); - SPI_push(); - foreach(cell, relid_list) { Oid relid = lfirst_oid(cell); @@ -2658,7 +2656,6 @@ schema_to_xml_internal(Oid nspid, const char *xmlschema, bool nulls, appendStringInfoChar(result, '\n'); } - SPI_pop(); SPI_finish(); xmldata_root_element_end(result, xmlsn); @@ -2822,8 +2819,6 @@ database_to_xml_internal(const char *xmlschema, bool nulls, nspid_list = database_get_xml_visible_schemas(); - SPI_push(); - foreach(cell, nspid_list) { Oid nspid = lfirst_oid(cell); @@ -2836,7 +2831,6 @@ database_to_xml_internal(const char *xmlschema, bool nulls, appendStringInfoChar(result, '\n'); } - SPI_pop(); SPI_finish(); xmldata_root_element_end(result, xmlcn); diff --git a/src/backend/utils/cache/plancache.c b/src/backend/utils/cache/plancache.c index c96a86500a..884cdab702 100644 --- a/src/backend/utils/cache/plancache.c +++ b/src/backend/utils/cache/plancache.c @@ -53,7 +53,6 @@ #include "access/transam.h" #include "catalog/namespace.h" #include "executor/executor.h" -#include "executor/spi.h" #include "miscadmin.h" #include "nodes/nodeFuncs.h" #include "optimizer/cost.h" @@ -878,7 +877,6 @@ BuildCachedPlan(CachedPlanSource *plansource, List *qlist, CachedPlan *plan; List *plist; bool snapshot_set; - bool spi_pushed; bool is_transient; MemoryContext plan_context; MemoryContext oldcxt = CurrentMemoryContext; @@ -926,22 +924,11 @@ BuildCachedPlan(CachedPlanSource *plansource, List *qlist, snapshot_set = true; } - /* - * The planner may try to call SPI-using functions, which causes a problem - * if we're already inside one. Rather than expect all SPI-using code to - * do SPI_push whenever a replan could happen, it seems best to take care - * of the case here. - */ - spi_pushed = SPI_push_conditional(); - /* * Generate the plan. */ plist = pg_plan_queries(qlist, plansource->cursor_options, boundParams); - /* Clean up SPI state */ - SPI_pop_conditional(spi_pushed); - /* Release snapshot if we got one */ if (snapshot_set) PopActiveSnapshot(); diff --git a/src/backend/utils/fmgr/fmgr.c b/src/backend/utils/fmgr/fmgr.c index 46a55ba7b9..3340b17d90 100644 --- a/src/backend/utils/fmgr/fmgr.c +++ b/src/backend/utils/fmgr/fmgr.c @@ -19,7 +19,6 @@ #include "catalog/pg_language.h" #include "catalog/pg_proc.h" #include "executor/functions.h" -#include "executor/spi.h" #include "lib/stringinfo.h" #include "miscadmin.h" #include "nodes/nodeFuncs.h" @@ -1878,25 +1877,16 @@ OidFunctionCall9Coll(Oid functionId, Oid collation, Datum arg1, Datum arg2, * the caller should assume the result is NULL, but we'll call the input * function anyway if it's not strict. So this is almost but not quite * the same as FunctionCall3. - * - * One important difference from the bare function call is that we will - * push any active SPI context, allowing SPI-using I/O functions to be - * called from other SPI functions without extra notation. This is a hack, - * but the alternative of expecting all SPI functions to do SPI_push/SPI_pop - * around I/O calls seems worse. */ Datum InputFunctionCall(FmgrInfo *flinfo, char *str, Oid typioparam, int32 typmod) { FunctionCallInfoData fcinfo; Datum result; - bool pushed; if (str == NULL && flinfo->fn_strict) return (Datum) 0; /* just return null result */ - pushed = SPI_push_conditional(); - InitFunctionCallInfoData(fcinfo, flinfo, 3, InvalidOid, NULL, NULL); fcinfo.arg[0] = CStringGetDatum(str); @@ -1922,8 +1912,6 @@ InputFunctionCall(FmgrInfo *flinfo, char *str, Oid typioparam, int32 typmod) fcinfo.flinfo->fn_oid); } - SPI_pop_conditional(pushed); - return result; } @@ -1932,22 +1920,12 @@ InputFunctionCall(FmgrInfo *flinfo, char *str, Oid typioparam, int32 typmod) * * Do not call this on NULL datums. * - * This is almost just window dressing for FunctionCall1, but it includes - * SPI context pushing for the same reasons as InputFunctionCall. + * This is currently little more than window dressing for FunctionCall1. */ char * OutputFunctionCall(FmgrInfo *flinfo, Datum val) { - char *result; - bool pushed; - - pushed = SPI_push_conditional(); - - result = DatumGetCString(FunctionCall1(flinfo, val)); - - SPI_pop_conditional(pushed); - - return result; + return DatumGetCString(FunctionCall1(flinfo, val)); } /* @@ -1956,8 +1934,7 @@ OutputFunctionCall(FmgrInfo *flinfo, Datum val) * "buf" may be NULL to indicate we are reading a NULL. In this case * the caller should assume the result is NULL, but we'll call the receive * function anyway if it's not strict. So this is almost but not quite - * the same as FunctionCall3. Also, this includes SPI context pushing for - * the same reasons as InputFunctionCall. + * the same as FunctionCall3. */ Datum ReceiveFunctionCall(FmgrInfo *flinfo, StringInfo buf, @@ -1965,13 +1942,10 @@ ReceiveFunctionCall(FmgrInfo *flinfo, StringInfo buf, { FunctionCallInfoData fcinfo; Datum result; - bool pushed; if (buf == NULL && flinfo->fn_strict) return (Datum) 0; /* just return null result */ - pushed = SPI_push_conditional(); - InitFunctionCallInfoData(fcinfo, flinfo, 3, InvalidOid, NULL, NULL); fcinfo.arg[0] = PointerGetDatum(buf); @@ -1997,8 +1971,6 @@ ReceiveFunctionCall(FmgrInfo *flinfo, StringInfo buf, fcinfo.flinfo->fn_oid); } - SPI_pop_conditional(pushed); - return result; } @@ -2009,22 +1981,12 @@ ReceiveFunctionCall(FmgrInfo *flinfo, StringInfo buf, * * This is little more than window dressing for FunctionCall1, but it does * guarantee a non-toasted result, which strictly speaking the underlying - * function doesn't. Also, this includes SPI context pushing for the same - * reasons as InputFunctionCall. + * function doesn't. */ bytea * SendFunctionCall(FmgrInfo *flinfo, Datum val) { - bytea *result; - bool pushed; - - pushed = SPI_push_conditional(); - - result = DatumGetByteaP(FunctionCall1(flinfo, val)); - - SPI_pop_conditional(pushed); - - return result; + return DatumGetByteaP(FunctionCall1(flinfo, val)); } /* diff --git a/src/include/executor/spi.h b/src/include/executor/spi.h index 1792fb1217..76ba394a2b 100644 --- a/src/include/executor/spi.h +++ b/src/include/executor/spi.h @@ -59,6 +59,13 @@ typedef struct _SPI_plan *SPIPlanPtr; #define SPI_OK_UPDATE_RETURNING 13 #define SPI_OK_REWRITTEN 14 +/* These used to be functions, now just no-ops for backwards compatibility */ +#define SPI_push() ((void) 0) +#define SPI_pop() ((void) 0) +#define SPI_push_conditional() false +#define SPI_pop_conditional(pushed) ((void) 0) +#define SPI_restore_connection() ((void) 0) + extern PGDLLIMPORT uint64 SPI_processed; extern PGDLLIMPORT Oid SPI_lastoid; extern PGDLLIMPORT SPITupleTable *SPI_tuptable; @@ -66,11 +73,6 @@ extern PGDLLIMPORT int SPI_result; extern int SPI_connect(void); extern int SPI_finish(void); -extern void SPI_push(void); -extern void SPI_pop(void); -extern bool SPI_push_conditional(void); -extern void SPI_pop_conditional(bool pushed); -extern void SPI_restore_connection(void); extern int SPI_execute(const char *src, bool read_only, long tcount); extern int SPI_execute_plan(SPIPlanPtr plan, Datum *Values, const char *Nulls, bool read_only, long tcount); diff --git a/src/pl/plperl/plperl.c b/src/pl/plperl/plperl.c index 461986cda3..9a2d0527f8 100644 --- a/src/pl/plperl/plperl.c +++ b/src/pl/plperl/plperl.c @@ -3057,12 +3057,6 @@ plperl_spi_exec(char *query, int limit) ReleaseCurrentSubTransaction(); MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - - /* - * AtEOSubXact_SPI() should not have popped any SPI context, but just - * in case it did, make sure we remain connected. - */ - SPI_restore_connection(); } PG_CATCH(); { @@ -3078,13 +3072,6 @@ plperl_spi_exec(char *query, int limit) MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - /* - * If AtEOSubXact_SPI() popped any SPI context of the subxact, it will - * have left us in a disconnected state. We need this hack to return - * to connected state. - */ - SPI_restore_connection(); - /* Punt the error to Perl */ croak_cstr(edata->message); @@ -3296,12 +3283,6 @@ plperl_spi_query(char *query) ReleaseCurrentSubTransaction(); MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - - /* - * AtEOSubXact_SPI() should not have popped any SPI context, but just - * in case it did, make sure we remain connected. - */ - SPI_restore_connection(); } PG_CATCH(); { @@ -3317,13 +3298,6 @@ plperl_spi_query(char *query) MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - /* - * If AtEOSubXact_SPI() popped any SPI context of the subxact, it will - * have left us in a disconnected state. We need this hack to return - * to connected state. - */ - SPI_restore_connection(); - /* Punt the error to Perl */ croak_cstr(edata->message); @@ -3382,12 +3356,6 @@ plperl_spi_fetchrow(char *cursor) ReleaseCurrentSubTransaction(); MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - - /* - * AtEOSubXact_SPI() should not have popped any SPI context, but just - * in case it did, make sure we remain connected. - */ - SPI_restore_connection(); } PG_CATCH(); { @@ -3403,13 +3371,6 @@ plperl_spi_fetchrow(char *cursor) MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - /* - * If AtEOSubXact_SPI() popped any SPI context of the subxact, it will - * have left us in a disconnected state. We need this hack to return - * to connected state. - */ - SPI_restore_connection(); - /* Punt the error to Perl */ croak_cstr(edata->message); @@ -3543,12 +3504,6 @@ plperl_spi_prepare(char *query, int argc, SV **argv) ReleaseCurrentSubTransaction(); MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - - /* - * AtEOSubXact_SPI() should not have popped any SPI context, but just - * in case it did, make sure we remain connected. - */ - SPI_restore_connection(); } PG_CATCH(); { @@ -3574,13 +3529,6 @@ plperl_spi_prepare(char *query, int argc, SV **argv) MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - /* - * If AtEOSubXact_SPI() popped any SPI context of the subxact, it will - * have left us in a disconnected state. We need this hack to return - * to connected state. - */ - SPI_restore_connection(); - /* Punt the error to Perl */ croak_cstr(edata->message); @@ -3694,12 +3642,6 @@ plperl_spi_exec_prepared(char *query, HV *attr, int argc, SV **argv) ReleaseCurrentSubTransaction(); MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - - /* - * AtEOSubXact_SPI() should not have popped any SPI context, but just - * in case it did, make sure we remain connected. - */ - SPI_restore_connection(); } PG_CATCH(); { @@ -3715,13 +3657,6 @@ plperl_spi_exec_prepared(char *query, HV *attr, int argc, SV **argv) MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - /* - * If AtEOSubXact_SPI() popped any SPI context of the subxact, it will - * have left us in a disconnected state. We need this hack to return - * to connected state. - */ - SPI_restore_connection(); - /* Punt the error to Perl */ croak_cstr(edata->message); @@ -3823,12 +3758,6 @@ plperl_spi_query_prepared(char *query, int argc, SV **argv) ReleaseCurrentSubTransaction(); MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - - /* - * AtEOSubXact_SPI() should not have popped any SPI context, but just - * in case it did, make sure we remain connected. - */ - SPI_restore_connection(); } PG_CATCH(); { @@ -3844,13 +3773,6 @@ plperl_spi_query_prepared(char *query, int argc, SV **argv) MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - /* - * If AtEOSubXact_SPI() popped any SPI context of the subxact, it will - * have left us in a disconnected state. We need this hack to return - * to connected state. - */ - SPI_restore_connection(); - /* Punt the error to Perl */ croak_cstr(edata->message); diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c index 91e1f8dd3f..77e7440002 100644 --- a/src/pl/plpgsql/src/pl_exec.c +++ b/src/pl/plpgsql/src/pl_exec.c @@ -1337,12 +1337,6 @@ exec_stmt_block(PLpgSQL_execstate *estate, PLpgSQL_stmt_block *block) * automatically cleaned up during subxact exit.) */ estate->eval_econtext = old_eval_econtext; - - /* - * AtEOSubXact_SPI() should not have popped any SPI context, but - * just in case it did, make sure we remain connected. - */ - SPI_restore_connection(); } PG_CATCH(); { @@ -1384,13 +1378,6 @@ exec_stmt_block(PLpgSQL_execstate *estate, PLpgSQL_stmt_block *block) /* Revert to outer eval_econtext */ estate->eval_econtext = old_eval_econtext; - /* - * If AtEOSubXact_SPI() popped any SPI context of the subxact, it - * will have left us in a disconnected state. We need this hack - * to return to connected state. - */ - SPI_restore_connection(); - /* * Must clean up the econtext too. However, any tuple table made * in the subxact will have been thrown away by SPI during subxact @@ -5587,8 +5574,6 @@ exec_eval_simple_expr(PLpgSQL_execstate *estate, * Without this, stable functions within the expression would fail to see * updates made so far by our own function. */ - SPI_push(); - oldcontext = MemoryContextSwitchTo(get_eval_mcontext(estate)); if (!estate->readonly_func) { @@ -5636,8 +5621,6 @@ exec_eval_simple_expr(PLpgSQL_execstate *estate, MemoryContextSwitchTo(oldcontext); - SPI_pop(); - /* * Now we can release our refcount on the cached plan. */ @@ -6281,8 +6264,6 @@ exec_cast_value(PLpgSQL_execstate *estate, ExprContext *econtext = estate->eval_econtext; MemoryContext oldcontext; - SPI_push(); - oldcontext = MemoryContextSwitchTo(get_eval_mcontext(estate)); econtext->caseValue_datum = value; @@ -6296,8 +6277,6 @@ exec_cast_value(PLpgSQL_execstate *estate, cast_entry->cast_in_use = false; MemoryContextSwitchTo(oldcontext); - - SPI_pop(); } } diff --git a/src/pl/plpython/plpy_exec.c b/src/pl/plpython/plpy_exec.c index fa5b25a5fa..697a0e1cc0 100644 --- a/src/pl/plpython/plpy_exec.c +++ b/src/pl/plpython/plpy_exec.c @@ -1103,8 +1103,6 @@ PLy_abort_open_subtransactions(int save_subxact_level) RollbackAndReleaseCurrentSubTransaction(); - SPI_restore_connection(); - subtransactiondata = (PLySubtransactionData *) linitial(explicit_subtransactions); explicit_subtransactions = list_delete_first(explicit_subtransactions); diff --git a/src/pl/plpython/plpy_spi.c b/src/pl/plpython/plpy_spi.c index b082d017ea..07ab6a087e 100644 --- a/src/pl/plpython/plpy_spi.c +++ b/src/pl/plpython/plpy_spi.c @@ -516,12 +516,6 @@ PLy_spi_subtransaction_commit(MemoryContext oldcontext, ResourceOwner oldowner) ReleaseCurrentSubTransaction(); MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - - /* - * AtEOSubXact_SPI() should not have popped any SPI context, but just in - * case it did, make sure we remain connected. - */ - SPI_restore_connection(); } void @@ -541,13 +535,6 @@ PLy_spi_subtransaction_abort(MemoryContext oldcontext, ResourceOwner oldowner) MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - /* - * If AtEOSubXact_SPI() popped any SPI context of the subxact, it will - * have left us in a disconnected state. We need this hack to return to - * connected state. - */ - SPI_restore_connection(); - /* Look up the correct exception */ entry = hash_search(PLy_spi_exceptions, &(edata->sqlerrcode), HASH_FIND, NULL); diff --git a/src/pl/plpython/plpy_subxactobject.c b/src/pl/plpython/plpy_subxactobject.c index 81fb3a3a4a..9f1caa87d9 100644 --- a/src/pl/plpython/plpy_subxactobject.c +++ b/src/pl/plpython/plpy_subxactobject.c @@ -7,7 +7,6 @@ #include "postgres.h" #include "access/xact.h" -#include "executor/spi.h" #include "utils/memutils.h" #include "plpython.h" @@ -213,12 +212,6 @@ PLy_subtransaction_exit(PyObject *self, PyObject *args) CurrentResourceOwner = subxactdata->oldowner; pfree(subxactdata); - /* - * AtEOSubXact_SPI() should not have popped any SPI context, but just in - * case it did, make sure we remain connected. - */ - SPI_restore_connection(); - Py_INCREF(Py_None); return Py_None; } diff --git a/src/pl/tcl/pltcl.c b/src/pl/tcl/pltcl.c index 20809102ef..b0d9e419bb 100644 --- a/src/pl/tcl/pltcl.c +++ b/src/pl/tcl/pltcl.c @@ -2182,11 +2182,9 @@ pltcl_returnnext(ClientData cdata, Tcl_Interp *interp, { HeapTuple tuple; - SPI_push(); tuple = pltcl_build_tuple_result(interp, rowObjv, rowObjc, call_state); tuplestore_puttuple(call_state->tuple_store, tuple); - SPI_pop(); } } else @@ -2249,12 +2247,6 @@ pltcl_subtrans_commit(MemoryContext oldcontext, ResourceOwner oldowner) ReleaseCurrentSubTransaction(); MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - - /* - * AtEOSubXact_SPI() should not have popped any SPI context, but just in - * case it did, make sure we remain connected. - */ - SPI_restore_connection(); } static void @@ -2273,13 +2265,6 @@ pltcl_subtrans_abort(Tcl_Interp *interp, MemoryContextSwitchTo(oldcontext); CurrentResourceOwner = oldowner; - /* - * If AtEOSubXact_SPI() popped any SPI context of the subxact, it will - * have left us in a disconnected state. We need this hack to return to - * connected state. - */ - SPI_restore_connection(); - /* Pass the error data to Tcl */ pltcl_construct_errorCode(interp, edata); UTF_BEGIN; @@ -3029,9 +3014,6 @@ pltcl_build_tuple_argument(HeapTuple tuple, TupleDesc tupdesc) * mess, there's no way to prevent the datatype input functions it calls * from leaking. Run it in a short-lived context, unless we're about to * exit the procedure anyway. - * - * Also, caller is responsible for doing SPI_push/SPI_pop if calling from - * inside SPI environment. **********************************************************************/ static HeapTuple pltcl_build_tuple_result(Tcl_Interp *interp, Tcl_Obj **kvObjv, int kvObjc, -- cgit v1.2.3 From 3887ba6dbb08f50c0ee6639a80e68ef697222457 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 9 Nov 2016 12:00:00 -0500 Subject: doc: Improve whitespace use in XSL --- doc/src/sgml/stylesheet-common.xsl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/stylesheet-common.xsl b/doc/src/sgml/stylesheet-common.xsl index e3841130eb..c23d38f128 100644 --- a/doc/src/sgml/stylesheet-common.xsl +++ b/doc/src/sgml/stylesheet-common.xsl @@ -77,7 +77,9 @@ - ?? + ? + + ? -- cgit v1.2.3 From 41124a91e61fc6d9681c1e8b15ba30494e84d643 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Wed, 9 Nov 2016 16:26:32 -0500 Subject: pgbench: Allow the transaction log file prefix to be changed. Masahiko Sawada, reviewed by Fabien Coelho and Beena Emerson, with some a bit of wordsmithing and cosmetic adjustment by me. --- doc/src/sgml/ref/pgbench.sgml | 26 +++++++++++++++++++------- src/bin/pgbench/pgbench.c | 20 ++++++++++++++++++-- 2 files changed, 37 insertions(+), 9 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/pgbench.sgml b/doc/src/sgml/ref/pgbench.sgml index 285608d508..3a65729bf3 100644 --- a/doc/src/sgml/ref/pgbench.sgml +++ b/doc/src/sgml/ref/pgbench.sgml @@ -614,6 +614,16 @@ pgbench options dbname + + + + + Set the filename prefix for the transaction log file created by + + + + @@ -1121,15 +1131,17 @@ END; With the , pgbench writes the time taken by each transaction to a log file. The log file will be named - pgbench_log.nnn, where - nnn is the PID of the pgbench process. - If the determines the minimum value a sequence can generate. If NO MINVALUE is specified, the defaults of 1 and - -263 for ascending and descending sequences, + the minimum value of the data type for ascending and descending sequences, respectively, will be used. If neither option is specified, the current minimum value will be maintained. @@ -118,7 +140,7 @@ ALTER SEQUENCE [ IF EXISTS ] name S class="parameter">maxvalue determines the maximum value for the sequence. If NO MAXVALUE is specified, the defaults of - 263-1 and -1 for ascending and descending + the maximum value of the data type and -1 for ascending and descending sequences, respectively, will be used. If neither option is specified, the current maximum value will be maintained. @@ -300,7 +322,7 @@ ALTER SEQUENCE serial RESTART WITH 105; ALTER SEQUENCE conforms to the SQL - standard, except for the START WITH, + standard, except for the AS, START WITH, OWNED BY, OWNER TO, RENAME TO, and SET SCHEMA clauses, which are PostgreSQL extensions. diff --git a/doc/src/sgml/ref/create_sequence.sgml b/doc/src/sgml/ref/create_sequence.sgml index 86ff018c4b..f1448e7ab3 100644 --- a/doc/src/sgml/ref/create_sequence.sgml +++ b/doc/src/sgml/ref/create_sequence.sgml @@ -21,7 +21,9 @@ PostgreSQL documentation -CREATE [ TEMPORARY | TEMP ] SEQUENCE [ IF NOT EXISTS ] name [ INCREMENT [ BY ] increment ] +CREATE [ TEMPORARY | TEMP ] SEQUENCE [ IF NOT EXISTS ] name + [ AS data_type ] + [ INCREMENT [ BY ] increment ] [ MINVALUE minvalue | NO MINVALUE ] [ MAXVALUE maxvalue | NO MAXVALUE ] [ START [ WITH ] start ] [ CACHE cache ] [ [ NO ] CYCLE ] [ OWNED BY { table_name.column_name | NONE } ] @@ -110,6 +112,21 @@ SELECT * FROM name; + + data_type + + + The optional + clause AS data_type + specifies the data type of the sequence. Valid types are + are smallint, integer, + and bigint. bigint is the + default. The data type determines the default minimum and maximum + values of the sequence. + + + + increment @@ -132,9 +149,8 @@ SELECT * FROM name; class="parameter">minvalue determines the minimum value a sequence can generate. If this clause is not supplied or is specified, then - defaults will be used. The defaults are 1 and - -263 for ascending and descending sequences, - respectively. + defaults will be used. The default for an ascending sequence is 1. The + default for a descending sequence is the minimum value of the data type. @@ -148,9 +164,9 @@ SELECT * FROM name; class="parameter">maxvalue determines the maximum value for the sequence. If this clause is not supplied or is specified, then - default values will be used. The defaults are - 263-1 and -1 for ascending and descending - sequences, respectively. + default values will be used. The default for an ascending sequence is + the maximum value of the data type. The default for a descending + sequence is -1. @@ -347,12 +363,6 @@ END; CREATE SEQUENCE conforms to the SQL standard, with the following exceptions: - - - The standard's AS data_type expression is not - supported. - - Obtaining the next value is done using the nextval() diff --git a/src/backend/catalog/information_schema.sql b/src/backend/catalog/information_schema.sql index 62ee2b4e0e..9a53003ecf 100644 --- a/src/backend/catalog/information_schema.sql +++ b/src/backend/catalog/information_schema.sql @@ -1531,8 +1531,8 @@ CREATE VIEW sequences AS SELECT CAST(current_database() AS sql_identifier) AS sequence_catalog, CAST(nc.nspname AS sql_identifier) AS sequence_schema, CAST(c.relname AS sql_identifier) AS sequence_name, - CAST('bigint' AS character_data) AS data_type, - CAST(64 AS cardinal_number) AS numeric_precision, + CAST(format_type(s.seqtypid, null) AS character_data) AS data_type, + CAST(_pg_numeric_precision(s.seqtypid, -1) AS cardinal_number) AS numeric_precision, CAST(2 AS cardinal_number) AS numeric_precision_radix, CAST(0 AS cardinal_number) AS numeric_scale, CAST(s.seqstart AS character_data) AS start_value, diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index b4c2425179..38be9cf1a0 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -169,6 +169,7 @@ CREATE OR REPLACE VIEW pg_sequences AS N.nspname AS schemaname, C.relname AS sequencename, pg_get_userbyid(C.relowner) AS sequenceowner, + S.seqtypid::regtype AS data_type, S.seqstart AS start_value, S.seqmin AS min_value, S.seqmax AS max_value, diff --git a/src/backend/commands/sequence.c b/src/backend/commands/sequence.c index e6f87543df..e0df642254 100644 --- a/src/backend/commands/sequence.c +++ b/src/backend/commands/sequence.c @@ -34,6 +34,7 @@ #include "funcapi.h" #include "miscadmin.h" #include "nodes/makefuncs.h" +#include "parser/parse_type.h" #include "storage/lmgr.h" #include "storage/proc.h" #include "storage/smgr.h" @@ -229,12 +230,13 @@ DefineSequence(ParseState *pstate, CreateSeqStmt *seq) memset(pgs_nulls, 0, sizeof(pgs_nulls)); pgs_values[Anum_pg_sequence_seqrelid - 1] = ObjectIdGetDatum(seqoid); - pgs_values[Anum_pg_sequence_seqcycle - 1] = BoolGetDatum(seqform.seqcycle); + pgs_values[Anum_pg_sequence_seqtypid - 1] = ObjectIdGetDatum(seqform.seqtypid); pgs_values[Anum_pg_sequence_seqstart - 1] = Int64GetDatumFast(seqform.seqstart); pgs_values[Anum_pg_sequence_seqincrement - 1] = Int64GetDatumFast(seqform.seqincrement); pgs_values[Anum_pg_sequence_seqmax - 1] = Int64GetDatumFast(seqform.seqmax); pgs_values[Anum_pg_sequence_seqmin - 1] = Int64GetDatumFast(seqform.seqmin); pgs_values[Anum_pg_sequence_seqcache - 1] = Int64GetDatumFast(seqform.seqcache); + pgs_values[Anum_pg_sequence_seqcycle - 1] = BoolGetDatum(seqform.seqcycle); tuple = heap_form_tuple(tupDesc, pgs_values, pgs_nulls); CatalogTupleInsert(rel, tuple); @@ -622,11 +624,11 @@ nextval_internal(Oid relid) if (!HeapTupleIsValid(pgstuple)) elog(ERROR, "cache lookup failed for sequence %u", relid); pgsform = (Form_pg_sequence) GETSTRUCT(pgstuple); - cycle = pgsform->seqcycle; incby = pgsform->seqincrement; maxv = pgsform->seqmax; minv = pgsform->seqmin; cache = pgsform->seqcache; + cycle = pgsform->seqcycle; ReleaseSysCache(pgstuple); /* lock page' buffer and read tuple */ @@ -1221,6 +1223,7 @@ init_params(ParseState *pstate, List *options, bool isInit, Form_pg_sequence seqform, Form_pg_sequence_data seqdataform, List **owned_by) { + DefElem *as_type = NULL; DefElem *start_value = NULL; DefElem *restart_value = NULL; DefElem *increment_by = NULL; @@ -1236,7 +1239,16 @@ init_params(ParseState *pstate, List *options, bool isInit, { DefElem *defel = (DefElem *) lfirst(option); - if (strcmp(defel->defname, "increment") == 0) + if (strcmp(defel->defname, "as") == 0) + { + if (as_type) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"), + parser_errposition(pstate, defel->location))); + as_type = defel; + } + else if (strcmp(defel->defname, "increment") == 0) { if (increment_by) ereport(ERROR, @@ -1320,6 +1332,20 @@ init_params(ParseState *pstate, List *options, bool isInit, if (isInit) seqdataform->log_cnt = 0; + /* AS type */ + if (as_type != NULL) + { + seqform->seqtypid = typenameTypeId(pstate, defGetTypeName(as_type)); + if (seqform->seqtypid != INT2OID && + seqform->seqtypid != INT4OID && + seqform->seqtypid != INT8OID) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("sequence type must be smallint, integer, or bigint"))); + } + else if (isInit) + seqform->seqtypid = INT8OID; + /* INCREMENT BY */ if (increment_by != NULL) { @@ -1352,12 +1378,34 @@ init_params(ParseState *pstate, List *options, bool isInit, else if (isInit || max_value != NULL) { if (seqform->seqincrement > 0) - seqform->seqmax = PG_INT64_MAX; /* ascending seq */ + { + /* ascending seq */ + if (seqform->seqtypid == INT2OID) + seqform->seqmax = PG_INT16_MAX; + else if (seqform->seqtypid == INT4OID) + seqform->seqmax = PG_INT32_MAX; + else + seqform->seqmax = PG_INT64_MAX; + } else seqform->seqmax = -1; /* descending seq */ seqdataform->log_cnt = 0; } + if ((seqform->seqtypid == INT2OID && (seqform->seqmax < PG_INT16_MIN || seqform->seqmax > PG_INT16_MAX)) + || (seqform->seqtypid == INT4OID && (seqform->seqmax < PG_INT32_MIN || seqform->seqmax > PG_INT32_MAX)) + || (seqform->seqtypid == INT8OID && (seqform->seqmax < PG_INT64_MIN || seqform->seqmax > PG_INT64_MAX))) + { + char bufx[100]; + + snprintf(bufx, sizeof(bufx), INT64_FORMAT, seqform->seqmax); + + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("MAXVALUE (%s) is out of range for sequence data type %s", + bufx, format_type_be(seqform->seqtypid)))); + } + /* MINVALUE (null arg means NO MINVALUE) */ if (min_value != NULL && min_value->arg) { @@ -1369,10 +1417,32 @@ init_params(ParseState *pstate, List *options, bool isInit, if (seqform->seqincrement > 0) seqform->seqmin = 1; /* ascending seq */ else - seqform->seqmin = PG_INT64_MIN; /* descending seq */ + { + /* descending seq */ + if (seqform->seqtypid == INT2OID) + seqform->seqmin = PG_INT16_MIN; + else if (seqform->seqtypid == INT4OID) + seqform->seqmin = PG_INT32_MIN; + else + seqform->seqmin = PG_INT64_MIN; + } seqdataform->log_cnt = 0; } + if ((seqform->seqtypid == INT2OID && (seqform->seqmin < PG_INT16_MIN || seqform->seqmin > PG_INT16_MAX)) + || (seqform->seqtypid == INT4OID && (seqform->seqmin < PG_INT32_MIN || seqform->seqmin > PG_INT32_MAX)) + || (seqform->seqtypid == INT8OID && (seqform->seqmin < PG_INT64_MIN || seqform->seqmin > PG_INT64_MAX))) + { + char bufm[100]; + + snprintf(bufm, sizeof(bufm), INT64_FORMAT, seqform->seqmin); + + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("MINVALUE (%s) is out of range for sequence data type %s", + bufm, format_type_be(seqform->seqtypid)))); + } + /* crosscheck min/max */ if (seqform->seqmin >= seqform->seqmax) { @@ -1590,8 +1660,8 @@ pg_sequence_parameters(PG_FUNCTION_ARGS) { Oid relid = PG_GETARG_OID(0); TupleDesc tupdesc; - Datum values[6]; - bool isnull[6]; + Datum values[7]; + bool isnull[7]; HeapTuple pgstuple; Form_pg_sequence pgsform; @@ -1601,7 +1671,7 @@ pg_sequence_parameters(PG_FUNCTION_ARGS) errmsg("permission denied for sequence %s", get_rel_name(relid)))); - tupdesc = CreateTemplateTupleDesc(6, false); + tupdesc = CreateTemplateTupleDesc(7, false); TupleDescInitEntry(tupdesc, (AttrNumber) 1, "start_value", INT8OID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 2, "minimum_value", @@ -1614,6 +1684,8 @@ pg_sequence_parameters(PG_FUNCTION_ARGS) BOOLOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 6, "cache_size", INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 7, "data_type", + OIDOID, -1, 0); BlessTupleDesc(tupdesc); @@ -1630,6 +1702,7 @@ pg_sequence_parameters(PG_FUNCTION_ARGS) values[3] = Int64GetDatum(pgsform->seqincrement); values[4] = BoolGetDatum(pgsform->seqcycle); values[5] = Int64GetDatum(pgsform->seqcache); + values[6] = ObjectIdGetDatum(pgsform->seqtypid); ReleaseSysCache(pgstuple); diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index cf97be512d..174773bdf3 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -3941,7 +3941,11 @@ SeqOptList: SeqOptElem { $$ = list_make1($1); } | SeqOptList SeqOptElem { $$ = lappend($1, $2); } ; -SeqOptElem: CACHE NumericOnly +SeqOptElem: AS SimpleTypename + { + $$ = makeDefElem("as", (Node *)$2, @1); + } + | CACHE NumericOnly { $$ = makeDefElem("cache", (Node *)$2, @1); } diff --git a/src/backend/parser/parse_utilcmd.c b/src/backend/parser/parse_utilcmd.c index 8d1939445b..0f78abaae2 100644 --- a/src/backend/parser/parse_utilcmd.c +++ b/src/backend/parser/parse_utilcmd.c @@ -469,7 +469,7 @@ transformColumnDefinition(CreateStmtContext *cxt, ColumnDef *column) */ seqstmt = makeNode(CreateSeqStmt); seqstmt->sequence = makeRangeVar(snamespace, sname, -1); - seqstmt->options = NIL; + seqstmt->options = list_make1(makeDefElem("as", (Node *) makeTypeNameFromOid(column->typeName->typeOid, -1), -1)); /* * If this is ALTER ADD COLUMN, make sure the sequence will be owned diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 9afacdb900..7364a12c25 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -15912,39 +15912,29 @@ dumpSequence(Archive *fout, TableInfo *tbinfo) PGresult *res; char *startv, *incby, - *maxv = NULL, - *minv = NULL, - *cache; - char bufm[100], - bufx[100]; + *maxv, + *minv, + *cache, + *seqtype; bool cycled; + bool is_ascending; PQExpBuffer query = createPQExpBuffer(); PQExpBuffer delqry = createPQExpBuffer(); PQExpBuffer labelq = createPQExpBuffer(); - snprintf(bufm, sizeof(bufm), INT64_FORMAT, PG_INT64_MIN); - snprintf(bufx, sizeof(bufx), INT64_FORMAT, PG_INT64_MAX); - if (fout->remoteVersion >= 100000) { /* Make sure we are in proper schema */ selectSourceSchema(fout, "pg_catalog"); appendPQExpBuffer(query, - "SELECT seqstart, seqincrement, " - "CASE WHEN seqincrement > 0 AND seqmax = %s THEN NULL " - " WHEN seqincrement < 0 AND seqmax = -1 THEN NULL " - " ELSE seqmax " - "END AS seqmax, " - "CASE WHEN seqincrement > 0 AND seqmin = 1 THEN NULL " - " WHEN seqincrement < 0 AND seqmin = %s THEN NULL " - " ELSE seqmin " - "END AS seqmin, " + "SELECT format_type(seqtypid, NULL), " + "seqstart, seqincrement, " + "seqmax, seqmin, " "seqcache, seqcycle " "FROM pg_class c " "JOIN pg_sequence s ON (s.seqrelid = c.oid) " "WHERE c.oid = '%u'::oid", - bufx, bufm, tbinfo->dobj.catId.oid); } else if (fout->remoteVersion >= 80400) @@ -15958,17 +15948,9 @@ dumpSequence(Archive *fout, TableInfo *tbinfo) selectSourceSchema(fout, tbinfo->dobj.namespace->dobj.name); appendPQExpBuffer(query, - "SELECT start_value, increment_by, " - "CASE WHEN increment_by > 0 AND max_value = %s THEN NULL " - " WHEN increment_by < 0 AND max_value = -1 THEN NULL " - " ELSE max_value " - "END AS max_value, " - "CASE WHEN increment_by > 0 AND min_value = 1 THEN NULL " - " WHEN increment_by < 0 AND min_value = %s THEN NULL " - " ELSE min_value " - "END AS min_value, " + "SELECT 'bigint'::name AS sequence_type, " + "start_value, increment_by, max_value, min_value, " "cache_value, is_cycled FROM %s", - bufx, bufm, fmtId(tbinfo->dobj.name)); } else @@ -15977,17 +15959,9 @@ dumpSequence(Archive *fout, TableInfo *tbinfo) selectSourceSchema(fout, tbinfo->dobj.namespace->dobj.name); appendPQExpBuffer(query, - "SELECT 0 AS start_value, increment_by, " - "CASE WHEN increment_by > 0 AND max_value = %s THEN NULL " - " WHEN increment_by < 0 AND max_value = -1 THEN NULL " - " ELSE max_value " - "END AS max_value, " - "CASE WHEN increment_by > 0 AND min_value = 1 THEN NULL " - " WHEN increment_by < 0 AND min_value = %s THEN NULL " - " ELSE min_value " - "END AS min_value, " + "SELECT 'bigint'::name AS sequence_type, " + "0 AS start_value, increment_by, max_value, min_value, " "cache_value, is_cycled FROM %s", - bufx, bufm, fmtId(tbinfo->dobj.name)); } @@ -16002,14 +15976,48 @@ dumpSequence(Archive *fout, TableInfo *tbinfo) exit_nicely(1); } - startv = PQgetvalue(res, 0, 0); - incby = PQgetvalue(res, 0, 1); - if (!PQgetisnull(res, 0, 2)) - maxv = PQgetvalue(res, 0, 2); - if (!PQgetisnull(res, 0, 3)) - minv = PQgetvalue(res, 0, 3); - cache = PQgetvalue(res, 0, 4); - cycled = (strcmp(PQgetvalue(res, 0, 5), "t") == 0); + seqtype = PQgetvalue(res, 0, 0); + startv = PQgetvalue(res, 0, 1); + incby = PQgetvalue(res, 0, 2); + maxv = PQgetvalue(res, 0, 3); + minv = PQgetvalue(res, 0, 4); + cache = PQgetvalue(res, 0, 5); + cycled = (strcmp(PQgetvalue(res, 0, 6), "t") == 0); + + is_ascending = incby[0] != '-'; + + if (is_ascending && atoi(minv) == 1) + minv = NULL; + if (!is_ascending && atoi(maxv) == -1) + maxv = NULL; + + if (strcmp(seqtype, "smallint") == 0) + { + if (!is_ascending && atoi(minv) == PG_INT16_MIN) + minv = NULL; + if (is_ascending && atoi(maxv) == PG_INT16_MAX) + maxv = NULL; + } + else if (strcmp(seqtype, "integer") == 0) + { + if (!is_ascending && atoi(minv) == PG_INT32_MIN) + minv = NULL; + if (is_ascending && atoi(maxv) == PG_INT32_MAX) + maxv = NULL; + } + else if (strcmp(seqtype, "bigint") == 0) + { + char bufm[100], + bufx[100]; + + snprintf(bufm, sizeof(bufm), INT64_FORMAT, PG_INT64_MIN); + snprintf(bufx, sizeof(bufx), INT64_FORMAT, PG_INT64_MAX); + + if (!is_ascending && strcmp(minv, bufm) == 0) + minv = NULL; + if (is_ascending && strcmp(maxv, bufx) == 0) + maxv = NULL; + } /* * DROP must be fully qualified in case same name appears in pg_catalog @@ -16033,6 +16041,9 @@ dumpSequence(Archive *fout, TableInfo *tbinfo) "CREATE SEQUENCE %s\n", fmtId(tbinfo->dobj.name)); + if (strcmp(seqtype, "bigint") != 0) + appendPQExpBuffer(query, " AS %s\n", seqtype); + if (fout->remoteVersion >= 80400) appendPQExpBuffer(query, " START WITH %s\n", startv); diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl index 242d3c078c..f73bf8974d 100644 --- a/src/bin/pg_dump/t/002_pg_dump.pl +++ b/src/bin/pg_dump/t/002_pg_dump.pl @@ -2494,6 +2494,7 @@ qr/CREATE TRANSFORM FOR integer LANGUAGE sql \(FROM SQL WITH FUNCTION pg_catalog catch_all => 'CREATE ... commands', regexp => qr/^ \QCREATE SEQUENCE test_table_col1_seq\E + \n\s+\QAS integer\E \n\s+\QSTART WITH 1\E \n\s+\QINCREMENT BY 1\E \n\s+\QNO MINVALUE\E @@ -2529,6 +2530,7 @@ qr/CREATE TRANSFORM FOR integer LANGUAGE sql \(FROM SQL WITH FUNCTION pg_catalog catch_all => 'CREATE ... commands', regexp => qr/^ \QCREATE SEQUENCE test_third_table_col1_seq\E + \n\s+\QAS integer\E \n\s+\QSTART WITH 1\E \n\s+\QINCREMENT BY 1\E \n\s+\QNO MINVALUE\E diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index abb4aab9f8..5f42bde136 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201701309 +#define CATALOG_VERSION_NO 201702101 #endif diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index f6c0f23982..41c12afd74 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -1766,7 +1766,7 @@ DATA(insert OID = 1576 ( setval PGNSP PGUID 12 1 0 0 0 f f f f t f v u 2 0 20 DESCR("set sequence value"); DATA(insert OID = 1765 ( setval PGNSP PGUID 12 1 0 0 0 f f f f t f v u 3 0 20 "2205 20 16" _null_ _null_ _null_ _null_ _null_ setval3_oid _null_ _null_ _null_ )); DESCR("set sequence value and is_called status"); -DATA(insert OID = 3078 ( pg_sequence_parameters PGNSP PGUID 12 1 0 0 0 f f f f t f s s 1 0 2249 "26" "{26,20,20,20,20,16,20}" "{i,o,o,o,o,o,o}" "{sequence_oid,start_value,minimum_value,maximum_value,increment,cycle_option,cache_size}" _null_ _null_ pg_sequence_parameters _null_ _null_ _null_)); +DATA(insert OID = 3078 ( pg_sequence_parameters PGNSP PGUID 12 1 0 0 0 f f f f t f s s 1 0 2249 "26" "{26,20,20,20,20,16,20,26}" "{i,o,o,o,o,o,o,o}" "{sequence_oid,start_value,minimum_value,maximum_value,increment,cycle_option,cache_size,data_type}" _null_ _null_ pg_sequence_parameters _null_ _null_ _null_)); DESCR("sequence parameters, for use by information schema"); DATA(insert OID = 4032 ( pg_sequence_last_value PGNSP PGUID 12 1 0 0 0 f f f f t f v u 1 0 20 "2205" _null_ _null_ _null_ _null_ _null_ pg_sequence_last_value _null_ _null_ _null_ )); DESCR("sequence last value"); diff --git a/src/include/catalog/pg_sequence.h b/src/include/catalog/pg_sequence.h index 350b286e45..ef15e68a57 100644 --- a/src/include/catalog/pg_sequence.h +++ b/src/include/catalog/pg_sequence.h @@ -8,23 +8,25 @@ CATALOG(pg_sequence,2224) BKI_WITHOUT_OIDS { Oid seqrelid; - bool seqcycle; + Oid seqtypid; int64 seqstart; int64 seqincrement; int64 seqmax; int64 seqmin; int64 seqcache; + bool seqcycle; } FormData_pg_sequence; typedef FormData_pg_sequence *Form_pg_sequence; -#define Natts_pg_sequence 7 +#define Natts_pg_sequence 8 #define Anum_pg_sequence_seqrelid 1 -#define Anum_pg_sequence_seqcycle 2 +#define Anum_pg_sequence_seqtypid 2 #define Anum_pg_sequence_seqstart 3 #define Anum_pg_sequence_seqincrement 4 #define Anum_pg_sequence_seqmax 5 #define Anum_pg_sequence_seqmin 6 #define Anum_pg_sequence_seqcache 7 +#define Anum_pg_sequence_seqcycle 8 #endif /* PG_SEQUENCE_H */ diff --git a/src/test/modules/test_pg_dump/t/001_base.pl b/src/test/modules/test_pg_dump/t/001_base.pl index 200455cd26..7b3955aac9 100644 --- a/src/test/modules/test_pg_dump/t/001_base.pl +++ b/src/test/modules/test_pg_dump/t/001_base.pl @@ -241,6 +241,7 @@ my %tests = ( 'CREATE SEQUENCE regress_pg_dump_table_col1_seq' => { regexp => qr/^ \QCREATE SEQUENCE regress_pg_dump_table_col1_seq\E + \n\s+\QAS integer\E \n\s+\QSTART WITH 1\E \n\s+\QINCREMENT BY 1\E \n\s+\QNO MINVALUE\E diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 9c99a451ba..c661f1d962 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1641,6 +1641,7 @@ UNION ALL pg_sequences| SELECT n.nspname AS schemaname, c.relname AS sequencename, pg_get_userbyid(c.relowner) AS sequenceowner, + (s.seqtypid)::regtype AS data_type, s.seqstart AS start_value, s.seqmin AS min_value, s.seqmax AS max_value, diff --git a/src/test/regress/expected/sequence.out b/src/test/regress/expected/sequence.out index d062e91d26..f339489151 100644 --- a/src/test/regress/expected/sequence.out +++ b/src/test/regress/expected/sequence.out @@ -28,6 +28,23 @@ CREATE TABLE sequence_test_table (a int); CREATE SEQUENCE sequence_testx OWNED BY sequence_test_table.b; -- wrong column ERROR: column "b" of relation "sequence_test_table" does not exist DROP TABLE sequence_test_table; +-- sequence data types +CREATE SEQUENCE sequence_test5 AS integer; +CREATE SEQUENCE sequence_test6 AS smallint; +CREATE SEQUENCE sequence_test7 AS bigint; +CREATE SEQUENCE sequence_testx AS text; +ERROR: sequence type must be smallint, integer, or bigint +CREATE SEQUENCE sequence_testx AS nosuchtype; +ERROR: type "nosuchtype" does not exist +LINE 1: CREATE SEQUENCE sequence_testx AS nosuchtype; + ^ +ALTER SEQUENCE sequence_test5 AS smallint; -- fails +ERROR: MAXVALUE (2147483647) is out of range for sequence data type smallint +ALTER SEQUENCE sequence_test5 AS smallint NO MINVALUE NO MAXVALUE; +CREATE SEQUENCE sequence_testx AS smallint MAXVALUE 100000; +ERROR: MAXVALUE (100000) is out of range for sequence data type smallint +CREATE SEQUENCE sequence_testx AS smallint MINVALUE -100000; +ERROR: MINVALUE (-100000) is out of range for sequence data type smallint --- --- test creation of SERIAL column --- @@ -445,13 +462,16 @@ SELECT * FROM information_schema.sequences regression | public | sequence_test2 | bigint | 64 | 2 | 0 | 32 | 5 | 36 | 4 | YES regression | public | sequence_test3 | bigint | 64 | 2 | 0 | 1 | 1 | 9223372036854775807 | 1 | NO regression | public | sequence_test4 | bigint | 64 | 2 | 0 | -1 | -9223372036854775808 | -1 | -1 | NO - regression | public | serialtest1_f2_foo | bigint | 64 | 2 | 0 | 1 | 1 | 9223372036854775807 | 1 | NO - regression | public | serialtest2_f2_seq | bigint | 64 | 2 | 0 | 1 | 1 | 9223372036854775807 | 1 | NO - regression | public | serialtest2_f3_seq | bigint | 64 | 2 | 0 | 1 | 1 | 9223372036854775807 | 1 | NO - regression | public | serialtest2_f4_seq | bigint | 64 | 2 | 0 | 1 | 1 | 9223372036854775807 | 1 | NO + regression | public | sequence_test5 | smallint | 16 | 2 | 0 | 1 | 1 | 32767 | 1 | NO + regression | public | sequence_test6 | smallint | 16 | 2 | 0 | 1 | 1 | 32767 | 1 | NO + regression | public | sequence_test7 | bigint | 64 | 2 | 0 | 1 | 1 | 9223372036854775807 | 1 | NO + regression | public | serialtest1_f2_foo | integer | 32 | 2 | 0 | 1 | 1 | 2147483647 | 1 | NO + regression | public | serialtest2_f2_seq | integer | 32 | 2 | 0 | 1 | 1 | 2147483647 | 1 | NO + regression | public | serialtest2_f3_seq | smallint | 16 | 2 | 0 | 1 | 1 | 32767 | 1 | NO + regression | public | serialtest2_f4_seq | smallint | 16 | 2 | 0 | 1 | 1 | 32767 | 1 | NO regression | public | serialtest2_f5_seq | bigint | 64 | 2 | 0 | 1 | 1 | 9223372036854775807 | 1 | NO regression | public | serialtest2_f6_seq | bigint | 64 | 2 | 0 | 1 | 1 | 9223372036854775807 | 1 | NO -(9 rows) +(12 rows) SELECT schemaname, sequencename, start_value, min_value, max_value, increment_by, cycle, cache_size, last_value FROM pg_sequences @@ -462,18 +482,21 @@ WHERE sequencename ~ ANY(ARRAY['sequence_test', 'serialtest']) public | sequence_test2 | 32 | 5 | 36 | 4 | t | 1 | 5 public | sequence_test3 | 1 | 1 | 9223372036854775807 | 1 | f | 1 | public | sequence_test4 | -1 | -9223372036854775808 | -1 | -1 | f | 1 | -1 - public | serialtest1_f2_foo | 1 | 1 | 9223372036854775807 | 1 | f | 1 | 3 - public | serialtest2_f2_seq | 1 | 1 | 9223372036854775807 | 1 | f | 1 | 2 - public | serialtest2_f3_seq | 1 | 1 | 9223372036854775807 | 1 | f | 1 | 2 - public | serialtest2_f4_seq | 1 | 1 | 9223372036854775807 | 1 | f | 1 | 2 + public | sequence_test5 | 1 | 1 | 32767 | 1 | f | 1 | + public | sequence_test6 | 1 | 1 | 32767 | 1 | f | 1 | + public | sequence_test7 | 1 | 1 | 9223372036854775807 | 1 | f | 1 | + public | serialtest1_f2_foo | 1 | 1 | 2147483647 | 1 | f | 1 | 3 + public | serialtest2_f2_seq | 1 | 1 | 2147483647 | 1 | f | 1 | 2 + public | serialtest2_f3_seq | 1 | 1 | 32767 | 1 | f | 1 | 2 + public | serialtest2_f4_seq | 1 | 1 | 32767 | 1 | f | 1 | 2 public | serialtest2_f5_seq | 1 | 1 | 9223372036854775807 | 1 | f | 1 | 2 public | serialtest2_f6_seq | 1 | 1 | 9223372036854775807 | 1 | f | 1 | 2 -(9 rows) +(12 rows) SELECT * FROM pg_sequence_parameters('sequence_test4'::regclass); - start_value | minimum_value | maximum_value | increment | cycle_option | cache_size --------------+----------------------+---------------+-----------+--------------+------------ - -1 | -9223372036854775808 | -1 | -1 | f | 1 + start_value | minimum_value | maximum_value | increment | cycle_option | cache_size | data_type +-------------+----------------------+---------------+-----------+--------------+------------+----------- + -1 | -9223372036854775808 | -1 | -1 | f | 1 | 20 (1 row) -- Test comments diff --git a/src/test/regress/sql/sequence.sql b/src/test/regress/sql/sequence.sql index 4b9824c3cc..0fbd255967 100644 --- a/src/test/regress/sql/sequence.sql +++ b/src/test/regress/sql/sequence.sql @@ -19,6 +19,19 @@ CREATE TABLE sequence_test_table (a int); CREATE SEQUENCE sequence_testx OWNED BY sequence_test_table.b; -- wrong column DROP TABLE sequence_test_table; +-- sequence data types +CREATE SEQUENCE sequence_test5 AS integer; +CREATE SEQUENCE sequence_test6 AS smallint; +CREATE SEQUENCE sequence_test7 AS bigint; +CREATE SEQUENCE sequence_testx AS text; +CREATE SEQUENCE sequence_testx AS nosuchtype; + +ALTER SEQUENCE sequence_test5 AS smallint; -- fails +ALTER SEQUENCE sequence_test5 AS smallint NO MINVALUE NO MAXVALUE; + +CREATE SEQUENCE sequence_testx AS smallint MAXVALUE 100000; +CREATE SEQUENCE sequence_testx AS smallint MINVALUE -100000; + --- --- test creation of SERIAL column --- -- cgit v1.2.3 From ae0e550ce1703621efb3b75f4558df253cef5bca Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Sun, 12 Feb 2017 20:22:06 -0500 Subject: doc: Remove accidental extra table cell --- doc/src/sgml/catalogs.sgml | 1 - 1 file changed, 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 7d1c90a311..96cb9185c2 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -5777,7 +5777,6 @@ seqtypid oid pg_type.oid - Data type of the sequence -- cgit v1.2.3 From 7ada2d31f47f1d378ae7d4ee2d044df1bb2b0c6a Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Mon, 13 Feb 2017 11:02:23 -0500 Subject: Remove contrib/tsearch2. This module was intended to ease migrations of applications that used the pre-8.3 version of text search to the in-core version introduced in that release. However, since all pre-8.3 releases of the database have been out of support for more than 5 years at this point, we expect that few people are depending on it at this point. If some people still need it, nothing prevents it from being maintained as a separate extension, outside of core. Discussion: http://postgr.es/m/CA+Tgmob5R8aDHiFRTQsSJbT1oreKg2FOSBrC=2f4tqEH3dOMAg@mail.gmail.com --- .gitattributes | 1 - contrib/Makefile | 1 - contrib/tsearch2/.gitignore | 4 - contrib/tsearch2/Makefile | 20 - contrib/tsearch2/data/test_tsearch.data | 508 ---- contrib/tsearch2/expected/tsearch2.out | 2977 ------------------------ contrib/tsearch2/sql/tsearch2.sql | 339 --- contrib/tsearch2/tsearch2--1.0.sql | 576 ----- contrib/tsearch2/tsearch2--unpackaged--1.0.sql | 144 -- contrib/tsearch2/tsearch2.c | 542 ----- contrib/tsearch2/tsearch2.control | 7 - doc/src/sgml/contrib.sgml | 1 - doc/src/sgml/filelist.sgml | 1 - doc/src/sgml/textsearch.sgml | 83 - doc/src/sgml/tsearch2.sgml | 203 -- src/test/modules/test_parser/test_parser.c | 1 - src/tools/msvc/Mkvcbuild.pm | 2 +- 17 files changed, 1 insertion(+), 5409 deletions(-) delete mode 100644 contrib/tsearch2/.gitignore delete mode 100644 contrib/tsearch2/Makefile delete mode 100644 contrib/tsearch2/data/test_tsearch.data delete mode 100644 contrib/tsearch2/expected/tsearch2.out delete mode 100644 contrib/tsearch2/sql/tsearch2.sql delete mode 100644 contrib/tsearch2/tsearch2--1.0.sql delete mode 100644 contrib/tsearch2/tsearch2--unpackaged--1.0.sql delete mode 100644 contrib/tsearch2/tsearch2.c delete mode 100644 contrib/tsearch2/tsearch2.control delete mode 100644 doc/src/sgml/tsearch2.sgml (limited to 'doc/src') diff --git a/.gitattributes b/.gitattributes index 4dfc13112e..bdbcdb560a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -13,7 +13,6 @@ README.* conflict-marker-size=32 # Certain data files that contain special whitespace, and other special cases *.data -whitespace -contrib/tsearch2/sql/tsearch2.sql whitespace=space-before-tab,blank-at-eof,-blank-at-eol contrib/pgcrypto/sql/pgp-armor.sql whitespace=-blank-at-eol doc/bug.template whitespace=space-before-tab,-blank-at-eof,blank-at-eol src/backend/catalog/sql_features.txt whitespace=space-before-tab,blank-at-eof,-blank-at-eol diff --git a/contrib/Makefile b/contrib/Makefile index 25263c0be9..9a74e1b99f 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -47,7 +47,6 @@ SUBDIRS = \ test_decoding \ tsm_system_rows \ tsm_system_time \ - tsearch2 \ unaccent \ vacuumlo diff --git a/contrib/tsearch2/.gitignore b/contrib/tsearch2/.gitignore deleted file mode 100644 index 5dcb3ff972..0000000000 --- a/contrib/tsearch2/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Generated subdirectories -/log/ -/results/ -/tmp_check/ diff --git a/contrib/tsearch2/Makefile b/contrib/tsearch2/Makefile deleted file mode 100644 index 36dcedc688..0000000000 --- a/contrib/tsearch2/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# contrib/tsearch2/Makefile - -MODULES = tsearch2 - -EXTENSION = tsearch2 -DATA = tsearch2--1.0.sql tsearch2--unpackaged--1.0.sql -PGFILEDESC = "tsearch2 - backward-compatible text search functionality" - -REGRESS = tsearch2 - -ifdef USE_PGXS -PG_CONFIG = pg_config -PGXS := $(shell $(PG_CONFIG) --pgxs) -include $(PGXS) -else -subdir = contrib/tsearch2 -top_builddir = ../.. -include $(top_builddir)/src/Makefile.global -include $(top_srcdir)/contrib/contrib-global.mk -endif diff --git a/contrib/tsearch2/data/test_tsearch.data b/contrib/tsearch2/data/test_tsearch.data deleted file mode 100644 index 29a26f2428..0000000000 --- a/contrib/tsearch2/data/test_tsearch.data +++ /dev/null @@ -1,508 +0,0 @@ -\n -\n -\n -\n -\n -\n -\n -\n -\n i8 hy qo xa jl wr le l5 ja jx zf ro vw wd wa cc mm wh fn yd td l8 ec rv th oc ix ir sm y4 gh pr qg ue cx ww zv c9 zv tx eo f5 gd km b9 wb rm ym yl xj u7 xz uk iq tm ux di if uc hc ge -\n gr ty ph jh po wa iw ag wq r3 yd ow rb ip et ej yl a9 dk pu y6 su ov hf xe qe sd qr zt kp ml ea tp pg dq e3 s3 hh gn hz j7 hb qs qd v0 v4 w0 nu ee wk ez un rd sz wx e7 pn yf gh uh ki kx rb qv f1 bh sr yj ry r2 -\n q1 q8 wp w9 vs ww rq de qt wo qp sa rv mc sn u8 yl -\n hv ra sa fr qs ps 4w z5 ls wt ad wy q6 zg bd vt wa e4 ft w7 ld es yg et ic pm sw ja qv ov jm ma b3 wu wi qy ug hs wh ex rt tj en ur e2 ut gv as ui dy qy du qo gv cy lx kw xm fl x2 hd ny nu hh dt wg wh rs wb wz yy yu tj ha ak rw sw io h1 ux ku v6 wc qa rv xb s8 qd f2 zo k2 ew w4 yh yu yi -\n rs tt gp qh wt q6 lg zh vr b8 uy uu lh px jm ww qe xu fp fd rs qu ki dr fn gq gw jv oq zt 2r lc ke wg l9 x3 x5 7g vs ar e7 u2 s8 t0 av dj kl nm u2 zp gf yw ee oc tw a1 -\n qs uz wr gq q9 rl e0 pe dj a9 hp qw aw er kq pp uu pl zo wp fr r6 ej pv u5 hh av lw ko qc pn qj ez n8 wn eu tq -\n po h9 rd qs hr la u0 um me wp 0p rl mv rc ab r0 fe fj fk qn jh iy cn lb bl ln b5 ll yg yh qt qp uz od dq as gn cr qa wa cu fy zy vo xk eq vg mr ns yy t7 yi op th yo ov pv em tc hg az io s5 ct os wu lq dr mp hk si gx -\n hm k5 pw a5 qh nb q3 ql wr wt z7 oz wu wh kv q8 c3 mt mg hb a3 rz pz uo y1 rb av us ek dz q0 d3 qw j2 ls wy qq jf ng eo gl ed ix em he qt du hp jc f2 m9 qp hb l4 gy zf l6 qr dn cp x1 oh qk kk s3 hy wg zs ot wj sl oz ie e9 ay it u5 ai hm gh py hz qk ki h8 ja zu qb ei vc qj hg ev h6 yh u0 tb id -\n qg d1 bt c5 r3 iv g6 d7 rc ml gk uh yn y0 zo uh qd wh ib uo u4 om qg ql yz -\n hb a3 q5 pl yj lo qy ki sy fo rj kk zq dl wn 7a zi wn wm yr w3 tv r1 -\n ft k6 iz qn qj q2 q3 bl zd av ro wo lk tg ea ew ed y1 ia yl ic g6 po aw sc zm qn gl wq qw zr jp wt j5 gs vt qt yc rr op yw tl ye hr i8 tb uu j0 xd lz vu nl qd fu wg pf wj bt ee wh t2 tp sz um oo tg ha u4 f5 sw pq pr ju qk mh ki zb vj ob cx df hj ef cj q6 u9 tv rv o4 sy ru fq ir -\n ps ko uk tz vv um t9 uk k2 ja o6 ob -\n qs nb gh ld q7 jc sp el w0 py qx i2 qe la rl qw tu ti dq ue iv oi wa qr ed t3 fg oa of rr fv qz xn wu wq te hx -\n yb ty pq az fi qg qn la bu ji lg wg q8 mi cv rl up lg om oq ym pv in aq gg js ha on ww qr bj vn pv he b5 mh qe cc mk qt rb eu qy rw tr qo ec op sn oh e2 ao iv e4 hy dt s6 qt p1 hb ih qs wg x1 bd l1 t1 ro r9 uv wb aw gu os t0 ah e0 s0 hj pe or qj zz ql fd ks qv bq qm bg ec ry oj u8 u0 yj ru r1 yx o7 -\n z4 wr qz cg nq ir bb gb w7 e5 zc pj e9 px uo fp ts aq db q9 iy qe zv xu a9 l1 mb qw tc qu fi hw ur de e4 hk lj wo wf fi ep rl wh vh ek vp oi sv rh ay hj px aa er tv do ir -\n tr o9 gb tt pp qa qs a5 ps rf q1 kj by ub ru ox co o8 ny wp wa ws rd kk b1 zc rl rz uo ts ig fh db qm q0 bg rr fu ld lr wb en nd cw vr hy rn qr en em au p8 so oh ut hz gq wp ow be ky wj dw t1 pl er wc ot na r9 wl ou un um wx iq sc e8 sn re rr f7 hz h4 ce wz qx wx kp px tl tx ai wq hf ec 6u rz og yt ok yy yp -\n sa pp a7 qm qh of je qj lo ph wt h0 ji cg z8 2v xs zl mo ik hm on tu d8 av ot pn iv ez ja qn pq wy 7r mq qu p1 tu p6 ti ur pj uy ui qo i9 qa nj xm s1 ya fb 7j ro wn t6 wz yu iq yi go en pb aj f5 hf ug uh hk av pr wl wz im ja v9 u2 ks it br wv wn se ia o5 ox ei r2 ig aj sp -\n sa tn z8 ew uo eh g8 zt wy 27 ff uh te en pd eh hv 2e wh ty oi sw xx 2p qs mx wb q3 rl eq aa eu -\n d4 ef ta zq j2 em c0 vv wf kj dw uk ql y9 rn -\n sq nm kl w8 ur kz c1 pc y1 g4 oi jv wr zy ew by se ec yn ti gq gt rd l5 ej yp tk da qz qx ir wm on q2 to ew -\n rd gu z2 kj qk bl 6d wy nw xq iu 8t ri uc kq nx ql oa vi kd o6 -\n ra gr he wy q0 ow ti ia pb ha qr lv ms qu pu qw qr ml qt ep sv i5 of fm oe nl xh x1 xz u4 ha ao fc ug pw nh n9 qv kh vx uq w1 u0 ei if -\n q1 d2 qz zd jd qb wj nt ah mj ea ed y1 et fj qe en b8 ty iv ht fv tn tm sg jb ky ai en us tl ud iu zj ql u1 ci ru iw tw -\n fr ub h9 pd ub jk vh z6 wu wh wp 5z yt w9 w0 uy om tl rc r6 ax d7 et y2 tw dz se vf ii m3 lf b4 jf vr qw qy uf es qp en tl to ye ue ph e3 uy i0 jl pz oe qo zp wp ft ka zf qd wd kr qf l9 mm wf qx ef t3 x8 ex rg ev s8 ys it da rw al hn tc f6 fv nd nc ad fj nr x0 bx yq ti rx ok tb hx o8 dp -\n o0 jq un xu q8 wo qq gg ta oj ec az dl bl wb -\n o9 ij pq gu gp nv qk gg la q4 nw bo z8 9a iw wu q8 eh wi nt jk ut ys c1 r5 up y1 yl py oy ht gd td db qn cz qw lp re c7 dh j5 ia bz dj qr qt wd wf qi rt sv ul uz tl ta yr e4 tm sg pc jv hc hv lc xg xm br vf r8 na wl ou td wc up rj s8 e8 ir ys ii qk p0 lt ho wb x8 bv lw w1 rz ew aa rv ry gx o8 -\n tt hn gn un db fu uq qf d4 q3 pp ji lf wu bx q8 hx kb ny t5 bn hb ex yf ef yj g1 g2 to yk g3 ej sk hy dv qc gj qv sy bg wr na wy bx z0 rc rm ml ug te qp i5 ue oj s4 im oq qt gx sa gt l4 sv at v3 bq mv wd x3 80 x8 aq xk rg yp en gs us dq ak tz al tx o2 dg f9 kv or h4 jy k1 jo h8 kp lt os kh as tn eu ul tm su an tw sp -\n za yi pe sh pv y4 y5 hy th jg qy qt ke ti ue qk yy ie cq wl p0 lw mf er w5 -\n k9 bt xu kc me is o5 z9 kb gv ur rc oe sk qn ve wi mm rn eu to ue uy qa xf by t1 td t7 aw up yf pr dk cg zr sc 3d at rw ec rl st zo rn do -\n o9 z5 wy vi ya ea ee fo gf va ov ww rr wr lb ro qq vr gj nw ru ym iv s4 hu tm wo wp zs br fs wg ej du y1 yt yu e7 eb em dd pq v7 cr um ae oz 0z kc tq rw zl rt wb y9 xv tm tq di eo te gc -\n tt un qs qn a7 qh je qj k0 o1 wr q6 wy ab q9 qm wr ea er eh pi hi sc hs m6 w1 bv lo zr tn yk ep op es ve xx sb ux hg sa gq qp wd n2 zh wf xf wj y3 wl e7 os u4 on ip kn ko qp s7 ly zn ba wu u4 kh f4 zo y9 q6 oh iw tq -\n qa a4 gu a7 cp z1 he ma q7 lu dp w7 ea rc ee d8 y4 tw ez im ae bv ii qe vb zt lc lv wm ro lk qr hp re tw yv es fp as zu oe qu qi bp wg cp p7 v4 ek rd wc ar rj tj e8 od e0 pm h2 h4 in qf wu wi 19 bj rl rc ee yj et tw ep -\n gv qd kj cd t3 c3 ih ws rg mc rx lh fd g8 gh cc vw b7 qe at j7 qo ws wg oy t6 t9 go eb e8 us u5 rq oe zj jy oz cj wb be ei pm og se w4 yu xw su yx if -\n o9 ub rd hw gs z3 ql nq ru wg jc 1t kv mr zm ah dd jk w8 ej aq ig y8 pp fj li wq jj cc qr no wy wu en bx yr qy oo es fy pd tk ix ph yr sf vx pn p2 jq fs ed oy yk os ie s9 u5 ak ud gd uf kb xc u1 xm eu xw 19 wn vh w1 to ee er aa rb rn ru an r1 ei -\n se kl 7h b6 xs ym tp an ta qb gn uo pt xi cl qp qy op vr ym ri ti tl i5 e1 e4 i9 ff i5 qp jx ht ql uo en pe ku h7 iw wn w4 ey ia si -\n ql xt wi k6 ew sf eg up eh oy sq ja g9 i3 qe cv l1 qq bv w2 la eu wg ec ef oh fs tb pc xd qs nl qu fn dy oi iu yf re fc hj hk xv zn zz w1 ew -\n po al hm qk jt cd ju nm li rs w9 ev ut ea 2f r4 d6 ey im pa nu wr m4 is bc xz w3 eu tb ha ft p4 ti to hr dy af i6 iz r4 jb x7 wj xg na rf gi at pn gd re wq qz ze bo wc vz sm zo my ye u7 oh dk w5 is yx tw fe dp -\n jl za gk cm wu vq jc zc iu mb oe fo fp ic sc 2l hy qr eb p5 pf dq pa fy lc td sz oo aw u1 rj fl tz nx aq xx oz xb 55 y0 -\n uq wr lh jv ri i7 ss qo gy bt s3 u1 dy ox hg it -\n ps hr lf jx bn qq up eh ab yl pn jg ng bz gd qr yw i9 j8 zi 3v oz at hd cx oj u9 rt uz ro ov -\n sq ga ny se cj id rg r3 pk kv ee sh ek dk sz pp q0 mn az kp ei qi ry em ph p9 gw hc m0 cp ea mn yf t1 5y wx ol e6 ec u2 e7 uh uj uk av ql lw qx zr qv mw qg cq ww wb pw tu w2 mf ut gk af yo ie ob -\n hn um a6 q7 af du r4 up tp ej sk lo le m8 rp eu ei qi ky op of tp ur oj hu tb dy qu gt tf oz wc s7 e7 ua pw ax nb wx wy fj wn 18 wv es yq ok w4 uz yx yc -\n pa qg qh q4 fv qz kx q6 cp gb c6 pr eh id in qw we bk wn qq b6 qy qu es ic s1 og gn wp op qf ic ro os yp rj fj ag oc ay da fv wl qp f1 yx n7 ea w2 ly yj iq iw rm o5 -\n o9 ps d3 lp wr qc md e5 rk w0 pm gx lf ku qt qp to tc pk fb tb qi lh nt yd vt ot ra tg gd zx wx vj rq cr hm ma jp vg u8 rt ei it -\n dx dv h9 rf qf uw a8 qh uv k3 ri is yr r3 eq uu tz yn y6 qc ps jf wq xe wx lc qr j4 ku xx nb 4z sr tr uq p6 uz of i6 s1 fs pj tc hu qu hz f1 hp lj s4 qx tg yp gs ob tz ds sw pm ug hm ip ql le vl wq tb xv eq w2 yg w4 st o6 -\n qd q4 pa z6 qz ia 70 r3 mb iu es r5 gh t9 cj vz qw mb ko vt qr qt gh qo ty eb kq n1 xb ef rp ek gu rg s7 rj sn ai hg o1 uj pr jt fg v0 tq tx ww bj bm ct w1 zi rn ox iw ri -\n al rd w8 vp yd yk r0 pi po se sr qa l0 qk ir e9 hm kc rz aa w6 -\n un pq qd a8 z2 qk z5 ws bi xy qx wg wp t4 mj gv qm rg c6 w7 w9 es y1 g2 ej yz gg qc qn wq qw m9 wx qe kr 27 fp fq m7 xp 3p qr rr tr ij il eh au s1 uc fx ut qu sj j8 j9 ya nr rz wg wh eg x8 sl t7 yu vf ay ds ap re dh qg qh qj hz qk zz qx k3 cy iq ox qv eu nx n6 6r lq n0 y0 uq tb sy iw fm an -\n yv dc qs gm q2 cv ok wt b2 cj wu mr zj kn e5 iu pz r8 pe fp ot tq a9 y5 sz ez cl wq qq wv a7 ln ky jd qe qr yx rm qi ea ln te y9 ev en eh iv tx e3 as tn j8 wf xh co fl nc wk xz es rx ee wh ub aq u1 ar e7 up it iu o2 wl ko jo cu pc wo al hm uq rn ul yz ro -\n pw na wu jd yf oe qr xr sk wa hw ql wg x6 s9 u7 am -\n uv tr ub k7 qg he u6 jt gs z3 by tn bi av z7 jc ck q7 2n ny cx km mk rf pj xi lh sf up yj to ia ab tq fq pm fd qc qv ps su qw fu xu cm zb bc qr qt tn ei rw gl p1 xi qo tt ed ef ri iz yw oh tc uy tv as qu l4 qr t4 wx e5 ae op oa em tz gd dq rw ug dr ux qj be ko cg nl je aj xw q1 vv ax rl w2 yt aa u0 eu ah -\n dc ph sq jt ql un q5 cg lk w9 ur uy pz uo sx qv qq cc ln fu ym ho su pn qa bq pd wj wj yk ou wl rk o2 pt uc km ja wm ry rm ob -\n gb pw qf we q3 ls q4 sy bl lg q8 t3 wl rg ed io ef if oi hp lo kw wy qw ei yz rt es p6 fp hi qo bn qw wg cy np uv yy oa uo ir of em ug x9 qh nj n8 ea u8 er w6 -\n ij dg cd lw gk wu zl dd eb eq sg ia am in wq xt nk wr xj qq p5 pd pk as sd fn lj jw fk l9 nt wl oo fj sb u4 gs fx hg o1 dr fb hj h8 xc yq ch er e2 aa af ah ob -\n a2 o0 hn pd iz hw jg q1 jl qz ip le me wi bb r3 z7 g1 eh td sw g9 qq c9 vy ud qo es ec tj uw dq ur hj dy oe zp lk l5 fl wj ys t2 ej t4 ek rs sl yu oa u3 gd pm rw h1 pr h2 py wl 2p s7 wq 6r mi 10 ox o6 -\n i3 qw ee ur cy nx r2 wj t2 ub ir aj cl qm u0 oz -\n qd qn un qz xy nq an kg hc c6 w8 93 eq ts g9 wy mg w3 rb 3f wf rw kt op es ef at em s6 pc wg bw x1 xl wg hl yk yo eb ud hm hl py wb u4 zp bj bm se sr sy ox am -\n rc ix qs ls qy at ut pk yo ys ec hs lq xv ks -\n yb al zf ws cn ac ih th ww vb kt b3 xo qe qi te ea p8 tn qd ci ix xk pk bg rc tl f4 wb rb ru -\n iy qd a5 jq jw qh sw fv oz cj hc qq ya ee yn pr av or us iv fa qb q9 bh ns d0 qe i1 b0 fh qy qu qi ry os ul hq ri ix e1 ao p0 qt sf qi uh ll ko lx nz sg jz hq sh p8 x3 wg rd sx yo yp u3 pv rq ds tc rr wx lr xb wn ep hh bk yw q6 og yr yg si tq do if -\n hv qa qf jg he q1 kj qz bh lr kn rj th kz ef eh av pp i1 ar gl ur lr bz xp yr ze qt tn es fl hw s5 qa ed t4 wz sx rg sv e9 fz hf al h1 av bg ym ee yg -\n k8 nn jy q4 wd lf xu q9 a1 4v yd mb r6 yh pb ta g6 dn d3 pl j1 jk wc cn wy 26 rr te ti fa e4 uy fb gr hb kd lc qf p5 wh au fa iv xo hf ot eg ra wv tp ec yo ah iu pw hj ac h3 py k2 u1 wb rl rz yt er w6 ru af yo ep -\n qd uq qh qm q3 vg qc c5 rd vp ut eq on yn ii xp up r8 d0 sz qx ue pl lx qe wr qr lm nh qt ha qo ki ri e2 tx iv ao s3 ow kp xf rh ya r2 rk cw nt by wd j8 t1 hk y1 ns t6 wc ev sq rq yf ux aw ch qs u2 zn sm rt wb bk yq dh 8w w3 rc yg o3 yi ox ov ir -\n u0 q7 qb ml or nu b5 1l xb tr tp in qt hz so v6 dq o2 qh wl nb rv fw -\n ss jr zf zh xt oy hy aw y8 js ob wq ny or vy fi en tb qi j9 gt ib ot oy rd e5 y6 tg th pt gq wz rt rl ew fm ie ri ir ro ah -\n o0 qj h9 wy ee g9 gk jd fg qt 3d fu ru iz tl fd tv ad hl wp oo wf nb ez sv tl f4 dr oy rp -\n ak il k6 qh q2 vd k3 zd bo lj k7 km 5c ut rz yd up ua is r0 qn zq wq j1 qe cv pw fu md bw yw qq ra rw qu ex ik at y0 ru ti yw fz ic ao ow gm jc i7 nf p4 fj xg kr br xk bs mb pk hl wl ta ez sv e9 us om rw ap gq wl k2 qz h8 gu kf et ru tq ag uz rp -\n yb az dd fu rf hw qg we u9 o3 q5 q6 ag c2 o7 wa kh w8 vo mc yg tu ua uh ta tw ih hu fj su bg ww bh kw ry ru wy ky wu wi fw 20 b9 qo ik oa ev hw s1 e1 e3 fc uu s5 tn qy hz jc do ou jq gb kf pf xl x3 yv lz iq eb e8 os sn fx dw qg ql wc ka n8 gf ly se tv yk di si o7 r2 rp -\n il mj vi sd ia y6 wq rm p5 ux ho nr ef ej wq iq fn -\n ft cs uo io er ic tw ig mm c9 xk ab ze uw i5 s1 e4 pl ui f2 lj p4 sf x4 kz ej ez eb ov of rw dy av qh f0 h5 ki qx cx eb og gk oz uc -\n ul io zd kn w9 y3 wt qq wp jl i9 jk ca h5 wx wb tm do -\n iy hv cs a2 ee yz y6 gk kq em qy uq ts w0 rq rr vt pb nc q5 -\n qn q3 vt vu yk ej fp tw zm qq qy y9 hh wo wg rh ep x5 wk mr el l9 av hz w5 -\n hq qz wy cx rh ur w9 e8 r4 fq im fj gj dm qn gl jn iz l1 yh mz rw e2 qo wh nt wk zw t7 e5 iq fh eb sn ud az uv fh sv dq q1 ku zs eb ue xq rn o6 do -\n ub lo sq wr d1 mt o7 ts t5 rd xe iu yg ot gg se pp qc js lu xt j3 j4 wt pc vz 5o yr qw zw qr eu db sy eb em fo i0 ad gw m9 ig ih lc od n4 pg rx bi ni kq wl aw e7 az jo mk bo wb ei mi ep wb eq di do -\n q1 ub xt db wt ws ik pl ee or to ej ic is fr jk ls c9 qq yg qt eo rw tp p8 dy pz gm hz or xs bt x8 t4 t8 s7 oj lt wv vx u7 w4 et ox yo -\n po o9 ih dx qa rf qf pd d2 kl ad lh kb bd qm bb b1 z8 ew d6 yg d7 ym ti eh ic iv oi y6 sz dx qn ut qm gz pj zw jj 4d bk wb lm xb ke yx oo qp yb yn en fo yw fp e4 aa fd jz qu gw qa zs nl v9 wf qt qi vg ni wx hk 9f sz tg t0 ga de re io av h2 jt x0 h4 wx wc fg rb rn nc yz iy zp ds ep zw pr xv rz yh yk zp do hc ep -\n hb ty z2 qz qz zh gw mg kb ve zz ti tp py el jp tg qc ar qv gx la qr cn lr nd ng ve qt 6g ml op pd uq uw eh i8 uy dt ho j8 wp wd qe xm w0 x4 qk el e9 pb sm pn tc gt ce oj jr mi ds wb ym ew u8 -\n ij yb hn u7 cd gj co dp lp b2 r5 ed ti pn qx g0 jb jn jj we bl ri ot pi rb yc sv ty oh ph hh e4 hy sd wp ll ft l7 wh ca ys wf wb t7 sv uo sb sn ha pb sw de un qc bz wo en as tb eu af eo -\n d2 k0 wr q4 q5 c2 sj iv pm g8 m1 l1 5s ij aa lb xm vf ej ta ar th od sm cw gy bu qd q1 u8 ry rn -\n qa ux q3 mj ex yu zx rk gi rl ya is py am tw ja js db ps dn qb qn gn lc pe qq vr qr eo qi ec oa ev uz yq of in ho qo jj jk wk wd zp wf lz t8 tk ha pv fz pn ug o2 pe uk kv gq v7 oi qv wv dj tv fn fw -\n dx a3 k5 um uq jd og nn q5 qx cu wp rd ws d6 px ac oe rb up tp ej ek ih ff qc gj qm xk b4 dz jg sq jh eu yx eo re es ul yw tp i6 pj ho qi qf sn og xo yv pk wj wb go ar uo eb ir iy pq uh qg h6 vt wv sn n0 rx af uz hx eo -\n yv ub ty gn gu fu dm ca q2 d4 cn ad iw k6 bf zl zz 2o w7 uo ee yk ix g3 am fw oi jo se ha vs qn iy qq 24 bl j6 g4 cw jv 1l ei qy ke j4 qi ep of ao hh tb gm sh lh vc uf vu wd p6 xm qt kh rk l9 s4 wh mr t4 oi rf iq op ox u4 e9 fk u5 it re uk f0 kb nd qk ce jp lr cy js qd qb sb tq n7 n8 ed ue tn ox o6 id r2 it -\n qa pa jd qn qg jt gh q5 lg ag qv ah qn vr da rh w7 b2 rz rx d6 d7 eg eh yl a9 ek dl tw sc hp ha su gz lo qe le ns kt qy qi 1h kp mz qu es yb yn p6 eh fs ok as im dy px gq qp qs l6 iv rl zw dr 4r hi wj rp t6 go s8 e8 at e9 f3 ak dg f9 qh pt dz ww rv wb oc pv be wq cs q1 xr xx eq yr u9 sr tb yl tq if hc ig -\n a5 co dh bt lw ck lh w7 3e mp r3 rz yf yh uh eh td y8 fg pa ar va dm su q9 d5 qw re vh he jc 1g ib xz qq qw yg vt rn rb cb ry ym em i7 hr ff f2 qp rd lx wg lb kh va jv qi xd wh wc el un sz tf gu oz ae e9 e0 iu dr io dt fb dh jo um wx s5 oa kx ly rn oc zy f3 hb tt wb u9 oz hx if ig -\n ak o0 qd q7 eq g1 y2 pt dk g8 qb vs qe dh 5i pt yh qo ul tp oj sp oq di uh zg xn rx tp tf ie f6 cg rv zm xw zq 5f md sr yk ru ro -\n a2 tt ub rs ij ml ow pe el gd va ue zm sa pq lc yw qi qw lv ep qo uj ym tl ye hj s6 uf qp 82 fk y1 wl oi t8 fk pb tx o1 sk lm oo xv n2 ad fk n6 dp on q6 rv -\n qf jf kk nm oz q7 b2 xo fw kj rh ua oe yl gh vd qe gn wb pt wi z0 se gj 48 of i5 oh so hz wp ae wg nc kg xf ev pv ov au iy az f7 qb q5 eq yr tv yy ol ry o4 oc di ep -\n po o9 dc a5 jd z1 sq ws b7 ti r9 sl ez aw tg zm si ng qe ky b5 pp eb od jl ff oe ce qp gy yv qk r4 xf kw iw sn tx gg uh cq ql qa 2s mt eq rb dp -\n qs qz cd dl se q0 lv eu yi rw qo uh uj ul en tx wo qd e6 pv gg je zx kp qc q3 ye en -\n un qs qh se ws lf so eq yf ef y3 g4 zb hs q0 no qw j2 y0 uu fb di f1 kq oa ul t3 ot fh ak yf fv dt f8 jo sx wx at wn cs lq zc -\n ub qa qs ik pw uq a6 pd dm d1 qm d2 qk cv zd bi wd ne ah qb kg kh ij 1p rk w9 wt r5 d5 px uf eh yk oy pm i2 hp st qn si qm zw we ls px lr ri qr sr db hp qu xk fy os eg en uc ur i7 sa hp vn qs kw dn od rh xj w9 wk ph ap yh el oi oo e7 gp ay s0 f4 gf az jy qk ql qx 1k v9 qc jq zy n5 kg hd ww wv bj hj ur er rn ry eo o7 -\n df a6 dn je ql no q6 ox wo zl bn rh ya mv e0 yn pr gd pi y8 i4 c7 g1 j6 wo rv eu xg eo c0 yx ea sv os wp qw wl ou un t6 u2 os of f6 dt f0 jt wc ja ae qv rm ds pq y7 qk ck aa ux -\n db iz jk zd wy wh c3 zk 2o rj hw vp on ed ac to g2 r0 id ta th qb dm pj m8 np oe pu bb tc gh ml rq uf tu eh ye tx gv pk jv j8 lk xs kd fi mx be wd t3 mr wk wl td eb ie tz dw rw pm re fb dj h6 ql wz wx qx qv u3 vz xe ex 2w ty ew xm oz an -\n ty a6 we wi ro lj bn rh r5 g4 aw jd q0 gz xy m6 wu qq et oo ex qp tr y0 fi au e3 oj gm px lg wk tu ek tg u2 ov em dg uk nd qj cy hp wq mi bj q4 ia fm r1 ei ie ux -\n tr qg h7 qk jl kc jr am mo w8 e8 td gl kw sd jo qr vl gs qe b9 mm fh eo uh ft ik e2 i0 uu ff qu f2 jq v2 wg kg ek aq wm yi yo s8 e8 sq ab cw wt ck pb pn xl bj yq wm ew xq su r2 -\n qk wa q6 jj ws ut gd gf ly ec pj sa pd wl e5 wc da kx zk zz zb wv rm te -\n jq uz nv ql as jx z8 q7 o7 yt rl ea e0 ym y2 pr ia sz sq sr qq qr vk oe pe lr bl ll rm yx y0 eg ti e1 ue uu ui jx zd oq kd rg lv lb r1 fx ro me ts ay f6 fc io qg py qj qk qs ky qh y9 ok o4 am -\n qh dl jt wy a2 yk y2 i4 zq kq we bb dg m6 qq zw rt ta tc ff xs xd qf g0 1d yg du wz iy sw tc dd hj hk mh ov zi wn hk ee yj af -\n ra ak uw q3 cb ji wy fw gw t4 mu ts qw ww rj vo rl yd ug d9 gj i3 zw qw wy md qq bv rn qy pd tl ic p9 hr dr hh ui sf f1 i6 ws cy es ef t5 kr ek oo t7 ec e8 u4 od dq ji ch jr zm jy q1 zp yq 2e og yo tm -\n tr tt qa qd jf pg qh jr sw ao q3 qz za wt js bl vw q9 ws uu w0 ya pl yf rx ee tu r7 gg dv it lo ww up js qq qe lz eu qy rr yb ri ay ye ta tv im sh ss uk qd qf bw ro sl wl t5 e5 um th ha fx re ii fv je hk ot cq km h8 ks bk vl qn xe te tt rl u7 iq ry ag dp o8 -\n sa z1 qj q2 nn wr z5 mq xu q7 gv t6 w7 r4 c1 mb sd ed ym ot ta ht ts tw gd tf g8 se ar gh fh qv qn zm hs qw qe oq wc xj vz xl wo rn i2 sr rq at yq uw s1 tx hy fm pc wo hv gy vu wd lc ul p8 wk wk el oz oa rh gp pn gf fx fc f7 rr dy x9 uk f0 py wl v7 cr ch qs wv nz lv lu 5o xe ym ly er yi ia gl ox r1 dp -\n uv qd hm qf gp k9 kj we lf bs ej 2i el wl t7 rj w0 rz yf ys r8 tp py tq tw dl im qc db qq sd ry c9 oe if qw aw qu uh tt p5 p7 p8 oj zi oe qu qi lk j9 sk zs ka lc wh wk zq mq vh t2 ej r9 mr ez t6 e5 op rk ga pb dq ap f9 py qk qz wc kd pv bd sm dr u7 mf o3 yk di r2 -\n po a3 uw q5 q7 ck kb zj td zz yf jd wq xh ld qr w4 p1 ij fu tp qq sv y2 yt t6 e5 op dw iu pw jp ka qv 4u qf rm vb w6 -\n fy a6 qg cs z3 ql dc jz wy me cj o6 ba kv wp w8 ea r5 uo fw ib ig g7 gg sy bg qr cb cq ro xl xv ex tt ru pd hg im oq gq ao rl pl aq sz t7 e6 os uf ug gg pr ql qz vt mj px wb ci qf ov be bg ww mp mi rz u7 w3 ei yc -\n gh cm ca rg uy pm y7 g8 lx yc qi re uh yn uq eh tz ph wo cr sv fp kh sl oi sx ov ga iu h1 je fd rv qv wi jy yy ry o4 tq si -\n qd iz qh q3 cg lf wy xa ez eq om ug eg yj fo fp yz qx qe wb or jg xb c9 p4 tj y0 iz tc oj tb i6 p1 ka zf qe yp wj mv ra ez rd uh pt zl lm sz wc lr bq oc zq sr af gl ei ux it -\n a2 qa h9 qh q3 fn kx ve wz us sj yz fd g7 vh c9 xq xj ln tz wp wf kg kk by vf j9 5y un yi e6 rh sn tx hj kn rb -\n un qs fi qm jk js bd o8 bx vt eq ya xp yg pz ym dj fp tp ta oy dl qc cx qq m1 rt wn d0 wm yr qw aq qt tb ha p1 uk yn ef tj gv im sd hk pz jx zi wa wf ba l0 wd mq ej wv t5 ek iq pv ov f3 em ak rq hn hh f7 uk qh ot ju ng ji h7 wz cr v9 bj bk rc er ia is iw ei id ov pu hc aj -\n pw gm qd jf u6 z3 jl q4 wt bi lr id wa wz w8 ya ev ew r6 yn ee io r8 ip ej td im si j2 jo d7 m4 pv iv yg qi il ti i6 ta ib ap fb hz wd vu wf wh kt kh og kk nm rp ti ek ns t7 y5 wc ae ir pv hf ub wc ho wb wn mi rn w6 yk tm te rp -\n o9 un h0 a6 pf iq xi tg w8 z7 r5 om oq eg or y4 sr fh zv vw zq tc ws rq db rw eo ym tl fv i9 pz jx j7 hx oi qf x2 l9 x3 qj by to el sx ys yd ao az uj hl dl tj nz wn kg kh on wv w1 w5 gl ei -\n gv az ql nm rc r7 yl ja zn q9 xw no iz qt pk x4 l4 tg u3 of zk wc go qb mo eq u7 tv rm ig fe -\n o0 ik qd um qg k9 mb bu wy bx ny ws hm ea mb iu pe eg ey sh uh g2 ic iv aq td qx ja qb ha 2j lp xr wc vl wn wi sl tx qt rq ec vw of uw sm ic qy j7 ns hb 2q kw wf vp 1f x1 5c zs y1 rg tg oz e7 fh eb ie up e0 ap ve wq zz cr wz h8 wv go ly fk az pr rz h4 ew w2 ok w5 ia si ro am -\n dv d2 cd qc zt 25 xp wd te es sh eo wn f4 wo tv oc -\n uv qd qn hr bj b1 mw lg io sh lo qq xh m6 28 rn xx p7 im qt jn jw bm qf r1 mn ny ed em ii dd wn cz ds vc wb hh q4 yw ur rt ie o6 ux r2 -\n rs dh z1 jr cb vq r3 eq om y1 sg r9 if tw qb qn m2 vy dc b0 ik fa ib aa jz qu sg qs 1s be of w9 oz sv t9 oc pv rw dd o2 dh lq ka lu 1m qk q4 y0 ye yq w2 w6 si ob it -\n o9 db iz fi qn qj mw 3v wp li e3 km zz yy mo ya rz mv yg r7 yh pc or r9 pm a0 td ih db lt pg jf gl re ww qw m4 j5 xi wi yd nh yg qr rm et ey ug re rr y9 y0 sb tu od ay of iv oh i8 ok uu sf gq lg wa nn uj qs cu kd xb wg cs 3k yv hk pn ii hh cw km wz wx wc n3 jr wm qn bd zo mo jo wm q6 w3 rt an hx ah am uc dp -\n ps pt vq kc bs vu vo xu ee ib hv wj x3 nu ud yf qh wb lb gz -\n ra o9 qs ty rd ps db pu qj u6 k9 nb qk oj ql wt jx bo ri xo o0 mk rh bm mj ut mb rc yn et yl pm ih i1 g9 qm xq oq bj wt jp xu pt bc eo ep qi ky sv uk fy ri i5 im dy hk ui tm f2 uf ug qd kf nc s3 fs 12 t2 ro du wk ek yt ej t7 s7 oc ay s9 pv fl s0 gf tx fc ac py v5 qk ce qz cr jp ck hp u3 zm xr ii yv ea cf hj ye w3 tp do tw ux -\n o9 qa ik wt kc q8 wk sp yy w8 w0 ys ea om tu yz pn fe ae g9 ps g0 i4 qb fk qn qm ut j1 d6 4d cb vj vk xy j5 be wi ve qq gf qr j3 ug qo p4 sm s2 ut fd pl qt jz ui qy qu qa nk fj iz xh wk iv qz fb ro x6 ti sl rs hc oi wm us ai dw o1 hh ab qh qj ju ng wl zr vk tw 5a vc hk md yr u7 w2 yt tn eu ul ah -\n uv ra qs ty jh q4 o4 bc vr o9 rg jz mc r4 ui g1 ey g3 sj am sq fj qn it xt ln dl jh b9 g8 dv qt yz ea ue ss w9 wj kk bi ym tg t0 ob ys iu uj qk nf v6 nj ox qb wy mw 1n pq eq w1 rx rp ge -\n a2 pa e4 xy yd sj vs jj xj lm qy qp ri ux p8 pj tv xs wd wf oz gd sw rw uj uk qj k1 xx um eu bx my em ey -\n az h0 qz iq bl kb xp yf y8 qn rv hx oc re gt k2 bo qg cf rl -\n yv uj d2 mq hx ws w7 mv yn r8 ab an ae jn xw al up be qr zr ep re qo ec ur ap hp pn wp i9 rf wf vo qk t8 eb yd uk kv ww wx wt ox kh mi eq yj oz fn ie am rp -\n ik df qg jg k9 wy kc ro wi ve bb rl ew io or eh sq oi qc qe d7 m4 pu gd db oo yv yq ix eh fl pg ib hu pl cr fr xd cy ke mx yh wk ag hf hk qg sj we mz gp u4 ak ma rz rv af ox di yx ob -\n hn pa pw qd il qh q1 z3 wr t3 wo ws vu uu ld pe fo dj ot dx pp vl qq rr ls j4 fs dl ve c6 rq ln xk ec rt ty ik y0 tu yq fz s5 sd sh jn wa uj ws lx qr ca rz wd nu ek yy yi y6 uo os up f4 fz qk h8 qc wr at 18 ca ww rv sy ox o6 -\n dv wu wo uo m2 we rp b6 qe ik e1 bq w8 x5 ez fh u4 iy jy wu li f5 u8 w3 yl te -\n sa ds qd no q5 ra jd qo ru r7 uo ar ud on ak fv dg wl qx qv ye yl ep ge -\n ss rf qn bu gj rj uo yz tf m1 kw zr oo y0 pf tc dy qu v6 xh t4 oo um df je qh dz v8 ho wn wo 0w dj rv o3 du ro -\n ij uj k7 me lg ih hv ws rj pl sd uo y1 yk d0 pt y4 g4 ou tw sq td fj ha qm qq 4a kw d7 xy m5 bx c9 yx nw tr qo uj fu en p6 s1 ht tb qo zp kq x2 wk wj wk yt wz sz ae iw ay fk ao ug pq qg k1 ql xx qc cl qk 56 bn oj yt et ut uy tw ir yc -\n k5 pg cp z5 wr no zd tk ej an qx gj i3 su we up 3q yq fx ib tv qp ik wj yf u1 os rk jt qo qx n9 w1 rb -\n k9 uv gs wr 3b mh km bm we w9 es or yk r0 g5 aq gf nq qv ll m5 yd zq qt qp sv ed p5 of eh i7 pz hl sg jn wa m0 nm kf w8 wj de e7 ar iy pn ly wn fx w3 rb ey am -\n o9 d2 vg gk ex rf rc hy qm j4 ga qw rm ls yl cm en tl tp fp tb i3 qy qo j9 vn zf wf qg mb kj qi jb mq wl rj s8 lw um zt wb f4 xw f9 -\n ra go ls qx wi c6 b0 rw g1 yz fe g8 ow qq ra mz ex oa fu iz tl uc e1 p6 x1 tf rh tk fz ap hl qh k3 xb mw zm yb yw q5 aa rp -\n yu qx sc xe j2 oq gs i6 i9 l0 -\n yv ss tt gu fi qj bt ql ls io nw gk hl up zv gl ni xt wy dz qe ud nw rw qu uw e4 qy px qf zw za ty ek t7 pv dg ho wn uq rx yx ep -\n ga 1w ld wy o7 xr pk r9 g6 hu jg lx sd no xt wr zy ku l2 nw 9r rt i5 to tp tc s6 f1 ud ko xb rj qy es t0 f4 fx ii rr hm hj fb ji oi n1 vk ci 9e mt yc 2r tv gk yp ux -\n hb hn k0 wy m4 w7 rc ts y6 j3 qe ve qy rt so di qo dp lk xf mq wl em f5 pr wl wn 3k ew yt w4 ri -\n qa ss lq wr bx t3 r5 ed eg sx dn we 7n ra qe b9 rm wd rw eo oa ri e1 e2 ut ap hu qo ws uz ai tz nl cu wq ln wn ie aj -\n yb rs un hm dg qm qk ao mw fn kv ur uo pj e9 sf ia tp tw a0 td sx fg su xq m1 om na vk wy xk em l1 z0 nh b0 mz qy p2 ru au iv p9 pz ug lz xn wf xg fk zu wj wd u1 e9 tl ak hf sw o1 pw dt gq v5 lm h7 nn nl wq uq zt o3 ad ry id ig -\n a3 pp dv qs gn u6 jy kk io dt wt ck rg ua yd ya yh ax ac y1 pe pc pv fp fw dc qv zb dn q0 ju jj m1 ui lx qe qr t9 ja he wi vw m7 l1 jn qe wa qt xg rq qy yi qu p3 yb ed en tz so s3 tb ho fm px gw zo lx wf sm mc dq wj yg l4 uv yk tp t5 wz ol fk f4 pe kc dj wz qb zm wi br zk ww ty 6i vm eb lt eq w2 ey yp yz o6 ei -\n fu ga io nt wp jz yf rv oe eh pt dz ih sx qx g0 qm hf xe lz gc d5 bh 2z cn d9 1r sz li bv qe 6d bb er xv yx p2 ea tj p5 ay uq dq pg oh qt s6 px sh ko qa nn oa bq cs kk hh cr wg tu y4 t6 e5 oz th sn ov u5 dw qg dh uk n1 zr qv 3d n4 yx xe wv eb h4 yo ro hx o8 rp -\n ra gu hm a7 jw qm qh jr gs k0 ql xt q5 dt ru wy k5 wh fw lu kb am m6 bx vy qq ev rk 2s mc yd es io pc pv g2 ek tq tw in ih ae qc d5 ui qe wt qq m8 vr nb ee hu rw rr tt ed fi em e1 e3 hh hi sh zp qo wp l4 ws qf qf pg eg to un gu u1 t9 ox e7 u4 od ds de hh py ql h7 gy js vz gf y8 uq se do ro rp -\n qs qg gk ta bf r3 hw r7 r9 sh ua g3 sq td g7 ha lu qw xr wy wu rz ko bb i6 uy as di qi za hv jw rf 1f 2u va ap qi rc du wk yt t7 u2 ob re ax v8 cg qb wy wn kg pn yi rn ru -\n gr ra jq qf go ga jh gs q3 tm q8 k7 o8 mj ym er ip ua ej hy i1 dv qb vg cv m5 wy xk g5 wi ng w3 3w ud rw ug ep hq ta fc fd aa i5 hx qp wp v7 qs l6 l9 l0 jn ty t6 ie rj tk od ys on pq tc zl nh qc xv wc cu ks ei lm vm cj yi ad r1 si sp -\n qa hm a3 ac q9 na rj if qw rr vw tj ib su qu wo dp j0 wf pf 2y wk ym ra wb ae ga gs f8 gq im ar pb ec f9 yu rm -\n t8 ej an y4 td ez ln z9 lj qy sm uw dq us cp nu tg vn -\n qa ds k5 hw k8 k0 ql hl 1r wi fw c6 w7 mz rj xy r4 e0 ym yh eg r0 us fs ib oi qv q0 ww lp gm ln bx nc qi l1 qe wg ea qo eb tk eg to ur jc oe dp hv wa 2q nk at rg wf wg ca xk jn pk yg er ot uv wb aq ol wx e6 ev sv uo vf eb ah ud da pe qg jr kn ju ng ae wv n3 iw ly kf cl pb wv tt vn eb vm u7 ew aa w6 rm gx r2 o8 -\n q4 q6 vk d6 eg pc pv r0 tw i3 q0 we tw sm e4 ow sn kg up hm qx zv nz wm u9 ul ri do -\n po uv dc qs qd hm qg q2 jj sw kl me q8 xa wa xf z5 yd r4 rq sf px ti ia r9 yl dj dk ek qx i2 sr qv qb lb wi nf wu qe tv fh qo rr yb fy eb ri ai ok qt pc ud qi qa ws qs lc zh nc x2 cs t2 di ke wk sz oc yp s9 ys ai ln wz cg wc wv os qb f2 ec y8 dg wm rz yr ee rn sy du su eu fm ei o6 dp hc -\n ft qj q6 wq up ut lg er uw db ll ws of og e4 1i r0 wx fh th vf re hm zi -\n a3 ty pw ph cg uo r7 oi q0 lb c0 vl xx mh hu b9 qy tt sv p5 eh to hh ow tm oe si sk oi gt kq cu vi j5 wf el tf yu u3 ya uj dy qh ql ct wc el y0 o3 o6 if ge -\n a3 dd by wt lf 2v bl 7c bn cf yo go yf ii et ey yl aq aw g8 ho i3 qb dn qn lu vf vg 2k le ml wy t0 4h xk qw b7 bb eu xr qu tt y0 os sn tl pf og tz tx pl ss us xd cu oa qd xn ke qf vp kh ny wk r8 ej rd t7 sc e6 rh ud tx al gg re hj ux qj gw xx zv xm iy ca vb yw en oh u9 aa w5 w6 ul oc an uc -\n qa un dn hq hw d1 jr jy kk kl wt kz zf z7 cm q7 me xp wp mj rh ue e7 ys rz eq ew ed xp ee yj y1 to fw aq po i2 jn li on m2 na vq wu ck er yu db yi gl ty eh uw fp tx e2 fs uu sf jx oe jb qp cy bn qd wg 3z nu j9 mr t6 gp pv ha tl ai fx uf fc kx qh gw xb zt qv qb ir cq vb y9 ct ol fn ah hx sp -\n db wt cm ch jc wi dd ys on td po y8 q0 wq kt eu tc tv or fr s3 na e7 uf gg re f5 tt aa tb ie -\n a5 jw qh q1 qj oj xy my b6 es yg yl y5 zm pv qw qt qo ea ri ao in s4 gv i0 ad lh wa qf gm rk vs oy r0 ez ab lm qs qh ry ox -\n ga ca z6 nr wo rg bm vu uu rj e0 ui io pe eh d9 ab tw fe tf fk wy ln md rk sk qq qw hy kp dc qy y0 p5 p6 p7 ic pg e4 jb ge wp qa bn xf ks zi oy e5 um wx ie yp fv je ng oj ja v9 bp qv er an pu -\n a4 jw a8 o1 q3 un gh le nq q8 ig rg rl ea io er tu fp dk hy sq ae lo qq wt wy 5i xj cw dz oo qo yn ty y9 y0 sb ef tj uw ta ur i8 tb oq af hz qi wo sk zs qs vi wf kt nb y1 oy wk r0 ol ex ec tg t9 eb ap fv qk ji cr s6 et xq bf ep mi ax rt yu iq af am yc -\n gr uv hv tr a3 qs lo u7 jy qz kl z6 gk gz ag kn rg k0 w7 wr rl pj ii yh up ac ot d0 g6 td y6 fr se pp dc g9 cl gx qw pl m1 ii qr vz oy nf eu eo p1 os y0 ri ix au uc ai fx p9 nm jj lz pa kw ul rg gw qh 4m eo qc l5 rp oy ej un yy yu t8 sv ud fx ac dy av gq qj ve sl bu th u3 rn nz n5 zm yz bd tx el ex n9 es rt rz rx ol sy rn yo -\n ph cb jx wu ib vb ih ty oy tl vu -\n df qd k7 z2 q2 ju jz zf cm mw yr gu rx yh ym ef pc qx jd q0 ow pw wt rj xo mf xl qq qw ud tw ku ik oa od ti hk f1 xs qd wf dm s2 ph xo ou sx ae iw t9 eb u3 rk ak hf dw ax oe zl zz wm sm el cx cw lq za tu yw rx yu rn fn yi ei -\n a6 ql wr jx z7 wu xi ym fo if a0 dv ww lx zv dk tu sn hh ff hu zo ws rf wf aa ni kq uv t7 um go e8 ob sm tz hl uc zz ol lr kc n6 bk ry if -\n qs gm tn rp iu pi qe ec to l1 wh ra wl it kx fd vx q1 ri -\n gu pw qg we d4 ws q4 cn q5 me qv zj zl ex wr xo yg r7 eg et ey us iv po aw se cx az lb nz nc qq ew rs rq yx ep tj uq eh fz hg gv jc di wp sa nc ya cs fv qz ti wn aw e7 ox u6 pn re o2 hm fv qg hl dl v9 qv tz 6y rl ye rx ur tn eo -\n gr qj z6 ld tm jw hc ed y5 se ke ht tn jb 12 yt ek ao io wv ew ey fm tw ir -\n gv fr ak o0 gb rd dv gu qf qg qh jg ux qj ph k0 oj wa jz bi ja eg c1 fe qn b5 rs rg b1 vo z7 us d5 r5 ii tu yh y1 or ek sl pm hy dc th sy ww ze vb wt m6 iv mj qe 6f qt gk tq ru yq au ap dr hh qy sf qa ik kt wd rz ej t3 ot ej ub wx oz th s7 t0 ag ga pv em fz sw o1 ip qh nd h5 et ho cu yz tq wq wn et tn yo si ov a1 -\n ij rs rd qd pd qg qh z4 ql ip nq q5 xu bz lh o7 my 3w xe ws 2p w9 rk es er d8 pu y6 qc gl bv qa rv qt rb os ru fn qy qu hx or wa qs at zg mx xo bg yh ec os eb hd rw dw ip vw ki ok qx cu wb sn wm yj o3 tm ei ah -\n tr rd pq qd um qj u7 q3 cf db k4 gl mr gw c3 bs k8 vi 4v kz cg rz et ey tp fq y5 el gd dx qx hp mn cz wq xh m4 av t0 vz m6 qw tv rq ei il sb tk eg uq tc wo qo zs rd nx 2y fo j5 l0 l1 hy vy t3 t4 yt va y5 rg e7 uo ox at ir ys hd uf fx re rr ac kc cq qk cw h6 kp xn zu bd cz ca pn pq w1 rx vc w6 yo is fw ir ov -\n ra ij a3 qs qn jf qm nv cs cv kz q5 um q7 q8 km ya ys rx yn d8 sh pt fe se js ue rk m7 wp et ei qy to tz s4 af 3i lc wk ej hc ex t7 oc sm s0 tl fx re fb jr jp qc kc jr cc w3 yl oc ob ep -\n sa yb qn k8 lf d1 c3 wp vr wl yd iu kb sz g7 mn jm lz sd m3 lv qq j1 ex qo ry ru em pk i3 hi rf fk nc wd vu yt td sc tg s9 tz tx dh x9 qh ku dz my yr w3 oj se ei gz tq hx ah fe -\n w9 rl rc or fq a9 pp db gj hs lc qr ec p4 ph hb x1 ez u5 qx ea 6t tn -\n a3 qa dd qf qn qm qj vj wi ag wo ig e3 wz r6 d7 ax pe rb ey r9 is ot tq oy if hy se qx ar qb vd qw qe np xy nd wi in gj qu y9 ev ti tb qt px ud wo ll cy wd hw kh fp wk wg wh ym vo ub rd t7 iq yo ox eb yp ys au u6 rq ii io pe qh nz vl be n8 wv hk og rc er yu u0 rn yl is do eo -\n dd nn oc el yu tl rc rv r7 y2 hi qc qm wu cq qw xc kp tr fu ib zi qu wp vi ci qj nu zw t1 wl fh ev os f4 f6 f7 cd zc qx zy wu bs qn u0 -\n o9 gr fu a7 qk xu q7 wp el yu fp ou y5 pm pp qm jm st op uc fx tn hl zs kp bq p7 hi ys qj ki qc qa n6 oj ey w6 yk si -\n q7 xo sr he uu sd s6 gy ws iz fk sw al v6 lq fh ie oh uz pu -\n of ch zj rk rx rc g8 i1 jk tv ul fi e1 ic sp in jl jv j7 nm rp r8 go hf wx tb oz tw it -\n se kc tj rx yh eh td pa zb qv c8 j5 ri eq b9 rm ik ev ul ti p6 en ok tn wp jm ws ke br wj rp en gd rq f6 ac ab zc rz ew tb ro -\n qd wr d6 i3 j1 ww if qt yn fd e4 qf j5 yh t8 u1 ev qc wv pw u7 oj ok yz tw o8 -\n un pa jd qh qm dz pi z3 ny gs k0 wt xy z6 cj k5 gl bz d1 fq ye yr rh t7 ot if g6 im pa ps fk zw lx kr lv na wu vw eo cm te qo tr ec ty sb y0 i5 to ye so tc i5 sg ct qs sc ws qr xj 2u n5 rl cw dw ys qj yn qc y1 sl t9 ox sv s8 ya s0 tl ys rw tx fx ds rr cq cd ql qa au qg vg rx yt iq tn yl uz ei si r2 ob -\n rs gm qk gg m3 rj eq mv yf sk gx ve eh iv i7 n3 pb uf gh uj tg ox ww bg oz su o6 -\n ih a3 uj rd qs df h0 jd d2 kj q2 ap wr ol nw bz q7 fq ir ra w0 eq ya r6 d6 eg ej pn py pp sr qb jb wq ni xe we lm be xo w2 qo mj qr hp tr qo qp ef of yw ai e2 i8 fb tm do dp i8 l4 wd p5 sn pf gr vs rx kz vh t2 wj ot ar t9 at ir dw qj nc cw fd sx qc mz lt pv br wv dr q3 yq vn ye yw dk oh rc w3 yt se ov ge -\n ds h0 jw he qh jr jt ql me na ah xa tf wt pj pk om 97 rc yg ym oe yj eg dl fe sz g9 lo qq qw wx rr c8 ns vq m7 xl gs vr qw qr kq qi qo eb tk ue dw e2 i8 i9 hy hh qt f1 pc vv qs bn ij i0 uj xh wk qz ns ej un oi yi rh od ha tl re tc o1 uh ac ip qj we qh lq eb w3 w4 ia oz eu ri uz ep -\n fr ij dl qk z3 qz wt z9 gq mr wo zz rd dd rz ee sw pp g0 sy vg ww iu pz uo cb t9 ld qr ei yx rw es ts zi wp wd gw wj hf r4 tt x8 wl t6 hc gp eb aj ai iu o2 nh qv ey kg dp wq f5 rt cg yw sr tb gj rb fm ro ah ig -\n ss ux q4 ji xa mj mi ld rl pj r4 rx yg ti ix a9 ig gj j1 ww ii qe j3 mz vl qq ye m8 b8 yl qp ik ki eg uq fi ok fb oq fm sf oe hp v4 nk wg mx kt vs j8 yn wc wj ot td wn iw os u4 tl u5 rq de io x9 dl ql n2 ji wb if hx -\n iy jk ql q4 wt kz fb q7 vy w8 ur ax uf ym yl py ou dl pm in sq ho j1 qr ls j6 ic sl ko xq jm qe qr qt yc es ry pf he i8 s3 pj tm oe qo lk wp j9 nb yd bu rs e5 yi ar rh ga ud al hh oe wx s6 do kv be pm w5 fn ey du do pu -\n po rd qs il rf uq of jf nf ih w8 b0 ur pl us ed tz tu ef rb sj tf ff i1 pa dv ue fk m2 qr wt j5 c0 vw xo b7 p7 qt zi wo gt qa oo qd bt nt zq x8 ou e5 u2 fj s0 yd sw re cd qx jp wc ja ga jt bh hm eq rv hx gx -\n iy ij pa gy qd qg he d2 qk d4 qz q5 nw wu am qm ft w7 rq yn ef oe pv ek fq sk y4 ts am fd qx q9 jf ju wy lm zw wp er sy qu oa ta jl ss gq fe wa p2 kq ws 2w dn xz t2 ej rp rs yy e6 iq ag e0 u5 tx dd pq fv jr qj ku oj ql wz fd s5 nj qx zt 2d qb nx pm ce f9 w3 er tw rp aj it -\n ss qa gm dm qk c1 jd k0 t8 mk rk tk om yn tz r7 px ac av ot ts if fw ez y6 se qb ha su qn cl wx we qr zt kr mj rc qo es tj ym iz tk i6 fa i7 s3 pl jx du qu j0 ws v2 wj ys yv wd s4 wf nm dt wv ub ez ta wx e0 fz al ap qg wr ar wb fj n4 cz qg wq fc mp yq ev se eu am gx dp te -\n a2 hv yb nv h7 jt lw xt lu wp yr rg 2o 93 uo pr ej ez jb lz ww az oq bk b5 wi qw rq hs te ea es ed fu ti uz fd tm jc do qi j9 zs j0 wd xv iz hr wx el ns oi t8 sc t9 sb fk hg cd rb mz wn 4i ov ln yr oz tm ro o8 -\n iy pi jt kz st tm rh ya b2 om ef eh tp el in sc qc g0 ps zq nu pq j3 oe a7 ja js ng tc qe pp eo em fc s3 hh i9 jl qy i8 lk wa ae 1p vh ox rk em hf dd jt rn tq is oc o6 so pi -\n qs gm qn gw qb cx w8 ur yp up uy ek ez ar sy qb hd bx rl qt yi nw tt eb fl eh pg oh ib qy qi bp jz lf eo ph wh oy y5 om az tc ab wv wb kg ww eq ok aa uy w6 ag ig pi -\n ra a5 db co qn d4 bu qz kx me nr q8 my lp t8 gu rk yn et y1 ej g7 yq d0 j6 b6 qq rn kw ei yc uq e2 s3 oj s6 jl kf rl ny wg mw t2 co el yy ez eb e0 al qg km k3 n2 zr tk qb n5 n7 et tm ul -\n po uv a2 o0 rd hq dh hw a8 d4 wi z2 vt ww kb d7 pv tq fa ta dl oi y6 im ff ae qv sy si wq pq bn b3 lm b5 wi ku qu ru ul ri tx fb ss or sk wp qd w7 kh nn es hy wh rp um sx e6 rh rk pn sq rr hm dt ip dh pt wl h8 qc vj ly bq zn gd wn q4 hj yq xb mf ok tm ge -\n ub pa fy qf dh qj q4 wt mw cm k5 gw kb el w7 w8 mx ya ii dj dk gd dc gh st qb iu jk qr bz vz ab b5 mf pu qe xd nq eo yb pd i6 ue dq e1 qo wo sp 1o n1 4v at qf fi of xj rj dq ew nm x5 wh na ub e5 um sb ob em pb re ip x9 h7 zv xm bw 1v mp zr w3 xm ee yg rv rt ia is ro ep -\n cm bp hc rx y4 sr q9 jj rt qo uk ev to ff so bg eg y4 l0 go os ay tx qh hl qc wb -\n z3 nn o9 xf fs gd g8 ns ec p0 tb wf uv iw jt wr dq bj u7 e2 -\n da td ta tw tf tt ay dq sf gi ae rl e1 gk af dp -\n un fi dx wt m5 vo ys j3 i5 ad nr wj mn tg ox bs ia -\n hv yv qa qf dg qj do ek w0 is sl ez sr i2 ww we rl vr qw y9 tu p5 uc hj i6 ud ws l5 qf xh kh lg wf wj uv tf t7 e7 dt qz ka xn cx xe fn it -\n iy yv rs qg uw oh q3 lr vq bz ab zm wa ds b1 w9 rl rz uy wy om uo ef fo py tw fe qx i2 qc qb qn ww vg ke wr j5 j6 oy qq ng sl qw mj yh xf yk xg qu te p2 ft y9 uk ym uq so fx ff fn qy du f1 na lh wo qo ge sk v1 wg mc dq wj iv 1h pk 3s ej oy ek td ex ae yi t9 go e8 rk tz ud rq ax hz dk qz kf wm yq cy w4 h6 rt ry tn r1 gz ux pi -\n d2 we aa cb o3 xi tu ti gd wq pl xg wc lm de e4 sj hc ic wc ra wc go o1 dd ip wl in wx js tv rx yi yc -\n ty um hq co ux ql q6 wi bc kn q0 r7 yz ib pm g7 po qv re we bh 8j ru xo ra eu ud qy uh ec ty ry yn hw sm e1 pg p0 dr qy oe lc x1 kt xz pl t4 el t5 ex sn us dq rq ao f8 pr md ql v7 v0 n7 kh vn wm u7 cz et w6 gl yo ei di tw -\n ub jg ph q1 q2 d4 q4 qz kl ld cn ji z9 ro ek gb w7 rh pk ea ax yj pv ot yl an sl y5 po im i1 zb fj qb i4 gl xq si m1 jj lb l2 ul sn ue s1 ta hg zu lh nd j9 ci qu wd bh ef ro ra aq t7 ex t8 od en fz fc df h7 qz n1 v9 zy 4a tc bb ea yw mf ia yp eo aj rp ob -\n o9 qa h9 dn vo a7 qj jt ji ne kc cj zh wo q0 w7 e5 ui vi ya wy c2 r6 ui px yh y2 to pr ab dj pn a9 pm tw sq ig hi bg ni ry lv wy ic bc li qe im dv rq xy ki i5 fa fv uu af do vv za l3 bi kd nx w8 nt lh hu ra ub un ec rj ua fk ir s0 f4 uf dw jt k2 kz ml cl km wv vv es tb yj w5 rn yo af ru ah ig -\n vg wr zh wo on ew ef ae ha id uf eg p9 ef gi al ng 16 rz o3 -\n qs jw qh cv z4 ok wt k8 kg km wa uu us sj oy iv tw jm c9 fo nd 20 qw w3 yi re qo yv rr op qp ue oh s3 uu px jc hx wf v2 br ep wg pz t7 t9 au re zj kn xc bo kg 1v hb wv tt u9 gj yu ry iw dp o8 -\n qd wi ij rh ef fe jm kw xj wh uk ef ti e2 j8 ou xo ny wh rp wj ub s7 pb nb qv ev o6 o7 yx -\n gr dc ft qs gm qd dn k6 lo k9 nb as zg bz lw ui ee g4 dl qv q9 lu jg rx w4 yj ep oo sv uq hq yw ao fc e3 ui dy du sk gc gy qs l7 kz ed ej wl un yu wm oc gq qk qc ks tk ti eq em ly vl er ry sy yo ro eo -\n lq d7 i4 7w y0 qt gw ch o6 eo -\n fr hb dc o0 yb hn gi jh sw kj we o1 vg nm q8 bz zk bf ml ev ed r8 iv ht fg th qv vz d3 ng xj 0h 42 ew vt yg qr qt ha qu hs qp ij yn eg of tl p8 fz oh iv jl ss dy zu or sk uj co kt rp wb wx fg ev t9 rj yp u5 us ys ak rw al io kc dt jr hl ln wl wz gy wy qv qb mu hd ky ku zp ww yw rl oh ee w4 yz -\n fu dg qf pg jg o1 dc by q4 st t3 lj ve jr am 2i rz ea lh pl ed pz y4 g8 i2 db g0 fj q9 qn bl en hr m8 qw rn qt yi ei yk qu xi uh fy yn ix uy gn jx f2 gr fi x2 zo pl vh ek sz u1 s7 ya em u5 da re f7 hl qh ju oz ar zb ci tk ob n7 vh og w1 ok er o5 ri ro tw rp it -\n gv ra fr ub h0 hm pf qj kk zf zh rj eq d7 oe eh ib oi gg i4 jd ph nu gc qw rr m3 vj ry is dk qi rm qy qu ep p3 ed pd ta s3 tc fd sa im ow jc oe qi j0 gt bm vm zf nj rg w7 x2 nr wf hi rp wk co t6 t7 e6 ag eb u3 e9 f4 om o2 dk h4 gq jo cr oz ka kx rn wn do ep wb vn ef rz ew yi r2 ro so ob -\n ft a4 qs pq iz pd u5 cs q3 qz ra rh w7 rk mv kv ee y1 to dj sj ta pn oi tf i2 th q0 vx vf ww 2l cb wt yq ku ye gs qe w4 qy qi xi tt es qp ed ef ti i7 tc pl jz ho zo qi za fy zy rk x2 r3 ht yv ex op ae iq u2 ag pb of dd h4 lq wx cy cu zy wm ry ef dj vx st ia ey te -\n rs al qd uq ga qj sw we pa bi ba e4 yy mo d6 er et ti rb py ek am ib fe y7 fh jv mn qe qr oe c0 l1 qi mh 44 xe ei ev hq ix e1 pg pj ui hp pm fr qs kd nk 1v wj fa wf yt t5 vp ex wx fh pn ug fc pq io gh dg oy nf v6 bt jo qz gu me wm n7 br tx mt q1 su eu di uz am if uc aj -\n da a6 q1 ph uv oj ji mp t5 mi rj cf jl w0 pk ew ii rv oe r9 ic id sl se su q9 vd we j3 ac d9 yw ew w3 y0 tk ao hr in e4 hu du qu jb wp cr qs v9 p5 vi xm kf s1 ea t2 wh y1 co iq yo au iy on ds fx yf qa zv qv f1 y8 wm u8 rc o3 -\n iu r5 el dz rt m9 hb lc x2 zp aw uz -\n k0 px qe qr i2 yz qo ap t1 ou n4 -\n qg q1 wr wt wu 5x ij rg lq eg ia r9 is dl aw g9 xx w2 qt au i7 us jc f2 ge qa gt l7 lb mc x3 3p tz u6 kx f8 fb ku ag hd oj o3 fn tw -\n ds rs k5 go qg ga qj gs by q3 xy q6 k5 4k o8 ws td mo w8 th ys eq pk yf r5 uo rb r9 td y8 tg ho qn gz li m0 oq kw qr g1 wy iv b7 vt qr qu ti to ta ut sa i0 pl oq sd ho qa gy qq l4 ks fu wg qg kj eh ez yu tf s7 os s9 ya em pq tc fv qg ve sx af ci ah qj bj df ry rl wm zy tv ol ey ox ri ie tq ir yc -\n ak ra yb ds gt fy qh d3 ql jk jl ni zs q5 zf lf so wo mu yt wa w8 kl ue e7 2d mb yn tu ac pv id pm sq sw jo dv jd jg qq qw qe wr j5 wu 1h b6 vr yf cx lz rn ho gh qi es ev ty p7 fx fs s5 pl sf lh sh i8 qa xs 1o kq zg qh wk fs vo wl ez iq uo tj u3 gs ii je jr hk ql xx 1j v8 nz kf vz ww yw yt w4 rb ol o4 rn ux ig sp gc -\n yv fr qa rd gm ps jd a8 qh ls vg q5 lg eh z0 vt mi vy rg lp ex ew d6 yg rv oe fs sz g6 sy ha cx qq wy j6 dk hr l1 qe gl ex ln uk sv ty at ru uc ts hi hl lg jv qi vc m0 fy xg qg eo hf mu mo kz ot np oy na el yy wz fh gp up ir e9 s9 f4 gf pw uh uj jr ab qh uc wl ce qz h8 v9 wv ie 37 eu gf yv 1m ma yw wm oh dk sr oc ei o8 -\n qn cd zf y4 oi dv xq q0 lc av cw ki xd lx qi gn bh em uf we ja ox iw qb wn my zs y9 ux -\n qd qf we ls lf k4 eg bc e5 rl ea r4 oq er ip g2 yl ot iv ps gx qr wy xj vz xl bx 3o qr eu qi uj p7 uc ph in pk qt i4 gq wp v6 kw kd xk zw 11 yj wj rd oz th yo eb ya tl au tx qj wl dz wz cg zv qa rb wm 7a zs vj yw ee eo -\n jd go qg d2 ji qn wa bf t8 ys eq ui d6 ed yn r7 is qb q9 lp lz qe c0 wu tx wa te qp 64 uq in qt qy wp j0 lz l5 og ca sz un ec rh pb pw h2 kv aw wy qf 16 rw ew tb aj -\n a2 gr qs fu db qn q1 uc jr qk cn q6 b2 ne lg q7 q8 wi wp b1 ec rk yj pc fo iv sk gk jb qm zw m1 wx zt xy wy em 41 ee gh xg cn yv qp sn od ao pj fs ut s5 tb ad jc j9 xa uj ws kf wg vp nv fa wk mq x6 vh wv t4 ex iq 7r y6 sv ox ev eb rj rk em aj pq gh f8 th os sb mt ak q1 xr yw ti ee tb as ox o5 yo gx uc -\n qj lp z0 aj wp vr wa bb xt w9 ya on ew ym ia ix pt tw dz jo ae cc qe lc qr cn b3 c0 ib ml qi uj qp pf p8 e1 s3 tn ui sg pn i8 hb ij qw pd ld fo ap ty ro 3b r0 sz ie gp rj e9 fk gd pw rr uj cf qz zr rq 4p kp pr vj w5 iq ey rn ie eo ir pi -\n gr rs gy pw qd ga jj z3 kj ql nn bg dm zz uy pl e0 lh ef oe am y5 fd qx hi uw i4 q9 hs jb vd cx ni qw wx zt qr d0 wi 43 w3 cc b9 qa rw oa ev ry p4 en tk ti yq pd i5 og ic ye so tc de pj ff hl oe sj qs wf v9 xn gm wg xm 1f ph dr vg wk ns t6 um oa e8 sb t0 gs sm fx o1 de h1 uk qh zj zk ng ct gp nx xe 3z wm rz yk tn ro -\n qa pt k7 og kl wy rp hx wp wa ui mx eb 95 ac eg dj yz aq in ih i2 q0 cz cb dg xi cq jc qe qt es ed sb en iz fp ta fc tv tn gw ka i0 lz sd il qf 1s iz qf nc xj xk ep r7 rp gu t7 wc t0 en tl iy iu pw kn km ql ct qp ch fl wm n6 rw eo qm vx ty ee ru ig -\n um rf qd db qf od d1 mb u0 le xu wy q6 mt bc qw cm uu us r5 uf or tq ek sx i1 it la cb ax t0 wu ab 1t qq g6 ko g7 mk qr ey ha ea qp y0 en ue tv ho i6 i8 sp xs qf v9 jl kt rk qy ot 14 na ub aq op yo en tk ob on tx f0 qk jp vi iw tj x9 zi n6 wo wb se aa ag oc gx -\n iy ub gy pt pd qf me xp w7 rj tk r4 rx ui ii r7 us pb pt g5 fw gf dm wi w1 eu re tq oo es pd ri tl og s2 fx ap ok i4 di lh f2 1i vm cp bh wj wx of on tx dt h2 hl qk wq qz lr tl f3 ce kp yr yg ro yx -\n k6 qf cp la wp gv es pl uo eg am tf y7 i3 hd jk we d7 rl b8 gg ug es rt p5 eg em tz ow 3y eo wg t1 lc wk ol tj en ak fc f6 df gt ol qc rn tz wv rx di ov -\n pp qd iz qm vk jg r4 pl ym y7 sc qn jf qy rq p8 yr di qu hb wd rf ks gw qg s1 x7 ec ae iw eb ai sq v8 h8 le ea vh yw yp -\n ik a7 cp sq q1 lq ql wa qz lr zh rp ra gb w9 ys ui ym px up r9 pr ek qv qb hs bg wt ku pu dc p1 qo ik uk y9 y0 en hr tx ts pk jl ce lj l5 p6 v4 wk nu vg oy aq aw rg os az uj kc py ql oj qc pc fj jr bf cx es vn q4 y0 og w2 ue u8 is ag ie yc -\n rs dd ik k5 hm dg k7 go q1 qk wt q7 wi ws t6 k0 go ii ee io ym ey sl sz sw jg si d3 qq qw nh lp cc kw xt m3 ip ln nf zm qq tc ex ry at iz p7 ux of he og dq e1 i7 pj sp s4 ok qt gt sd xf ow qr pd hd wj qh x3 yb lx wx um e6 t8 s7 uo u2 it sw pm rr qg h3 aq ze h8 ks zb kb bh ec wb vb w2 oj af -\n ak ds dh jg cp ws q5 nq wy su q7 kb o7 ys sf et r9 ta sq y6 dn sy cx na j6 jc qi qw qe qr rb tn 3g eo uh tr ft ri uw of i5 ue ta fs s3 uy as ss qu ns lj wp zf wg sm x1 ix mc va mi rx ej yy y4 t7 ex u1 y6 u3 up en au ds ap kv qh kn gw k1 zv eu lu kh tx qk dr dh wm ti h5 o4 w6 yk af fq so aj -\n uv sa hb ps q4 as wi ej qm zc yd yn fp y3 td hy ue qw qy es tu uq tx e3 jz ud sv l6 fu xh dq wk wx yi dj qz v0 qd ga mp wm yy tn fn yx -\n qh qz ar qq ma kq rx qa st ei -\n dc df il he c1 jt qn yd yn pe et pn pi d7 ke g2 j6 rl sk ng z0 m8 mh qw j1 eu qu rr es ec uk ev ul pf e4 sg jv m9 qf vd wk gu rh e9 f3 on qv vj dh aa ru ux yx o8 a1 -\n ra qs h0 qh bf q3 dv bl mr if ws df ev b2 pl om tz ax yk ta y7 aw dn zr ax qt m5 xx wp qy qi qo at ti p7 tv i0 fm qu sh so lk qp hb p5 xk ib vd hk t2 np ek yt um u1 ir sm yf ug az qj v5 wr fg zv af qv ck ay cs ww pq wn w1 yh as yk ei -\n tr yb df um qf iz k7 q3 we cb cj ne zg a2 e6 ya r3 ut on rq io ow qx ja qv cx cv bh vj qr lv pc 3a rm ep uk ed ev au p8 so fx p0 ts e4 fb hj qt dy px sf f1 zo vx qa wa sa qs vm wf xg kf fz r3 bu t1 tu ez t7 va e6 fl tz uf gg io qg qj h5 zz nh qz zt et ba lu tq vz xe bb md u7 oh 5k rv rt tb yu tn ah -\n gr da ty qj by we ls av kc qc wi wo xr mx cm yg oe xs pr ua pt dk oy hp qm qq zw vk xi ln he rx ko dz yt qe tv eu qi yz tt y9 ev ry ym ay uq pg oj aa s4 sg hp f1 qu wp qa bn vi os iz hw kt t2 rs wl r0 ez rf pv hs om dd f8 uj dj pt dk km k1 qz qx wc n3 nl wv qn zo vx ww dr yr oj r1 tq -\n ra jg jr ao c1 wh rj fp gz iy lo gc dh qw qr 8p eo ev fu tl i5 uy uu ui qp mb hk yt ou aq oi e9 ip dt k2 qx vb mf id -\n rd h0 qn ql la vg qz lw q8 ra sp ts pr av qc vs vg ku am z0 lo ry ev eh i8 aa pl dt du i4 zs w7 wj xl yg yh ra ex u4 pn lw gu pc on n9 n0 wm em tn -\n gb ik rd ql xi bd yr e3 qq w7 ex rz on ui yg ax fo pv ab ta jp qw xi wi qw qe fh mz eo gk qu uj ed ev en fo ux ye fv jv ws lx kr kf n5 qj ea s4 vh ez um tj ir od ga tk f5 dh uk pr pt in v9 js sv qb zn wb vl zj wm ca mu zs ef rl yw u8 er id uz ah -\n iy ij ub qs lo ql jk dv h0 wy cm q7 wu eh fq w8 hm w9 mv yd rz rx rv r9 eh pr ek dk el hi qc sy i4 qq lp jj we m2 g2 fo j5 wy m6 ve tx yg w3 rv rn rq qy hs tt y9 ry ym eh to e1 ur ff hk do wa kq jw p7 yp ky r2 wx oy uv ra yt t6 yy sz t7 wc s0 of ds om kx ng ql qz vj wt wb ly wm lw dh md ew w3 tv er as yu an gz si ro do -\n o9 k7 q2 dt 1i wa uu t8 ut mv ef uo g3 gj hp jn nt cm rj ms wi b6 im qu eo yc ex qp eg sf do sh i8 ih qa wa wf kd yo xj ql wf ek un wx t7 s8 rj f9 qh qk k1 lq h7 in nj um bu qv ov n7 bh bn 3z w1 yt et o4 gl -\n a2 gt rs ty rd rf qd qn jf qh k8 q1 qj ql d4 cg wt q5 z7 lr wf wu q7 sd yg yh g1 eg to el ih sw tf fg qx dv q0 wq qq uu px vl xi js jd ze la ud qy rr ky ft i5 em p8 p9 i8 hg im as jz tn qo ul wg 8d vs ap mq x6 no t3 ub wl tf iw rh ox ua pv ir us pb tx pw dg h1 uk ux cr sz ko wx jw vl rc tv af du ei -\n gm we cm jx lf vq vw kb wk e3 df r3 r4 ew yf ti id fe fr su xr sl jg rq rw uq tp ss qy ws od nv wg ro t7 ar th ak da yf sw io jt cq v9 kb iy u9 -\n qd ga q1 h8 xt um wt nq wy wp a2 rg w7 hm cf tj ut r3 ch oe r8 pa qb jb zw mm wq pl m2 wr wy mh hi ei qy nw uf yv s1 fx ut sa tb ss hl qu qi zp nf zd ar l5 5h gm vo ix xk wk wf vf el r0 sx e6 uo rj f3 em dd uh qj cf wz n9 ga tc qk mu rt ye w4 o4 ad ag -\n qa jr kz c3 c6 vp e0 ng wu ug ty uk tu to hr sp ud m0 ar pa qf wf kr fi ya kk wl xs ed mp x6 ub gu fh rj e9 ya om wl vj ha ex y0 id -\n qm q2 oh cd q7 kk ld ys yd rv yk id wt qy iz ri fi i5 ic e1 ht 5z iq ha ai sq pn al gh un kt wq mi dr ax u8 u9 gk ru ov hc ep -\n iy sa un h9 rf fi he uc u6 cd q6 wu zl zz rk lf yd rx d7 ef er rb d9 r9 im hu zv ps qb jf qm m8 qq ji g2 kt qq ew la xy qo es ft ik tl ye ur as tb m9 i8 qa ka qs bm zg ix ya kl t1 wj r9 oi um aw yo ie ys yf hg gq nh zc sb nw qf xm bc xr bj es rx w3 yj iq tm di gx o7 pi aj sp -\n il qf pd k6 h0 na is q8 4p zl jl z5 hm ec io sf dk if gd qw 1a ld lf qr yx re tq y9 pd iz yw sa wp bn jq w6 v3 x2 br ta yi ha en o1 io ip pr kp nl lt kd eu kf kn n8 zs rx ux -\n ih db gm jd wr zj xp vp qb c8 pc g7 uf uz p7 sh or xh xm wh mt no fh dh wv tk li qm vb ms if -\n he ql wi bn c1 rc ip ia av or y8 mx yr dx ex gz 1p ic wf aj kn 51 bj wn o6 -\n hb ty dv gu ps qj ls qz ch q8 zh xp bs vt rh oe ot pb y5 y6 fr ih sc q0 re zx lm id xp yy qr ry ay p6 he dq s4 ff qt sd vx jb qo qp gb ws wd sd co fp kg s1 nm rp cu 8l y2 tf ev sn au us fz hj qg wc u4 au qh wv bn eq r1 -\n uw vr eq rx et rb fa ek id qx ui kr wn uf p4 tl au hw tx im sf yd dz bo wb xw -\n uv yb ik qd gm gp k8 qk ao z6 ps mw zf jc eg a1 wa 7c zz rh yi lf pl r7 yh d8 g2 r0 tq su cz pl qe qe wr wv ku ho qt yv uj ij es ec ik yn ym uw tl sm he p8 fa ho wo gy ws zf bw nb 5q ql t1 ro rp ej xg uv el l8 rd wz rg go rh sv fh ya it pn hd ao az tc dr ac dy ot sj nd qz ok um ol sx xb wb wi n8 ji rz yr sr h6 et o3 ru rm pi -\n rs fi ag c3 lw ys ef sg qu qi uq eh e4 gy qt ya ro hx oa f5 1j qa cl wq rl yh pu -\n ub rd qd fi jl zk oq r8 y1 tp sl i2 qn sd cq 6d mj w3 p6 ta fm bo nv qi wh yj e0 ao uh kn h6 r2 -\n pa q1 fm c4 ig ex 2a yi mx ek ez dv jf qw qe 4s xt ld dh qq mg qr yc eh s4 hj yy s9 pv rr uj or qj cd wc ly x0 wv hh ye ew yh rb yk o5 tm -\n pp q3 mw rd up td j2 lv af ih hb ee xh yy ua ug aa tb -\n sa tr ds az qd fi dn hw qg dh qh nt z3 qz ad q7 q8 tf vu ue mx vp lg tz er yj to hy fr sw th qn hf gx jj pz wt lb cm m7 wi b7 vr lo yl qi ry ef sn uq ri fx oh i3 sd i4 ho vb wa qa ik uk ar hw l8 ya cw s4 wg r7 ot wk gu u1 fh th rk en sm u5 iy iu re pr hk qg kn gq cf h8 nj ct gp wb qg hj wm cy ok er tv u0 sy fq o6 gx eo sp ob -\n yv ak ra co wy zj e7 ew tl fo ek ez im q0 jm bj lc tc rm ec ou bn sd os x2 lh wj ot oi y6 e6 yp ob sq p0 js qh el bg rr rc xw pu -\n ih um k9 q4 ls jx ej om sf uh dz oi qx cl zm qw qr zc qe i2 i6 uu qp wp ws qd sd fj mx qk yn wj ub gu ar pn rr qg ln dl al vg mf w6 -\n tr ub ds jd gp qk jk d4 kv xo ws gi yo sj sl el tw i3 ow qe zx nx b4 qq ee eu uf uh ex p2 rr ea ry ef eb y0 en ri eh e1 oh fx fv sj jn xf qd vi l7 wg x7 r9 uv ek yt ns aw sx sc vf tk ud ds o2 pr kv ab gt v6 un qz wr wv rb os ie u4 rm zm rw n8 vc za q3 zu yy o3 yi ag pu -\n ij a4 uj gg jy dt 1w rj a6 r3 ii pe r0 ej ta ts ff i2 ho ov wq kw qt ot m7 qq xv ei lv kr yx yb ri fa ur de pj hi si jq wg r4 x5 hj oy 5i u3 tx tc nd v6 oz wc qv bz qb qj zl dg ed ka vh w3 yt ey w6 -\n o9 ft az ps hq uq a8 ql we wg z8 ye wk bf rs wa c6 dd ys rl wy om pe ix y3 g4 dz gf se tg pa va jn jj al qw sf ma j5 qy wu xo dc rn se eu xb nw qu qi p4 ef ru sm eh im ad gm jv pm zd g0 wg qf ai qz ym qc t8 op iw ox ay tc av k1 ko vj qb zo wq bg q2 n0 rl yt as rn uz -\n o0 jq qf he qh 7k q7 kb wp z5 tl ew yg et or ez jp g9 jv gk lx vk vw qa qo ou qg kk xz rx ro wc oa us ip x0 ku hp jr o3 -\n pp dc gi fi qf ql by we la za un qz q4 jc zh wp kg o9 qm bm wt r6 rw eg ix yl tp am a0 aw i2 fh th xq gc eq xx yr qt xt nw qu ri uq tl ue pj p0 hk vx uf jm kq ws jl 1s p6 ca he x2 wk wd r5 wg bi hk ro wj t5 sx fh sv e9 ya aj e0 pn ao ug ac kn h4 gq nj wr cu qs kf vx xw k1 og yt u0 yk di gx dp -\n ij tt ss dv a6 uz gp qk cv lq ql un kz gj wt 4y fq lh z0 6h w8 vp r5 ee tu sg pv y4 pu a0 tg q9 gx qq qw we la wx rr ls zy qu wi xz wi xp ew qe rn ei qo uj rt fy ik tr tk sn pf i5 tc sp s4 in gv i0 f1 si ks nl kw bq co mu eh oi ec sc wc u3 ga fk sm om kb wx bo zj hv y0 en og q6 er tv tb rb u0 w6 tm -\n rd ik og q3 q5 cn xp c4 ig mi e0 rc ym id tq ou po gg qb sy ob hf pk xr qr up j6 ng xx b0 qt tm eo vw ux yw pg tc e4 gy p4 xv pd wg n5 r3 wk iv rl ht oy uv ub wc ar t9 ga s0 em pw x0 pt bw wm vv vm yg fn ad af do -\n hb gg kl q5 t1 mi a4 b9 r4 ee up pr g8 gl q0 cc kr c9 vq yy wa qy mz ty yn yq ai og tx tn nd ws 1d ky x3 sz td gu t8 op gs tz de av sk cy zm be wv qk og uc -\n ph qj d2 cd q3 q4 bi wp vt oq y1 ps cw kn gz ij p4 sp e4 wa sx nj v1 w7 me s7 e9 tc k1 lw zc vj wb kb tw a1 aj -\n rd a5 hq qg qh q1 la cd kl mp k8 mj vy zz yu ut uo sg yz hi gk sy q0 m1 qw b5 wi dz qw rc aw eu zr uk ti em yr lk kw nz wg fx tt wg x7 rr lm ju th tt eq oc o6 yx ro o8 a1 -\n a2 dm wy ej rh rg pe a9 oi y7 zt vk ga yf pp fh ml tj p5 sn tk im jv v6 lb pf zq ty wh t5 go sv e8 it f7 ac p9 cu bw kg qk f7 w2 ee w5 tq ep -\n vs rg rj 70 ys nq uf ex hh jn kg ep e0 -\n df q3 u9 4j bn rj hw up td i1 dc hi zb wv g3 l1 rz qt qy ty tj ef eg sm tx ap in px af mx r3 r6 t2 t4 rd uo e0 iy ii hh qg zc v8 qc ch px zy zi ye og er tm if -\n gr pq se pp qe lq n1 cy qb wb ey -\n gn q1 ji sc nh pe yh qt ss rw hf kx zm o6 gx -\n ih hq ap bl wi wa 3y er pc eh r0 yl ta ts dl gg fg i3 hp kw lc ls rl ff wg qi uh uk yq yw ok as wp gc jm qf to yy fh e9 pn qk sz nm li q1 vb yq 2e rl tv sy di sp -\n o0 yb rf k6 qm fv q5 wi rp fe dd mz rz ee oe ia yz am ig hp fj su gl li we m2 ls xj md z0 uh yx eb eh ur i8 qy oe i5 jv ce jb jm lx ci kg oy hx e5 u2 tj kv qh qo af pv tz az vg yz ri ge -\n il z2 d2 cs ba wa 1o ys uy ed er d8 sh qx sr qb jd ov xq nu lv g1 ku pu rp qq ee et rm eu xh i5 p8 tx in i3 bn v3 ap r5 oy sl oo dr dg hj lm sk ff vu cu wb kd zi wv mu w3 yt ok rv ol ey yx ah -\n hv qs qf qm ph o2 zg wa a2 rl pz ow uo y2 ey ix us d0 ek fg ww j4 wv a7 wu qq bv te tr uj tp ue ts do qo cy uk j5 ra ou aq iq ev tj u5 tx gg df dg h4 qj nc wz v7 im kv jt y8 rz w2 yy ei ge -\n gt df jg og k0 lq kz zf iw kh gv cn ur ea eq yj ix id ez dc qc st wq c0 lm fa lj qq mj sw qe xb rq qo yn ru at e1 p9 s2 ts i8 s5 i3 sf jc xa hb qq gy wd p5 ow qf wj 1n wd qz yt ex e5 op at sm ud tz yf tc ax f7 dg qg ve sl qv tl wn 2h ky yv f8 rt o8 -\n yv dh kj jl wr bi kc 4j nt c5 lo z3 e8 on or g3 g8 uq sr qm hd ll c0 nh ws qu ug ph tc dy oq qo nj wn t7 ox zj qj km h6 ql zz qz qc hn bk fw ob -\n un qs fu k7 co je gp o1 bg dt vu rj mc rz rx r6 g3 ek qc uw fj li qw lx ku z9 br b6 mz yc ty tl yw yr e4 i3 ud ce i9 rf bq xh ep wf ej rp vu wl u1 os ay of pm ap gg dt hk ab ql lq qs lt nl u3 wq n7 bm ef xm yh w6 ru fq tw -\n qn ql zj rg ed hs we qw re p7 p8 yr tv pc lh gx i8 wp lz cp fv lq 1b di -\n rs q0 is q0 yq rl qq vr qa v3 tu in h7 zy u9 o7 -\n ak pw af z0 wa jh tf to ey r0 ta fe tf th wt m5 xu wu xo l1 b7 bv se w4 qi es dy ge vm l8 nv kh l0 j9 bi ez iw ux f0 gt zc 6w bf cx u8 w6 yo o6 fw -\n gb py gp pg ql um lf bx 6q ra w7 vu rj ui pj tl ii y1 r8 ac eg yl sj tp y3 py im tg zv ll ip wt av fq qw dc yu ei uf qu tw y9 em tl dq fx hk oq zu jx hb i9 bq iz mv wd qi pz lx ek aw fg ag u4 ir tk of pb az f7 qk wz le qs wy wn dd bj mp yq zb cj rc tm yo -\n o0 pa rd qf dn qg nm ji q6 cm wp ec uu ax r8 us aq se lt g1 bl vz jc mj 6h ul en yw e1 ye tc i0 do ge gn v2 r1 7f ed x8 rf oo t8 s9 u5 pb fc ug ab pt uc ce qz h8 u3 ga fl bc yq iq iw id o6 r2 te -\n da qf qn nm wr jw c2 ig rj vi ys pl ed ii ax y1 ua r9 ia ab tq an ek dl sl jo g8 qv gl hs jf d5 c7 kt ix wy wi lm qq dz ko qe ff pp qt eu wd yx ec p3 rt ty ik p6 pg i6 e3 sa fd dr gb jl as hz qo j9 j0 qs qd wf p7 br wd x5 hh t2 wl el rf tg ar rj tk em ud pn rw av iq nl qv x8 ov wq wb ec vm y0 w2 ew 5k w6 fw a1 -\n a4 a5 z1 hr qk q3 mq zf qc wu q8 bd a3 xf es yn rc et d8 pr or yl fa se dv dn uy vz wu en mb qa tj uq i6 pl du jx f1 xs qs qd vm kd wh kh mv ed rp ra co tg oa u2 ie ha ir da pr cq uv oy qb qb eo ye rz rc ei o6 id if do yc -\n a3 qs w0 po gx xo nh uj e1 lz os wj uv ud tx rr gq ve ch cs xz -\n rs rd qs qd hw dh q1 ql dy mj zz ws vy pj ea mv yf om uf sg up ix pb ab ej tq fr tg i3 nq dn gc wx wc qr wr lv t0 cm wy wu ko qe qr ze ug oa ed ym p5 os uq i5 tp pg s1 ok tb fb vx ns p1 wa qs ka qd lx ps sf zy kj pl ro rs rd wz tf ev gp e8 u3 ir od f5 dh qj cw qk h6 ql ad qb fj tl al zs h2 rl eq rx er w5 eu yz id pu hc te -\n pa pw q0 j4 4f vq iv yu ry fa e4 kw xm wj t3 ff ye w1 oj rv ul ep -\n mw di ec wt rx ko i4 qo l3 iq iw py yl -\n dx uv yv qa a3 ij ps qh cg qc 2n mc ut eq rz ea kc pz sj dl in db fk su qm qe m2 we 1q ke bn xy wb yq qq vr qw xc gf tn re oo qo p8 iv e2 tc e3 e4 fv ff pl sd qy qi si pm ws aa zw bu hu fn yj pv hs gh o2 dy ln v5 qv zb tk bq 4p qj zz wv 8n h1 y9 wm yw og mf u7 tq ov sp -\n iy wt gx nd t5 r0 yl tg dc qc th wt ld nd zn tc ke qu qo qp tk fa hh gn qp 5d qd ar rg oz fh at ag 2a kd nx w2 w5 -\n il wp ee ym tu ef dl el qb cl qn ob qe qr m6 mf xq i2 ud fn pc gw m9 l7 fs eh gi e7 fk ys pe ok 30 mf o8 uc -\n db k5 a7 je kj q3 we wt q6 ie ck kg kn gr lf pj io us id in aq jo qx su hd qm li qq jm we lc wc ld d8 lb xj em rn j4 tq oo eg fl sm he ue so hg ok dt qt px j7 qi nz l9 lj x8 wj na wk ez ex rg e7 u5 h4 kn ad cu oa qd do q4 eq w3 yy sy su r1 ri if yc uc -\n fr ij ft db dn gp qh ph ga gg kl ws wt ab q8 lk wp mi w9 tj pk yf ew us yz id gd y7 dc qx hp qv gl jn q0 d4 qw re qe cn cq pt wu lm dk cz yy qr rq te qi rr ea ex ye dq ok qt sg qa nk wh kt bt bd lh wf yb hu mw el rd rg sb em on qg wq oj zx wv n4 bo qf bf re wb ev yq cl yu w5 ad ag id a1 gc -\n qd ga dt ej kn r5 ax sg ix av am pu g5 ez fe qm nu ii zt wr vm qt jd im rb ml yx yv ru hr gv ad dw x4 ub hx wz rd ol oo rg yf f6 ii uk dl wl yw yu ie id -\n yv ga z3 kk pp nq le lr bp qc t1 c2 hc vt lw jl w9 ur r3 ys iu es rc ud yj r9 pb ot ta sk gj jm gx xg sf lc qr ht ve sl g7 qy nw vw fo ta tb oq fn qu hc qp op nz ow wk zo zq yg wg ke yt kr yy sb tk rk tz iu o1 rr pr qh dl dz p0 n2 ci cd vn ms aa yh ry w6 af du gc -\n q1 q2 mw bz k6 xf ec r4 rx yg ed d8 pv is sj qc q9 zw vf gc cv d0 xx rn ex uj ij ts ff ge i8 zg 4m x5 vh oy yy y5 wc tg at fk ob fv f9 ql bo k4 zn be ww ea ry tv w4 ru ov -\n tu g2 tq od pk tm fi y1 uf ku wn ew -\n hn rd um qg qh ph aa zn bc gv w8 rj ea es ui r7 r8 ey fw gh jf qn nu cc la 3u bx ve po et ei eo ea qp ul i7 uu fb dt pz qt m0 zf l8 kl t1 ej wj oy rf th rk gs sm em ap hg o2 ub wx ka hd br q2 hg y8 2e eq tb an oc it ep -\n tt um gi qh lp kl lw q4 q6 ro b1 if y6 qc th g0 q9 qm j2 we xt xi nd is ng bc ku yw sm ye dw e2 f1 lg qo wp zs gy l4 ul lv w9 br xl ql vf as yg y3 t5 wc ec iy hf ii f6 re hk qz jp oo qx xv v0 f2 vx cd vb yq ew zu yx -\n lo dt mt z3 rg av us pa xq wq qe qt yx y9 ev ry tk hu oe oi r1 x7 wk wz td jy ww qc 15 ba hd mo 72 -\n rs wi vt rh we jl ur tz tw ht y8 fh i3 qb qm b3 qy ep op yn tu ay hw fd ug qp qd x7 rs yy td u1 t9 sm uh dz ql 4t rw kn wn rc eo -\n a4 d3 kk q5 q6 wf wg m4 2b vt w7 ur uo rw pc g2 sl if y8 fj va rr ld wn xo qw rn ml la qu ep re rr qo pd eg og e1 i8 ui qt px i4 jc gq oe jj ws ks qd ul zh sm ql tf go e8 os tk rk ay us u6 dw dh pr qh qj oy lm jo cf ff wm br k1 en og aa rb yj o4 te -\n gb dd rd qd a6 qj wt jx z7 xi q7 kv mv uy pk or yk ek el y5 ez aq dx qx th sy jn pe vl fp xz m8 ng jh kq qr la ei ft p9 oh hj tn ho ud xs wa jq vn il zg p7 fp ic qk wd bu e7 go hd sq ug hh ip sl ch qc px wb hs rq qh bk rl ef yw dj yt u9 st yi uz uc a1 -\n al tt pq um uq un z6 wa vu w8 ed sj r0 tq pu dz qx js 2j na ip vz lm qq qe yj ud ei wf qi te p1 tu il tk iv dy i0 xg sf ix gp fk ai qk lm ql ch fg qv vg yw rl yy rv rm pu uc -\n rd ik qd jd ph gs k0 q3 qz ia q6 af q7 6w o8 tg gi sj ou aq po ja qn q0 qq gc nh xt wr fs m8 qw aq wp ho qu rm uh tr qp tj ay fi ao i8 aa i9 tv gv qu wo lj vv wa jm qa qd uk nl g0 qd dn gw ic kj nn wf wg dy ej rp y3 rs yy e6 wc oz fh eb e0 of hd yf uf ab gw qc ww yc cd ji y7 n0 wn cg u0 rn yp ie a1 dp o8 -\n ih h9 qd xy yl ez g9 lt qm on vg rz dx qa kt p1 ex yx od uz e3 in sd oi qa cq dr me e8 ua ya dk wx bw 4p vx tn o7 pu -\n ij rf fu jt gs ld wu q7 wo 2u bd k8 rg ws gi oe yl if sq hi dn jn qw bb cb wt lm sq vr qe wd qi qo tk uc hp i0 pa w6 fo v4 1m x3 t7 u5 sq ai hg ap hm ju fl tz wv w4 ry tn yl fm gl ox -\n ql db ch wu rl ih qc 25 pb qq ty yw fp ao qy sp p1 qa rf iu rw qg uk gq km dz wv at qb jo eq ur rb ad -\n ra fy gm iz qk qz lf bp kc is nr ws mb es tl d6 up pe ey sj fq ek iv pn sx ly qb jd jn q0 lp m2 xl fs xx rs b8 hu rn ey qo yn vr yq ti ai e1 so ts jz du do oi sa i0 oq ow 3r 1a wg qg r4 yv ty r8 wk wl s7 u4 of e0 gd pm yf h1 qg n1 lt sv qb eu wi mt ez w2 yk su dp -\n rd qf dz aa bo pd pm qw rn tf ah wu -\n dv jd he jt ao ql zg eq pz oe dj fw sc fj nt io fi lr xu ns mx qe qt ei rq qy yw em e1 i8 sp in jv wo ci fa x6 fn ej wv oi um yi oc au yf mt te az yv yi ad yo yz -\n uv hv ty qd nv jh qj q1 ql we fv vj q7 gw fe wl vu w8 vp 6k yd r7 yj ia ey ot g6 dx tf im hu ae qx fh g9 lt fk hs su ov lo m1 cn t9 wi ki qq qw xx aw nw es yb vw yn ry pd ix em so ut oj du f2 lj qp jm w5 xh ny wg 13 wc qc ek el sx oo th uo yo at pv hn uh hj qj zk ql lw zx qv cc wv en yr w3 yy st uy iq ox an ah -\n rs pa df il iz qm k9 bu jz bi ji du lo ts yy xf cv gu mc ii rc up d8 ey pv av to yz y5 fd qx sc qv jv qr xt bk m6 ot md qq qt rq ex p2 yn sb xc fp pj qt ce wp i8 j0 wa qd gn ps qf be qt wh ky l9 za wg rp wn fh rh em vn dh vw ng wz k2 h8 f1 nz zh yz q5 zy e1 yh ad tw ep te -\n db wr a1 pk ew uu r7 d0 pn fe g8 la qq b0 ef os oh pk fm wa wk yi ev ua sq wu n5 tw xc er -\n fr yb qa hm d2 q2 o1 oj ox dm oc km kj r5 px rb et r9 y7 vs pj q0 4s lm gs qe eu ep ti tl of jq od pf nr nb ea ej yy rh u2 iu qh dk zl qv xe 2k vc vb zb xn yg gk ox tw -\n hb dc rd fu gu zd js wu xa c5 a5 w9 vo r5 d0 g3 oy ib i3 ha jm nu rr wy m8 b9 ws qu ru eg uz hw eh hg sp sf do jb zp dp nz wj wv fg ae ah ob hd dd gh dz qz xc s5 vi je n4 re bh ma wm as st tn yl -\n wi w7 om ug qa x4 yn r7 gi re n3 n7 -\n ij pa qd qg uw qm d4 z4 q8 t3 mu yp uf ia pm ez ih qn jm nu kw qr j6 qw bb yu qs rw wf re qi ru jz tm gw lh ce xa wa xs bn nl ys t1 wk r9 gu oc u4 hd ku hp yg rv as yz ro dp -\n qa ss dc gu qk cs cv fv nm z7 lf z8 xs a3 rk r5 ys eg y1 po qv cl jf xq vh we wr qr b3 bv yr wf re qo sv tj ti eg dq ic fx jz du qp vb ws nl wd wj zq vg t2 zf wb tf yu ex yi yp tj en ua ud dr pe qk dl lq qz h7 ol v8 cy vi wv ck el gd q2 yw w1 ye xq w5 gk o6 ob -\n k5 hr jk ju k3 jq q6 zg wi id bb wa gb rf rq g1 et ot pn ht dc ww c7 we c8 cm xo mg w1 1l yx tr p2 oo lm ry ao e2 i9 e4 hi tn di lv cp ca 2u t2 no ub ex rk ys pw qg av py ql qo lb pn eo wb er tb yk ie id r2 tw o8 -\n ra qa qd ph jh d2 dx d4 2z jl q5 ld cm wu wi 2t wa dd lg ui to id in uq ww rr g2 wu rl qe 1l qi qo ec yn ed uw p8 ut sj ig p4 zh xm p8 vs vg x7 ot cu l6 sx gu yp t0 gs az pe nf wl qz nj lr cy wr qv tj s7 u2 ly be br ym w2 af ri it ob -\n pp uv nb bu kz wi ah z3 c6 rg la vi oe ia ot pm dv gk 2h xq kq xg bv qr b9 j2 ec od ay p8 qi wp zd ay kg ea mq 6b qc un fh tl u5 av ub ji zx k2 wc zy 1x kc ah rw vc wv yq zv e2 rp -\n a2 iy sa ft pp un qn qz ol lf mg wo fr vu ya rk w0 pj pl el dz i3 jd su ob c8 pb id b9 ep yn ru tk s3 sh sj xa l3 wa nj ke kr ic xl bd ej rg yo f3 al f5 sw re uh h2 av cq bo vk kf bd mu wq wm ew ue tv ol tb o3 ul ov -\n iy a5 gu q2 se ls dt zf o4 dm ez jj uu ik ue w0 ya ea on ui tu rv y1 et r9 tq y5 ht dc fg i2 vs q0 av iv ku in il en ri p7 uc e2 ut sp fv qt gn f2 wo qa op v7 ws l6 wh ys zq t2 wc y3 sx yi t9 t0 ys of rq ug o2 av kn h5 ju ji ko v0 nz wn kf te dw u8 yt fn r1 ie yc it -\n qj lw ji eq oe g7 jf jc yr qo v7 p7 wd ma xg wz qb u7 w3 -\n un ol eh g5 px b8 rr og gn mx yf wv sl on jo uz -\n qg qm q5 wy eg ri bm mz d5 rx pt ek fs pi td ez ho gh q0 ll pl kq wr d0 l1 qq ko er qt wf ei p2 ru uq ye tx s4 hc zd vn ps ix zo wk t4 y3 xh ez rf u3 up ys ou xx zv qa wb at rm eu qj wv za zs eq zy rv ry tn tq yc ob -\n ss qa pp rd jf a7 lp h8 um kc q7 wl rg r3 w0 wy tl a0 ih ly qm qq m1 xg qr up ja b8 yh dc lx rw ep ea ev ay ux to p7 tp tb i3 qu gq do gr za l8 rj og oy ub e5 ae tg t0 sq tx hj ad gs it vc bh yb eb w2 yg u9 w5 fn iw si di ah hc -\n ub o0 ik ps qd q1 ga lp cf kl m3 z7 q8 ue ee ud ix g5 ib gd aq fh th pa qc pg ue ur xw ww qr vj m5 jd ng in tx ff xc er qs eo qi p2 ky pd eg e1 yr ut ib oj tb hi hk ho gm qu qi hc ou gn wg sn wh ix wj wj wg ot ra wl un e5 rg s7 t0 oc sm tz hf fx pw x0 wz th qv 1z zt n4 qb cl wn xq xr y8 rt y9 rz tb iw vb -\n qn lq bu eg iw wi 2u 3q t6 k0 yd sd ed rb eh ek yz if pu y6 in fr qc qm ob il ma b4 en wu dk nh b8 qe bb mj ws qt ho yl ug qi ea tw uh p5 eg tl yr i9 pl lh ce i7 wp qa xs dn 7p kr p7 eo vs mb pk ni yh ef rp wj ej y2 iq y6 u5 em e0 ii md jy lw nj fh bq pc xm km q2 wv k1 rx u7 ut et gj tb iw gx fw yx -\n qa pw k6 qn qg qh as u0 dy q7 wp hv 4z 4c w0 d7 et aw wr bl mx md j6 an wi qt lc yx ec tj ri fx ht in gm ua qo qa ik ys eq n7 wh rs wz wx e7 eb ak gg ip sj py ka rl su ag -\n gb uz q2 qz wr q4 z7 ia ad je am my vi zc mx ym r7 yk ua pt g6 hy y7 sx ih qx pa hp jd sy gk nh no qq yg rc 3s qy ep p2 yb oa tr eb p5 en ic yr dw tc in hk qt zp i8 lz ks ci lg x3 wd xa x5 zf yt y5 op u1 oa iw fh oc rk ay pb pw uh qg zj qh h5 nf cd nv qx kp qs qb 6q cl kh xe u7 ew tv sr as rt o4 ey tn is -\n hv qj bo ru z9 t3 lj q9 rg vi rj sd r8 g2 sj yl aq fe po pa qv jf dm qq re we la wt wu qq vt gj ei yz rr xk uk pf so pk im zu ua sg j8 sk zd sd xz zw kl wk ol um yi rh sv u5 pb tz dl oi wz h7 s8 qf wn cx f4 mo wb ed oh ee er ry eu ei oc fw -\n hn a7 cv q6 cj q8 fs jv rl qq qi od dt l5 co qr zq ex u2 ah on pr wx kp wb yh gx -\n gv qg je zg jc q8 fr r6 yn ii g1 pe sj ta el jo sr jv ni jj zr bj ns qr qi ur hz vu wh cs ep s3 hu ez rh u2 t0 dw uj oi wx n5 18 bf wb yq oh ov -\n gb dc um jr mn we bl t6 vi pj c4 d7 rb ia yz tf qn dm ke xb ft au ix tv xd qq xg rx x6 vg r8 wz op h3 qj qx lr xv qc kc wp lq ea rn rm ri eo yx -\n ra gt qs dv ik gn co qg qm qj cb qz z6 wt ji q6 dy qc b4 ws ds vu cf on yg d8 eh py hu tg qc hp qn d3 wq c9 pv pr or qq ml eo ug tw es il os fi uw to em ic oj ho px wo qp m0 qa 1o ks 7o cp wh wk wg x5 ee yn bi ef wj ns r0 ez um u1 iw eb ir fk ov s0 fl hn h1 pr x0 ux cd wz aq jp im k4 qv bp wm n0 vm u7 w4 gj tm uz te a1 -\n gv il ps db he nb wr ql kc zh tf mp lw ab us pn a0 tg pa th ps hf wc 1a yw l1 fs eq wp qr rw yv tr eh so i0 qf wf l7 wg na ou ah ay f4 io ip f8 cd h7 nj rn wb qb qn wp oj w3 w6 di id pu eo -\n hm fu pd qk bi wf q6 wu b2 q7 q8 oc lj c3 o7 6s a1 jh rg rj 2s z7 ya id ez fr gh vl cl zq hd jh xh ru c0 bz wu dl qw km kp b9 rn eu yc yc p4 ru tk ux fo ue p9 iv tv s5 do l4 cu rg w6 os fi 4b uz l7 ld l8 fx jb ee wx rp ek tg e8 uf de qh hz h4 qj gq nb wx qc sv go wm zi zo tc 3k ez ec rz ye oh ck w2 sy ia gk rm ei si dp -\n gi go z5 qz wj mg kl yh g5 y6 g9 xt p6 eh ap sa qu dw j8 ql yg aw t7 ir zj v5 v7 ba tw yq cz gc -\n ps qn z3 sw gs q4 ie gx ye wz r3 us ef d9 pb y6 tg y8 qb gc ww az c8 cb lv wy a9 qq qw l2 c8 qu uf yx qo ic de ut e4 uu tb fn oe dp wa uj bq sg mx lv v3 ya xk wd by n7 ra cp gu va yo u2 sv rk ir ya hf kc kp bo qb gp qb yc ku q3 dj o3 ey ad si o7 tw ge uc -\n dx yv ij pw a8 qm ph k9 dz q1 q3 cn wo wp my el bb uo on eh id yz am fe hy sw ha m8 vg wt vl wm qq w3 ls gj yx eo ef en ta e3 i3 zu hl m0 wd co zy l9 nn ea yj e5 rg gi fg gp u4 ir tl tz pm dd kc p9 zx sx qc qv kf ln on qm lw vn vm ew yg se as is di ro gc -\n al tt gu qf qj xo q8 c4 ws e5 ur vp ea rz g3 fw sx th db kq wt sv tb ad hv 1u gt ss xk wj qj pk rp e7 ha fk f6 dr rr hk dk nf qo lr ka ie fk cz yz q3 ym ks gl -\n gb q1 qk we q3 q4 t1 ox di ny wa ws gi ea rx yg r6 io ow y1 d8 ey ab g3 is ek pu ez dx qx th i3 jv fk io xh wt oe kr nd md pb vz wi ro se b9 tr yb p4 i5 p7 ux fp p8 sp in ok hj qy hz wa qq zd qd 3t wj aa dy 5u el yi uo go t0 u5 tl dq gd rw uf gg kb ux dj qj go wb zm lu tx vc es ev ry zt w2 tp w5 tn o6 -\n ra ps hm qf qg 4q we ql q4 z6 d1 wp vt xs tg 2s e7 r3 ys oq ef c4 av dj pn aw sr th hf gx uy wr ac zv m6 wn ko c5 qt qy yl yc uh od ri uq fz dq i7 tx fc aa uu qt oe i5 ge ce wa gb vn xg wh og ya xk fs ea yf zw wh ub 8x th iw rj ah ya e9 tl yd tx yf ii fc kx hl zj or qj mh ww kf zm lb ob qn wp ww wn ym u7 rv ie pu -\n uj by wo ml dl qx m3 8i 2y r1 u4 hj h6 qa xv rn rm -\n qa gm qh ql u9 ls e3 yk fa ts wr vj ac en id ud ke ye i7 fn tn f1 ks at me l8 kl hk lx rp ek l6 ek oi e6 wc u4 2h pn 9y zq qk ec cf yq oj vz tb o4 iw ox gx te -\n gt ub hn qd qf hq dh q1 lp qk by ql lq we wr sy wy lh z0 ge k9 w8 th vp pz yg ti fo tp r0 g4 yz ig sx im i1 jv qn q0 wq nu 5t pw wm id qe gg qt xg wh en uc tz pj e4 tv i0 ff qt gm dp jb qp cr gr qa nk ws os wf ne wd mm wf t1 vu wz t8 e7 t0 od hs dq df av km v5 kl we fg cx al ax yq y9 rx yh ul hc -\n we cf q4 cj bf ws ww yd tk ef ek y8 qv fk wt ko qe ep rt ik ut op mr j0 ej t3 s8 ir pt qk km ww cg wc gi lu n6 yr rc oc -\n ak ft gy rd hn a6 uq q2 q4 lr ia eg d1 eb on sj dj pn pp qv i4 hs gx ww xj m8 ko im rt fi tc uy tb pl qy pc uf kf kt mv l0 qj x7 oi um tf ap uk wl ql zb vj wv tk re y7 de q4 rc ad rm ul is fq yx r2 -\n dx gb he dl k9 z3 qk lf ad ch js o5 vq zl rj wr th r4 tu uo r8 fp ic g4 fs g6 im fr wq bg no wt dg ru ln rp wi yd qq xz ew i1 kq qt rq wg sb pj hk qu vx oi jm pa vi x5 wf ni ro ot oy di un y6 yo rh sb us tz ac f8 av ve h5 ji mj n2 ci rm yx ep rt 5f yq u9 rb hx aj -\n ih a3 pi mt do w7 zc nh qe wv rs xc qr ts ut pj im hp xa lv x3 ph tt sc od rr qh km oc rq xl vv jp ef st tw -\n ft ik a7 lp jz jx k4 hz wo bv q9 6t ur rl qx qv js dm fw wd re ea fd hu jc qu zo p3 lx pf wk vf fj 2o wx sb gs it ol yp di eo ro -\n tr ih yv h9 k5 qj qk nb wy gk q8 c6 mj a5 yi ur rl uy eq up yj r8 xs sh a0 ez oi y8 ly lp lx fu il ke zy cj sk xq 7u ey p2 uh yv qp rt y9 eg pf so ph tx tc uy e4 tn j0 gy ik vm ul mt nm mm x6 wj rp eh sb u4 ov of tc jt pt qj jy k1 s5 qs 7z do zi cx wq wb ma wm uw sr w5 o3 o5 su r1 yx fe -\n h9 gm cp z2 fb zd gk ve o7 mt bc wp bd rg w8 rk kx mv uu rb or yk eh r0 y5 ht pu tf se ar fj hp su m2 cb c9 c0 b3 ns qq qw rv gg ij y9 oa od of pg i7 hk do pm 2n sj wa vm zw vg yy om qx nj v9 f3 ee w5 w6 iw sp ep -\n da jq iz z1 ls z5 cg nt zk 1i gt w7 yi r3 xi yn pc fa ta ez in i2 qc uw si qw d5 kw il b4 fa ib 6c ud rq yl wg tr qp p4 ry sm ut s5 hi qi do j7 jn j0 qs iz p7 wg aq ex go ax ku nc h8 n3 v0 oc ah wm li zp rl h5 is eu o6 -\n gr sa a4 ik dm gp q1 wy m5 fw a2 t6 rj mx e9 et ej q0 ot wu em fa mj qe yg gh j2 te tj p5 eg tk em ao i7 di lh ce m9 wa wf ys eq l1 t3 ej el tf t9 rk u4 ay rw gg dr hj ac qg zk h6 dz ok zy kc mr fz iu hk yj oz ey ag id r2 ov -\n ds fu ps vo qf qn pf a8 ph q4 cb le cn h0 jg gy b9 rk r3 pj yd oe ht pu ig ez hu q0 qm nu ww qw ow xh b4 is a0 qq hu bb cn xh oo qo y0 fz ue e2 yr fa pj in sa hj ui sp nj zg wj ge wk xg ra ex vs oz eb pv tl iu x0 ln cq xx iq s7 wv qs zn tl wn lr yr r2 -\n az qf fi qn u5 we jq zh wh c4 sd is y5 po oq ki sz qe rm qy yv p4 ye tb ho 1u gq r3 pl td ov u4 hg ax zj wz wb vl vv se 5k eo -\n tt gi pd jk lq q6 wi gx mj ut ax av g7 qv zm lo 5y dh cw xo ve vy xg yv iv i0 qq xg jv l2 yi sc ga pv pn iu ug gy k3 cl oj tv yl di -\n qk se pd gt rz uu d6 io d7 tq gf em ym tu ib oe v3 si wa nm wf qf wg rk kz yd hl wx cy bp mx et -\n uv yb dd k9 ph q4 me o6 nr xa mu ld r5 rb g2 or fe pa fj hp db lu qn nr j2 bk kt xl rn wf qp tl uc dq i7 tx fv ar sf xm mx x1 zw yh lz mr t6 l9 s8 ah u6 x7 yw ol tq eo gc pi -\n ia ys jh wy sb i0 cp u3 ql kl kh -\n po al qg qm d2 gs q2 ap qz q4 q7 kv ah o8 rs 2o ex zx qw z5 r4 r8 y1 is ts y8 qc dm ll we wr sg lb jx wu jv qe ee qt qy es ed ym hw to tx hr s2 oh dt dy af di do qo oo qa w5 uz wh kh x1 t1 hk no r7 rp na sl ej op ev tj eb sb ya pb u5 tx ds o1 hz v6 lw jp k4 wv do wn aj bc wn yw to w4 as ey yk is ig rp o8 -\n rs ty q1 wt wy xp e3 wa yd d7 ht ts sz fk su kw xg bw cw qw oo fu od ix sd zu jn qd ci fi xh mo cp ev th ua e0 em kc lm cu u3 n8 xr yq ti yj fm yx tw -\n dx z2 ga kb yo sf sk fd gf i2 vs qw vj vw qe eu mz tu sp xs qs es t0 eb ak uh hl n1 v9 wv kd o8 rp -\n ds qn qj qz cb kz bo wi o6 z9 fq wq ml cv cb lf eq r6 r8 ic y4 am sz sx po jp g8 ze we wu en ew qw 3q lz kw tt ty ti e1 fx ut tc uu s6 ow gm sh qp vn l4 uj op xn wh qk wz rc wh uv um ar e7 uf az uh py h5 lq vt nz lu li lm dd rl er rt yu o4 eu yz if -\n iz ld me z1 y2 dj ar qb b4 l1 mz ij ry to ad xs sd wf eo hj wl ex ie u5 pr zj gt oi wc kg my ex zt ks yg eu aj -\n gr iy ft pq um qj dz wt gj cn ru kx q8 ws ue rk eb ee fo jb jf la ji ke qq qi qw rq yb qp il eb y0 iv ff zp l3 xm fi x4 r5 xd r0 ol wc t8 ae iw ox fk of pb qj ku xc ct wc ie xn zi wm rz w2 tb u0 su pi -\n a3 ss je un zf vq zg wo mx d7 pm gd vn eu yk tq ik fu ai qt qf j0 to yy at ii qk wz lw n7 ly -\n ub dz rv qt rm wg ea pc j9 qa mr h8 -\n a4 wp td ur px qq ki yx go wc tm an -\n po gn rf a5 uw qn q1 nn we is z8 wj ca t5 ij eb tz ef pr ix g3 ek ta a0 y6 sq pa wq cx kq qw we rt c0 mz pv py wi cw mj qt qu 0e oa tj ux pf to hr ao tx yr ts fd fv s5 ui qu j7 gw ug ss cy ks qf xm fk wg vp kt mv qj mn lk vh yt ol rf th os e8 tj ua rk on pq dg kv km wq kp ad bp os bw pb qh wp zs q6 u0 gj yu o5 if a1 -\n a6 he pu vd cd q4 jz z6 qc jc bz eh wi b1 ed ym eg fo us ib td y8 gj zm pk lc qr wu mh qw sw fi ue j0 xm kz y2 ev aj df h3 qk qc tx rr rl rc ut ad so ro tw -\n a2 pa we kk eg q7 lh zj wp 4z gi yd yg rc io ix r9 jb xe rr iz jd ij tz p9 qi l4 pa g9 s4 vi tj pb hd f4 qh qk lq qs hn ro -\n gn k9 qz aw wu ki yf e2 pk v8 xk wg 5y t3 sl u4 ya gd hn ql zr oj ig -\n fr qn qh ph k0 q2 nq wi zz rh rx ee ef uo d7 ix el fh qv dm vg px wi m8 qq gh ud qy ec fu yw uw sa tb lg us sk wf 5h qf sh vp wk qk zw qz wg aq of ys ak al f5 re f8 pr ku qx wc u1 lr qs wb f4 cw k2 ka hk mf yr w3 ro ir -\n gr qh ql wh kv e4 r4 oy lu qw pb et uj tb tn xs kr dt td t8 e8 uc xn eq yi af -\n q5 dt ed y1 am qv ut gx m7 yt rr yq yr dy sh mt wd wm th bv ym -\n tr qs ca lp uv q3 wu o5 c1 rx om ee er ta ou i1 jb rt ry os ti fc ss px jn gy jz vp ea tg ay rq u6 al de qc zt wn ez rz eq aa rm ox -\n uv ds h9 fy rf jq he qh h8 d4 wr wf du ck wi km yp ut rv io g1 rb av y4 tw a0 hy sz qx gh ha q9 qw ze 1a bz bx bv qw po ee wa qi xk ri i6 ic he tm hl sj jb qp wp jk qr kf sm l8 be x3 qu ql t1 x6 yy fg rj ua ug qg kv k1 wl v7 xx bj ae wc n3 zy tl tz zk wq re n0 yw oh yr oz eu fm do ux uc -\n hb pa h0 pg q3 mw q6 mg ls lh am sc gz al j2 wt t9 lm qe j2 rm re rr ry fl yw ux i0 2e eo bt vh ra ys sm pb on tx re ff wn wv tu rt ox ul ge -\n al fw zb p6 hi qy ay ou rg sx ag rz uy -\n bx wi kv t5 3w e7 sh ht ff nu la xo qi s3 uy jb 1o vm vi l6 be x3 ny pk aw u1 rk fx km qc be rw yn ey eo ro -\n dn q3 jf w0 e0 lh rv zv js j2 xg ld hr qe mk s3 dr kw kc dh h2 ql cg zv n3 ym yt aa as -\n ft ty qh pi d3 qz ip wu wi q8 wj a1 mg mj ut wt r3 om ua y3 ou fd dc zw lp xe cn dh ng qe kp qt mz xy ef ay od tz p8 i0 hu hz qo kw jw qf kf 3y v3 qj w0 ib ew t4 fg oc e9 ua ov hs u6 f9 h6 vj qv li wq iu yv xv rc w6 rm r2 -\n d2 d4 av jc si rs ut 5q pa mm s4 e5 tc km -\n qa uk uw qh vd d3 q4 xo wo c3 wl wa w7 w9 mc r4 y2 fp r0 tw fr g8 ae qn lp sd bk en vr gd hu j1 xv xg rw ep wh ed fu ul eb fl fz i7 ht jx ns v3 ll zs j0 op xf qf l6 l7 sn wk zw wg ej ti wb wz t6 oz rh rj uf je av dj h6 ql wl oi im v8 zr qv wn ku wb bj ef o4 yl r1 ei so if uc -\n k7 qg q5 kx oz mu wl ws rh b2 rk yh qn qe 6o 1y mj ei pf ye e1 dw hj hx do qo gc rh v2 zw x5 t1 t3 yu th e9 em au qh f0 qk km ql kp wc 5p vx bg ea ev wn wm w2 rx o3 yk ru -\n a2 gr qa az dd gm d1 k9 hr we bz lg mg ny wp xd mp yo pj uy xs ua pt g4 tw ez jv q9 qn wr nd md nf qq ng pi bb j2 eu yl ij ty sb os eh hw i6 hg m9 nj wa qs w5 qf nz zh hw wh be zo fs rz me yk y3 ub t7 t8 u2 u3 en ha fl yd hf qh qk wc 1x ze w1 oj ee w3 ey du uz id ah pi -\n o0 hn qz q5 du 2b bn a4 ex rj y1 dj yz y5 ig tf th js qv 5e gc j3 ls d8 yw bc cz tx mb wf ij ty ai as hi lh l3 qd n1 7o wf wh qh wg ot ra l6 el e5 s8 dg vm 3f 7o wn yw gj tb et -\n fu a5 cp ch hx hc rd eh tg g4 li sw sf il eg eb zj 2o cg ew uc -\n qs uc rp ml eb yd if pa c7 oq vk wu ot yq rl uz tv gb zu vm w0 sc tf ud qk wm ko yy er tw -\n ak tt ub pa pq il jq q1 hr k0 uv ql q3 kk zs q5 z8 is lh q8 w7 qw es ii av yz dl ht td g6 vs jm zr px bj no g2 g7 la c7 xt mz yx tt ym os to s1 ur ta s3 gv pl qy pc qi sh jv j8 gr l3 bi oa wd fy v1 s1 zp hg 5e t1 rp wj ms wl t5 el wm at fl e0 sq h3 dl qz ka ox tj qb wb wq re xr rx em ee yu ri do uc a1 rp -\n uv rs un qs um il ul q2 jy kl wo a1 rj tj pk ys r5 yn uo oe y4 ou y5 ar zb g0 qn gx zt lb 2v rm qy nw yz lm eg og i7 ht ss qy qi ge wf bw lv lb wh cs wk hf 7g yb zw hk ns ol rh th yo f4 rq dt uj qo fd wx nk rv ka fl mu rr q3 w2 oj ee rt w6 ru ul -\n qs hq qf qn d1 k0 q2 kk o5 na si bv mx e9 tz d8 tq dl ez qv jf zw ww wu xo b8 w3 i2 te p3 ry iz s1 ut as s5 hk wo wa l6 wh bq xk wj wd ra gu yi u1 t0 ak ai tc ax ip uj nf zb wv bd wo fz qm qj pe md ew rv gj ol yk tn iq yl is si ie r2 -\n d3 ni wr ws li mj ds sh sl qx vs rp ft ik e1 sd af ho xn wg zh 6b rp eb f3 u5 uf df py k1 wz vk vx k2 dg wm er rv rb -\n gv iz qz wo o7 k0 oq ti r9 us ib ps g0 jm jb tq ue iv pj sa cr kh t1 ot wc at e9 ys o2 ab qj ww za rn yi -\n ty qn qh nt ql la q3 kl wt q5 mp mg o9 ls lh g2 id ez sw hu qb cx nu pl kw wt vz v2 1t wi mh qr eu rm qs xb ei ij uj yb sn ai iv pj oj de hy gv ka lz pa nx wg wj kj cw zp wk eq wj t5 ns rf e6 rh ev t9 eb ir e0 sm gs gd ds f5 dd de fc f8 x0 lm qz wz xc wc kx cu wv ks lv kv wn pv ei wm ju ww yv zy yt e2 rb w6 oc o6 tq ig -\n q2 o2 gj q6 zh mg li mo vo ch tl ax ip ho wt ln ro wo qr tr tt os fo e1 de hg gb sk w5 fp kg mm l6 yi u3 fl hs u6 fx re rr dl wb jr el rw pm rt to rx w4 gx -\n yv a3 qa uz k9 q2 o2 bp ny zl mj vo tk rx ui g5 pu qx nr xt ls lm rv qs yv ik fz tv wp nk gn qh qi yf ek e5 pb au tc ac kc br qz 4t qc mx ly wm kb ez w2 u8 ei o8 -\n rf vo q1 o2 wt q5 oc 5l a2 es oe sc cx wq qw ky em tx rm p2 qo ft ed uq fs i9 ok lz v3 au xj v4 wf l3 eg ej ex wl rv qv ak mi hm cj yr w4 si -\n a2 tr qa fy qd qh oh kk lq dy bz oc jf wa k0 ip pm po qx wq cx we wr bj j5 yq qe gg rq rr oo ru od ix qt af or sj qo jn gr sk wd nc xo xl wk hh yf eh ek aq ex ar en tl ys rq pm ug ql qz wb wn yw og xw an if it -\n ak kk wu ig df w0 rb y4 fr gg i2 qb qw yf j1 ij ue s2 qt ad i5 yp ai l9 wk km ql zn yb ee rb ir te -\n hb q2 ld wt q7 qm km ws w7 vi iu yf rc g1 pr ot a9 pn dk ib qx hi qc jd jg hd q0 lo jj cn wb mz ec qp uj y9 tu yq au tp hg pl jx vx j7 wd 3y ca au rq kv qg dh k2 ok cf qa wv bp dw iu de k2 rt hj wm rc er o3 ey fn si if so -\n hq q1 qj d3 ws as ld mu rj ut d8 ey ou ib ez gf y7 qx qn vz qm vd zw ww d8 xu v1 av b6 mg gs bb g8 rm qy yv rr ry oa tb dt jb qa j0 qs l5 nz qg wj t4 td t6 eb ua s0 pn ii ac x9 qj k4 wc v9 s7 cj zy wo 10 hn yq vz u0 fm uz ux -\n ra pp qd d2 vg wj qn rh we ht st jm bv wu wi qy y9 de gw wa sb uz r1 qu pz ot td rg go e8 sn iy zr wc s0 ww ea pw ac w1 h4 w6 rp -\n fr uv pa jt qk q4 ls wu wi mt xa vu tk ed rb pe fp am sr hp qv m1 gc en yw qq qt ud ey eo p3 tj tu en ix ux ye tp ic s3 ad hz qi uf qa kr wh vd lk yn 5t u1 t0 od rr x9 f0 zj kn nk wp zs se rv ei pi -\n yb dc qs py qk nb jj ql q5 m3 mg zz e5 i1 zr lc zx xu vq hr xp by ei yx gl qi qp ij eg ai ph ap tn dy zp qp bp kf j5 ib vd eg el yy td yo ie ag em fc kx zx h8 wv sv q1 te wv vm yq ol if a1 -\n ak rd rf qg lo nq xy z6 q6 mw c1 cu z8 q7 vw wp a2 zz w8 yo uu ee yn r8 av yz y6 pp uq dv i3 db jv q9 2l xg rp ib lj sq tx tc mj mk qr rw te p3 ik eg pj e3 i9 im tb tn fm na j7 lj wa ct rg v4 he x3 kl bi r7 wj y1 l6 wk el 9f t6 gu tj od e9 tz re uh o2 zk ki cf lw jp sc s6 qs qb yn yw ms md w3 rv as yi ox du yz ir yc dp hc -\n yv gy ik k5 db gm ux qj gc w8 ea g6 dx po jb 5t pv wi rz qp jv v1 ea en al ii dt qj py w4 if ux -\n iy pd yg qq p4 in qa y1 yy ta fb zk s6 lu -\n ql ws rv yj jk ke lm ff lb fx s4 av uv wl n1 rv dp -\n qh d3 rs ih rc aq we 7y ud t3 h2 zt cu oc -\n fr hn k6 je q3 k4 tm zh lj aj li a4 t7 w7 kx ut pl rc ih th hp wq pl ls ma oe lf wu l1 ve lp qt qy fi ti he oh hi ow tm cu p7 nr va r3 tt wk wc s8 s0 hs al o2 hk x0 qj lm v5 wl qz 7u iw os lu ah wm hk e1 o4 rm fq ro so -\n gr sa rd um pf ca ga ql qz wt ld z6 vw kv my xt cn wr eq tk rw fe qx qc qr qr rk qa qi ex p3 p5 dq ff pc sh cy oq v4 wg s7 yo ya od rq dt un bw zm da bg q5 ru ah -\n iy qd k6 dm oj qz zd vr w0 r7 d8 et y1 eg yj gz qq p3 il i8 ge wp sx s1 wj t4 lm jo qp pw xc vm sr uz ig -\n qf u8 iq rg rk g4 im ih oq fp a0 ib tc uj tn nc kz ll u1 un qv ck lv vv -\n a4 df um jg z2 lq wr tn xu id wo ez rk rz ew d5 ti ix to r0 sk fa fe fr hp jk wr v1 ms wu jm rc qt tr ex uk vr to tz ut hl sg hb qd xn rj jc 6x mo wz rp ek y4 oi oo ae sc e9 od pn hg wz js qv ln ju rx yg as rn gk o7 so it -\n dd qs qd py k9 lw wt db gk zf nw wh t2 nf zx w8 rj ue w0 tl ew r6 ui ef g1 or ej pn an tq ou ff qv gj jv q0 kq 2l uo oe ku wi w1 tx qe rv 1l ws eu lv p4 ru fo tz ph ib fv uu i3 qu oe pc gw wp sx zd kw w6 bo w9 cs cw my qz zf rg fh e8 ay yd fz rw fc ng qz cg wx qp oz qv ly ha cx al iu rt mp e1 ee w4 tn ul ru o6 ag aj -\n qa kv e5 bm yj j4 m5 rj qe ri ht oi qf qe t6 e5 aw t8 wc dw un yb -\n iy jg jj d4 ju ol wy bs wk wq t8 a9 y4 ta dx jp qc q9 su si ut q0 m8 qq 4a zy qq zq sw po qe qt la sn p6 ht oj hy hh qi gc bn w7 au ya kz na oa ox ov je fb or wl qx ze cg we fk tz tv cg uq w6 -\n dx sa qa qs db go lo z3 lf ox jc wj 93 tj tk yf ii ef fo ua sk g5 fs el oi fr sx fg se q9 xq qw j3 m6 4h qq l2 eu rq qu qi hs uh p6 ix fx qa i0 wf ke bq ne wh xl ms un ex sx yi ua pb s0 rq ak ao fc pr qj cw wq vy wv u2 3h wq re yw eb gl eu -\n hv gv il jd go qh d3 we q4 q5 ej lk my gv a2 ds ex e7 rk yf fq pu ff qx ho js gx j1 qe kw gm ja ns wy ln d0 cz xt te xu tt sb em ix ic p0 i8 im ui di gt ws os r1 qy pz l6 e5 e6 op sb pv sn ii rr v6 ql le zy pv mt da yq ol w6 yz ag it -\n ds gy dg jf qg uw qj uv q5 q8 q8 q0 qm e5 rk yd tz pv if qn ju cv xy ki qw mj ls yv ru of tz so yr oq qi m9 sc kw qf zh jx wh kg l0 t1 pz un fj os ha sn f3 e0 om ab cw ct nj zy wn fl ww vn fn tn ie ov -\n co qh jr jj cb bi wt q6 ra qm zx ur r5 an fw tg jm re j5 vl em sl xz qe wa 45 nq ex yb ed ef ph e4 tb s6 qy lz cu gm pd gq mc ca 85 yf wf hu sb eb fk fv dt cq lq qz ww wv xr n0 eq ok er et iw r2 o7 fe it -\n ra dx og wd wt o9 i4 it pk qo ic dt hj jl oq sf lz wd ca hi fg f5 ap x9 gq nd iy q6 ep -\n o9 gb a5 z5 q6 wu w0 tl r9 dj if ts ig it zq ll qw qy ep ed ry p5 ut hh dr i0 hl qp qa zs wd ya ot xk s7 e9 om io dl ki k2 wv q1 5g er rb rm -\n qs w0 om ed tk ta th gf ii av og 6o ee o8 -\n po ty rf qf he qc hz bv c5 mi rh ew tu ef sh ix r0 d0 pb tq fw ig g8 fh i2 uw hf qw qr sf cn wi fh qt hp nq yv fy tj od ux ut fb hu tn qt oi qs oa xm dm fa nm qx yy oo ec ev ox sb ya rk ys ud jy dz zv qc zb qb 17 tb lt yt u9 w4 rb st et ry yl o5 di ux -\n gb gt az uk gm qh d3 bt qz wd ld o3 bl cm q7 ck k7 we b2 yd ua ew tl rq yg us tq js fk jb gz jn jg qq in qr rq qy 3h c0 qp p3 yn ef sb ym jl sf sh hv qd p4 fj od ix kz ni wj wz tg t0 tj e0 sm om kx ku cw nb zc aw cy qv bq kg wq pn zz wv cd wn yr se o6 pu eo gc -\n gy rf qf k6 qh qj cd q3 kk cj fw b4 fr mj w7 bm wr ya z7 wt w0 r8 ip ti is pn am y5 qb hs jf qw uu wr np qt wi 1t bx qq qw aw er cv qy rw eo oa iz fp iv qi jm nk kr qf xm eo nr w0 qj bi t3 uv wk ek wn ex e5 rf rh ga it f5 pm hm f8 qj gy jp le wc qc lt s9 zu lb q1 ju w1 uw w3 oj tn tq ir r2 te -\n ak gn pw a5 qh k9 qk nb 2l qz wr kv mt gt w7 zx ii oe ug ix sz qx qc ar sr zb su vg qe np yg qt yj sv uk pd uq pf of eh jb sk gy ws ke kd be og kh x6 me wz e6 yo iw ah rw pm rr qg pt lm ql qz wz qs ly wb wn q3 ry rx gj ia o5 ge -\n gy qm d3 q3 ia c1 ta ex e5 e8 eg sy cl jf qe nj nh m9 qa w6 ek iw kv qg ab n4 w5 iq do -\n uv gv qa un az jd qm eg iw nr q8 zj ny c5 vu rl rx yn et ia ua is ot pt hu im gj qv vv xe xu 1g wo vt qt st qy rw qi wh ft es uk hw to p8 fn f1 hp qu sk p2 l4 zf qf g0 fi l7 be ky iv mn xp nn dt 6b ro kw uv ra tp e6 sv s8 sn tl fz iy qh hz ve v8 h8 th wc wy qb xm s9 hs wq zs yq tu en zt w4 dp yc -\n sa ak gi hw qm gp pp qz fm id lh 6w 9h xr w7 ui ow rb oe ia us fq g5 y5 ig oi y6 im gk ze qe gn 3o ye xz jh qe db qi p1 ep re op te y9 os pj e3 p0 zp qa ih l4 cu zg wg sn rh lf fz ic kh ni wh vh wx th ag u3 f6 uj jy k3 2a wc kv lu wo hb eq w1 rb xw yo ei o7 gx -\n hw wd r0 g4 sz b7 pi sn tc in qt zs rg eu -\n iy hv hb hq qf z3 q2 xy ia tm zf jw wq a3 rk w0 d6 eg et is id po tg gh ob jj wr np no wt ja wy xl l2 yr bt bb tc cx qr rn qt lc rw sy ex y0 ru od to tz og ye hr pj de tv e4 qt ad oo qa jq fj l7 fo wh nt pl ro wl vp yy tf va e6 fg th ar s7 os at s8 re df pe hj f0 qx qc x7 lu nc zo tx bf ww pq cg w1 rx id pu it fe -\n qf fi ld 4o ge da t5 zz mo zx vi ui w9 pj rl rz r4 uy r5 sf av ot fq tw sc zv g9 qv i4 ut iy m1 wy xo b0 ud cb qy rq ug wg sn fo ix uc aa i9 ss sd di sh j8 qp xa ep w0 7h wl t7 iw tj ya hs e0 fz hd u6 hh dr dh uk qj qk wx ol xb qv wb s8 15 wy zm jr nx it eo fx ww yv zy to ee yu yi iq ey iw rm uz yz ob -\n po ra ik qd qf je jr lp wr ji ne wu 2b nt wa e7 rx xo ia yl ta ig ff hp gk jf q9 vd si gc qe rk wu by yg rq sb os fu eh em ux ic ao pj hy im du qy cr 2e l5 qw v1 lf mv wk rx dy rp ra rg gi eb hs au ap bo oa sn zi f6 h1 zt ey yz do -\n lj vu y2 if qr wr ta so kg 3k ol u1 f5 -\n yb cv mq lf zz ue ui dx qe tv qu ex tz hh tb dy ds wi rb -\n hb gt qa h9 rf qf qg k9 q1 lo q3 ql z8 gl zg q8 1t wo vr rg ez ws mo w9 yo r3 yd e9 ea sf a0 im tg y8 ar qb ni wr qr wv m4 ix d9 nz yq fa if yr bt qr mz qi ea rr xx at ic pk qi za hv kq w7 co xj wd x4 zs wh y3 rf ec u2 e9 ah s0 uh io un h8 iq zb bs nr be zo fz vb wm pr yw md rc ur er ia yl ox ei ux eo tw o8 -\n qs rd rh yf px ow d8 tq ig ih cl qr yw qq in qy wc ek rh ya qg cd x9 qm lq to o3 ul a1 -\n a2 cp as gk kc 4u zj z2 t5 zz rb eh sh sx fg fh g9 hp vd gx oq cv pe wv b8 qe cc nw tr il tl e3 qu l4 mt wk wh aq td e9 gf lm qz bu jq my wp vb dg y0 ye yq w6 r1 -\n ij az qn ph qv bv bf mz iu is y8 ar fh q9 hd qq ji sf ld up qo p2 at sp in qi nk l5 e0 rw px 5o ew oc te -\n qd jr pa q5 w9 hw hu y8 dn qn zw wr ma ei xl dq i7 i9 vb sx wf wh na wl um e7 s0 h2 nh nk fj yl wn iu u7 as ad ey so uc -\n la up ic g5 ay ic x8 u2 ar eb wb yr aj -\n po da rd un qg uw lq m2 4r wg q8 z9 t3 zc d9 ae st q0 li qw wt kr qu ry en sm qf kf kh ny yt gi rh u5 em tc kv h8 qx lr jq ef -\n a2 dh qh q1 h8 qz z6 kx z8 bv 3q df pk tl d8 tq tf g8 zb qw 5p zm qe cv yb ec uz iv e2 gq wp uh kq ws lc wk x3 t8 rj fc io je dk lr lt wv wt bw be eo q4 ye yy rv ok yo yp ir ig -\n rs ij ty ps ul wr bh kb rs z4 z8 er px uo up y1 rb fo jo gg dv ph q0 jn xw ww d8 rp yd yf tx b0 op yn of jl tm px fm jc zf qd pk wh rp uv tp t9 ir yf ug qg v5 ku qz fd k4 cu mw zn iu bg lq ly rv su -\n ik dm cm or tw pu lp eh qd kk j0 em ng tw -\n ra ds qk cg q7 k6 4p t6 yu lq go yd eq r8 fw am dm xy cm v1 cz eo qi ij yn eg hq tc sj qa i0 oa l6 p6 wj vd wf mr yt ex e6 yo t9 ev s8 en rr bq kg hb lm re bj ms w1 et du -\n o9 gy bl wz t8 hq iu ix av y5 y8 jn j1 np xt t9 vw qq 43 xv 9w yi ft es hy op lg vs hg wd ef wx ou ox sw dr ze xr st fm ah -\n a3 wr fb jc c2 w8 rx fe q9 hd xq qq wi te y9 e1 qt qi qs nl ca bh u2 md tv hx -\n ra jh q3 aa t3 1o eq lh rv fo us pt dj pm pi qn zr bj xj cm ix a0 ra hi eu nw yc p3 ru ri ue e2 jl hi wo g9 xn qh wl wx go yp rr dj nd ch u3 fj bd jy vn w1 ia ox tm uz -\n ma y5 bl qi g7 ri fl ap 7a yo ko rp -\n um dh pg wq r5 sf ia ta an hs ne q9 wt sh rk yi ym of tb oq do hv wj ic oi sc pe sc wq wb wm tq -\n hq qm gg q4 xu k4 k7 uo wt kb et fo ey aw g0 xi am in qy eo qi eb ay ue og nt kl wx s9 df qg f9 v6 rv sv pc tl my bj wb eq h4 o3 ri -\n tt uq qh nb qz wt jx ya on om io ow ha qp e2 fd e4 hp hx p2 vm xg xn ra l8 iu yf jr qh k4 1l oa tk zp yw rz sy ul yx eo ep -\n ij yb qs fi ul qk by ql jl wr bi q5 bl tm q7 xp k7 vy gi tj rl rx yf ym tu er r8 pe ip ej y4 fd jn gx zt vj xt xh rj kt cm ri nh zq pp eu rw to pg e1 ue dw e2 so tb gm qi jb nk gy pa v1 xj fl kh 3k kl ed wx wc ek yy ez wc iq yo u4 it tz ak hn f8 h2 hl uv wz h8 gi nk ch zt wt bw kn yq e1 tv zp ag it -\n qs rf jq go qh a8 jj xt le wu qq yd d6 d0 ff qc su c7 lc wp ty y0 tu ti pf ta aa ug vb rf vi rx no un yu e5 7r up rj ag ha hs fc wz bo qv bp tv ki er -\n yb qa rd lo ok q6 o6 ba r6 ow or yl am aq gd dc ho cx c0 wu g4 ib c5 ep re qo ed yw iv ta hy jc pn xs oq xn bo zg ps kr iz yp wh wj xl xo zq me na ek wl wx wc rg uo ir tk aj da rw hm io uj fb jt qj dk nh zx jp qx wc zr zy zu nc xe ww wv vn h2 q6 en ew ad yl af eo if r2 -\n yb o0 dn z1 q1 sw qk po xt wr ls z8 ox wu jd ro q8 lh mh wa rg ea c3 pz tu g2 is a9 pn tw po qc sa jj vh ax xh kt wy jv mg nh kp b9 qi od ht e4 uu ij qf sf nc wk ap wd qz rd yi s7 tk pb it f7 o2 f9 kv qh h4 ln wz gi qd zn xm sn 39 yn rz u7 yr yj is ie ag ir tw -\n po dx uv pq jd dm d2 hr cs gs d3 q4 wr np ab di mh vt w7 w0 on tl ia ta aq dz y7 i1 qc pg q0 wq j2 c7 la cb il wr lv na ru 4g vz nf bc sl qq qp qt ud rq wg ex uw he tz ye p9 ui ou vu at ix w0 vf bi ed yh wh ra y6 wc u1 t8 fj ov iy tz kx h1 hj jy qj h6 ng jo wz cj ie mt rq te n9 mp ma q5 8w rb o7 sp ob -\n po ra dg ca qj q2 is kn rd ws lq tu ym yl y5 tg pp qw we cn a7 1t jd m7 wo yr sz qa hp ei qy ec p4 hw p7 to au iv ht qt qy qo l4 lz xm wd yf wg ez s7 en f3 tx yf rr f8 cw ji qv yz ry ew w2 oj w4 w5 st rn oz ri o6 dp aj -\n gm jg ju q5 np lf q7 xo 1y qn k0 jo pp qx th q9 4f or ro pu bv eu nw uq ao dy jb j9 gr rd nz wj wj r9 co ta rk od dd gg hm df pr km ng oj qc sb qd wb tz cq ex wb vb ty eq tv iw -\n fu uq co qk jl cg ld lg wo vr gc bd rj r3 yd rz iu ew d6 io to sh y7 jp db dn qn qm si xg qr ls jo lr wy rk wn m7 qu bb es op qp ru en ta e3 in hy dy hl vc gc gt jw ke 2t wh rk lj hg oy e6 yo ev em fz rw pq re dg qk ku oi qz k4 qv li rq n8 ec 4d yb wb e1 iw id o6 ir do ux pi ep -\n a2 wu jd ef dc mn 5e qp pl xd ag ay -\n yv o9 al a5 uq qg jw pi z2 jt cd q5 3m zl ez vu rg jl rz yf ix sj fp d0 tq ff ha hs zw om ni m0 xg c8 a7 ki qw cc ei xg j3 tt tu il p6 ix tp tx ib sp hg p0 fc pj su qu jv sj lk qp ws qs gm 1x mx lv wj qu l1 dt wh wv un aq fg rg e6 uo ar ie up it sw tx f6 o2 h3 qc qa ho vj u3 kd zy n6 my ww vc lr em w2 se rt o4 yc a1 te -\n qs dv pa ty iz uw qh jj z3 tn jc eg e4 qq w7 ut r3 uu kb up g1 fo iv if fd gd sc qm qe xg ia wb he ky hu tv rw qu rr es p3 ue s2 as i0 dt qt hz jm j0 gy ci fi hw nv ea kk vf rx 68 ti rp wl oi vp at om io uk pt qh qj dl cf lr cx wq ku ki w2 yh af ul sp yc it -\n ub yb dc ty gm dm go nv we ql by la o1 ju o3 jx fm aj wa rg e4 vi a6 r4 xo tz oe ip pv dk tq a0 tf fg tg i4 pz sd ry ky mg hy g7 eu qy yi rw qp eg yw hw sm uc i7 dw fx s3 sf zo m9 xs vn rf ci nz kr qt 9y pj lk ee pz ef rk e0 fx uf az fc qg jr oy lq cg qp um ad wc zn bw n6 my xr mp tu en o3 iq ir ro -\n da a3 d1 jr dz ca ql nu q3 cf o6 nr mt lk yr rs lp w7 a5 pj ys ym r8 ey to fs dz im ih sw qx qv zn gl j1 xe lc zc vw 6a mh b9 qt rm re oo qp p4 tk ix p7 og tz yr sp aa hk ih lx qd mx 4n kk vf el oo td ae yo fj uf pr hl qj qk wr qc qv kf yz my wq hn zs dr ee u8 rv et ru ie ag tw -\n gt ph z0 zl mu ui av zm om ui vh qr he qr es fl ws w6 nc ra rk kp ol wm yu it -\n dd df jq jd ux ql el 3r ya uu iu ee eh g3 sj us ib pp qc jv hd bh zt uo d8 b3 xu bc rq te uh ex tt eb il qu pc ge sj qp ih xf 3r gr yh qx tu wl wn sz up ay it ab jt qz v7 wn li za 6o w3 fn yk eu ie gz ro -\n yv o9 qf eg eh mh jh rh r3 rv ix y3 a0 sr qc qq qr wr qe bx ki m8 mk qi lm uk eb ai ur e2 xd nc ca eo mb ed uv rs up ya of hn lw wz qz et qh wm zr rc o3 r2 -\n ss qg qj ph qk q2 cs z4 bi qc cj q8 qn w7 rj ys ea r4 uy om rc ii fp sj ej yz el qx qv zn gk q9 hs m2 ii d7 nk c8 j4 qq dl gg pp ei qo yc od fo eh fp ta hy ok tv uu us dp qf sb zg ks sg n3 wh x2 nr cs wk kh wk wf ew 7h 7k oy t6 gi rg yp s9 ya e9 pb tc dt dh hk h2 pt qh h7 wz n1 qv kd pm cc xr kp yk tm ge -\n gr qa ft tt gn qs pw pu ca ph ls cg cn zf bz q7 z0 c3 qn gv w7 rg ut e8 ii er ip sg y3 oy ek ht gd qx g0 qv db su jn qq lz uu jo ru an wi kn sq nh qr qy yx eo qi xz y9 ru pd au p7 dq he ut ok fd jz ui hc j9 l3 hb xd jq gy kh wf xs sl rs aq ez y4 ts um yi e0 gh dk py v5 qk ql ko jq wc nk v0 wb qv br iu wm 6p sr gk yp pu -\n o0 pa pf z3 jt jy z7 cm ne w8 yz fd fg zn qq ll vg wr wb ia xx yj ty eh e1 so ts tc s4 i0 tn wo wp wa op va wk x3 vg qx rs sn au f3 tz sq hn rr o2 fv un k2 vj ey dj -\n iy ra ij ty a4 un rf qg dm jr kj uv we cv gk wy z8 oc wp mo jl mz ev ch rv tu ax y2 g3 oy y6 im uq qv hp qb hd iy lp nk w2 bb ho ep tr os en sm p8 p9 hy ss ui gm qi oo vn ae qd w6 ps dn wd wg ro mr yt ol oz rg s7 u5 tl yd rr ax f0 cq ku qx ze n4 wn kt ca jy bg yc zs yw rz w1 eu rm r1 -\n hv fu ca q8 mt la r3 pl yh to sy yh tv x4 tg yp ov wn ze sp -\n o9 qa az gm qd pw hq pd ga qj cd q3 jk pd du c2 zk xf t8 eq om es rc ua y3 pu ig qx se qv db st qn ii lx qe wt xk nx ku br qe qr qt rm eu xf xb rn qu qi ep qo rr ex xk p5 ym fi uq to ux ix ai hj gn zi oq qf kd wf xn kr w8 rl kk mq rp rf u1 s7 oa fh e7 yp e0 pr ql sx ck ag kg kt mp eb rl em ee w6 du rm yz if ep -\n qs pi am 6a ut r4 ii sd ua ib y6 pa kt pb wm qq dz qt qp y0 he p8 ue tb qu or qo wh 40 8j t4 sl rf iu gh hh qc iq bf rl wm mf oh ew is dp -\n da ps a7 jr z4 q3 bu xt ip jx q6 np z7 bp lg bz ye wo ig bb ww rf om uu ef r8 ey pt ta pn y7 gg th dn pg qb ri qq 42 qw zw b9 b0 xb qt qy yl wh xk ft at yw i7 sp de pz fn qy si m0 ik wf sg cq ql hk er eg lc ek na wc iw ir rk ua e0 ak iu sw ap uj av ab hl uv zc qa wc jw vj qv vk ay kg xw on rw 1b wn rl eb vm eq h4 yt oz eu uz -\n a2 qa rd cp qh ub vg ws u0 4g bp da yo ev kz eq uu ee ef yk pr sj sk fe oi lt uy j2 gn io vk ns 27 ln wu ve yr l2 qu qi ry il ul tj eg ux i5 yr tx ph oj gq or zp wa qs gy iz v0 qt wj ic ca lh yb np ej sl td l8 yi iw s8 e8 ys yd sq al dt pt tg te yb eu tq r1 ir fe -\n tr h9 go qj b1 wu q7 zh el tg mb ys ed ii er r8 xs pe r0 sw db ov m7 d3 re no nu zr pq ji wr lc or qw ee yy qr rn rm re qi te ea qo yb yn y0 uz uq iz tl yr i0 fm wp qs qd he wk kl ew as hl ez oo ox ie s0 f4 dl wq wl ww lr qc qv vk mt my kn ep em yt sy am so rp sp -\n ak ft qg bg ji q6 bk xi tf mo ur pj yk ua qv wq gc qe ke ef eb tk uq i6 oh iv gb qs rx el yo rr pe wz wx ho xq tc mo yk du r1 tq oc hc te -\n ra ub qj jh jt dx ql q6 da tf r3 ew iu sg tp yl el gc 6n tt ry pd ye ff lz kt yp fl yf dl rn -\n o9 hb h9 qd dh qg q1 qj jy se q5 wt nr qv ge c5 el 6y uo rv ax pe et r0 fe y6 dx qx ha qq lo we zy v1 wy dl vr wa qr rm qi qp yn tz pg ph de p0 do qp wp wf bw xh ky xz wh hl to ek rd sv rj rq re h1 qg qh kl f1 zm 18 ez xe vm en 5j o3 rn fw fe it -\n db c2 bb o0 w8 kl kc y4 qx zm pk cw id ve mh lp rq jb fl x1 qi wd lx f3 cy bq dd ye fn ig diff --git a/contrib/tsearch2/expected/tsearch2.out b/contrib/tsearch2/expected/tsearch2.out deleted file mode 100644 index eef85ef09d..0000000000 --- a/contrib/tsearch2/expected/tsearch2.out +++ /dev/null @@ -1,2977 +0,0 @@ -CREATE EXTENSION tsearch2; --- Check whether any of our opclasses fail amvalidate -SELECT amname, opcname -FROM pg_opclass opc LEFT JOIN pg_am am ON am.oid = opcmethod -WHERE opc.oid >= 16384 AND NOT amvalidate(opc.oid); - amname | opcname ---------+--------- -(0 rows) - ---tsvector -SELECT '1'::tsvector; - tsvector ----------- - '1' -(1 row) - -SELECT '1 '::tsvector; - tsvector ----------- - '1' -(1 row) - -SELECT ' 1'::tsvector; - tsvector ----------- - '1' -(1 row) - -SELECT ' 1 '::tsvector; - tsvector ----------- - '1' -(1 row) - -SELECT '1 2'::tsvector; - tsvector ----------- - '1' '2' -(1 row) - -SELECT '''1 2'''::tsvector; - tsvector ----------- - '1 2' -(1 row) - -SELECT E'''1 \\''2'''::tsvector; - tsvector ----------- - '1 ''2' -(1 row) - -SELECT E'''1 \\''2''3'::tsvector; - tsvector -------------- - '1 ''2' '3' -(1 row) - -SELECT E'''1 \\''2'' 3'::tsvector; - tsvector -------------- - '1 ''2' '3' -(1 row) - -SELECT E'''1 \\''2'' '' 3'' 4 '::tsvector; - tsvector ------------------- - ' 3' '1 ''2' '4' -(1 row) - -select '''w'':4A,3B,2C,1D,5 a:8'; - ?column? ------------------------ - 'w':4A,3B,2C,1D,5 a:8 -(1 row) - -select 'a:3A b:2a'::tsvector || 'ba:1234 a:1B'; - ?column? ----------------------------- - 'a':3A,4B 'b':2A 'ba':1237 -(1 row) - -select setweight('w:12B w:13* w:12,5,6 a:1,3* a:3 w asd:1dc asd zxc:81,567,222A'::tsvector, 'c'); - setweight ----------------------------------------------------------- - 'a':1C,3C 'asd':1C 'w':5C,6C,12C,13C 'zxc':81C,222C,567C -(1 row) - -select strip('w:12B w:13* w:12,5,6 a:1,3* a:3 w asd:1dc asd'::tsvector); - strip ---------------- - 'a' 'asd' 'w' -(1 row) - ---tsquery -SELECT '1'::tsquery; - tsquery ---------- - '1' -(1 row) - -SELECT '1 '::tsquery; - tsquery ---------- - '1' -(1 row) - -SELECT ' 1'::tsquery; - tsquery ---------- - '1' -(1 row) - -SELECT ' 1 '::tsquery; - tsquery ---------- - '1' -(1 row) - -SELECT '''1 2'''::tsquery; - tsquery ---------- - '1 2' -(1 row) - -SELECT E'''1 \\''2'''::tsquery; - tsquery ---------- - '1 ''2' -(1 row) - -SELECT '!1'::tsquery; - tsquery ---------- - !'1' -(1 row) - -SELECT '1|2'::tsquery; - tsquery ------------ - '1' | '2' -(1 row) - -SELECT '1|!2'::tsquery; - tsquery ------------- - '1' | !'2' -(1 row) - -SELECT '!1|2'::tsquery; - tsquery ------------- - !'1' | '2' -(1 row) - -SELECT '!1|!2'::tsquery; - tsquery -------------- - !'1' | !'2' -(1 row) - -SELECT '!(!1|!2)'::tsquery; - tsquery ------------------- - !( !'1' | !'2' ) -(1 row) - -SELECT '!(!1|2)'::tsquery; - tsquery ------------------ - !( !'1' | '2' ) -(1 row) - -SELECT '!(1|!2)'::tsquery; - tsquery ------------------ - !( '1' | !'2' ) -(1 row) - -SELECT '!(1|2)'::tsquery; - tsquery ----------------- - !( '1' | '2' ) -(1 row) - -SELECT '1&2'::tsquery; - tsquery ------------ - '1' & '2' -(1 row) - -SELECT '!1&2'::tsquery; - tsquery ------------- - !'1' & '2' -(1 row) - -SELECT '1&!2'::tsquery; - tsquery ------------- - '1' & !'2' -(1 row) - -SELECT '!1&!2'::tsquery; - tsquery -------------- - !'1' & !'2' -(1 row) - -SELECT '(1&2)'::tsquery; - tsquery ------------ - '1' & '2' -(1 row) - -SELECT '1&(2)'::tsquery; - tsquery ------------ - '1' & '2' -(1 row) - -SELECT '!(1)&2'::tsquery; - tsquery ------------- - !'1' & '2' -(1 row) - -SELECT '!(1&2)'::tsquery; - tsquery ----------------- - !( '1' & '2' ) -(1 row) - -SELECT '1|2&3'::tsquery; - tsquery ------------------ - '1' | '2' & '3' -(1 row) - -SELECT '1|(2&3)'::tsquery; - tsquery ------------------ - '1' | '2' & '3' -(1 row) - -SELECT '(1|2)&3'::tsquery; - tsquery ---------------------- - ( '1' | '2' ) & '3' -(1 row) - -SELECT '1|2&!3'::tsquery; - tsquery ------------------- - '1' | '2' & !'3' -(1 row) - -SELECT '1|!2&3'::tsquery; - tsquery ------------------- - '1' | !'2' & '3' -(1 row) - -SELECT '!1|2&3'::tsquery; - tsquery ------------------- - !'1' | '2' & '3' -(1 row) - -SELECT '!1|(2&3)'::tsquery; - tsquery ------------------- - !'1' | '2' & '3' -(1 row) - -SELECT '!(1|2)&3'::tsquery; - tsquery ----------------------- - !( '1' | '2' ) & '3' -(1 row) - -SELECT '(!1|2)&3'::tsquery; - tsquery ----------------------- - ( !'1' | '2' ) & '3' -(1 row) - -SELECT '1|(2|(4|(5|6)))'::tsquery; - tsquery ------------------------------ - '1' | '2' | '4' | '5' | '6' -(1 row) - -SELECT '1|2|4|5|6'::tsquery; - tsquery ------------------------------ - '1' | '2' | '4' | '5' | '6' -(1 row) - -SELECT '1&(2&(4&(5&6)))'::tsquery; - tsquery ------------------------------ - '1' & '2' & '4' & '5' & '6' -(1 row) - -SELECT '1&2&4&5&6'::tsquery; - tsquery ------------------------------ - '1' & '2' & '4' & '5' & '6' -(1 row) - -SELECT '1&(2&(4&(5|6)))'::tsquery; - tsquery ---------------------------------- - '1' & '2' & '4' & ( '5' | '6' ) -(1 row) - -SELECT '1&(2&(4&(5|!6)))'::tsquery; - tsquery ----------------------------------- - '1' & '2' & '4' & ( '5' | !'6' ) -(1 row) - -SELECT E'1&(''2''&('' 4''&(\\|5 | ''6 \\'' !|&'')))'::tsquery; - tsquery ------------------------------------------- - '1' & '2' & ' 4' & ( '|5' | '6 '' !|&' ) -(1 row) - -SELECT '''the wether'':dc & '' sKies '':BC & a:d b:a'; - ?column? ------------------------------------------- - 'the wether':dc & ' sKies ':BC & a:d b:a -(1 row) - -select 'a' < 'b & c'::tsquery; - ?column? ----------- - t -(1 row) - -select 'a' > 'b & c'::tsquery; - ?column? ----------- - f -(1 row) - -select 'a | f' < 'b & c'::tsquery; - ?column? ----------- - t -(1 row) - -select 'a | ff' < 'b & c'::tsquery; - ?column? ----------- - f -(1 row) - -select 'a | f | g' < 'b & c'::tsquery; - ?column? ----------- - f -(1 row) - -select numnode( 'new'::tsquery ); - numnode ---------- - 1 -(1 row) - -select numnode( 'new & york'::tsquery ); - numnode ---------- - 3 -(1 row) - -select numnode( 'new & york | qwery'::tsquery ); - numnode ---------- - 5 -(1 row) - -create table test_tsquery (txtkeyword text, txtsample text); -\set ECHO none -alter table test_tsquery add column keyword tsquery; -update test_tsquery set keyword = to_tsquery('english', txtkeyword); -alter table test_tsquery add column sample tsquery; -update test_tsquery set sample = to_tsquery('english', txtsample::text); -create unique index bt_tsq on test_tsquery (keyword); -select count(*) from test_tsquery where keyword < 'new & york'; - count -------- - 1 -(1 row) - -select count(*) from test_tsquery where keyword <= 'new & york'; - count -------- - 2 -(1 row) - -select count(*) from test_tsquery where keyword = 'new & york'; - count -------- - 1 -(1 row) - -select count(*) from test_tsquery where keyword >= 'new & york'; - count -------- - 3 -(1 row) - -select count(*) from test_tsquery where keyword > 'new & york'; - count -------- - 2 -(1 row) - -set enable_seqscan=off; -select count(*) from test_tsquery where keyword < 'new & york'; - count -------- - 1 -(1 row) - -select count(*) from test_tsquery where keyword <= 'new & york'; - count -------- - 2 -(1 row) - -select count(*) from test_tsquery where keyword = 'new & york'; - count -------- - 1 -(1 row) - -select count(*) from test_tsquery where keyword >= 'new & york'; - count -------- - 3 -(1 row) - -select count(*) from test_tsquery where keyword > 'new & york'; - count -------- - 2 -(1 row) - -set enable_seqscan=on; -select rewrite('foo & bar & qq & new & york', 'new & york'::tsquery, 'big & apple | nyc | new & york & city'); - rewrite ------------------------------------------------------------------------------- - 'foo' & 'bar' & 'qq' & ( 'city' & 'new' & 'york' | 'nyc' | 'big' & 'apple' ) -(1 row) - -select rewrite('moscow', 'select keyword, sample from test_tsquery'::text ); - rewrite ---------------------- - 'moskva' | 'moscow' -(1 row) - -select rewrite('moscow & hotel', 'select keyword, sample from test_tsquery'::text ); - rewrite ------------------------------------ - 'hotel' & ( 'moskva' | 'moscow' ) -(1 row) - -select rewrite('bar & new & qq & foo & york', 'select keyword, sample from test_tsquery'::text ); - rewrite ---------------------------------------------------------------------------------- - 'citi' & 'foo' & ( 'bar' | 'qq' ) & ( 'nyc' | 'big' & 'appl' | 'new' & 'york' ) -(1 row) - -select rewrite( ARRAY['moscow', keyword, sample] ) from test_tsquery; - rewrite ---------------------- - 'moskva' | 'moscow' -(1 row) - -select rewrite( ARRAY['moscow & hotel', keyword, sample] ) from test_tsquery; - rewrite ------------------------------------ - ( 'moskva' | 'moscow' ) & 'hotel' -(1 row) - -select rewrite( ARRAY['bar & new & qq & foo & york', keyword, sample] ) from test_tsquery; - rewrite ---------------------------------------------------------------------------------- - 'citi' & 'foo' & ( 'bar' | 'qq' ) & ( 'nyc' | 'big' & 'appl' | 'new' & 'york' ) -(1 row) - -select keyword from test_tsquery where keyword @> 'new'; - keyword ----------------- - 'new' & 'york' -(1 row) - -select keyword from test_tsquery where keyword @> 'moscow'; - keyword ----------- - 'moscow' -(1 row) - -select keyword from test_tsquery where keyword <@ 'new'; - keyword ---------- -(0 rows) - -select keyword from test_tsquery where keyword <@ 'moscow'; - keyword ----------- - 'moscow' -(1 row) - -select rewrite( ARRAY[query, keyword, sample] ) from test_tsquery, to_tsquery('english', 'moscow') as query where keyword <@ query; - rewrite ---------------------- - 'moskva' | 'moscow' -(1 row) - -select rewrite( ARRAY[query, keyword, sample] ) from test_tsquery, to_tsquery('english', 'moscow & hotel') as query where keyword <@ query; - rewrite ------------------------------------ - ( 'moskva' | 'moscow' ) & 'hotel' -(1 row) - -select rewrite( ARRAY[query, keyword, sample] ) from test_tsquery, to_tsquery('english', 'bar & new & qq & foo & york') as query where keyword <@ query; - rewrite ---------------------------------------------------------------------------------- - 'citi' & 'foo' & ( 'bar' | 'qq' ) & ( 'nyc' | 'big' & 'appl' | 'new' & 'york' ) -(1 row) - -select rewrite( ARRAY[query, keyword, sample] ) from test_tsquery, to_tsquery('english', 'moscow') as query where query @> keyword; - rewrite ---------------------- - 'moskva' | 'moscow' -(1 row) - -select rewrite( ARRAY[query, keyword, sample] ) from test_tsquery, to_tsquery('english', 'moscow & hotel') as query where query @> keyword; - rewrite ------------------------------------ - ( 'moskva' | 'moscow' ) & 'hotel' -(1 row) - -select rewrite( ARRAY[query, keyword, sample] ) from test_tsquery, to_tsquery('english', 'bar & new & qq & foo & york') as query where query @> keyword; - rewrite ---------------------------------------------------------------------------------- - 'citi' & 'foo' & ( 'bar' | 'qq' ) & ( 'nyc' | 'big' & 'appl' | 'new' & 'york' ) -(1 row) - -create index qq on test_tsquery using gist (keyword gist_tp_tsquery_ops); -set enable_seqscan='off'; -select keyword from test_tsquery where keyword @> 'new'; - keyword ----------------- - 'new' & 'york' -(1 row) - -select keyword from test_tsquery where keyword @> 'moscow'; - keyword ----------- - 'moscow' -(1 row) - -select keyword from test_tsquery where keyword <@ 'new'; - keyword ---------- -(0 rows) - -select keyword from test_tsquery where keyword <@ 'moscow'; - keyword ----------- - 'moscow' -(1 row) - -select rewrite( ARRAY[query, keyword, sample] ) from test_tsquery, to_tsquery('english', 'moscow') as query where keyword <@ query; - rewrite ---------------------- - 'moskva' | 'moscow' -(1 row) - -select rewrite( ARRAY[query, keyword, sample] ) from test_tsquery, to_tsquery('english', 'moscow & hotel') as query where keyword <@ query; - rewrite ------------------------------------ - ( 'moskva' | 'moscow' ) & 'hotel' -(1 row) - -select rewrite( ARRAY[query, keyword, sample] ) from test_tsquery, to_tsquery('english', 'bar & new & qq & foo & york') as query where keyword <@ query; - rewrite ---------------------------------------------------------------------------------- - 'citi' & 'foo' & ( 'bar' | 'qq' ) & ( 'nyc' | 'big' & 'appl' | 'new' & 'york' ) -(1 row) - -select rewrite( ARRAY[query, keyword, sample] ) from test_tsquery, to_tsquery('english', 'moscow') as query where query @> keyword; - rewrite ---------------------- - 'moskva' | 'moscow' -(1 row) - -select rewrite( ARRAY[query, keyword, sample] ) from test_tsquery, to_tsquery('english', 'moscow & hotel') as query where query @> keyword; - rewrite ------------------------------------ - ( 'moskva' | 'moscow' ) & 'hotel' -(1 row) - -select rewrite( ARRAY[query, keyword, sample] ) from test_tsquery, to_tsquery('english', 'bar & new & qq & foo & york') as query where query @> keyword; - rewrite ---------------------------------------------------------------------------------- - 'citi' & 'foo' & ( 'bar' | 'qq' ) & ( 'nyc' | 'big' & 'appl' | 'new' & 'york' ) -(1 row) - -set enable_seqscan='on'; -select lexize('simple', 'ASD56 hsdkf'); - lexize ------------------ - {"asd56 hsdkf"} -(1 row) - -select lexize('english_stem', 'SKIES Problems identity'); - lexize --------------------------- - {"skies problems ident"} -(1 row) - -select * from token_type('default'); - tokid | alias | descr --------+-----------------+------------------------------------------ - 1 | asciiword | Word, all ASCII - 2 | word | Word, all letters - 3 | numword | Word, letters and digits - 4 | email | Email address - 5 | url | URL - 6 | host | Host - 7 | sfloat | Scientific notation - 8 | version | Version number - 9 | hword_numpart | Hyphenated word part, letters and digits - 10 | hword_part | Hyphenated word part, all letters - 11 | hword_asciipart | Hyphenated word part, all ASCII - 12 | blank | Space symbols - 13 | tag | XML tag - 14 | protocol | Protocol head - 15 | numhword | Hyphenated word, letters and digits - 16 | asciihword | Hyphenated word, all ASCII - 17 | hword | Hyphenated word, all letters - 18 | url_path | URL path - 19 | file | File or path name - 20 | float | Decimal notation - 21 | int | Signed integer - 22 | uint | Unsigned integer - 23 | entity | XML entity -(23 rows) - -select * from parse('default', '345 qwe@efd.r '' http://www.com/ http://aew.werc.ewr/?ad=qwe&dw 1aew.werc.ewr/?ad=qwe&dw 2aew.werc.ewr http://3aew.werc.ewr/?ad=qwe&dw http://4aew.werc.ewr http://5aew.werc.ewr:8100/? ad=qwe&dw 6aew.werc.ewr:8100/?ad=qwe&dw 7aew.werc.ewr:8100/?ad=qwe&dw=%20%32 +4.0e-10 qwe qwe qwqwe 234.435 455 5.005 teodor@stack.net qwe-wer asdf qwer jf sdjk ewr1> ewri2 -/usr/local/fff /awdf/dwqe/4325 rewt/ewr wefjn /wqe-324/ewr gist.h gist.h.c gist.c. readline 4.2 4.2. 4.2, readline-4.2 readline-4.2. 234 - wow < jqw <> qwerty'); - tokid | token --------+-------------------------------------- - 22 | 345 - 12 | - 1 | qwe - 12 | @ - 19 | efd.r - 12 | ' - 14 | http:// - 6 | www.com - 12 | / - 14 | http:// - 5 | aew.werc.ewr/?ad=qwe&dw - 6 | aew.werc.ewr - 18 | /?ad=qwe&dw - 12 | - 5 | 1aew.werc.ewr/?ad=qwe&dw - 6 | 1aew.werc.ewr - 18 | /?ad=qwe&dw - 12 | - 6 | 2aew.werc.ewr - 12 | - 14 | http:// - 5 | 3aew.werc.ewr/?ad=qwe&dw - 6 | 3aew.werc.ewr - 18 | /?ad=qwe&dw - 12 | - 14 | http:// - 6 | 4aew.werc.ewr - 12 | - 14 | http:// - 5 | 5aew.werc.ewr:8100/? - 6 | 5aew.werc.ewr:8100 - 18 | /? - 12 | - 1 | ad - 12 | = - 1 | qwe - 12 | & - 1 | dw - 12 | - 5 | 6aew.werc.ewr:8100/?ad=qwe&dw - 6 | 6aew.werc.ewr:8100 - 18 | /?ad=qwe&dw - 12 | - 5 | 7aew.werc.ewr:8100/?ad=qwe&dw=%20%32 - 6 | 7aew.werc.ewr:8100 - 18 | /?ad=qwe&dw=%20%32 - 12 | - 7 | +4.0e-10 - 12 | - 1 | qwe - 12 | - 1 | qwe - 12 | - 1 | qwqwe - 12 | - 20 | 234.435 - 12 | - 22 | 455 - 12 | - 20 | 5.005 - 12 | - 4 | teodor@stack.net - 12 | - 16 | qwe-wer - 11 | qwe - 12 | - - 11 | wer - 12 | - 1 | asdf - 12 | - 13 | - 1 | qwer - 12 | - 1 | jf - 12 | - 1 | sdjk - 12 | < - 1 | we - 12 | - 1 | hjwer - 12 | - 13 | - 12 | - 3 | ewr1 - 12 | > - 3 | ewri2 - 12 | - 13 | - 12 | + - | - 19 | /usr/local/fff - 12 | - 19 | /awdf/dwqe/4325 - 12 | - 19 | rewt/ewr - 12 | - 1 | wefjn - 12 | - 19 | /wqe-324/ewr - 12 | - 19 | gist.h - 12 | - 19 | gist.h.c - 12 | - 19 | gist.c - 12 | . - 1 | readline - 12 | - 20 | 4.2 - 12 | - 20 | 4.2 - 12 | . - 20 | 4.2 - 12 | , - 1 | readline - 20 | -4.2 - 12 | - 1 | readline - 20 | -4.2 - 12 | . - 22 | 234 - 12 | + - | - 12 | < - 1 | i - 12 | - 13 | - 12 | - 1 | wow - 12 | - 12 | < - 1 | jqw - 12 | - 12 | <> - 1 | qwerty -(133 rows) - -SELECT to_tsvector('english', '345 qwe@efd.r '' http://www.com/ http://aew.werc.ewr/?ad=qwe&dw 1aew.werc.ewr/?ad=qwe&dw 2aew.werc.ewr http://3aew.werc.ewr/?ad=qwe&dw http://4aew.werc.ewr http://5aew.werc.ewr:8100/? ad=qwe&dw 6aew.werc.ewr:8100/?ad=qwe&dw 7aew.werc.ewr:8100/?ad=qwe&dw=%20%32 +4.0e-10 qwe qwe qwqwe 234.435 455 5.005 teodor@stack.net qwe-wer asdf qwer jf sdjk ewr1> ewri2 -/usr/local/fff /awdf/dwqe/4325 rewt/ewr wefjn /wqe-324/ewr gist.h gist.h.c gist.c. readline 4.2 4.2. 4.2, readline-4.2 readline-4.2. 234 - wow < jqw <> qwerty'); - to_tsvector ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - '+4.0e-10':28 '-4.2':60,62 '/?':18 '/?ad=qwe&dw':7,10,14,24 '/?ad=qwe&dw=%20%32':27 '/awdf/dwqe/4325':48 '/usr/local/fff':47 '/wqe-324/ewr':51 '1aew.werc.ewr':9 '1aew.werc.ewr/?ad=qwe&dw':8 '234':63 '234.435':32 '2aew.werc.ewr':11 '345':1 '3aew.werc.ewr':13 '3aew.werc.ewr/?ad=qwe&dw':12 '4.2':56,57,58 '455':33 '4aew.werc.ewr':15 '5.005':34 '5aew.werc.ewr:8100':17 '5aew.werc.ewr:8100/?':16 '6aew.werc.ewr:8100':23 '6aew.werc.ewr:8100/?ad=qwe&dw':22 '7aew.werc.ewr:8100':26 '7aew.werc.ewr:8100/?ad=qwe&dw=%20%32':25 'ad':19 'aew.werc.ewr':6 'aew.werc.ewr/?ad=qwe&dw':5 'asdf':39 'dw':21 'efd.r':3 'ewr1':45 'ewri2':46 'gist.c':54 'gist.h':52 'gist.h.c':53 'hjwer':44 'jf':41 'jqw':66 'qwe':2,20,29,30,37 'qwe-wer':36 'qwer':40 'qwerti':67 'qwqwe':31 'readlin':55,59,61 'rewt/ewr':49 'sdjk':42 'teodor@stack.net':35 'wefjn':50 'wer':38 'wow':65 'www.com':4 -(1 row) - -SELECT length(to_tsvector('english', '345 qw')); - length --------- - 2 -(1 row) - -SELECT length(to_tsvector('english', '345 qwe@efd.r '' http://www.com/ http://aew.werc.ewr/?ad=qwe&dw 1aew.werc.ewr/?ad=qwe&dw 2aew.werc.ewr http://3aew.werc.ewr/?ad=qwe&dw http://4aew.werc.ewr http://5aew.werc.ewr:8100/? ad=qwe&dw 6aew.werc.ewr:8100/?ad=qwe&dw 7aew.werc.ewr:8100/?ad=qwe&dw=%20%32 +4.0e-10 qwe qwe qwqwe 234.435 455 5.005 teodor@stack.net qwe-wer asdf qwer jf sdjk ewr1> ewri2 -/usr/local/fff /awdf/dwqe/4325 rewt/ewr wefjn /wqe-324/ewr gist.h gist.h.c gist.c. readline 4.2 4.2. 4.2, readline-4.2 readline-4.2. 234 - wow < jqw <> qwerty')); - length --------- - 53 -(1 row) - -select to_tsquery('english', 'qwe & sKies '); - to_tsquery ---------------- - 'qwe' & 'sky' -(1 row) - -select to_tsquery('simple', 'qwe & sKies '); - to_tsquery ------------------ - 'qwe' & 'skies' -(1 row) - -select to_tsquery('english', '''the wether'':dc & '' sKies '':BC '); - to_tsquery ------------------------- - 'wether':CD & 'sky':BC -(1 row) - -select to_tsquery('english', 'asd&(and|fghj)'); - to_tsquery ----------------- - 'asd' & 'fghj' -(1 row) - -select to_tsquery('english', '(asd&and)|fghj'); - to_tsquery ----------------- - 'asd' | 'fghj' -(1 row) - -select to_tsquery('english', '(asd&!and)|fghj'); - to_tsquery ----------------- - 'asd' | 'fghj' -(1 row) - -select to_tsquery('english', '(the|and&(i&1))&fghj'); - to_tsquery --------------- - '1' & 'fghj' -(1 row) - -select plainto_tsquery('english', 'the and z 1))& fghj'); - plainto_tsquery --------------------- - 'z' & '1' & 'fghj' -(1 row) - -select plainto_tsquery('english', 'foo bar') && plainto_tsquery('english', 'asd'); - ?column? ------------------------ - 'foo' & 'bar' & 'asd' -(1 row) - -select plainto_tsquery('english', 'foo bar') || plainto_tsquery('english', 'asd fg'); - ?column? ------------------------------- - 'foo' & 'bar' | 'asd' & 'fg' -(1 row) - -select plainto_tsquery('english', 'foo bar') || !!plainto_tsquery('english', 'asd fg'); - ?column? ------------------------------------ - 'foo' & 'bar' | !( 'asd' & 'fg' ) -(1 row) - -select plainto_tsquery('english', 'foo bar') && 'asd | fg'; - ?column? ----------------------------------- - 'foo' & 'bar' & ( 'asd' | 'fg' ) -(1 row) - -select 'a b:89 ca:23A,64b d:34c'::tsvector @@ 'd:AC & ca'; - ?column? ----------- - t -(1 row) - -select 'a b:89 ca:23A,64b d:34c'::tsvector @@ 'd:AC & ca:B'; - ?column? ----------- - t -(1 row) - -select 'a b:89 ca:23A,64b d:34c'::tsvector @@ 'd:AC & ca:A'; - ?column? ----------- - t -(1 row) - -select 'a b:89 ca:23A,64b d:34c'::tsvector @@ 'd:AC & ca:C'; - ?column? ----------- - f -(1 row) - -select 'a b:89 ca:23A,64b d:34c'::tsvector @@ 'd:AC & ca:CB'; - ?column? ----------- - t -(1 row) - -CREATE TABLE test_tsvector( t text, a tsvector ); -\copy test_tsvector from 'data/test_tsearch.data' -SELECT count(*) FROM test_tsvector WHERE a @@ 'wr|qh'; - count -------- - 158 -(1 row) - -SELECT count(*) FROM test_tsvector WHERE a @@ 'wr&qh'; - count -------- - 17 -(1 row) - -SELECT count(*) FROM test_tsvector WHERE a @@ 'eq&yt'; - count -------- - 6 -(1 row) - -SELECT count(*) FROM test_tsvector WHERE a @@ 'eq|yt'; - count -------- - 98 -(1 row) - -SELECT count(*) FROM test_tsvector WHERE a @@ '(eq&yt)|(wr&qh)'; - count -------- - 23 -(1 row) - -SELECT count(*) FROM test_tsvector WHERE a @@ '(eq|yt)&(wr|qh)'; - count -------- - 39 -(1 row) - -create index wowidx on test_tsvector using gist (a); -set enable_seqscan=off; -SELECT count(*) FROM test_tsvector WHERE a @@ 'wr|qh'; - count -------- - 158 -(1 row) - -SELECT count(*) FROM test_tsvector WHERE a @@ 'wr&qh'; - count -------- - 17 -(1 row) - -SELECT count(*) FROM test_tsvector WHERE a @@ 'eq&yt'; - count -------- - 6 -(1 row) - -SELECT count(*) FROM test_tsvector WHERE a @@ 'eq|yt'; - count -------- - 98 -(1 row) - -SELECT count(*) FROM test_tsvector WHERE a @@ '(eq&yt)|(wr&qh)'; - count -------- - 23 -(1 row) - -SELECT count(*) FROM test_tsvector WHERE a @@ '(eq|yt)&(wr|qh)'; - count -------- - 39 -(1 row) - -select set_curcfg('english'); - set_curcfg ------------- - -(1 row) - -CREATE TRIGGER tsvectorupdate -BEFORE UPDATE OR INSERT ON test_tsvector -FOR EACH ROW EXECUTE PROCEDURE tsearch2(a, t); -SELECT count(*) FROM test_tsvector WHERE a @@ to_tsquery('345&qwerty'); - count -------- - 0 -(1 row) - -INSERT INTO test_tsvector (t) VALUES ('345 qwerty'); -SELECT count(*) FROM test_tsvector WHERE a @@ to_tsquery('345&qwerty'); - count -------- - 1 -(1 row) - -UPDATE test_tsvector SET t = null WHERE t = '345 qwerty'; -SELECT count(*) FROM test_tsvector WHERE a @@ to_tsquery('345&qwerty'); - count -------- - 0 -(1 row) - -insert into test_tsvector (t) values ('345 qwerty copyright'); -select count(*) FROM test_tsvector WHERE a @@ to_tsquery('345&qwerty'); - count -------- - 1 -(1 row) - -select count(*) FROM test_tsvector WHERE a @@ to_tsquery('copyright'); - count -------- - 1 -(1 row) - -select rank(' a:1 s:2C d g'::tsvector, 'a | s'); - rank ------------ - 0.0911891 -(1 row) - -select rank(' a:1 s:2B d g'::tsvector, 'a | s'); - rank ----------- - 0.151982 -(1 row) - -select rank(' a:1 s:2 d g'::tsvector, 'a | s'); - rank ------------ - 0.0607927 -(1 row) - -select rank(' a:1 s:2C d g'::tsvector, 'a & s'); - rank ----------- - 0.140153 -(1 row) - -select rank(' a:1 s:2B d g'::tsvector, 'a & s'); - rank ----------- - 0.198206 -(1 row) - -select rank(' a:1 s:2 d g'::tsvector, 'a & s'); - rank ------------ - 0.0991032 -(1 row) - -insert into test_tsvector (t) values ('foo bar foo the over foo qq bar'); -drop trigger tsvectorupdate on test_tsvector; -select * from stat('select a from test_tsvector') order by ndoc desc, nentry desc, word collate "C"; - word | ndoc | nentry ------------+------+-------- - qq | 109 | 109 - qt | 102 | 102 - qe | 100 | 100 - qh | 98 | 98 - qw | 98 | 98 - qa | 97 | 97 - ql | 94 | 94 - qs | 94 | 94 - qi | 92 | 92 - qr | 92 | 92 - qj | 91 | 91 - qd | 87 | 87 - qz | 87 | 87 - qc | 86 | 86 - qn | 86 | 86 - qv | 85 | 85 - qo | 84 | 84 - qy | 84 | 84 - wp | 84 | 84 - qf | 81 | 81 - qk | 80 | 80 - wt | 80 | 80 - qu | 79 | 79 - qg | 78 | 78 - wb | 78 | 78 - qx | 77 | 77 - wr | 77 | 77 - ws | 73 | 73 - wy | 73 | 73 - wa | 72 | 72 - wf | 70 | 70 - wg | 70 | 70 - wi | 70 | 70 - wu | 70 | 70 - wc | 69 | 69 - wj | 69 | 69 - qp | 68 | 68 - wh | 68 | 68 - wv | 68 | 68 - qb | 66 | 66 - eu | 65 | 65 - we | 65 | 65 - wl | 65 | 65 - wq | 65 | 65 - wk | 64 | 64 - ee | 63 | 63 - eo | 63 | 63 - qm | 63 | 63 - wn | 63 | 63 - ef | 62 | 62 - eh | 62 | 62 - ex | 62 | 62 - re | 62 | 62 - rl | 62 | 62 - rr | 62 | 62 - eb | 61 | 61 - ek | 61 | 61 - ww | 61 | 61 - ea | 60 | 60 - ei | 60 | 60 - em | 60 | 60 - eq | 60 | 60 - ew | 60 | 60 - ro | 60 | 60 - rw | 60 | 60 - tl | 60 | 60 - eg | 59 | 59 - en | 59 | 59 - ez | 59 | 59 - rj | 59 | 59 - ry | 59 | 59 - tw | 59 | 59 - tx | 59 | 59 - ej | 58 | 58 - es | 58 | 58 - ra | 58 | 58 - rd | 58 | 58 - rg | 58 | 58 - rx | 58 | 58 - tb | 58 | 58 - wd | 58 | 58 - ed | 57 | 57 - tc | 57 | 57 - wx | 57 | 57 - er | 56 | 56 - wm | 56 | 56 - wo | 56 | 56 - yw | 56 | 56 - ep | 55 | 55 - rk | 55 | 55 - rp | 55 | 55 - rz | 55 | 55 - ta | 55 | 55 - rq | 54 | 54 - yn | 54 | 54 - ec | 53 | 53 - el | 53 | 53 - ru | 53 | 53 - rv | 53 | 53 - tz | 53 | 53 - un | 53 | 53 - wz | 53 | 53 - ys | 53 | 53 - oe | 52 | 52 - tn | 52 | 52 - tq | 52 | 52 - ty | 52 | 52 - uq | 52 | 52 - yg | 52 | 52 - ym | 52 | 52 - oi | 51 | 51 - to | 51 | 51 - yi | 51 | 51 - pn | 50 | 50 - rb | 50 | 50 - ri | 50 | 50 - rn | 50 | 50 - ti | 50 | 50 - tv | 50 | 50 - um | 50 | 50 - ut | 50 | 50 - ya | 50 | 50 - et | 49 | 49 - ix | 49 | 49 - ox | 49 | 49 - q3 | 49 | 49 - yf | 49 | 49 - yl | 49 | 49 - yo | 49 | 49 - yr | 49 | 49 - ev | 48 | 48 - ey | 48 | 48 - ot | 48 | 48 - rc | 48 | 48 - rm | 48 | 48 - th | 48 | 48 - uo | 48 | 48 - ia | 47 | 47 - q1 | 47 | 47 - rh | 47 | 47 - yq | 47 | 47 - yz | 47 | 47 - av | 46 | 46 - im | 46 | 46 - os | 46 | 46 - tk | 46 | 46 - yy | 46 | 46 - ir | 45 | 45 - iv | 45 | 45 - iw | 45 | 45 - oj | 45 | 45 - pl | 45 | 45 - pv | 45 | 45 - te | 45 | 45 - tu | 45 | 45 - uv | 45 | 45 - ux | 45 | 45 - yd | 45 | 45 - yx | 45 | 45 - ij | 44 | 44 - pa | 44 | 44 - se | 44 | 44 - tg | 44 | 44 - ue | 44 | 44 - yb | 44 | 44 - yt | 44 | 44 - if | 43 | 43 - ik | 43 | 43 - in | 43 | 43 - ph | 43 | 43 - pj | 43 | 43 - q5 | 43 | 43 - rt | 43 | 43 - ub | 43 | 43 - ud | 43 | 43 - uh | 43 | 43 - uj | 43 | 43 - w7 | 43 | 43 - ye | 43 | 43 - yv | 43 | 43 - db | 42 | 42 - do | 42 | 42 - id | 42 | 42 - ie | 42 | 42 - ii | 42 | 42 - of | 42 | 42 - pr | 42 | 42 - q4 | 42 | 42 - rf | 42 | 42 - td | 42 | 42 - uk | 42 | 42 - up | 42 | 42 - yh | 42 | 42 - yk | 42 | 42 - io | 41 | 41 - it | 41 | 41 - pb | 41 | 41 - q0 | 41 | 41 - q7 | 41 | 41 - rs | 41 | 41 - tj | 41 | 41 - ur | 41 | 41 - ig | 40 | 40 - iu | 40 | 40 - iy | 40 | 40 - od | 40 | 40 - q6 | 40 | 40 - tt | 40 | 40 - ug | 40 | 40 - ul | 40 | 40 - us | 40 | 40 - uu | 40 | 40 - uz | 40 | 40 - ah | 39 | 39 - ar | 39 | 39 - as | 39 | 39 - dl | 39 | 39 - dt | 39 | 39 - hk | 39 | 39 - iq | 39 | 39 - is | 39 | 39 - oc | 39 | 39 - ov | 39 | 39 - oy | 39 | 39 - uf | 39 | 39 - ui | 39 | 39 - aa | 38 | 38 - ad | 38 | 38 - fh | 38 | 38 - gm | 38 | 38 - ic | 38 | 38 - jd | 38 | 38 - om | 38 | 38 - or | 38 | 38 - oz | 38 | 38 - pm | 38 | 38 - q8 | 38 | 38 - sf | 38 | 38 - sm | 38 | 38 - sv | 38 | 38 - uc | 38 | 38 - ak | 37 | 37 - aq | 37 | 37 - di | 37 | 37 - e4 | 37 | 37 - fi | 37 | 37 - fx | 37 | 37 - ha | 37 | 37 - hp | 37 | 37 - ih | 37 | 37 - og | 37 | 37 - po | 37 | 37 - pw | 37 | 37 - sn | 37 | 37 - su | 37 | 37 - sw | 37 | 37 - w6 | 37 | 37 - yj | 37 | 37 - yu | 37 | 37 - ag | 36 | 36 - am | 36 | 36 - at | 36 | 36 - e1 | 36 | 36 - ff | 36 | 36 - gx | 36 | 36 - he | 36 | 36 - hj | 36 | 36 - ib | 36 | 36 - iz | 36 | 36 - lm | 36 | 36 - ok | 36 | 36 - pk | 36 | 36 - pp | 36 | 36 - pu | 36 | 36 - sp | 36 | 36 - tf | 36 | 36 - tm | 36 | 36 - ay | 35 | 35 - dy | 35 | 35 - fu | 35 | 35 - ku | 35 | 35 - lh | 35 | 35 - lq | 35 | 35 - o6 | 35 | 35 - ob | 35 | 35 - on | 35 | 35 - op | 35 | 35 - pd | 35 | 35 - ps | 35 | 35 - si | 35 | 35 - sl | 35 | 35 - sx | 35 | 35 - tp | 35 | 35 - tr | 35 | 35 - w3 | 35 | 35 - y1 | 35 | 35 - al | 34 | 34 - ap | 34 | 34 - az | 34 | 34 - dc | 34 | 34 - dd | 34 | 34 - dz | 34 | 34 - e0 | 34 | 34 - fj | 34 | 34 - fp | 34 | 34 - gd | 34 | 34 - gg | 34 | 34 - gk | 34 | 34 - go | 34 | 34 - ho | 34 | 34 - jc | 34 | 34 - oa | 34 | 34 - oh | 34 | 34 - oo | 34 | 34 - pe | 34 | 34 - px | 34 | 34 - sd | 34 | 34 - sq | 34 | 34 - sy | 34 | 34 - ab | 33 | 33 - ae | 33 | 33 - af | 33 | 33 - aw | 33 | 33 - e5 | 33 | 33 - fk | 33 | 33 - gu | 33 | 33 - gy | 33 | 33 - hb | 33 | 33 - hm | 33 | 33 - hy | 33 | 33 - jl | 33 | 33 - jr | 33 | 33 - ls | 33 | 33 - oq | 33 | 33 - pt | 33 | 33 - sa | 33 | 33 - sh | 33 | 33 - sj | 33 | 33 - so | 33 | 33 - sz | 33 | 33 - t7 | 33 | 33 - uw | 33 | 33 - w8 | 33 | 33 - y0 | 33 | 33 - yp | 33 | 33 - dh | 32 | 32 - dp | 32 | 32 - dq | 32 | 32 - e7 | 32 | 32 - fn | 32 | 32 - fo | 32 | 32 - fr | 32 | 32 - ga | 32 | 32 - gq | 32 | 32 - hh | 32 | 32 - il | 32 | 32 - ip | 32 | 32 - jv | 32 | 32 - lc | 32 | 32 - ol | 32 | 32 - pc | 32 | 32 - q9 | 32 | 32 - ds | 31 | 31 - e9 | 31 | 31 - fd | 31 | 31 - fe | 31 | 31 - ft | 31 | 31 - gs | 31 | 31 - hl | 31 | 31 - hs | 31 | 31 - jb | 31 | 31 - kc | 31 | 31 - kw | 31 | 31 - mj | 31 | 31 - q2 | 31 | 31 - r3 | 31 | 31 - sb | 31 | 31 - sk | 31 | 31 - ts | 31 | 31 - ua | 31 | 31 - yc | 31 | 31 - zw | 31 | 31 - ao | 30 | 30 - du | 30 | 30 - fw | 30 | 30 - gj | 30 | 30 - hu | 30 | 30 - kh | 30 | 30 - kl | 30 | 30 - kv | 30 | 30 - ld | 30 | 30 - lf | 30 | 30 - pq | 30 | 30 - py | 30 | 30 - sc | 30 | 30 - sr | 30 | 30 - uy | 30 | 30 - vg | 30 | 30 - w2 | 30 | 30 - xg | 30 | 30 - xo | 30 | 30 - au | 29 | 29 - cx | 29 | 29 - fv | 29 | 29 - gh | 29 | 29 - gl | 29 | 29 - gt | 29 | 29 - hw | 29 | 29 - ji | 29 | 29 - km | 29 | 29 - la | 29 | 29 - ou | 29 | 29 - r0 | 29 | 29 - w0 | 29 | 29 - y9 | 29 | 29 - zm | 29 | 29 - zs | 29 | 29 - zy | 29 | 29 - ax | 28 | 28 - cd | 28 | 28 - dj | 28 | 28 - dn | 28 | 28 - dr | 28 | 28 - ht | 28 | 28 - jf | 28 | 28 - lo | 28 | 28 - lr | 28 | 28 - na | 28 | 28 - ng | 28 | 28 - r8 | 28 | 28 - ss | 28 | 28 - xt | 28 | 28 - y6 | 28 | 28 - aj | 27 | 27 - ca | 27 | 27 - cg | 27 | 27 - df | 27 | 27 - dg | 27 | 27 - dv | 27 | 27 - gc | 27 | 27 - gn | 27 | 27 - gr | 27 | 27 - hd | 27 | 27 - i8 | 27 | 27 - jn | 27 | 27 - jt | 27 | 27 - lp | 27 | 27 - o9 | 27 | 27 - ow | 27 | 27 - r9 | 27 | 27 - t8 | 27 | 27 - u5 | 27 | 27 - w4 | 27 | 27 - xm | 27 | 27 - zz | 27 | 27 - a2 | 26 | 26 - ac | 26 | 26 - ai | 26 | 26 - cm | 26 | 26 - cu | 26 | 26 - cw | 26 | 26 - dk | 26 | 26 - e2 | 26 | 26 - fc | 26 | 26 - fg | 26 | 26 - fl | 26 | 26 - fs | 26 | 26 - ge | 26 | 26 - gv | 26 | 26 - hc | 26 | 26 - hi | 26 | 26 - hx | 26 | 26 - jj | 26 | 26 - jm | 26 | 26 - kg | 26 | 26 - kk | 26 | 26 - kn | 26 | 26 - ko | 26 | 26 - kt | 26 | 26 - ln | 26 | 26 - mx | 26 | 26 - pg | 26 | 26 - r4 | 26 | 26 - t6 | 26 | 26 - u1 | 26 | 26 - u4 | 26 | 26 - vi | 26 | 26 - vr | 26 | 26 - w1 | 26 | 26 - w9 | 26 | 26 - xk | 26 | 26 - xs | 26 | 26 - zf | 26 | 26 - bb | 25 | 25 - dm | 25 | 25 - dw | 25 | 25 - e8 | 25 | 25 - fb | 25 | 25 - gw | 25 | 25 - h8 | 25 | 25 - hf | 25 | 25 - hg | 25 | 25 - hn | 25 | 25 - hv | 25 | 25 - i0 | 25 | 25 - i3 | 25 | 25 - jg | 25 | 25 - jo | 25 | 25 - jx | 25 | 25 - kq | 25 | 25 - lw | 25 | 25 - lx | 25 | 25 - o3 | 25 | 25 - p7 | 25 | 25 - pf | 25 | 25 - pi | 25 | 25 - pz | 25 | 25 - r2 | 25 | 25 - r5 | 25 | 25 - t9 | 25 | 25 - u7 | 25 | 25 - ve | 25 | 25 - vu | 25 | 25 - y5 | 25 | 25 - y8 | 25 | 25 - zt | 25 | 25 - an | 24 | 24 - bj | 24 | 24 - dx | 24 | 24 - fm | 24 | 24 - fz | 24 | 24 - gb | 24 | 24 - gi | 24 | 24 - gp | 24 | 24 - hr | 24 | 24 - hz | 24 | 24 - i5 | 24 | 24 - jq | 24 | 24 - kb | 24 | 24 - ke | 24 | 24 - kf | 24 | 24 - kp | 24 | 24 - lv | 24 | 24 - lz | 24 | 24 - o8 | 24 | 24 - r1 | 24 | 24 - s7 | 24 | 24 - sg | 24 | 24 - u3 | 24 | 24 - vj | 24 | 24 - vt | 24 | 24 - w5 | 24 | 24 - zj | 24 | 24 - be | 23 | 23 - bi | 23 | 23 - bn | 23 | 23 - cn | 23 | 23 - cy | 23 | 23 - da | 23 | 23 - e6 | 23 | 23 - fa | 23 | 23 - js | 23 | 23 - ki | 23 | 23 - kz | 23 | 23 - li | 23 | 23 - mt | 23 | 23 - mz | 23 | 23 - nu | 23 | 23 - o2 | 23 | 23 - p5 | 23 | 23 - p8 | 23 | 23 - r7 | 23 | 23 - t0 | 23 | 23 - t1 | 23 | 23 - t3 | 23 | 23 - vm | 23 | 23 - xh | 23 | 23 - xx | 23 | 23 - zp | 23 | 23 - zr | 23 | 23 - a3 | 22 | 22 - bg | 22 | 22 - de | 22 | 22 - e3 | 22 | 22 - fq | 22 | 22 - i2 | 22 | 22 - i7 | 22 | 22 - ja | 22 | 22 - jk | 22 | 22 - jy | 22 | 22 - kr | 22 | 22 - kx | 22 | 22 - ly | 22 | 22 - nb | 22 | 22 - nh | 22 | 22 - ns | 22 | 22 - s3 | 22 | 22 - u2 | 22 | 22 - vn | 22 | 22 - xe | 22 | 22 - y4 | 22 | 22 - zh | 22 | 22 - zo | 22 | 22 - zq | 22 | 22 - a1 | 21 | 21 - bl | 21 | 21 - bo | 21 | 21 - cb | 21 | 21 - ch | 21 | 21 - co | 21 | 21 - cq | 21 | 21 - cv | 21 | 21 - d7 | 21 | 21 - g8 | 21 | 21 - je | 21 | 21 - jp | 21 | 21 - jz | 21 | 21 - lg | 21 | 21 - me | 21 | 21 - nc | 21 | 21 - p4 | 21 | 21 - st | 21 | 21 - vb | 21 | 21 - vw | 21 | 21 - vz | 21 | 21 - xj | 21 | 21 - xq | 21 | 21 - xu | 21 | 21 - xy | 21 | 21 - zb | 21 | 21 - bv | 20 | 20 - bz | 20 | 20 - cj | 20 | 20 - cp | 20 | 20 - cs | 20 | 20 - d8 | 20 | 20 - ju | 20 | 20 - k0 | 20 | 20 - ks | 20 | 20 - ky | 20 | 20 - l1 | 20 | 20 - lb | 20 | 20 - lj | 20 | 20 - lu | 20 | 20 - nm | 20 | 20 - nw | 20 | 20 - nz | 20 | 20 - o7 | 20 | 20 - p6 | 20 | 20 - vh | 20 | 20 - vp | 20 | 20 - vs | 20 | 20 - xb | 20 | 20 - xr | 20 | 20 - z3 | 20 | 20 - zv | 20 | 20 - bq | 19 | 19 - br | 19 | 19 - by | 19 | 19 - cl | 19 | 19 - d2 | 19 | 19 - f1 | 19 | 19 - f4 | 19 | 19 - gf | 19 | 19 - hq | 19 | 19 - k9 | 19 | 19 - ka | 19 | 19 - kd | 19 | 19 - kj | 19 | 19 - md | 19 | 19 - mi | 19 | 19 - ml | 19 | 19 - my | 19 | 19 - nj | 19 | 19 - ny | 19 | 19 - o1 | 19 | 19 - s4 | 19 | 19 - s8 | 19 | 19 - t5 | 19 | 19 - u0 | 19 | 19 - xl | 19 | 19 - zg | 19 | 19 - zi | 19 | 19 - a5 | 18 | 18 - b9 | 18 | 18 - bh | 18 | 18 - bx | 18 | 18 - d3 | 18 | 18 - fy | 18 | 18 - g2 | 18 | 18 - i4 | 18 | 18 - i6 | 18 | 18 - i9 | 18 | 18 - jw | 18 | 18 - lk | 18 | 18 - mb | 18 | 18 - mv | 18 | 18 - nd | 18 | 18 - nr | 18 | 18 - nt | 18 | 18 - t2 | 18 | 18 - xf | 18 | 18 - xv | 18 | 18 - zc | 18 | 18 - zd | 18 | 18 - a7 | 17 | 17 - bc | 17 | 17 - bd | 17 | 17 - ce | 17 | 17 - cf | 17 | 17 - cr | 17 | 17 - g9 | 17 | 17 - j0 | 17 | 17 - j5 | 17 | 17 - mp | 17 | 17 - mr | 17 | 17 - mw | 17 | 17 - nk | 17 | 17 - no | 17 | 17 - o0 | 17 | 17 - o4 | 17 | 17 - s0 | 17 | 17 - s1 | 17 | 17 - t4 | 17 | 17 - u9 | 17 | 17 - vf | 17 | 17 - vx | 17 | 17 - x3 | 17 | 17 - xi | 17 | 17 - xn | 17 | 17 - xz | 17 | 17 - zl | 17 | 17 - zn | 17 | 17 - a0 | 16 | 16 - bu | 16 | 16 - bw | 16 | 16 - ci | 16 | 16 - ck | 16 | 16 - d0 | 16 | 16 - d4 | 16 | 16 - d6 | 16 | 16 - f5 | 16 | 16 - g1 | 16 | 16 - gz | 16 | 16 - h4 | 16 | 16 - jh | 16 | 16 - l4 | 16 | 16 - lt | 16 | 16 - mg | 16 | 16 - mh | 16 | 16 - mo | 16 | 16 - ni | 16 | 16 - nl | 16 | 16 - nq | 16 | 16 - p2 | 16 | 16 - u8 | 16 | 16 - v9 | 16 | 16 - vl | 16 | 16 - vo | 16 | 16 - xp | 16 | 16 - y3 | 16 | 16 - y7 | 16 | 16 - z7 | 16 | 16 - za | 16 | 16 - zx | 16 | 16 - bf | 15 | 15 - bp | 15 | 15 - cc | 15 | 15 - g0 | 15 | 15 - j2 | 15 | 15 - j9 | 15 | 15 - l6 | 15 | 15 - le | 15 | 15 - ll | 15 | 15 - m8 | 15 | 15 - ma | 15 | 15 - mu | 15 | 15 - nf | 15 | 15 - r6 | 15 | 15 - s5 | 15 | 15 - vd | 15 | 15 - vk | 15 | 15 - xa | 15 | 15 - xw | 15 | 15 - y2 | 15 | 15 - z8 | 15 | 15 - ze | 15 | 15 - zu | 15 | 15 - a6 | 14 | 14 - bk | 14 | 14 - bt | 14 | 14 - c0 | 14 | 14 - f8 | 14 | 14 - g3 | 14 | 14 - g4 | 14 | 14 - g7 | 14 | 14 - h6 | 14 | 14 - h7 | 14 | 14 - h9 | 14 | 14 - i1 | 14 | 14 - k1 | 14 | 14 - k2 | 14 | 14 - k6 | 14 | 14 - k7 | 14 | 14 - mc | 14 | 14 - nn | 14 | 14 - p9 | 14 | 14 - u6 | 14 | 14 - xd | 14 | 14 - z6 | 14 | 14 - zk | 14 | 14 - a4 | 13 | 13 - a9 | 13 | 13 - bm | 13 | 13 - cz | 13 | 13 - f2 | 13 | 13 - f3 | 13 | 13 - f6 | 13 | 13 - g6 | 13 | 13 - h2 | 13 | 13 - j1 | 13 | 13 - k5 | 13 | 13 - m1 | 13 | 13 - mf | 13 | 13 - mq | 13 | 13 - np | 13 | 13 - nx | 13 | 13 - o5 | 13 | 13 - p0 | 13 | 13 - p1 | 13 | 13 - s6 | 13 | 13 - s9 | 13 | 13 - v6 | 13 | 13 - va | 13 | 13 - vc | 13 | 13 - xc | 13 | 13 - z0 | 13 | 13 - c9 | 12 | 12 - d1 | 12 | 12 - h0 | 12 | 12 - h1 | 12 | 12 - j8 | 12 | 12 - k4 | 12 | 12 - l5 | 12 | 12 - l9 | 12 | 12 - m2 | 12 | 12 - m6 | 12 | 12 - m9 | 12 | 12 - n7 | 12 | 12 - nv | 12 | 12 - p3 | 12 | 12 - vq | 12 | 12 - vy | 12 | 12 - x1 | 12 | 12 - x2 | 12 | 12 - z5 | 12 | 12 - c1 | 11 | 11 - c3 | 11 | 11 - ct | 11 | 11 - f9 | 11 | 11 - g5 | 11 | 11 - j6 | 11 | 11 - l8 | 11 | 11 - n1 | 11 | 11 - v7 | 11 | 11 - vv | 11 | 11 - x5 | 11 | 11 - x8 | 11 | 11 - z2 | 11 | 11 - b0 | 10 | 10 - b2 | 10 | 10 - b8 | 10 | 10 - c6 | 10 | 10 - f0 | 10 | 10 - f7 | 10 | 10 - h5 | 10 | 10 - j3 | 10 | 10 - j4 | 10 | 10 - j7 | 10 | 10 - l7 | 10 | 10 - m0 | 10 | 10 - m7 | 10 | 10 - mm | 10 | 10 - mn | 10 | 10 - n8 | 10 | 10 - v1 | 10 | 10 - x0 | 10 | 10 - x6 | 10 | 10 - x7 | 10 | 10 - x9 | 10 | 10 - a8 | 9 | 9 - b1 | 9 | 9 - b4 | 9 | 9 - b5 | 9 | 9 - b6 | 9 | 9 - ba | 9 | 9 - bs | 9 | 9 - c5 | 9 | 9 - d5 | 9 | 9 - k8 | 9 | 9 - l0 | 9 | 9 - m5 | 9 | 9 - mk | 9 | 9 - ms | 9 | 9 - n3 | 9 | 9 - n4 | 9 | 9 - n6 | 9 | 9 - ne | 9 | 9 - v0 | 9 | 9 - v3 | 9 | 9 - v5 | 9 | 9 - v8 | 9 | 9 - b3 | 8 | 8 - b7 | 8 | 8 - c2 | 8 | 8 - c7 | 8 | 8 - c8 | 8 | 8 - d9 | 8 | 8 - k3 | 8 | 8 - l3 | 8 | 8 - m3 | 8 | 8 - m4 | 8 | 8 - n0 | 8 | 8 - n5 | 8 | 8 - v4 | 8 | 8 - x4 | 8 | 8 - z1 | 8 | 8 - z9 | 8 | 8 - l2 | 7 | 7 - s2 | 7 | 7 - z4 | 7 | 7 - 1l | 6 | 6 - 1o | 6 | 6 - 1t | 6 | 6 - 2e | 6 | 6 - 2o | 6 | 6 - c4 | 6 | 6 - h3 | 6 | 6 - n2 | 6 | 6 - n9 | 6 | 6 - v2 | 6 | 6 - 2l | 5 | 5 - 2u | 5 | 5 - 3k | 5 | 5 - 4p | 5 | 5 - 18 | 4 | 4 - 1a | 4 | 4 - 1i | 4 | 4 - 2s | 4 | 4 - 3q | 4 | 4 - 3y | 4 | 4 - 5y | 4 | 4 - 1f | 3 | 3 - 1h | 3 | 3 - 1m | 3 | 3 - 1p | 3 | 3 - 1s | 3 | 3 - 1v | 3 | 3 - 1x | 3 | 3 - 27 | 3 | 3 - 2a | 3 | 3 - 2b | 3 | 3 - 2h | 3 | 3 - 2n | 3 | 3 - 2p | 3 | 3 - 2v | 3 | 3 - 2y | 3 | 3 - 3d | 3 | 3 - 3w | 3 | 3 - 3z | 3 | 3 - 4a | 3 | 3 - 4d | 3 | 3 - 4v | 3 | 3 - 4z | 3 | 3 - 5e | 3 | 3 - 5i | 3 | 3 - 5k | 3 | 3 - 5o | 3 | 3 - 5t | 3 | 3 - 6b | 3 | 3 - 6d | 3 | 3 - 6o | 3 | 3 - 6w | 3 | 3 - 7a | 3 | 3 - 7h | 3 | 3 - 7r | 3 | 3 - 93 | 3 | 3 - 10 | 2 | 2 - 12 | 2 | 2 - 15 | 2 | 2 - 16 | 2 | 2 - 19 | 2 | 2 - 1b | 2 | 2 - 1d | 2 | 2 - 1g | 2 | 2 - 1j | 2 | 2 - 1n | 2 | 2 - 1r | 2 | 2 - 1u | 2 | 2 - 1w | 2 | 2 - 1y | 2 | 2 - 20 | 2 | 2 - 25 | 2 | 2 - 2d | 2 | 2 - 2i | 2 | 2 - 2j | 2 | 2 - 2k | 2 | 2 - 2q | 2 | 2 - 2r | 2 | 2 - 2t | 2 | 2 - 2w | 2 | 2 - 2z | 2 | 2 - 3b | 2 | 2 - 3f | 2 | 2 - 3h | 2 | 2 - 3o | 2 | 2 - 3p | 2 | 2 - 3r | 2 | 2 - 3s | 2 | 2 - 3v | 2 | 2 - 42 | 2 | 2 - 43 | 2 | 2 - 4f | 2 | 2 - 4g | 2 | 2 - 4h | 2 | 2 - 4j | 2 | 2 - 4m | 2 | 2 - 4r | 2 | 2 - 4s | 2 | 2 - 4t | 2 | 2 - 4u | 2 | 2 - 5c | 2 | 2 - 5f | 2 | 2 - 5h | 2 | 2 - 5p | 2 | 2 - 5q | 2 | 2 - 5z | 2 | 2 - 6a | 2 | 2 - 6h | 2 | 2 - 6q | 2 | 2 - 6r | 2 | 2 - 6t | 2 | 2 - 6y | 2 | 2 - 70 | 2 | 2 - 7c | 2 | 2 - 7g | 2 | 2 - 7k | 2 | 2 - 7o | 2 | 2 - 7u | 2 | 2 - 8j | 2 | 2 - 8w | 2 | 2 - 9f | 2 | 2 - 9y | 2 | 2 - foo | 1 | 3 - bar | 1 | 2 - 0e | 1 | 1 - 0h | 1 | 1 - 0p | 1 | 1 - 0w | 1 | 1 - 0z | 1 | 1 - 11 | 1 | 1 - 13 | 1 | 1 - 14 | 1 | 1 - 17 | 1 | 1 - 1k | 1 | 1 - 1q | 1 | 1 - 1z | 1 | 1 - 24 | 1 | 1 - 26 | 1 | 1 - 28 | 1 | 1 - 2f | 1 | 1 - 30 | 1 | 1 - 345 | 1 | 1 - 37 | 1 | 1 - 39 | 1 | 1 - 3a | 1 | 1 - 3e | 1 | 1 - 3g | 1 | 1 - 3i | 1 | 1 - 3m | 1 | 1 - 3t | 1 | 1 - 3u | 1 | 1 - 40 | 1 | 1 - 41 | 1 | 1 - 44 | 1 | 1 - 45 | 1 | 1 - 48 | 1 | 1 - 4b | 1 | 1 - 4c | 1 | 1 - 4i | 1 | 1 - 4k | 1 | 1 - 4n | 1 | 1 - 4o | 1 | 1 - 4q | 1 | 1 - 4w | 1 | 1 - 4y | 1 | 1 - 51 | 1 | 1 - 55 | 1 | 1 - 56 | 1 | 1 - 5a | 1 | 1 - 5d | 1 | 1 - 5g | 1 | 1 - 5j | 1 | 1 - 5l | 1 | 1 - 5s | 1 | 1 - 5u | 1 | 1 - 5x | 1 | 1 - 64 | 1 | 1 - 68 | 1 | 1 - 6c | 1 | 1 - 6f | 1 | 1 - 6g | 1 | 1 - 6i | 1 | 1 - 6k | 1 | 1 - 6n | 1 | 1 - 6p | 1 | 1 - 6s | 1 | 1 - 6u | 1 | 1 - 6x | 1 | 1 - 72 | 1 | 1 - 7f | 1 | 1 - 7j | 1 | 1 - 7n | 1 | 1 - 7p | 1 | 1 - 7w | 1 | 1 - 7y | 1 | 1 - 7z | 1 | 1 - 80 | 1 | 1 - 82 | 1 | 1 - 85 | 1 | 1 - 8d | 1 | 1 - 8i | 1 | 1 - 8l | 1 | 1 - 8n | 1 | 1 - 8p | 1 | 1 - 8t | 1 | 1 - 8x | 1 | 1 - 95 | 1 | 1 - 97 | 1 | 1 - 9a | 1 | 1 - 9e | 1 | 1 - 9h | 1 | 1 - 9r | 1 | 1 - 9w | 1 | 1 - copyright | 1 | 1 - qwerti | 1 | 1 -(1146 rows) - -insert into test_tsvector values ('1', 'a:1a,2,3b b:5a,6a,7c,8'); -insert into test_tsvector values ('1', 'a:1a,2,3c b:5a,6b,7c,8b'); -select * from stat('select a from test_tsvector','a') order by ndoc desc, nentry desc, word collate "C"; - word | ndoc | nentry -------+------+-------- - b | 2 | 3 - a | 2 | 2 -(2 rows) - -select * from stat('select a from test_tsvector','b') order by ndoc desc, nentry desc, word collate "C"; - word | ndoc | nentry -------+------+-------- - b | 1 | 2 - a | 1 | 1 -(2 rows) - -select * from stat('select a from test_tsvector','c') order by ndoc desc, nentry desc, word collate "C"; - word | ndoc | nentry -------+------+-------- - b | 2 | 2 - a | 1 | 1 -(2 rows) - -select * from stat('select a from test_tsvector','d') order by ndoc desc, nentry desc, word collate "C"; - word | ndoc | nentry ------------+------+-------- - a | 2 | 2 - foo | 1 | 3 - bar | 1 | 2 - 345 | 1 | 1 - b | 1 | 1 - copyright | 1 | 1 - qq | 1 | 1 - qwerti | 1 | 1 -(8 rows) - -select * from stat('select a from test_tsvector','ad') order by ndoc desc, nentry desc, word collate "C"; - word | ndoc | nentry ------------+------+-------- - a | 2 | 4 - b | 2 | 4 - foo | 1 | 3 - bar | 1 | 2 - 345 | 1 | 1 - copyright | 1 | 1 - qq | 1 | 1 - qwerti | 1 | 1 -(8 rows) - -select to_tsquery('english', 'skies & books'); - to_tsquery ----------------- - 'sky' & 'book' -(1 row) - -select rank_cd(to_tsvector('Erosion It took the sea a thousand years, -A thousand years to trace -The granite features of this cliff -In crag and scarp and base. -It took the sea an hour one night -An hour of storm to place -The sculpture of these granite seams, -Upon a woman s face. E. J. Pratt (1882 1964) -'), to_tsquery('sea&thousand&years')); - rank_cd ------------ - 0.0555556 -(1 row) - -select rank_cd(to_tsvector('Erosion It took the sea a thousand years, -A thousand years to trace -The granite features of this cliff -In crag and scarp and base. -It took the sea an hour one night -An hour of storm to place -The sculpture of these granite seams, -Upon a woman s face. E. J. Pratt (1882 1964) -'), to_tsquery('granite&sea')); - rank_cd ------------ - 0.0238095 -(1 row) - -select rank_cd(to_tsvector('Erosion It took the sea a thousand years, -A thousand years to trace -The granite features of this cliff -In crag and scarp and base. -It took the sea an hour one night -An hour of storm to place -The sculpture of these granite seams, -Upon a woman s face. E. J. Pratt (1882 1964) -'), to_tsquery('sea')); - rank_cd ---------- - 0.2 -(1 row) - -select headline('Erosion It took the sea a thousand years, -A thousand years to trace -The granite features of this cliff -In crag and scarp and base. -It took the sea an hour one night -An hour of storm to place -The sculpture of these granite seams, -Upon a woman s face. E. J. Pratt (1882 1964) -', to_tsquery('sea&thousand&years')); - headline --------------------------------------------- - sea a thousand years,+ - A thousand years to trace + - The granite features of this cliff -(1 row) - -select headline('Erosion It took the sea a thousand years, -A thousand years to trace -The granite features of this cliff -In crag and scarp and base. -It took the sea an hour one night -An hour of storm to place -The sculpture of these granite seams, -Upon a woman s face. E. J. Pratt (1882 1964) -', to_tsquery('granite&sea')); - headline -------------------------------------------- - sea a thousand years, + - A thousand years to trace + - The granite features of this cliff -(1 row) - -select headline('Erosion It took the sea a thousand years, -A thousand years to trace -The granite features of this cliff -In crag and scarp and base. -It took the sea an hour one night -An hour of storm to place -The sculpture of these granite seams, -Upon a woman s face. E. J. Pratt (1882 1964) -', to_tsquery('sea')); - headline ------------------------------------- - sea a thousand years, + - A thousand years to trace + - The granite features of this cliff -(1 row) - -select headline(' - - - -Sea view wow foo bar qq -YES   -ff-bg - - -', -to_tsquery('sea&foo'), 'HighlightAll=true'); - headline ------------------------------------------------------------------------------ - + - + - + - + - Sea view wow foo bar qq + - YES  + - ff-bg + - + - + - -(1 row) - ---check debug -select * from public.ts_debug('Tsearch module for PostgreSQL 7.3.3'); - ts_name | tok_type | description | token | dict_name | tsvector ----------+-----------+-----------------+------------+----------------+-------------- - english | asciiword | Word, all ASCII | Tsearch | {english_stem} | 'tsearch' - english | blank | Space symbols | | {} | - english | asciiword | Word, all ASCII | module | {english_stem} | 'modul' - english | blank | Space symbols | | {} | - english | asciiword | Word, all ASCII | for | {english_stem} | - english | blank | Space symbols | | {} | - english | asciiword | Word, all ASCII | PostgreSQL | {english_stem} | 'postgresql' - english | blank | Space symbols | | {} | - english | version | Version number | 7.3.3 | {simple} | '7.3.3' -(9 rows) - ---check ordering -insert into test_tsvector values (null, null); -select a is null, a from test_tsvector order by a; - ?column? | a -----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - f | - f | - f | - f | - f | - f | - f | - f | - f | - f | 'a':1A,2,3B 'b':5A,6A,7C,8 - f | 'a':1A,2,3C 'b':5A,6B,7C,8B - f | 'bar':2,8 'foo':1,3,6 'qq':7 - f | '345':1 'copyright':3 'qwerti':2 - f | '7w' 'ch' 'd7' 'eo' 'gw' 'i4' 'lq' 'o6' 'qt' 'y0' - f | 'ar' 'ei' 'kq' 'ma' 'qa' 'qh' 'qq' 'qz' 'rx' 'st' - f | 'gs' 'i6' 'i9' 'j2' 'l0' 'oq' 'qx' 'sc' 'xe' 'yu' - f | 'ap' 'i2' 'k0' 'n4' 'ou' 'px' 'qe' 'qo' 'qr' 't1' 'yz' - f | 'cb' 'ib' 'ih' 'jx' 'oy' 'ph' 'tl' 'ty' 'vb' 'vu' 'wu' - f | 'cp' 'i0' 'ia' 'jh' 'kh' 'kl' 'ql' 'sb' 'u3' 'wy' 'ys' - f | 'cy' 'ey' 'gr' 'lq' 'n1' 'pp' 'pq' 'qb' 'qe' 'se' 'wb' - f | 'ja' 'k2' 'ko' 'o6' 'ob' 'ps' 't9' 'tz' 'uk' 'um' 'vv' - f | '5e' 'a2' 'ag' 'ay' 'dc' 'ef' 'jd' 'mn' 'pl' 'qp' 'wu' 'xd' - f | '7a' 'ap' 'bl' 'fl' 'g7' 'ko' 'ma' 'qi' 'ri' 'rp' 'y5' 'yo' - f | 'a4' 'an' 'go' 'ki' 'px' 'qq' 'td' 'tm' 'ur' 'wc' 'wp' 'yx' - f | 'aa' 'ah' 'bo' 'dz' 'pd' 'pm' 'qf' 'qw' 'rd' 'rn' 'tf' 'wu' - f | 'aj' 'ar' 'ay' 'eb' 'g5' 'ic' 'la' 'u2' 'up' 'wb' 'x8' 'yr' - f | 'aw' 'dz' 'el' 'hb' 'iu' 'lc' 'm9' 'r5' 'rt' 'uz' 'x2' 'zp' - f | 'dz' 'ea' 'h8' 'j9' 'mr' 'pc' 'qa' 'qt' 'rm' 'rv' 'ub' 'wg' - f | 'ew' 'fi' 'g2' 'ku' 'od' 'pk' 'tm' 'tq' 'tu' 'uf' 'wn' 'y1' - f | 'gi' 'n3' 'n7' 'om' 'qa' 'r7' 're' 'ug' 'w7' 'wi' 'x4' 'yn' - f | '3k' 'f5' 'if' 'kg' 'lj' 'ol' 'qr' 'so' 'ta' 'u1' 'vu' 'wr' 'y2' - f | '70' 'e0' 'ep' 'ex' 'hh' 'jn' 'kg' 'nq' 'rg' 'rj' 'uf' 'vs' 'ys' - f | 'ag' 'al' 'ay' 'fw' 'hi' 'ou' 'p6' 'qy' 'rg' 'rz' 'sx' 'uy' 'zb' - f | 'cm' 'dm' 'eh' 'em' 'ik' 'j0' 'kk' 'lp' 'ng' 'or' 'pu' 'qd' 'tw' - f | 'di' 'ec' 'i4' 'iq' 'iw' 'ko' 'l3' 'mw' 'py' 'qo' 'rx' 'wt' 'yl' - f | '5q' 'av' 'd2' 'd4' 'e5' 'jc' 'km' 'mm' 'pa' 'rs' 's4' 'si' 'tc' 'ut' - f | '6o' 'av' 'ed' 'ee' 'gf' 'ii' 'o8' 'og' 'om' 'qs' 'ta' 'th' 'tk' 'w0' - f | '7y' 'aq' 'cu' 'd3' 'h2' 'ih' 'oc' 'qh' 'rc' 'rs' 't3' 'ud' 'we' 'zt' - f | 'b7' 'eu' 'g4' 'hw' 'in' 'pi' 'qt' 'r0' 'rg' 'sn' 'sz' 'tc' 'wd' 'zs' - f | 'fb' 'in' 'iy' 'lu' 'p4' 'pd' 'qa' 'qq' 's6' 'ta' 'y1' 'yg' 'yy' 'zk' - f | 'at' 'ec' 'hs' 'ix' 'ks' 'lq' 'ls' 'pk' 'qs' 'qy' 'rc' 'ut' 'xv' 'yo' 'ys' - f | 'az' 'bl' 'dl' 'ec' 'gg' 'jq' 'o0' 'oj' 'q8' 'qq' 'ta' 'un' 'wb' 'wo' 'xu' - f | 'c0' 'd4' 'dw' 'ef' 'em' 'j2' 'kj' 'ql' 'rn' 'ta' 'uk' 'vv' 'wf' 'y9' 'zq' - f | 'gn' 'gx' 'hf' 'ji' 'kx' 'nh' 'o6' 'pe' 'q1' 'qt' 'rw' 'sc' 'ss' 'yh' 'zm' - f | 'h7' 'in' 'is' 'o7' 'q0' 'qa' 'qq' 'rl' 'rs' 'tu' 'u9' 'v3' 'vr' 'yq' 'zy' - f | 'ae' 'af' 'ay' 'da' 'dp' 'dq' 'e1' 'gi' 'gk' 'rl' 'sf' 'ta' 'td' 'tf' 'tt' 'tw' - f | 'aj' 'cl' 'cy' 'ee' 'i3' 'ir' 'nx' 'oz' 'qm' 'qw' 'r2' 't2' 'u0' 'ub' 'ur' 'wj' - f | 'av' 'dp' 'ff' 'fx' 'jk' 'ke' 'lb' 'lm' 'n1' 'ql' 'rv' 's4' 'uv' 'wl' 'ws' 'yj' - f | 'b8' 'eh' 'g5' 'gn' 'jo' 'mx' 'og' 'ol' 'on' 'px' 'rr' 'sl' 'un' 'uz' 'wv' 'yf' - f | 'bt' 'dy' 'gy' 'hg' 'i7' 'it' 'jv' 'lh' 'ox' 'qo' 'ri' 's3' 'ss' 'u1' 'uq' 'wr' - f | 'ef' 'ej' 'fn' 'ho' 'ia' 'il' 'iq' 'mj' 'nr' 'p5' 'rm' 'sd' 'ux' 'vi' 'wq' 'y6' - f | '25' 'cd' 'd2' 'dv' 'eo' 'es' 'f4' 'oc' 'qc' 'sh' 'te' 'tv' 'wd' 'wn' 'wo' 'xp' 'zt' - f | '2y' '8i' 'by' 'dl' 'h6' 'hj' 'm3' 'ml' 'qa' 'qx' 'r1' 'rm' 'rn' 'u4' 'uj' 'wo' 'xv' - f | 'ad' 'bs' 'dx' 'fi' 'i5' 'ia' 'j3' 'm5' 'mn' 'nr' 'ox' 'tg' 'un' 'vo' 'wj' 'wt' 'ys' - f | 'am' 'hw' 'jd' 'na' 'oe' 'pw' 'ql' 'qr' 's9' 'sk' 'u7' 'wa' 'wg' 'wu' 'x6' 'xr' 'yf' - f | 'de' 'mc' 'q1' 'q8' 'qp' 'qt' 'rq' 'rv' 'sa' 'sn' 'u8' 'vs' 'w9' 'wo' 'wp' 'ww' 'yl' - f | 'aa' 'af' 'ee' 'hb' 'ih' 'j2' 'lv' 'mw' 'pp' 'q3' 'rd' 'tb' 'td' 'ua' 'ug' 'up' 'xh' 'yy' - f | 'an' 'cp' 'dq' 'ej' 'ez' 'lj' 'ln' 'nu' 'qy' 'sm' 't8' 'td' 'tg' 'us' 'uw' 'vn' 'y4' 'z9' - f | 'ca' 'do' 'h5' 'i9' 'io' 'jk' 'jl' 'kn' 'qq' 'tm' 'ul' 'w9' 'wb' 'wp' 'wt' 'wx' 'y3' 'zd' - f | '16' 'ae' 'al' 'ef' 'eg' 'ew' 'gi' 'ha' 'id' 'ng' 'o3' 'on' 'p9' 'rz' 'uf' 'vg' 'wo' 'wr' 'zh' - f | 'ca' 'fu' 'hv' 'la' 'mt' 'ov' 'pl' 'q8' 'r3' 'sp' 'sy' 'tg' 'to' 'tv' 'wn' 'x4' 'yh' 'yp' 'ze' - f | 'cv' 'ds' 'dx' 'dy' 'ex' 'hh' 'lf' 'mq' 'qe' 'qu' 'rb' 'tb' 'tv' 'tz' 'ue' 'ui' 'wi' 'yb' 'zz' - f | 'ec' 'fd' 'gm' 'it' 'iu' 'kx' 'l1' 'pi' 'q1' 'qe' 'qs' 'ra' 'ri' 'rp' 'tn' 'to' 'vx' 'wh' 'wl' - f | 'eq' 'g7' 'jc' 'jf' 'ji' 'lw' 'ma' 'oe' 'p7' 'qb' 'qj' 'qo' 'u7' 'v7' 'w3' 'wd' 'wz' 'xg' 'yr' - f | 'a2' 'cs' 'ee' 'em' 'gk' 'hv' 'iy' 'kq' 'nc' 'pb' 'q5' 'qy' 'rq' 'rr' 'ts' 'uq' 'vt' 'w0' 'y6' 'yz' - f | 'al' 'fh' 'fk' 'gy' 'he' 'ie' 'iz' 'lq' 'oh' 'pu' 'q7' 's6' 'sd' 'sr' 'sw' 'uu' 'uz' 'v6' 'ws' 'xo' - f | 'aw' 'bm' 'dw' 'e5' 'ht' 'j4' 'kv' 'm5' 'oi' 'qa' 'qe' 'qf' 'ri' 'rj' 't6' 't8' 'un' 'wc' 'yb' 'yj' - f | 'az' 'bl' 'bo' 'cf' 'gt' 'h0' 'hx' 'iq' 'k2' 'kb' 'oc' 'qg' 'qn' 'qz' 're' 'rl' 'rv' 'xp' 'y8' 'yf' - f | 'bj' 'dq' 'e2' 'ec' 'fs' 'g8' 'gd' 'iw' 'jt' 'nn' 'ns' 'o9' 'p0' 'tb' 'u7' 'uv' 'wf' 'wr' 'xf' 'z3' - f | 'bs' 'ee' 'gz' 'hv' 'ib' 'kc' 'lb' 'nu' 'ps' 'pt' 'qh' 'ud' 'vo' 'vq' 'vu' 'wb' 'wj' 'x3' 'xu' 'yf' - f | '2o' 'a5' 'cg' 'ch' 'cp' 'eb' 'eg' 'eh' 'ew' 'fu' 'g4' 'hc' 'hx' 'il' 'li' 'rd' 'sf' 'sw' 'tg' 'uc' 'zj' - f | '6d' '8t' 'bl' 'gu' 'iu' 'kd' 'kj' 'kq' 'nw' 'nx' 'o6' 'oa' 'qk' 'ql' 'rd' 'ri' 'uc' 'vi' 'wy' 'xq' 'z2' - f | 'a3' 'ch' 'cs' 'e1' 'gq' 'gx' 'lz' 'nh' 'os' 'po' 'qs' 'rr' 'tx' 'ud' 'uj' 'uv' 've' 'w0' 'wj' 'xo' 'xz' - f | 'aa' 'al' 'e9' 'hm' 'ir' 'kc' 'l0' 'pi' 'po' 'qa' 'qk' 'r0' 'rd' 'rz' 'se' 'sr' 'vp' 'w6' 'w8' 'yd' 'yk' - f | 'am' 'bv' 'dt' 'dy' 'ed' 'gx' 'm7' 'mt' 'q5' 'qv' 'rr' 'sh' 'th' 'ut' 'wd' 'wm' 'y1' 'ym' 'yq' 'yr' 'yt' - f | '4f' 'e4' 'ep' 'fa' 'ff' 'iv' 'j4' 'kw' 'oj' 'pa' 'pw' 'q0' 'rv' 'ry' 't3' 'ul' 'vq' 'w1' 'wj' 'xm' 'ye' 'yu' - f | '7a' 'a3' 'dl' 'fo' 'hb' 'ki' 'kk' 'lo' 'pl' 'q5' 'qy' 'r1' 'rj' 'sy' 'tv' 'w3' 'wm' 'wn' 'yj' 'yr' 'zi' 'zq' - f | '1l' 'b5' 'dq' 'fw' 'hz' 'in' 'ml' 'nb' 'nu' 'o2' 'or' 'q7' 'qb' 'qh' 'qt' 'rv' 'so' 'tp' 'tr' 'u0' 'v6' 'wl' 'xb' - f | '5y' 'aw' 'e2' 'gd' 'gn' 'hn' 'ig' 'k9' 'ki' 'oj' 'pk' 'ql' 'qz' 'sl' 't3' 'u4' 'v8' 'wg' 'wu' 'xk' 'ya' 'yf' 'zr' - f | '6t' 'a9' 'db' 'ea' 'ec' 'ez' 'fq' 'gj' 'hb' 'hs' 'lc' 'or' 'p4' 'ph' 'pp' 'qr' 'qx' 'rc' 'rl' 'tn' 'u5' 'w9' 'x1' - f | 'bt' 'c5' 'd1' 'd7' 'g6' 'gk' 'ib' 'iv' 'ml' 'om' 'qd' 'qg' 'ql' 'r3' 'rc' 'u4' 'uh' 'uo' 'wh' 'y0' 'yn' 'yz' 'zo' - f | '1b' 'cp' 'di' 'ed' 'fv' 'gx' 'hs' 'i8' 'lh' 'lq' 'lz' 'p7' 'p8' 'pc' 'ql' 'qn' 'qw' 're' 'rg' 'tv' 'we' 'wp' 'yr' 'zj' - f | '1i' 'db' 'e4' 'er' 'fh' 'ft' 'hm' 'lg' 'll' 'of' 'og' 'q6' 'qj' 'r0' 're' 'th' 'up' 'ut' 'uw' 'vf' 'wq' 'ws' 'wx' 'zi' - f | 'a0' 'ck' 'fp' 'g4' 'ib' 'ih' 'im' 'iq' 'kz' 'll' 'lv' 'nc' 'oq' 'qf' 'qv' 'rg' 'rk' 'tc' 'tn' 'u1' 'u8' 'uj' 'un' 'vv' - f | 'ak' 'ar' 'dg' 'ds' 'ep' 'fv' 'ge' 'jd' 'no' 'on' 'q5' 'qd' 'qo' 'qv' 'qx' 'r7' 'ra' 'ru' 'sa' 'ud' 'uo' 'wl' 'ye' 'yl' - f | 'av' 'es' 'fl' 'gt' 'he' 'it' 'kp' 'mu' 'nc' 'ol' 'om' 'ph' 'qr' 'ra' 'rk' 'ui' 'vh' 'w6' 'wm' 'ws' 'yu' 'z0' 'zl' 'zm' - f | 'da' 'e5' 'ec' 'gd' 'gf' 'jj' 'kx' 'ly' 'pd' 'pj' 'q6' 'qk' 'rm' 'sa' 'te' 'ut' 'wa' 'wc' 'wl' 'ws' 'wv' 'zb' 'zk' 'zz' - f | '12' 'ao' 'ed' 'ek' 'ew' 'ey' 'fm' 'gr' 'hc' 'ht' 'io' 'ir' 'jb' 'jw' 'ke' 'ld' 'qj' 'se' 'tm' 'tn' 'tw' 'wv' 'y5' 'yt' 'z6' - f | '1p' '51' 'aj' 'av' 'bj' 'bn' 'c1' 'dx' 'ex' 'gz' 'he' 'ia' 'ic' 'ip' 'kn' 'mx' 'o6' 'or' 'ql' 'rc' 'wf' 'wi' 'wn' 'y8' 'yr' - f | 'av' 'ej' 'el' 'ep' 'fp' 'hh' 'hz' 'l9' 'mr' 'q3' 'qn' 'qq' 'qy' 'rh' 'tw' 'vt' 'vu' 'w5' 'wg' 'wk' 'wo' 'x5' 'y9' 'yk' 'zm' - f | 'b6' 'bq' 'dv' 'e1' 'ez' 'f5' 'fh' 'ik' 'iy' 'jy' 'li' 'm2' 'qe' 'rp' 'te' 'u4' 'u8' 'uo' 'w3' 'w8' 'we' 'wo' 'wu' 'x5' 'yl' - f | 'cq' 'er' 'hy' 'ie' 'jg' 'ke' 'lw' 'mf' 'p0' 'pe' 'pv' 'qk' 'qt' 'qy' 'sh' 'th' 'ti' 'ue' 'w5' 'wl' 'y4' 'y5' 'yi' 'yy' 'za' - f | 'a3' 'bh' 'c2' 'ca' 'e1' 'fb' 'fe' 'hd' 'hx' 'jc' 'md' 'nl' 'q9' 'qi' 'qq' 'qs' 'qt' 'rx' 'te' 'tv' 'u2' 'w8' 'wi' 'wr' 'xq' 'y9' - f | 'af' 'dt' 'e4' 'e8' 'eq' 'et' 'gr' 'kr' 'kv' 'lu' 'oy' 'pb' 'qh' 'ql' 'qw' 'r4' 't8' 'tb' 'td' 'tn' 'uc' 'uj' 'wh' 'xn' 'xs' 'yi' - f | 'au' 'bo' 'dz' 'ek' 'eq' 'et' 'fa' 'hw' 'id' 'im' 'kr' 'p4' 'qx' 'rb' 'rx' 'sf' 'tl' 'tx' 'uf' 'ui' 'uw' 'vr' 'wb' 'wn' 'xw' 'yd' - f | 'ay' 'bg' 'bp' 'cm' 'eg' 'ev' 'ff' 'go' 'hc' 'hl' 'jj' 'l0' 'os' 'q9' 'qc' 'qh' 'qo' 'rt' 'rx' 'so' 'sr' 'to' 'tx' 'uk' 'wb' 'y4' - f | 'd6' 'e4' 'ev' 'fd' 'i3' 'if' 'j1' 'j5' 'o8' 'oj' 'ok' 'pw' 'qc' 'qd' 'qf' 'qt' 't8' 'tw' 'u1' 'u7' 'wr' 'wv' 'ww' 'yh' 'yn' 'yz' - f | '1j' 'ag' 'c3' 'cl' 'e4' 'ef' 'eh' 'f5' 'fi' 'gy' 'hx' 'lw' 'oa' 'pu' 'qa' 'qi' 'qt' 'qu' 'rl' 'ro' 'rs' 'sg' 'uq' 'wq' 'ya' 'yh' 'ys' - f | 'a7' 'ah' 'cj' 'co' 'cv' 'dt' 'ex' 'fs' 'gx' 'hn' 'jv' 'kp' 'l5' 'od' 'on' 'pr' 'q6' 'q8' 'qi' 'qq' 'qr' 'rl' 'u2' 'wb' 'wx' 'yh' 'zq' - f | 'bg' 'eh' 'eq' 'gg' 'gh' 'gm' 'gx' 'i7' 'iv' 'm3' 'mv' 'n3' 'o6' 'ox' 'oz' 'pb' 'qk' 'rj' 'rs' 'sk' 'su' 'tg' 'uf' 'uj' 've' 'ww' 'yf' - f | 'cd' 'dl' 'e6' 'en' 'eu' 'gg' 'je' 'kp' 'lv' 'pv' 'q0' 'q3' 'qc' 'qd' 'qo' 'qs' 'qz' 'rw' 'se' 'tx' 'uh' 'uj' 'ul' 'wo' 'ye' 'yi' 'zx' - f | 'd6' 'do' 'e4' 'eg' 'hm' 'i3' 'kg' 'nz' 'ow' 'pc' 'pv' 'q0' 'q4' 'q6' 'qx' 'r0' 'ri' 'sm' 'sn' 'tw' 'u9' 'ul' 'up' 'vk' 'we' 'wm' 'zv' - f | 'a1' 'cd' 'cl' 'd8' 'ek' 'ig' 'ih' 'in' 'lq' 'o3' 'ow' 'px' 'qg' 'qm' 'qq' 'qr' 'qs' 'qy' 'rd' 'rh' 'to' 'tq' 'ul' 'wc' 'x9' 'ya' 'yf' 'yw' - f | 'aa' 'as' 'cg' 'dh' 'dn' 'dr' 'e0' 'h2' 'hr' 'j2' 'jf' 'js' 'kc' 'kw' 'ld' 'lh' 'mk' 'n3' 'q3' 'qe' 'ql' 'rv' 's3' 'w0' 'xg' 'ym' 'yt' 'zv' - f | 'ap' 'ca' 'dt' 'dx' 'ep' 'f5' 'fg' 'gq' 'hi' 'hj' 'i4' 'ic' 'it' 'iy' 'jl' 'lz' 'nd' 'o9' 'og' 'oq' 'pk' 'q6' 'qo' 'ra' 'sf' 'wd' 'wt' 'x9' - f | 'e2' 'ef' 'ev' 'fe' 'ij' 'j8' 'jm' 'kw' 'nb' 'ny' 'o6' 'o7' 'ou' 'pb' 'qd' 'qv' 'rh' 'rp' 's7' 'ti' 'ub' 'uk' 'wh' 'wi' 'wj' 'xj' 'xo' 'yx' - f | '3d' 'ad' 'dr' 'ee' 'ez' 'f4' 'fd' 'fg' 'fu' 'g9' 'gk' 'h9' 'hl' 'iz' 'jd' 'nb' 'o0' 'oo' 'oy' 'qj' 'qt' 'rp' 'ru' 'sv' 'tl' 'tv' 'wf' 'wp' 'wy' - f | '5t' 'al' 'db' 'dt' 'dx' 'ea' 'en' 'g6' 'gc' 'gm' 'gy' 'if' 'ii' 'ik' 'jb' 'jv' 'k5' 'po' 'pv' 'py' 'qj' 'qp' 'rz' 'ux' 'v1' 'w4' 'w8' 'wi' 'yv' - f | 'a1' 'b0' 'd0' 'db' 'ef' 'er' 'ev' 'ew' 'fe' 'fm' 'g8' 'la' 'n5' 'oh' 'os' 'pk' 'pn' 'qq' 'r7' 'sq' 'tw' 'ua' 'uu' 'wa' 'wk' 'wr' 'wu' 'xc' 'yi' - f | 'ab' 'c1' 'cl' 'd3' 'do' 'e5' 'e8' 'eg' 'ek' 'ex' 'gy' 'ia' 'iq' 'iw' 'jf' 'kv' 'm9' 'n4' 'nh' 'nj' 'q3' 'qa' 'qe' 'qg' 'qm' 'sy' 'ta' 'w5' 'w6' - f | 'av' 'bh' 'cd' 'cw' 'dv' 'em' 'gn' 'iw' 'ja' 'ki' 'lc' 'lx' 'my' 'oi' 'ox' 'q0' 'qb' 'qi' 'qn' 'uf' 'ux' 'we' 'wn' 'xd' 'xq' 'y4' 'y9' 'zf' 'zs' - f | 'ch' 'e1' 'fi' 'g8' 'go' 'hf' 'i1' 'ic' 'in' 'it' 'j7' 'jk' 'jl' 'jv' 'nm' 'of' 'oz' 'r8' 'rc' 'rk' 'rp' 'rx' 'sp' 'tb' 'tv' 'tw' 'ul' 'wx' 'zj' - f | 'an' 'dh' 'do' 'hs' 'hv' 'ia' 'ic' 'ne' 'of' 'oi' 'oq' 'pe' 'pg' 'q9' 'r5' 'rk' 'sc' 'sf' 'sh' 'ta' 'tb' 'tq' 'um' 'wb' 'wj' 'wm' 'wq' 'wt' 'yi' 'ym' - f | 'bb' 'bq' 'c2' 'cw' 'cy' 'db' 'dd' 'f3' 'fl' 'fn' 'id' 'ig' 'jb' 'kc' 'kl' 'lp' 'lx' 'mh' 'o0' 'pk' 'qi' 'qx' 'rq' 've' 'w8' 'wd' 'x1' 'y4' 'ye' 'zm' - f | 'c7' 'eb' 'er' 'gb' 'if' 'ko' 'ml' 'oq' 'ot' 'pa' 'qk' 'qs' 'rl' 'rp' 'sc' 'tf' 'tv' 'tw' 'uc' 'ud' 'uz' 'vk' 'vm' 'w0' 'wm' 'wu' 'yd' 'yq' 'yy' 'zu' - f | '27' '2e' '2p' 'aa' 'eh' 'en' 'eq' 'eu' 'ew' 'ff' 'g8' 'hv' 'mx' 'oi' 'pd' 'q3' 'qs' 'rl' 'sa' 'sw' 'te' 'tn' 'ty' 'uh' 'uo' 'wb' 'wh' 'wy' 'xx' 'z8' 'zt' - f | '3k' 'di' 'dp' 'em' 'ew' 'f5' 'hb' 'hn' 'j3' 'k0' 'lk' 'm4' 'mq' 'pr' 'qe' 'qo' 'qy' 'rc' 'ri' 'rt' 'so' 'ts' 've' 'w4' 'w7' 'wl' 'wn' 'wy' 'xf' 'y6' 'yt' - f | '3v' 'ab' 'at' 'bn' 'bz' 'cx' 'eh' 'gd' 'hd' 'hr' 'i9' 'j8' 'jg' 'jx' 'lf' 'ng' 'oj' 'ov' 'oz' 'pn' 'ps' 'qq' 'qr' 'ro' 'rt' 'u9' 'up' 'uz' 'yl' 'yw' 'zi' - f | '6n' 'da' 'dl' 'dx' 'el' 'ew' 'ff' 'fl' 'gc' 'iu' 'jh' 'jt' 'kt' 'lz' 'pd' 'q6' 'qj' 'ql' 'r3' 'ra' 'rn' 'ry' 'sg' 'tf' 'tp' 'tt' 'ub' 'ye' 'yf' 'yl' 'yp' - f | 'a3' 'ai' 'at' 'd7' 'eu' 'fu' 'gd' 'ii' 'ik' 'j0' 'je' 'lw' 'ly' 'mx' 'n7' 'pm' 'qf' 'qk' 'qt' 'ss' 'to' 'tq' 'un' 'vn' 'vq' 'wo' 'wz' 'yk' 'yy' 'zf' 'zg' - f | 'aa' 'ch' 'cm' 'db' 'dd' 'e7' 'eu' 'f5' 'fr' 'gg' 'ie' 'jc' 'kt' 'na' 'on' 'or' 'po' 'q0' 're' 's3' 'tb' 'tc' 'td' 'tt' 'tv' 'uf' 'wi' 'wq' 'wt' 'y8' 'ys' - f | 'ac' 'al' 'b3' 'bg' 'ci' 'cn' 'ea' 'f4' 'ih' 'ix' 'kt' 'p8' 'pk' 'qd' 'qe' 'qi' 'rb' 'rc' 'ru' 'te' 'th' 'tl' 'tn' 'vb' 'wb' 'ws' 'ww' 'xk' 'xo' 'yb' 'zf' - f | 'az' 'eq' 'fe' 'go' 'gv' 'ig' 'iz' 'ja' 'l4' 'mo' 'nm' 'no' 'of' 'pk' 'q9' 'qb' 'ql' 'qt' 'r7' 'rc' 'rm' 'tg' 'tv' 'u3' 'u7' 'wc' 'x4' 'xw' 'yl' 'zk' 'zn' - f | 'c8' 'db' 'dh' 'fh' 'g7' 'gm' 'if' 'ih' 'jd' 'li' 'ms' 'mt' 'no' 'or' 'p7' 'pc' 'qb' 'qm' 'sh' 'tk' 'uf' 'uz' 'vb' 'vp' 'wh' 'wr' 'wv' 'xh' 'xm' 'xp' 'zj' - f | '5s' 'aa' 'ar' 'bu' 'c2' 'cw' 'd2' 'ej' 'g8' 'gy' 'ij' 'iv' 'k0' 'l1' 'lb' 'm1' 'od' 'pm' 'q1' 'q4' 'q5' 'qd' 'rn' 'ry' 'sj' 'sm' 'ta' 'th' 'u8' 'vf' 'wr' 'xm' - f | '6d' 'ao' 'bo' 'cq' 'e0' 'fi' 'fm' 'h6' 'i2' 'jl' 'kn' 'mj' 'nv' 'oq' 'p6' 'qd' 'qi' 'qn' 'r2' 'r8' 'rd' 'sd' 'sl' 'ta' 'tp' 'ub' 'uh' 'w3' 'wh' 'y1' 'yj' 'zk' - f | 'aa' 'cb' 'd2' 'dd' 'de' 'e4' 'gd' 'go' 'hc' 'ic' 'in' 'ip' 'js' 'lm' 'o1' 'o3' 'pl' 'ra' 'rx' 'sj' 'ti' 'tu' 'tv' 'wc' 'we' 'wl' 'wq' 'wx' 'xg' 'xi' 'yc' 'yi' - f | 'ad' 'ai' 'ak' 'df' 'ee' 'fr' 'gg' 'i2' 'i5' 'ig' 'ij' 'ir' 'j1' 'kk' 'km' 'l9' 'qb' 'ql' 'qt' 'qw' 'rb' 's2' 'te' 'ue' 'w0' 'wk' 'wu' 'y4' 'yb' 'yf' 'yp' 'zn' - f | 'ap' 'aw' 'ba' 'cz' 'dw' 'eh' 'g5' 'g9' 'gc' 'gi' 'go' 'ir' 'j8' 'kl' 'mg' 'p6' 'ql' 'qu' 'qz' 'sa' 't7' 'tw' 'v5' 'v7' 'wj' 'xt' 'y6' 'yg' 'yh' 'yq' 'z5' 'zj' - f | 'bp' 'cy' 'd6' 'd7' 'em' 'et' 'gf' 'gt' 'hl' 'ib' 'io' 'kz' 'mx' 'nm' 'oe' 'pd' 'qf' 'qk' 'rk' 'rz' 'se' 'si' 'tq' 'tu' 'uu' 'v3' 'wa' 'wf' 'wg' 'wx' 'yd' 'ym' - f | '15' '72' 'av' 'ba' 'dt' 'ev' 'hd' 'hu' 'jy' 'lo' 'mo' 'mt' 'oe' 'oi' 'pa' 'qc' 'qe' 'qt' 'r1' 'rg' 'ry' 'td' 'tk' 'us' 'wk' 'wq' 'ww' 'wz' 'x7' 'xq' 'y9' 'yx' 'z3' - f | '3q' 'an' 'cp' 'ej' 'fx' 'gj' 'i3' 'ib' 'ik' 'jt' 'k5' 'n9' 'no' 'os' 'pg' 'qo' 'qp' 'qx' 'rb' 'rk' 'su' 'tk' 'tv' 'u1' 'up' 'w1' 'we' 'wj' 'wr' 'yf' 'yq' 'z5' 'zd' - f | 'ak' 'dx' 'eb' 'es' 'eu' 'fd' 'ga' 'gf' 'hl' 'i2' 'kb' 'kd' 'mz' 'n1' 'o8' 'qe' 'qs' 'qw' 'rp' 'sf' 'sk' 'sp' 't0' 'tu' 'uh' 'v9' 'vj' 'vs' 'vw' 'wv' 'xs' 'yo' 'z2' - f | '25' 'ad' 'ao' 'at' 'ch' 'db' 'dz' 'eq' 'fp' 'gq' 'ih' 'iu' 'jo' 'km' 'p1' 'pb' 'qa' 'qb' 'qc' 'qg' 'ql' 'qq' 'qy' 'rb' 'rf' 'rl' 'rw' 'sp' 'ty' 'uk' 'ur' 'wu' 'wv' 'yw' - f | '1o' '3w' 'aw' 'be' 'bx' 'e7' 'eo' 'ey' 'ff' 'fx' 'ht' 'jb' 'km' 'kv' 'l6' 'la' 'nu' 'ny' 'pk' 'qc' 'qi' 'rk' 'ro' 'rw' 's3' 'sh' 't5' 'u1' 'uy' 'vi' 'vm' 'wi' 'x3' 'xo' 'yn' - f | '2a' '5d' 'ag' 'ar' 'at' 'dc' 'fa' 'fh' 'gn' 'gx' 'hh' 'iy' 'kd' 'ke' 'ld' 'nd' 'nx' 'oz' 'qc' 'qd' 'qo' 'qp' 'qu' 'r0' 'rg' 't5' 'tc' 'tg' 'th' 'tk' 'w2' 'w5' 'wt' 'yl' 'zn' - f | '30' 'cl' 'dl' 'e7' 'ee' 'ef' 'eh' 'el' 'fk' 'fn' 'fs' 'gi' 'gw' 'i2' 'il' 'l7' 'm6' 'm9' 'mf' 'o8' 'ob' 'ok' 'pc' 'pe' 'qb' 'qe' 'qn' 'qr' 'tu' 'uc' 'ud' 'wp' 'xq' 'ym' 'ys' - f | '5o' 'ar' 'at' 'az' 'bf' 'bv' 'e0' 'ew' 'fh' 'hd' 'ij' 'in' 'is' 'iu' 'ji' 'l5' 'ld' 'mz' 'nk' 'oc' 'p2' 'ph' 'px' 'q9' 'qi' 'qn' 'qo' 'qq' 'qv' 'rw' 'sf' 'sp' 'te' 'up' 'y8' - f | 'a2' 'bx' 'e4' 'em' 'eu' 'ey' 'gd' 'jj' 'k1' 'lm' 'my' 'oz' 'p8' 'pa' 'pj' 'qj' 'qp' 'qy' 'ri' 'rw' 'sj' 'sw' 'tv' 'uj' 'uk' 'um' 'ux' 'vs' 'wd' 'wf' 'xj' 'xs' 'xx' 'xy' 'yd' - f | 'ab' 'at' 'cr' 'e9' 'g0' 'gv' 'ib' 'iv' 'iz' 'jb' 'jm' 'k0' 'kh' 'o2' 'o7' 'oq' 'ot' 'pj' 'ps' 'qj' 'qz' 'r9' 'rn' 'sa' 't1' 'ti' 'tq' 'ue' 'us' 'wc' 'wo' 'ww' 'yi' 'ys' 'za' - f | 'by' 'c1' 'da' 'ec' 'ej' 'ew' 'g4' 'gq' 'gt' 'ir' 'jv' 'kl' 'kz' 'l5' 'nm' 'oi' 'on' 'pc' 'q2' 'qx' 'qz' 'rd' 'se' 'sq' 'ti' 'tk' 'to' 'ur' 'w8' 'wm' 'wr' 'y1' 'yn' 'yp' 'zy' - f | 'd8' 'dm' 'eg' 'et' 'ge' 'gz' 'i8' 'ig' 'il' 'iy' 'jo' 'k6' 'lm' 'oj' 'p3' 'pw' 'qd' 'qp' 'qq' 'qz' 'r7' 's1' 'sr' 'sx' 't4' 'uz' 'vm' 'vr' 'w0' 'wj' 'wp' 'xc' 'y1' 'yj' 'zd' - f | '0w' 'bu' 'df' 'dj' 'du' 'dy' 'dz' 'gj' 'ho' 'je' 'kw' 'm1' 'o3' 'oo' 'pf' 'qh' 'qn' 'qu' 'rf' 'rj' 'ro' 'rv' 'ss' 't4' 'tc' 'tf' 'um' 'uo' 'v6' 'v8' 'wn' 'wo' 'xh' 'y0' 'yz' 'zr' - f | 'a3' 'do' 'ef' 'hp' 'ih' 'im' 'jp' 'km' 'lv' 'mt' 'nh' 'oc' 'od' 'ph' 'pi' 'pj' 'qe' 'qh' 'qr' 'rq' 'rr' 'rs' 'sc' 'st' 'ts' 'tt' 'tw' 'ut' 'vv' 'w7' 'wv' 'x3' 'xa' 'xc' 'xl' 'zc' - f | 'ad' 'aj' 'ar' 'b4' 'dj' 'eo' 'eu' 'ex' 'gt' 'hj' 'ie' 'ij' 'iz' 'kg' 'ks' 'l1' 'ld' 'me' 'my' 'mz' 'oi' 'pr' 'qb' 'ry' 'sd' 'to' 'u5' 'wc' 'wf' 'wl' 'xs' 'y2' 'yg' 'z1' 'zj' 'zt' - f | '2y' 'a3' 'ac' 'ae' 'ar' 'dp' 'ec' 'f8' 'f9' 'ga' 'gq' 'gs' 'hm' 'ib' 'if' 'im' 'j0' 'na' 'pb' 'pf' 'q9' 'qa' 'qu' 'qw' 'ra' 'rj' 'rm' 'rr' 'su' 'tj' 'vw' 'wb' 'wf' 'wk' 'wo' 'ym' 'yu' - f | '5y' 'a2' 'by' 'c9' 'e6' 'fd' 'fn' 'g7' 'h9' 'hj' 'j9' 'kg' 'kk' 'kn' 'kx' 'ln' 'q3' 'qa' 'qh' 'rb' 'rh' 'sj' 'sn' 'tx' 'tz' 'un' 'us' 've' 'vf' 'vh' 'wf' 'wp' 'wz' 'xj' 'xq' 'yi' 'yz' - f | 'a1' 'aj' 'bi' 'cd' 'cw' 'd2' 'e4' 'e9' 'gz' 'ij' 'k1' 'kb' 'kn' 'lw' 'me' 'nj' 'oq' 'p4' 'ph' 'ps' 'q3' 'q4' 'qj' 's7' 'sp' 'sx' 'tc' 'tw' 'v1' 'vj' 'vt' 'w7' 'wa' 'wb' 'wp' 'y1' 'zc' - f | 'a9' 'av' 'aw' 'dj' 'e0' 'ej' 'er' 'eu' 'ez' 'fr' 'gq' 'hh' 'hp' 'ko' 'kq' 'lw' 'n8' 'pe' 'pl' 'pn' 'pp' 'pv' 'q9' 'qc' 'qj' 'qs' 'qw' 'r6' 'rl' 'tq' 'u5' 'uu' 'uz' 'wn' 'wp' 'wr' 'zo' - f | 'dq' 'ed' 'el' 'fg' 'fv' 'gh' 'hx' 'i2' 'iv' 'jc' 'la' 'ld' 'nb' 'oa' 'of' 'oi' 'py' 'q7' 'qe' 'qr' 'qs' 'qw' 'qx' 'qz' 'rl' 'rr' 'sp' 't3' 'te' 'ti' 'tu' 'ue' 'w0' 'wa' 'wq' 'wu' 'xn' - f | '4p' 'bw' 'cq' 'dk' 'dr' 'dx' 'e3' 'e8' 'ex' 'ez' 'g9' 'h9' 'ih' 'in' 'kt' 'lt' 'me' 'o7' 'od' 'oi' 'on' 'p1' 'pu' 'qa' 'qd' 'qm' 'rz' 'sd' 'tn' 'ua' 'uz' 'vg' 'vx' 'wx' 'xy' 'ya' 'yl' 'yx' - f | '6b' 'af' 'd3' 'df' 'dg' 'ds' 'e1' 'eb' 'er' 'f3' 'ft' 'ho' 'ik' 'k1' 'k2' 'li' 'mj' 'ni' 'py' 'qx' 'rb' 'rp' 'rv' 'sd' 'sh' 'sl' 'u5' 'uf' 'vk' 'vs' 'vx' 'wg' 'wm' 'wr' 'ws' 'wz' 'xn' 'zh' - f | '4u' 'a3' 'ck' 'dw' 'e5' 'fu' 'ij' 'iu' 'jd' 'jp' 'ka' 'kb' 'ld' 'op' 'p1' 'po' 'pw' 'q5' 'q7' 'qf' 'qq' 'qr' 'qv' 'rm' 'sv' 't6' 'td' 'tp' 'uw' 'vb' 'w4' 'w6' 'wq' 'xh' 'y2' 'yf' 'yt' 'zj' 'zz' - f | '7n' 'ai' 'aj' 'ap' 'b9' 'bx' 'cu' 'dn' 'e1' 'e2' 'ed' 'eg' 'eo' 'hu' 'ie' 'ln' 'lq' 'nl' 'oa' 'qa' 'qe' 'qo' 'r5' 'ra' 'ri' 'rm' 'rw' 'ss' 'sx' 't3' 'tz' 'ut' 'uz' 'wd' 'we' 'wn' 'wq' 'wr' 'ws' - f | '8p' 'ao' 'aq' 'c1' 'dh' 'dt' 'e9' 'eo' 'ev' 'fp' 'fu' 'gc' 'gz' 'hk' 'i5' 'id' 'ip' 'iy' 'jg' 'jr' 'k2' 'lo' 'mb' 'mf' 'oi' 'ou' 'qp' 'qr' 'qw' 'qx' 'ra' 'rj' 'tl' 'ui' 'uu' 'uy' 'vb' 'wh' 'yt' - f | 'a7' 'bq' 'el' 'ey' 'fp' 'fu' 'fx' 'gr' 'hi' 'hl' 'jm' 'ki' 'kp' 'n6' 'o9' 'oj' 'op' 'ou' 'p7' 'pm' 'pp' 'q7' 'qa' 'qc' 'qj' 'qk' 'qm' 'si' 'st' 'tn' 'uc' 'w6' 'wp' 'xu' 'y5' 'yk' 'ys' 'yu' 'zs' - f | 'ah' 'ai' 'b8' 'ci' 'd2' 'ea' 'ed' 'en' 'et' 'fj' 'fv' 'ht' 'iu' 'iv' 'iw' 'jb' 'jd' 'ky' 'mj' 'nt' 'q1' 'qb' 'qe' 'ql' 'qz' 'ru' 'sg' 'tl' 'tm' 'tn' 'tw' 'ty' 'u1' 'ud' 'us' 'wj' 'y1' 'zd' 'zj' - f | 'bf' 'cf' 'cg' 'cj' 'ef' 'ej' 'ek' 'ep' 'fk' 'gi' 'ik' 'ir' 'j0' 'km' 'ko' 'lu' 'mr' 'n6' 'oc' 'op' 'pt' 'q4' 'qe' 'qk' 'qv' 'rc' 'rt' 's8' 't3' 'tk' 'ut' 'wc' 'we' 'ws' 'wt' 'ww' 'y8' 'yd' 'yr' - f | '1d' 'a2' 'af' 'bb' 'dd' 'dg' 'dl' 'du' 'ee' 'ff' 'g0' 'hj' 'hk' 'i4' 'iy' 'jt' 'kq' 'm6' 'mh' 'ov' 'qf' 'qh' 'qq' 'rt' 'sw' 'ta' 'tc' 'we' 'wn' 'wy' 'wz' 'xd' 'xs' 'y2' 'yg' 'yj' 'yk' 'zi' 'zq' 'zw' - f | '1l' 'e2' 'e4' 'eo' 'ep' 'fd' 'ha' 'hp' 'hx' 'io' 'iu' 'jr' 'jx' 'k4' 'l8' 'nb' 'oa' 'om' 'on' 'ow' 'p2' 'qh' 'qp' 'qz' 'ra' 'rz' 'sy' 'tk' 'tt' 'ul' 'uq' 'vm' 'wt' 'xg' 'xn' 'ya' 'yf' 'yw' 'yx' 'zp' - f | '2l' '55' 'aq' 'aw' 'cm' 'dq' 'eb' 'fl' 'fo' 'fp' 'fy' 'gk' 'hy' 'ic' 'iu' 'jc' 'jl' 'lc' 'mb' 'nx' 'oe' 'oo' 'oz' 'p5' 'pa' 'pf' 'qr' 'rj' 'sc' 'sz' 'td' 'tz' 'u1' 'vq' 'wu' 'xb' 'xx' 'y0' 'za' 'zc' - f | '4z' 'a2' 'eg' 'f4' 'g9' 'gi' 'hd' 'hn' 'ij' 'io' 'ix' 'iz' 'jb' 'jd' 'kk' 'l4' 'lh' 'lq' 'p9' 'pa' 'pb' 'q7' 'qh' 'qi' 'qk' 'qs' 'r9' 'rc' 'ro' 'rr' 's4' 'tj' 'tz' 'vi' 'we' 'wp' 'xe' 'yd' 'yg' 'zj' - f | '7h' 'an' 'b6' 'cl' 'e1' 'e4' 'en' 'ey' 'ff' 'gn' 'h7' 'ht' 'i5' 'i9' 'ia' 'iw' 'jx' 'kl' 'ku' 'op' 'pe' 'pt' 'qb' 'ql' 'qp' 'qy' 'ri' 'se' 'si' 'ta' 'ti' 'tl' 'tp' 'uo' 'vr' 'w4' 'wn' 'xi' 'xs' 'ym' - f | '7k' 'et' 'ew' 'ez' 'g9' 'gk' 'he' 'hp' 'ip' 'jp' 'jq' 'jr' 'jv' 'kb' 'kk' 'ku' 'lx' 'o0' 'o3' 'oa' 'or' 'ou' 'q7' 'qa' 'qf' 'qg' 'qh' 'qo' 'ro' 'rx' 'tl' 'us' 'vk' 'vw' 'wc' 'wp' 'x0' 'xz' 'yg' 'z5' - f | 'a5' 'ab' 'ad' 'ao' 'b6' 'ea' 'es' 'ez' 'gm' 'gv' 'i0' 'in' 'jw' 'lh' 'lm' 'my' 'oj' 'ox' 'oy' 'pv' 'q1' 'qf' 'qh' 'qj' 'qo' 'qs' 'qt' 'qw' 'r0' 'ri' 'rk' 'ry' 's4' 'vs' 'wa' 'xy' 'y5' 'yg' 'yl' 'zm' - f | 'aa' 'ap' 'b0' 'c6' 'e1' 'ex' 'fe' 'fu' 'fz' 'g1' 'g8' 'go' 'hl' 'iz' 'k3' 'ls' 'mw' 'mz' 'oa' 'ow' 'p6' 'q5' 'qh' 'qq' 'qx' 'ra' 'rh' 'rp' 'rw' 'tf' 'tk' 'tl' 'uc' 'wi' 'x1' 'xb' 'yb' 'yw' 'yz' 'zm' - f | 'ad' 'as' 'dn' 'dq' 'e7' 'ei' 'ey' 'fj' 'h2' 'hu' 'hw' 'i7' 'i9' 'iu' 'jr' 'ma' 'na' 'nh' 'nk' 'pa' 'q5' 'qd' 'qn' 's0' 'so' 'sx' 'u7' 'uc' 'um' 'vb' 'w9' 'wf' 'wh' 'wl' 'wn' 'wr' 'xl' 'y8' 'yl' 'zw' - f | 'ae' 'ai' 'di' 'ea' 'eb' 'ec' 'gw' 'h8' 'hb' 'iw' 'iz' 'jf' 'jg' 'ks' 'le' 'p8' 'pl' 'pp' 'qd' 'qg' 'qm' 'qn' 'qu' 'qy' 'r4' 'rf' 'rq' 's1' 'sc' 'sq' 'v8' 'vh' 'vk' 'wd' 'x7' 'y7' 'ym' 'yp' 'yr' 'yw' - f | 'ah' 'aw' 'e5' 'en' 'ew' 'fi' 'fm' 'gq' 'gt' 'hy' 'ib' 'ie' 'ir' 'j9' 'jr' 'js' 'ny' 'ob' 'or' 'ot' 'oy' 'pt' 'qi' 'rd' 'ri' 'rl' 'ro' 'rt' 'ss' 'tb' 'tg' 'th' 'vy' 'wq' 'wz' 'xt' 'y6' 'y8' 'zf' 'zh' - f | '43' '9w' 'ah' 'av' 'bl' 'dr' 'ef' 'es' 'fm' 'ft' 'gy' 'hg' 'hq' 'hy' 'iu' 'ix' 'j1' 'jn' 'lg' 'np' 'o9' 'op' 'ou' 'ox' 'qq' 'st' 'sw' 't8' 't9' 'vs' 'vw' 'wd' 'wx' 'wz' 'xr' 'xt' 'xv' 'y5' 'y8' 'yi' 'ze' - f | '5f' '5i' 'ak' 'cg' 'dh' 'di' 'dk' 'eq' 'f6' 'g1' 'g8' 'ie' 'md' 'o0' 'oj' 'oq' 'pt' 'q7' 'qb' 'qd' 'qe' 'qo' 'ro' 'ru' 'rv' 'rx' 'sp' 'sr' 'tf' 'tp' 'uh' 'ul' 'vs' 'xn' 'xw' 'y2' 'yh' 'yk' 'zg' 'zm' 'zq' - f | 'ab' 'ac' 'b9' 'br' 'c8' 'eh' 'en' 'eq' 'ev' 'ew' 'f6' 'gd' 'ik' 'j5' 'jm' 'kc' 'ke' 'ok' 'p6' 'pa' 'qv' 'ri' 'rm' 'ro' 'rp' 'rq' 'rx' 'rz' 'se' 'tb' 'td' 'ti' 'tj' 'tn' 'ul' 'wj' 'wp' 'ws' 'yh' 'zb' 'zc' - f | 'ak' 'at' 'cs' 'di' 'dt' 'ef' 'eq' 'f1' 'f8' 'fb' 'fh' 'fv' 'g4' 'hs' 'j2' 'jo' 'kq' 'lf' 'lq' 'no' 'oa' 'ot' 'q0' 'qh' 'qs' 'qw' 'se' 'so' 'sx' 't3' 'ul' 'un' 'uu' 'wn' 'ws' 'wx' 'y0' 'y3' 'yf' 'zb' 'zc' - f | 'bq' 'cc' 'cg' 'dc' 'fu' 'ho' 'ja' 'jt' 'km' 'lk' 'ln' 'o2' 'ob' 'ou' 'pd' 'ph' 'pn' 'pt' 'pz' 'q5' 'qa' 'ql' 'qq' 'qv' 'rk' 'rm' 'ry' 'sq' 'su' 'sx' 'uc' 'un' 'uo' 'ur' 'uy' 'w9' 'wj' 'wl' 'wm' 'yk' 'ym' - f | 'ca' 'cm' 'cr' 'eh' 'fd' 'fp' 'g8' 'ga' 'gh' 'h1' 'iu' 'je' 'jy' 'kh' 'lx' 'o4' 'oi' 'ov' 'ph' 'pm' 'qi' 'qv' 're' 'rg' 'rv' 'ry' 'si' 'sl' 'sv' 'sx' 'tq' 'tz' 'uh' 'uq' 'uy' 'wi' 'wo' 'y7' 'yc' 'yn' 'yy' - f | '1u' '5k' 'ax' 'az' 'c4' 'eo' 'fi' 'gq' 'hg' 'ho' 'is' 'jq' 'ki' 'oq' 'ov' 'p4' 'pl' 'po' 'qe' 'qf' 'qn' 'qy' 'r3' 'rm' 'sd' 'se' 'sz' 'tb' 'td' 'u4' 'u5' 'vl' 'vv' 'wb' 'we' 'wh' 'wz' 'y5' 'ye' 'yv' 'zh' 'zj' - f | '4r' 'ae' 'd9' 'da' 'ef' 'em' 'en' 'gi' 'h8' 'jq' 'kf' 'kh' 'kr' 'kv' 'li' 'lq' 'lr' 'm2' 'ny' 'po' 'q0' 'q8' 'qf' 'qg' 'qu' 'qw' 'qx' 'rd' 'rh' 'ry' 'sm' 'st' 't3' 'tc' 'u5' 'un' 'uw' 'wg' 'wt' 'yt' 'z9' 'zc' - f | '4t' 'ay' 'b3' 'dz' 'eo' 'ep' 'fd' 'fh' 'ht' 'hw' 'i3' 'jl' 'kn' 'op' 'qb' 'qd' 'ql' 'qm' 'qp' 'qy' 'rc' 'rh' 'rs' 'rw' 'sm' 't9' 'td' 'tu' 'tw' 'tz' 'u1' 'ug' 'uh' 'ur' 'vt' 'we' 'wi' 'wn' 'x7' 'y8' 'yn' 'yy' - f | '5z' 'ai' 'al' 'ax' 'cd' 'dr' 'e1' 'ep' 'fi' 'gh' 'gk' 'ha' 'hc' 'ht' 'i5' 'ic' 'id' 'iq' 'iz' 'kk' 'kt' 'ld' 'mi' 'oh' 'ov' 'pn' 'q2' 'q7' 'qm' 'qy' 'ri' 'ru' 'rv' 'sq' 'u8' 'u9' 'un' 'wq' 'wt' 'yd' 'yk' 'ys' - f | 'ar' 'c3' 'c6' 'e0' 'e9' 'ed' 'ex' 'fh' 'fi' 'gu' 'ha' 'hr' 'id' 'jr' 'kk' 'kr' 'kz' 'm0' 'mp' 'ng' 'om' 'pa' 'qa' 'qf' 'rj' 'sp' 'to' 'tu' 'ty' 'ub' 'ud' 'ug' 'uk' 'vj' 'vp' 'wf' 'wl' 'wu' 'x6' 'xs' 'y0' 'ya' - f | 'bt' 'dg' 'dz' 'e4' 'ek' 'ep' 'fi' 'gk' 'gl' 'gu' 'hl' 'ho' 'io' 'ls' 'ni' 'nw' 'pv' 'px' 'qe' 'qf' 'qj' 'ql' 'qu' 'qy' 'rw' 'rx' 'ss' 't7' 'tt' 'ty' 'ud' 'up' 'uq' 'uw' 'wn' 'wy' 'xt' 'yv' 'yx' 'za' 'zv' 'zw' - f | '2o' '6t' 'a7' 'bv' 'di' 'dm' 'ea' 'eo' 'fd' 'fj' 'ft' 'fw' 'gs' 'hu' 'hz' 'ik' 'it' 'jc' 'js' 'jx' 'jz' 'k4' 'lp' 'lx' 'ol' 'p3' 'pf' 'q9' 'qu' 'qv' 'qx' 're' 'rl' 'ro' 'sb' 'ur' 'vf' 'wd' 'wk' 'wo' 'wx' 'yp' 'zo' - f | '5y' 'av' 'ax' 'cl' 'cw' 'dh' 'di' 'g7' 'ga' 'gi' 'gx' 'gy' 'i0' 'iu' 'iv' 'jk' 'jv' 'k3' 'l2' 'lo' 'lq' 'mj' 'oj' 'pd' 'pn' 'pv' 'q6' 'qq' 'qv' 'sc' 'tt' 'tv' 'ug' 'ut' 've' 'vy' 'wi' 'xg' 'xo' 'yi' 'yl' 'yv' 'zm' - f | 'ab' 'av' 'c9' 'cs' 'cx' 'dy' 'e4' 'eb' 'ej' 'er' 'ez' 'f0' 'f2' 'ft' 'gk' 'h5' 'i5' 'ic' 'ig' 'io' 'ki' 'kz' 'lj' 'mm' 'of' 'og' 'ov' 'oz' 'p4' 'pl' 'qh' 'qx' 'rw' 's1' 'sf' 'tw' 'uc' 'ui' 'uo' 'uw' 'x4' 'xk' 'ze' - f | 'ac' 'bv' 'd2' 'de' 'e8' 'ea' 'go' 'gw' 'h4' 'ht' 'iy' 'jm' 'ot' 'pp' 'pw' 'pz' 'qd' 'qn' 'qu' 'qy' 'r1' 'ra' 'rg' 'rh' 'rp' 's0' 'sb' 'sn' 'st' 'td' 'uz' 'vg' 'w1' 'w6' 'wa' 'wc' 'we' 'wi' 'wj' 'wu' 'ww' 'y9' 'zr' - f | 'ak' 'bg' 'bj' 'bn' 'co' 'e6' 'e7' 'ec' 'ek' 'el' 'ew' 'ez' 'fo' 'im' 'jm' 'js' 'lc' 'lh' 'ob' 'oi' 'os' 'ot' 'ou' 'p0' 'pu' 'q0' 'qh' 'ra' 'rc' 'rm' 'rr' 'sd' 'sq' 'tc' 'tl' 'wj' 'wy' 'x2' 'xw' 'y6' 'yp' 'yv' 'zj' - f | 'ao' 'ei' 'ep' 'fc' 'fm' 'gr' 'ha' 'he' 'i5' 'ia' 'if' 'kh' 'lv' 'ml' 'ms' 'n9' 'nh' 'nl' 'oe' 'of' 'ow' 'pb' 'pu' 'pw' 'q0' 'qr' 'qt' 'qu' 'qv' 'qw' 'ra' 'sv' 'ti' 'u0' 'u4' 'ug' 'uq' 'vx' 'w1' 'wy' 'x1' 'xh' 'xz' - f | 'cr' 'd3' 'e5' 'ei' 'fb' 'gd' 'gx' 'hm' 'it' 'jp' 'ku' 'lf' 'lh' 'lp' 'ma' 'md' 'nt' 'o9' 'ot' 'pk' 'pm' 'ps' 'qc' 'qi' 'qp' 'qt' 'ra' 'rk' 'rq' 'rt' 'tb' 'tc' 'tg' 'to' 'u8' 'vg' 'vj' 'vt' 'w0' 'wr' 'wx' 'yd' 'zx' - f | '3p' '5x' 'ag' 'au' 'aw' 'dl' 'eg' 'f2' 'f8' 'fb' 'fn' 'g9' 'ge' 'gt' 'hd' 'i7' 'ia' 'ij' 'is' 'jc' 'ku' 'kx' 'l7' 'lb' 'lq' 'mc' 'o3' 'oj' 'q1' 'qa' 'qg' 'qt' 'r9' 'rg' 'tw' 'tz' 'u6' 'us' 'w2' 'wr' 'wt' 'wu' 'x3' 'xx' - f | 'aa' 'al' 'ay' 'c1' 'ca' 'de' 'ea' 'ee' 'eq' 'er' 'ez' 'fc' 'gy' 'i1' 'jb' 'jn' 'jz' 'lp' 'o5' 'om' 'os' 'ou' 'ox' 'px' 'q3' 'qc' 'qs' 'rm' 'rq' 'rt' 'rx' 'ry' 'rz' 'ss' 'ta' 'tg' 'ti' 'tr' 'u6' 'uv' 'vp' 'wn' 'wu' 'zt' - f | 'ak' 'bg' 'bk' 'du' 'eb' 'ef' 'el' 'ft' 'gb' 'gc' 'hc' 'ho' 'i6' 'iv' 'ji' 'ke' 'mo' 'oc' 'oh' 'pe' 'pj' 'q6' 'qe' 'qg' 'qs' 'qv' 'r1' 'rr' 'rx' 'tc' 'te' 'tf' 'tk' 'tq' 'ua' 'uq' 'ur' 'wq' 'wx' 'wz' 'xi' 'xq' 'yk' 'yo' - f | 'al' 'ar' 'cl' 'dl' 'dz' 'ej' 'fj' 'gu' 'i2' 'i6' 'ih' 'jx' 'k9' 'ln' 'ls' 'mf' 'mx' 'oi' 'om' 'pn' 'q4' 'qd' 'qe' 'qg' 'qk' 'qp' 'qr' 'qw' 'qx' 'rr' 'sd' 'sf' 'ub' 'uh' 'um' 'uu' 'vg' 'w6' 'wj' 'wp' 'ws' 'yn' 'zc' 'zm' - f | 'am' 'aw' 'ay' 'bj' 'df' 'eb' 'eo' 'eq' 'et' 'ey' 'f9' 'fo' 'g0' 'gg' 'h4' 'hq' 'in' 'k4' 'k7' 'kb' 'kl' 'my' 'nt' 'o3' 'og' 'pc' 'q4' 'qg' 'qi' 'qm' 'qy' 'ri' 'rv' 's9' 'sv' 'tl' 'ue' 'uo' 'v6' 'wb' 'wt' 'wx' 'xi' 'xu' - f | 'bs' 'cd' 'ci' 'cq' 'dd' 'el' 'ev' 'f4' 'f6' 'f7' 'fh' 'fu' 'hi' 'ib' 'kp' 'nn' 'nu' 'oc' 'os' 'qc' 'qj' 'qm' 'qn' 'qu' 'qw' 'qx' 'r7' 'rc' 'rv' 't1' 'tl' 'tr' 'u0' 'vi' 'wl' 'wp' 'wu' 'xc' 'y2' 'yu' 'zc' 'zi' 'zw' 'zy' - f | 'bt' 'c9' 'db' 'dy' 'ee' 'ej' 'eo' 'et' 'fr' 'gm' 'hz' 'ic' 'ik' 'is' 'jk' 'ls' 'lt' 'oj' 'or' 'ox' 'p8' 'pl' 'pz' 'q1' 'qq' 'qt' 'rw' 's7' 't4' 't8' 'to' 'tp' 'u7' 'ub' 'vx' 'w4' 'ws' 'wt' 'wv' 'x8' 'xs' 'xt' 'yg' 'yo' - f | 'cm' 'd2' 'en' 'ex' 'f4' 'f9' 'fp' 'ga' 'gk' 'hy' 'i3' 'j4' 'j9' 'jb' 'kj' 'ls' 'lw' 'mb' 'mq' 'o9' 'qg' 'qi' 'qm' 'qo' 'qw' 'qy' 'rc' 'rf' 'rj' 'rm' 's8' 'tb' 'tl' 'tp' 'um' 'vg' 'vn' 'wb' 'wf' 'wl' 'xw' 'yl' 'zf' 'zt' - f | '28' 'b1' 'bj' 'bm' 'cz' 'dd' 'ds' 'ed' 'em' 'hh' 'hr' 'ie' 'ii' 'im' 'io' 'jn' 'jw' 'lg' 'lo' 'm6' 'mn' 'mw' 'ny' 'o6' 'p7' 'q4' 'qd' 'qf' 'qn' 'qq' 'qt' 'r1' 'r2' 'rn' 'rt' 'sh' 'ur' 'uv' 'ux' 'vc' 'wb' 'wn' 'xh' 'xx' 'yw' - f | '2e' 'al' 'am' 'bt' 'eo' 'ff' 'fl' 'ge' 'gz' 'h0' 'hb' 'i0' 'j2' 'lh' 'lm' 'ls' 'mg' 'mw' 'on' 'ox' 'pa' 'pb' 'pg' 'q3' 'q6' 'qe' 'ra' 're' 'rm' 'rr' 'rt' 'ry' 'sc' 'sm' 't9' 'tu' 'tx' 'ul' 'ux' 'vh' 'wn' 'wt' 'wv' 'ys' 'yw' - f | '3d' 'at' 'aw' 'bt' 'by' 'cg' 'dk' 'do' 'ec' 'eu' 'gv' 'is' 'k9' 'kb' 'kc' 'me' 'mm' 'o5' 'oe' 'pr' 'qa' 'qn' 'rc' 'rl' 'rn' 'rw' 'sc' 'sk' 'st' 't1' 't7' 'td' 'to' 'ue' 'up' 'ur' 'uy' 've' 'wi' 'xf' 'xu' 'yf' 'z9' 'zo' 'zr' - f | '40' '6a' '8j' 'am' 'bf' 'dp' 'dz' 'ew' 'gh' 'he' 'hh' 'ib' 'ii' 'iq' 'is' 'iu' 'kt' 'mf' 'oh' 'or' 'p8' 'pa' 'pb' 'pi' 'qc' 'qo' 'qp' 'qq' 'qs' 'qt' 'qu' 'r4' 'rf' 'rl' 'sd' 'sl' 't4' 'tb' 'ua' 'ue' 'ut' 'wh' 'wm' 'y0' 'y6' - f | 'a0' 'a6' 'aa' 'bk' 'dk' 'dv' 'e8' 'ff' 'fo' 'go' 'hh' 'hl' 'hu' 'if' 'jx' 'kc' 'kq' 'lr' 'lx' 'n6' 'ni' 'ob' 'ol' 'ql' 'rf' 'ry' 'sm' 'sn' 't7' 'tu' 'tz' 'uc' 'um' 'uv' 'wf' 'wr' 'ws' 'wu' 'ww' 'xi' 'ym' 'z7' 'zo' 'zv' 'zz' - f | 'as' 'dj' 'dq' 'e3' 'ej' 'es' 'fn' 'fp' 'fu' 'ga' 'hb' 'hy' 'jz' 'l6' 'mp' 'ps' 'q4' 'qd' 'qm' 'qw' 'qy' 'qz' 'sa' 'sv' 'td' 'tn' 'tu' 'tx' 'ud' 'ue' 'uq' 'uv' 'v0' 'wi' 'wk' 'wm' 'wx' 'xh' 'y3' 'yd' 'yi' 'yn' 'yx' 'yy' 'zc' - f | 'cx' 'dg' 'do' 'dt' 'e7' 'ek' 'ez' 'fn' 'hj' 'hv' 'i2' 'i6' 'is' 'it' 'ka' 'kh' 'l5' 'lg' 'p5' 'qa' 'qf' 'qj' 'qw' 'qz' 'rl' 'sl' 'sr' 't7' 'tf' 'tu' 'uc' 'ud' 'uv' 'vr' 'w0' 'we' 'wf' 'wj' 'ws' 'ww' 'xe' 'xh' 'xn' 'y9' 'yv' - f | '2a' '4s' 'c4' 'cd' 'dh' 'dv' 'eh' 'ek' 'ew' 'ex' 'ez' 'fm' 'hh' 'hj' 'ig' 'jf' 'ld' 'ly' 'mg' 'mx' 'o5' 'or' 'pa' 'pv' 'q1' 'qe' 'qj' 'qq' 'qr' 'qw' 'rb' 'rr' 's4' 's9' 'tm' 'uj' 'wc' 'wv' 'x0' 'xt' 'yc' 'ye' 'yh' 'yi' 'yk' 'yy' - f | 'a2' 'a9' 'ac' 'bw' 'cu' 'dm' 'e8' 'ee' 'ej' 'ep' 'f7' 'fh' 'ga' 'go' 'im' 'it' 'jv' 'kg' 'lb' 'ml' 'oi' 'p5' 'p9' 'pe' 'pf' 'pp' 'qk' 'rg' 'rh' 'sn' 'sv' 't5' 'tj' 'tk' 'tq' 'ty' 'v6' 'vk' 'w2' 'w5' 'wh' 'wy' 'y7' 'yf' 'zq' 'zt' - f | 'aa' 'am' 'av' 'dt' 'du' 'eh' 'em' 'ev' 'ex' 'gu' 'h0' 'i4' 'i8' 'ku' 'la' 'lo' 'lw' 'n0' 'n9' 'on' 'pc' 'pl' 'pn' 'pr' 'q8' 'qc' 'ql' 'qn' 'qz' 'ra' 'rd' 'ry' 'sp' 'tn' 'ts' 'u4' 'vg' 'vs' 'w7' 'wj' 'wm' 'xl' 'yg' 'yh' 'z0' 'zs' - f | 'ak' 'ar' 'cm' 'cq' 'da' 'df' 'e3' 'ew' 'fe' 'fr' 'gm' 'id' 'io' 'iy' 'jg' 'jt' 'jx' 'kb' 'lf' 'nv' 'od' 'qy' 'r3' 'r4' 'ro' 'rq' 'rw' 'sl' 'ss' 'su' 'sw' 't7' 'th' 'ti' 'tp' 'u9' 'uq' 'v9' 'vq' 'vw' 'we' 'wg' 'wk' 'ws' 'xr' 'yf' - f | 'bv' 'cv' 'dy' 'ec' 'ef' 'eg' 'eh' 'eu' 'ew' 'fc' 'fn' 'fs' 'g9' 'hj' 'hk' 'i3' 'iu' 'ja' 'k6' 'l1' 'la' 'nl' 'oh' 'oi' 'oy' 'pc' 'qe' 'ql' 'qq' 'qs' 'qu' 're' 'sf' 'sq' 'tb' 'up' 'w1' 'w2' 'wg' 'wi' 'xd' 'xt' 'xv' 'yf' 'zn' 'zz' - f | '3y' 'ak' 'am' 'b8' 'cp' 'd7' 'df' 'di' 'eg' 'em' 'en' 'eo' 'es' 'f6' 'fc' 'gg' 'gt' 'gv' 'hd' 'i3' 'jk' 'k6' 'la' 'lc' 'ol' 'ov' 'ow' 'p5' 'pl' 'qc' 'qf' 'rl' 'rn' 'rt' 'rx' 't1' 'tf' 'tj' 'tz' 'ug' 'uo' 'we' 'wg' 'wk' 'wp' 'wv' 'y7' - f | '5g' 'a5' 'dj' 'dl' 'dr' 'e9' 'ed' 'ep' 'er' 'gb' 'hh' 'hl' 'i0' 'if' 'ig' 'io' 'it' 'k2' 'ki' 'll' 'o9' 'om' 'ot' 'p5' 'q1' 'q6' 'qa' 'qp' 'qw' 'qy' 'r9' 'rb' 'rm' 'ry' 's7' 'tl' 'ts' 'ut' 'w0' 'wd' 'wu' 'wv' 'xk' 'ya' 'z5' 'zq' 'zs' - f | '5l' 'a2' 'ak' 'au' 'cj' 'cx' 'ed' 'eg' 'ej' 'em' 'es' 'ex' 'fs' 'ft' 'hm' 'i9' 'ky' 'l3' 'lz' 'mi' 'o2' 'oc' 'oe' 'ok' 'p2' 'q1' 'q5' 'qo' 'qv' 'qw' 'rf' 'rm' 'rv' 'sc' 'si' 'tx' 'uq' 'v3' 'v4' 'vo' 'w4' 'wf' 'wl' 'wq' 'wt' 'xj' 'yr' - f | 'al' 'ar' 'av' 'bg' 'bh' 'bz' 'e9' 'ed' 'ee' 'ef' 'eh' 'es' 'fl' 'fz' 'gl' 'h1' 'he' 'hf' 'hv' 'hw' 'i1' 'jg' 'kj' 'kn' 'kz' 'lr' 'pp' 'q1' 'qa' 'qf' 'qt' 'qz' 'rg' 'rj' 's5' 'sv' 'sx' 't4' 'th' 'tn' 'ur' 'wz' 'xp' 'yg' 'ym' 'yr' 'ze' - f | 'az' 'cx' 'dm' 'do' 'dq' 'e2' 'e5' 'e8' 'eb' 'fh' 'fj' 'fq' 'gj' 'gl' 'hq' 'im' 'iq' 'iz' 'jn' 'ku' 'l1' 'mz' 'nt' 'o6' 'q1' 'qn' 'qo' 'qz' 'r4' 'rh' 'rn' 'rw' 'sn' 'sv' 't7' 'ud' 'ue' 'ur' 'uv' 'w9' 'wh' 'wk' 'wy' 'xq' 'yh' 'zs' 'zw' - f | '18' 'bf' 'bj' 'cs' 'dw' 'el' 'ep' 'ez' 'fr' 'g1' 'gv' 'hu' 'hz' 'ii' 'jc' 'je' 'jj' 'jo' 'jv' 'n5' 'ni' 'ns' 'oh' 'oi' 'ov' 'pe' 'q8' 'qg' 'qi' 'qr' 'r6' 'rh' 's3' 'sj' 'sr' 't0' 'ta' 'u2' 'uj' 'ur' 'vu' 'wb' 'wh' 'wx' 'yn' 'yq' 'zg' 'zr' - f | '2s' 'a5' 'aw' 'b5' 'b7' 'ce' 'cq' 'dc' 'dp' 'eb' 'eq' 'ez' 'ff' 'gg' 'gy' 'iw' 'jd' 'jl' 'kw' 'ky' 'mt' 'ng' 'o9' 'od' 'oe' 'po' 'pp' 'qa' 'qe' 'qk' 'ql' 'qp' 'r4' 'r9' 'rb' 'si' 'sl' 'sn' 'sq' 'tg' 'ti' 'tx' 'uh' 'ws' 'xf' 'yv' 'z1' 'zm' - f | '4j' 'bi' 'bk' 'c0' 'c5' 'dh' 'dy' 'e8' 'fw' 'g3' 'g8' 'h6' 'hd' 'hn' 'jl' 'kc' 'kj' 'km' 'll' 'lo' 'nh' 'nj' 'nt' 'ob' 'on' 'oq' 'or' 'ox' 'ph' 'qc' 'qj' 'ql' 'qm' 'qo' 'qu' 'qz' 'sr' 't7' 'tc' 'ug' 'uq' 'wn' 'wr' 'ws' 'yv' 'z3' 'zj' 'zz' - f | '6w' 'af' 'ak' 'b7' 'bf' 'bi' 'bv' 'cx' 'dy' 'es' 'ey' 'ez' 'f0' 'fe' 'fw' 'ge' 'gt' 'iw' 'j9' 'jh' 'kh' 'l0' 'l1' 'l8' 'm5' 'nv' 'o6' 'pw' 'qi' 'r0' 'se' 'ta' 'tf' 'th' 'to' 'u8' 'ux' 'vm' 'w4' 'w6' 'wa' 'wt' 'wu' 'xo' 'xu' 'yo' 'z0' 'zc' - f | 'au' 'bl' 'c4' 'd7' 'dc' 'dm' 'ea' 'eo' 'ft' 'gb' 'h3' 'ia' 'ix' 'jr' 'kc' 'ke' 'lq' 'lr' 'mn' 'op' 'pj' 'qc' 'qj' 'qn' 'qq' 'qx' 'r8' 'rb' 'ri' 'rm' 'rn' 'rx' 't6' 'tf' 'tv' 'um' 'vg' 'vi' 'we' 'wp' 'wz' 'x6' 'xb' 'xd' 'xg' 'xv' 'yx' 'yz' - f | '7r' 'a8' 'aa' 'ag' 'bo' 'bp' 'c7' 'd0' 'd6' 'e5' 'er' 'fc' 'ff' 'go' 'ha' 'hs' 'jj' 'jq' 'ki' 'lc' 'le' 'no' 'pf' 'qc' 'qh' 'qq' 'qs' 'qv' 'rf' 'rj' 'rx' 'su' 'ta' 'ti' 'tu' 'tv' 'ty' 'ug' 'un' 'up' 'vb' 'vi' 'wp' 'wu' 'wz' 'xt' 'y0' 'yd' 'yu' - f | 'at' 'b7' 'be' 'c3' 'cc' 'cd' 'cj' 'e8' 'eb' 'ei' 'fd' 'g8' 'gh' 'go' 'gv' 'if' 'ih' 'j7' 'jy' 'kj' 'lh' 'mc' 'oe' 'og' 'oy' 'oz' 'pm' 'qd' 'qe' 'qo' 'rg' 'rq' 'rx' 'se' 'su' 't3' 't6' 't9' 'u5' 'us' 'vw' 'w4' 'wb' 'wg' 'ws' 'xw' 'yu' 'yx' 'zj' - f | '18' 'a6' 'af' 'ax' 'du' 'dy' 'e7' 'ei' 'ej' 'es' 'eu' 'fj' 'gt' 'hn' 'hu' 'ky' 'le' 'lo' 'm8' 'nb' 'of' 'oj' 'ok' 'op' 'oz' 'pw' 'q7' 'qi' 'qu' 'r4' 'rp' 's7' 'sk' 'tb' 'tf' 'tp' 'ua' 'um' 'up' 'ur' 'uz' 'w4' 'wc' 'wn' 'wv' 'wx' 'wy' 'yc' 'yq' 'yx' - f | '82' 'a2' 'ad' 'dp' 'el' 'ep' 'fk' 'gd' 'hj' 'ij' 'lc' 'lm' 'lv' 'ml' 'n2' 'n6' 'o1' 'oi' 'on' 'oo' 'ow' 'pb' 'pe' 'pq' 'q6' 'qi' 'qo' 'qp' 'qw' 'rs' 'rv' 's6' 'sa' 'sk' 't8' 'tl' 'tt' 'tx' 'ub' 'ue' 'uf' 'uj' 'va' 'wl' 'xv' 'y1' 'ye' 'ym' 'yw' 'zm' - f | 'a3' 'b9' 'c0' 'cg' 'ct' 'cu' 'dy' 'eh' 'el' 'ge' 'gt' 'hh' 'hu' 'if' 'j5' 'kq' 'lb' 'mh' 'o3' 'o6' 'oe' 'oi' 'ow' 'p5' 'ph' 'pw' 'q0' 'qh' 'ql' 'qy' 'r7' 'si' 'sk' 'sv' 'tf' 'tm' 'to' 'tt' 'ty' 'u3' 'uj' 'uo' 'vi' 'vl' 'wc' 'wf' 'xx' 'y0' 'ya' 'yu' - f | 'ad' 'am' 'av' 'ax' 'dl' 'dt' 'dw' 'ej' 'ez' 'f6' 'fe' 'g5' 'ga' 'gv' 'hr' 'hx' 'id' 'ie' 'ii' 'im' 'ix' 'jd' 'kn' 'ml' 'nu' 'ol' 'oo' 'pu' 'qd' 'qm' 'qt' 'r5' 'rb' 'rd' 'rg' 'ru' 'sg' 'ub' 'uk' 'vm' 'wl' 'wr' 'wz' 'x4' 'yf' 'yu' 'yv' 'yw' 'yx' 'zt' - f | 'bw' 'ci' 'cp' 'cu' 'cw' 'd7' 'e0' 'e3' 'em' 'ev' 'fi' 'fk' 'fm' 'fu' 'ht' 'ix' 'jn' 'kc' 'kw' 'lm' 'mo' 'n8' 'od' 'oo' 'q1' 'qd' 'qw' 'rs' 'sd' 'su' 'sz' 'th' 'ti' 'ts' 'tw' 'ty' 'u3' 'ua' 'wa' 'wt' 'wy' 'xg' 'xh' 'xp' 'xr' 'yd' 'yj' 'yq' 'yx' 'zu' - f | '1d' 'a4' 'ai' 'av' 'b9' 'be' 'c9' 'cc' 'cy' 'de' 'ee' 'g8' 'gg' 'gl' 'gs' 'gu' 'hb' 'kl' 'kr' 'ky' 'mi' 'mz' 'nd' 'og' 'op' 'pr' 'q0' 'q5' 'qk' 'qy' 'r4' 'sk' 'sz' 't1' 't8' 'td' 'tn' 'tx' 'ty' 'tz' 'uc' 'up' 'vq' 'wa' 'ws' 'wv' 'x3' 'yn' 'yq' 'yy' 'zm' - f | '1u' 'ad' 'al' 'c4' 'cz' 'db' 'dk' 'dr' 'e5' 'e7' 'ea' 'f6' 'fk' 'fw' 'g3' 'gl' 'gt' 'gu' 'ha' 'hk' 'hv' 'ie' 'ka' 'kq' 'ks' 'lr' 'nf' 'pk' 'q3' 'q8' 'qf' 'qj' 'qo' 'rp' 'rr' 'rz' 'ss' 'sv' 'sx' 'tb' 'th' 'tt' 'ur' 'vp' 'wj' 'ws' 'wt' 'xk' 'xo' 'ym' 'yz' - f | '3f' '93' 'am' 'an' 'at' 'bj' 'bm' 'bw' 'c6' 'eb' 'ef' 'em' 'eq' 'es' 'g9' 'hc' 'hl' 'hm' 'kg' 'kt' 'mg' 'nq' 'op' 'ox' 'pc' 'py' 'qd' 'qn' 'qz' 'rb' 'rw' 's6' 'se' 'sr' 'sy' 'ts' 'u4' 'ud' 'un' 'w3' 'w8' 'wb' 'wf' 'wg' 'wy' 'x1' 'xl' 'xy' 'yk' 'yo' 'zp' - f | 'a6' 'ad' 'aj' 'b1' 'bz' 'cd' 'df' 'ed' 'eg' 'eh' 'ev' 'fi' 'fo' 'gj' 'h3' 'he' 'ib' 'j0' 'jc' 'jz' 'kz' 'lc' 'mh' 'pk' 'pu' 'q4' 'qc' 'qk' 'qr' 'qw' 'rc' 'rl' 'ro' 'rr' 'so' 'sw' 'td' 'tw' 'tx' 'ue' 'us' 'ut' 'vd' 'wi' 'wu' 'xm' 'y2' 'y8' 'ym' 'z6' 'zm' - f | '2e' '3y' 'ap' 'as' 'bl' 'di' 'dl' 'e9' 'eh' 'er' 'ff' 'fg' 'fh' 'gc' 'gg' 'hp' 'hq' 'i3' 'ih' 'jm' 'kw' 'lc' 'li' 'ls' 'nm' 'ok' 'pc' 'pn' 'q1' 'qf' 'qi' 'qk' 'r0' 'rl' 'sp' 'sy' 'sz' 'ta' 'to' 'ts' 'tv' 'uh' 'uk' 'vb' 'wa' 'wg' 'wi' 'wp' 'yl' 'yq' 'yw' 'yy' - f | '2h' '9y' 'ac' 'at' 'cf' 'e3' 'e6' 'ec' 'ek' 'en' 'f1' 'fa' 'fn' 'gm' 'gx' 'hk' 'i7' 'id' 'iw' 'ke' 'kl' 'ks' 'l6' 'l8' 'ls' 'lx' 'me' 'o4' 'oi' 'oj' 'ox' 'pn' 'qa' 'qh' 'qk' 'ql' 'rp' 'tb' 'te' 'tn' 'ts' 'u4' 'u9' 'ud' 'vj' 'vz' 'wc' 'wr' 'ye' 'yk' 'yq' 'zq' - f | '4j' 'af' 'ap' 'bn' 'ch' 'dc' 'df' 'e0' 'ef' 'eg' 'er' 'g3' 'hh' 'hi' 'hw' 'i1' 'if' 'ii' 'in' 'iy' 'l1' 'mx' 'og' 'px' 'q3' 'qc' 'qg' 'qt' 'qy' 'r3' 'r6' 'rd' 'rj' 'rz' 'sm' 't2' 't4' 'td' 'tj' 'tm' 'tx' 'ty' 'u9' 'uo' 'up' 'v8' 'wv' 'ye' 'zb' 'zc' 'zi' 'zy' - f | 'a0' 'ai' 'bx' 'ca' 'e2' 'eb' 'ed' 'eg' 'eh' 'eo' 'et' 'hn' 'ix' 'jh' 'ki' 'lm' 'lw' 'm8' 'mb' 'mh' 'mk' 'nc' 'o3' 'o9' 'of' 'qc' 'qe' 'qf' 'qh' 'qi' 'qq' 'qr' 'qz' 'r2' 'r3' 'rc' 'rh' 'rs' 'rv' 'sr' 'uk' 'up' 'ur' 'uv' 'wm' 'wr' 'wz' 'xd' 'y3' 'ya' 'yv' 'zr' - f | 'ab' 'ae' 'al' 'am' 'an' 'ap' 'be' 'd2' 'eb' 'ec' 'ep' 'eq' 'fn' 'hp' 'hx' 'i9' 'ie' 'jn' 'kh' 'kv' 'mi' 'mq' 'mv' 'ox' 'oz' 'pn' 'qk' 'qo' 'qr' 'r8' 're' 'rf' 'rp' 't8' 'uj' 'uk' 'up' 'ur' 'vo' 'w7' 'wf' 'wp' 'ws' 'wt' 'ww' 'wx' 'xw' 'yd' 'yj' 'yn' 'yv' 'zr' - f | 'ax' 'ch' 'de' 'dl' 'e1' 'el' 'fl' 'fo' 'fp' 'fx' 'gb' 'gj' 'gx' 'hg' 'ho' 'hs' 'ip' 'jr' 'kg' 'l6' 'li' 'ln' 'mg' 'mm' 'mo' 'o2' 'os' 'pm' 'q2' 'q6' 'qr' 're' 'ro' 'rr' 'rt' 'rw' 'rx' 'sk' 'tl' 'to' 'tr' 'tt' 'u3' 'u6' 'vo' 'w4' 'w5' 'wb' 'wo' 'wt' 'yi' 'zh' - f | 'bl' 'bn' 'cy' 'ea' 'ed' 'ef' 'ei' 'em' 'er' 'es' 'fp' 'gb' 'hi' 'hp' 'if' 'io' 'ir' 'kw' 'lg' 'lo' 'ls' 'n8' 'nj' 'np' 'oa' 'of' 'oi' 'p6' 'pw' 'q3' 'q4' 'q8' 'qf' 'qh' 'qo' 'qw' 'rg' 'rt' 'sy' 't3' 'u8' 'ug' 'uo' 'uv' 'w6' 'we' 'wg' 'wl' 'wy' 'x9' 'yy' 'yz' - f | '16' '64' 'aj' 'aw' 'bf' 'c0' 'ca' 'd2' 'd6' 'ec' 'ed' 'eq' 'ew' 'go' 'h2' 'in' 'is' 'j0' 'jd' 'ji' 'kv' 'l5' 'lp' 'lz' 'og' 'pb' 'pw' 'q9' 'qb' 'qe' 'qf' 'qg' 'qn' 'qp' 'qt' 'qy' 'r7' 'rh' 'rw' 'sz' 't8' 'tb' 'te' 'tx' 'ui' 'un' 'uq' 'wa' 'wp' 'wu' 'wy' 'yn' 'ys' - f | '1m' '1s' 'aa' 'b0' 'be' 'cb' 'dc' 'dd' 'dh' 'eq' 'fa' 'ib' 'if' 'ik' 'it' 'jr' 'jz' 'ka' 'lq' 'lu' 'm2' 'o2' 'ob' 'oc' 'of' 'om' 'oz' 'pv' 'q4' 'qb' 'qk' 'qn' 'qs' 'qu' 'r3' 'r9' 'rs' 'rw' 'sg' 'si' 'sv' 't9' 'tw' 'vq' 'vy' 'w2' 'w6' 'w9' 'y0' 'y1' 'ye' 'yq' 'z1' - f | '1o' 'a0' 'aa' 'bd' 'bj' 'ch' 'cm' 'dj' 'e2' 'eq' 'eu' 'fj' 'fo' 'g9' 'go' 'hi' 'ia' 'ix' 'jh' 'jl' 'jy' 'lh' 'nd' 'nw' 'ox' 'p3' 'pi' 'pm' 'pt' 'q3' 'qh' 'qn' 'ra' 'ri' 'rr' 'ru' 'rv' 't3' 'tm' 'u3' 'ue' 'us' 'uz' 'vn' 'w1' 'wl' 'wo' 'wx' 'xj' 'xn' 'yc' 'yp' 'zr' - f | '3b' 'am' 'aq' 'ar' 'bm' 'de' 'e7' 'ed' 'eh' 'es' 'ey' 'fx' 'g5' 'gf' 'gs' 'hl' 'i7' 'iy' 'jn' 'k9' 'kf' 'km' 'll' 'ly' 'm0' 'm5' 'mh' 'nm' 'nq' 'of' 'or' 'p5' 'pn' 'pz' 'qp' 'qt' 'qv' 'r0' 'rb' 'sg' 'sv' 'uv' 'w3' 'w8' 'w9' 'wa' 'we' 'wj' 'wn' 'wr' 'yd' 'yk' 'zq' - f | '4t' 'a3' 'ac' 'au' 'bp' 'br' 'e5' 'ei' 'ek' 'ez' 'fz' 'g5' 'gn' 'ik' 'k9' 'kb' 'kc' 'lm' 'ls' 'ly' 'mj' 'mx' 'nk' 'nr' 'ny' 'o2' 'o8' 'pb' 'pu' 'q2' 'qa' 'qc' 'qh' 'qi' 'qs' 'qx' 'qz' 'rv' 'rx' 'tc' 'tk' 'tv' 'u8' 'ui' 'uz' 'vo' 'w2' 'wm' 'wp' 'xt' 'yf' 'yv' 'zl' - f | 'aa' 'ab' 'ag' 'ar' 'az' 'bp' 'bx' 'cx' 'eb' 'eh' 'ek' 'eo' 'eq' 'ez' 'fl' 'gm' 'gw' 'hd' 'ib' 'ig' 'jz' 'kg' 'lf' 'nw' 'oh' 'ok' 'om' 'oy' 'pg' 'ph' 'pi' 'qb' 'qi' 'qn' 'qs' 'qt' 'qy' 'rl' 'sy' 'tc' 'tt' 'up' 'ur' 'uy' 'w6' 'w8' 'wb' 'wh' 'wv' 'ww' 'y5' 'yi' 'yp' - f | 'aa' 'af' 'ah' 'am' 'as' 'cd' 'ch' 'dd' 'dg' 'dr' 'e2' 'eb' 'eq' 'er' 'fb' 'fj' 'fk' 'fn' 'fx' 'gk' 'gs' 'h8' 'hg' 'hj' 'ia' 'ij' 'in' 'jw' 'l9' 'lj' 'lw' 'nk' 'nt' 'o1' 'ob' 'oo' 'p5' 'pd' 'pk' 'qq' 'sb' 'sd' 'sg' 'u4' 'wl' 'wq' 'wr' 'wu' 'xc' 'xj' 'xt' 'yq' 'zl' - f | 'ad' 'ao' 'au' 'az' 'ci' 'dj' 'dv' 'e1' 'ei' 'ej' 'em' 'eq' 'fa' 'fi' 'fj' 'fn' 'fw' 'he' 'i8' 'in' 'io' 'jd' 'jt' 'jv' 'lr' 'mt' 'mx' 'ns' 'nt' 'oc' 'oe' 'oi' 'pz' 'qe' 'ql' 'qt' 'qy' 'rq' 'sc' 'sp' 'te' 'um' 'wo' 'wv' 'x6' 'xu' 'yf' 'yi' 'yo' 'yv' 'yw' 'yz' 'zg' - f | 'ag' 'ay' 'b6' 'bk' 'c6' 'cp' 'da' 'ea' 'eh' 'es' 'f1' 'fj' 'fv' 'gb' 'gn' 'ic' 'id' 'in' 'iq' 'iw' 'kx' 'ly' 'n7' 'o5' 'oc' 'og' 'op' 'os' 'pa' 'pr' 'q4' 'q6' 'qf' 'qg' 'qh' 'qp' 'qq' 'qu' 'qw' 'qy' 'qz' 'rj' 'rm' 'ro' 's1' 'w2' 'we' 'wl' 'wn' 'wp' 'yj' 'yp' 'yx' - f | 'ah' 'bg' 'bw' 'ca' 'cn' 'cy' 'da' 'dq' 'dt' 'eq' 'ex' 'fe' 'ff' 'ga' 'gr' 'kv' 'ld' 'my' 'od' 'oq' 'p3' 'p5' 'pc' 'pf' 'q5' 'qa' 'qc' 'qi' 'ql' 'qr' 'qx' 'qz' 'rd' 'rk' 'rq' 'ru' 'rw' 's7' 'sa' 'sh' 'tk' 'um' 'un' 'v4' 'vw' 'wg' 'wr' 'wt' 'xt' 'ya' 'yo' 'z6' 'zm' - f | '2r' '7g' 'a1' 'ar' 'av' 'b8' 'dj' 'dr' 'e7' 'ee' 'fd' 'fn' 'fp' 'gf' 'gp' 'gq' 'gw' 'jm' 'jv' 'ke' 'ki' 'kl' 'l9' 'lc' 'lg' 'lh' 'nm' 'oc' 'oq' 'px' 'q6' 'qe' 'qh' 'qu' 'rs' 's8' 't0' 'tt' 'tw' 'u2' 'uu' 'uy' 'vr' 'vs' 'wg' 'wt' 'ww' 'x3' 'x5' 'xu' 'yw' 'zh' 'zp' 'zt' - f | '4u' 'a2' 'aq' 'as' 'b8' 'bu' 'cc' 'cp' 'cv' 'dg' 'e3' 'e9' 'eh' 'fg' 'fh' 'g9' 'gf' 'gk' 'gx' 'hp' 'il' 'jq' 'kc' 'l4' 'lm' 'mt' 'my' 'nw' 'oq' 'pe' 'qe' 'qu' 'qz' 'r1' 'rb' 'sh' 'sx' 't5' 'td' 'tl' 'tr' 'vb' 'vd' 'w6' 'wh' 'wk' 'wp' 'wv' 'y0' 'ye' 'yq' 'z2' 'zj' 'zz' - f | 'a1' 'a5' 'aw' 'b5' 'cd' 'dz' 'em' 'eq' 'eu' 'fx' 'gk' 'hi' 'hq' 'ju' 'k8' 'kl' 'kw' 'la' 'lk' 'lm' 'm1' 'mj' 'mp' 'nz' 'o6' 'o8' 'oc' 'q0' 'q1' 'qg' 'qh' 'qw' 'rc' 'rd' 'ro' 'rr' 'sg' 'sy' 'th' 'ti' 'tt' 'uk' 'uo' 'ut' 'vy' 'wg' 'wi' 'x7' 'yr' 'yu' 'yx' 'yz' 'zr' 'zz' - f | 'a6' 'au' 'aw' 'bj' 'bn' 'cy' 'dg' 'e3' 'ei' 'ek' 'em' 'et' 'ex' 'fi' 'fm' 'g4' 'gm' 'gz' 'hp' 'ia' 'ie' 'jd' 'lg' 'lj' 'm6' 'mi' 'nd' 'oj' 'oo' 'ov' 'px' 'q0' 'q4' 'qj' 'qp' 'qq' 'r1' 'r5' 'rh' 'ro' 'tg' 'tr' 'tu' 'ty' 'u2' 'uk' 'ux' 'we' 'wi' 'wk' 'wq' 'wu' 'xy' 'y0' - f | 'a9' 'aa' 'aq' 'ay' 'bb' 'cg' 'db' 'de' 'do' 'e4' 'e5' 'e9' 'ek' 'ep' 'er' 'fi' 'fp' 'gb' 'hj' 'hk' 'hw' 'ir' 'iy' 'l1' 'lj' 'mb' 'nq' 'oi' 'pj' 'px' 'q9' 'qe' 'qu' 'qw' 'qz' 'rh' 'rl' 'sv' 'tc' 'ts' 'tv' 'uo' 'ur' 'vh' 'vp' 'w7' 'wf' 'wh' 'wo' 'wr' 'xu' 'z4' 'zc' 'zv' - f | 'au' 'cm' 'dj' 'e1' 'eh' 'ey' 'f3' 'fd' 'fg' 'fv' 'hn' 'i0' 'ia' 'jt' 'jy' 'k2' 'll' 'ne' 'o0' 'o2' 'op' 'pa' 'pf' 'qq' 'qx' 'rr' 'rs' 's4' 'sn' 'so' 'sq' 'tc' 'tn' 'ts' 'ty' 'tz' 'un' 'va' 'vg' 'vj' 'w8' 'wa' 'wb' 'wk' 'wo' 'wp' 'wr' 'x3' 'xx' 'yj' 'yz' 'z3' 'z7' 'zn' - f | '1f' '2u' 'ap' 'as' 'ax' 'bb' 'bf' 'cg' 'di' 'du' 'g3' 'g7' 'gk' 'ha' 'hv' 'hw' 'i6' 'jw' 'kg' 'ko' 'lu' 'ob' 'pn' 'qb' 'qg' 'qi' 'qs' 'qw' 'r3' 'r7' 'r9' 'rc' 're' 'rf' 'rn' 'ru' 'rz' 'sh' 'sq' 't7' 'ta' 'td' 'u2' 'ua' 'uy' 'v8' 'va' 'wk' 'wn' 'wu' 'wy' 'xr' 'yi' 'yt' 'za' - f | '1p' 'a7' 'ae' 'b2' 'dd' 'ef' 'eh' 'el' 'em' 'eo' 'fc' 'g0' 'hf' 'hh' 'i8' 'i9' 'in' 'is' 'iy' 'j3' 'ja' 'jl' 'js' 'jt' 'kz' 'lk' 'ng' 'nu' 'o6' 'oc' 'oe' 'om' 'ox' 'pi' 'pp' 'pq' 'ps' 'qc' 'qe' 'qy' 'rh' 'rk' 'rn' 's3' 'sc' 'so' 'st' 'tc' 'tm' 'tp' 'tq' 'vh' 'wa' 'ya' 'zq' - f | '1w' '2r' '9e' '9r' 'ci' 'es' 'f1' 'f4' 'fb' 'fx' 'g6' 'ga' 'gk' 'hj' 'hm' 'hu' 'i5' 'ii' 'jg' 'ji' 'ko' 'ku' 'l2' 'ld' 'lx' 'mt' 'n1' 'no' 'nw' 'o7' 'oi' 'pk' 'qy' 'r9' 'rj' 'rr' 'rt' 's6' 'sd' 't0' 'tc' 'to' 'tp' 'tv' 'ud' 'ux' 'vk' 'wr' 'wy' 'xb' 'xr' 'xt' 'yc' 'yp' 'zy' - f | '2k' '4s' 'd2' 'dk' 'dm' 'ea' 'ej' 'ep' 'et' 'eu' 'fr' 'gk' 'gs' 'hm' 'iu' 'jq' 'kj' 'km' 'lm' 'nb' 'nr' 'o1' 'oc' 'od' 'of' 'oj' 'ox' 'pf' 'pj' 'px' 'q0' 'q2' 'qa' 'qe' 'qh' 'qv' 'r5' 'r9' 'rb' 'rh' 'ti' 'tl' 'tw' 'u2' 'vb' 'vc' 'vs' 'xe' 'xn' 'y7' 'yb' 'yg' 'yy' 'zb' 'zl' - f | '48' 'ae' 'au' 'az' 'b2' 'di' 'ep' 'eq' 'ev' 'f7' 'fw' 'gh' 'gj' 'gn' 'hz' 'i5' 'iy' 'jf' 'kg' 'kj' 'kk' 'nc' 'nm' 'o4' 'oc' 'oe' 'of' 'oh' 'ol' 'ov' 'oz' 'pt' 'pv' 'q5' 'q7' 'qb' 'qe' 'qf' 'rh' 'ry' 'se' 'so' 'tv' 'ua' 'vd' 'wb' 'wg' 'wi' 'wp' 'xf' 'xo' 'yl' 'yr' 'yy' 'z0' - f | '70' 'ai' 'bj' 'bm' 'cj' 'ct' 'eb' 'ef' 'ek' 'es' 'fg' 'gh' 'gu' 'hg' 'ia' 'iu' 'iw' 'jt' 'ko' 'kq' 'mb' 'n1' 'o1' 'ox' 'pa' 'pr' 'q4' 'qd' 'qo' 'qr' 'qt' 'qw' 'qz' 'r3' 'r5' 'rg' 'ri' 'rj' 'rn' 'rp' 's7' 'sn' 't9' 'tq' 'tx' 'ty' 'uj' 'v0' 'vt' 'vz' 'w1' 'ww' 'xb' 'z6' 'zi' - f | 'a1' 'aa' 'c1' 'd7' 'dc' 'df' 'dh' 'e4' 'e9' 'ec' 'es' 'et' 'eu' 'ev' 'f3' 'g2' 'gu' 'he' 'il' 'j1' 'j6' 'jt' 'jv' 'ke' 'm8' 'm9' 'mh' 'ng' 'o8' 'on' 'pe' 'pf' 'pi' 'pn' 'qf' 'qn' 'qu' 'qv' 'qw' 'rh' 'rl' 'rr' 'ru' 'sg' 'sk' 'uk' 'ul' 'ux' 'vd' 'vj' 'wk' 'yd' 'yn' 'yx' 'z0' - f | 'af' 'bq' 'c9' 'cg' 'eg' 'ei' 'eq' 'ez' 'fo' 'fp' 'gl' 'i6' 'it' 'iz' 'jg' 'ka' 'lf' 'lm' 'lr' 'mv' 'oc' 'oj' 'om' 'or' 'p1' 'p4' 'pt' 'q3' 'qd' 'qe' 'qh' 'qx' 'ra' 'rd' 'sr' 'sz' 'tb' 'tc' 'tj' 'ug' 'uh' 'ux' 'wb' 'wc' 'wj' 'wy' 'xa' 'xb' 'y0' 'yj' 'yp' 'yz' 'zf' 'zl' 'zq' - f | '1a' '4p' 'bn' 'br' 'dk' 'ec' 'en' 'eu' 'gd' 'h0' 'ha' 'hm' 'if' 'il' 'io' 'ip' 'is' 'iz' 'jl' 'jq' 'k6' 'kd' 'kf' 'kn' 'kp' 'ld' 'lf' 'lt' 'n8' 'na' 'nl' 'o1' 'pd' 'pr' 'q8' 'qf' 'qr' 'qw' 're' 'rx' 'sa' 'sf' 'ta' 'tq' 'ux' 'v3' 'w6' 'wp' 'x2' 'y9' 'yi' 'yw' 'yx' 'z5' 'zl' 'zs' - f | '1y' '5p' '6o' 'au' 'b2' 'bg' 'do' 'dw' 'e1' 'e9' 'ea' 'ei' 'em' 'ev' 'f0' 'gc' 'hj' 'hx' 'k7' 'km' 'kp' 'kx' 'mj' 'mu' 'o3' 'oz' 'pf' 'q5' 'qe' 'qg' 'qh' 'qk' 'ql' 'qn' 'qo' 'rh' 'rk' 'ru' 'rx' 't1' 't3' 'th' 'v2' 'vx' 'w2' 'wc' 'wl' 'wm' 'wn' 'ws' 'x5' 'ye' 'yh' 'yk' 'yu' 'zw' - f | '2b' '3f' '5e' '7o' 'a4' 'ai' 'as' 'bc' 'bn' 'cz' 'd8' 'dg' 'dj' 'du' 'e5' 'el' 'et' 'ex' 'gc' 'gj' 'hi' 'hn' 'ig' 'ij' 'j3' 'js' 'l3' 'l6' 'lh' 'ls' 'mb' 'n1' 'o0' 'ot' 'q5' 'qd' 'qh' 'qv' 'qz' 'ra' 'rj' 's8' 'tb' 'tf' 'th' 'tx' 'ty' 'vm' 'wf' 'wg' 'wh' 'wn' 'y1' 'y5' 'yw' 'yz' - f | '2j' 'ai' 'al' 'ch' 'dy' 'dz' 'ed' 'ei' 'fg' 'fk' 'gp' 'i0' 'il' 'ip' 'iv' 'ix' 'js' 'lm' 'na' 'p1' 'pq' 'pu' 'qe' 'qi' 'qk' 'ql' 'qq' 'qv' 'qx' 'r0' 'rl' 'rm' 'rv' 'sf' 'sj' 'te' 'tk' 'tq' 'tt' 'tu' 'uc' 'ud' 'um' 'un' 'uq' 'vg' 'vu' 'vz' 'w8' 'wa' 'wf' 'xg' 'yj' 'yw' 'yy' 'z6' - f | '4c' '4z' 'ag' 'ak' 'an' 'as' 'aw' 'bl' 'd7' 'dy' 'e7' 'eb' 'ec' 'eq' 'et' 'fx' 'gg' 'gm' 'ht' 'hv' 'ik' 'in' 'ip' 'j6' 'k6' 'ka' 'lc' 'md' 'mx' 'n7' 'pw' 'py' 'q7' 'qa' 'qg' 'qh' 'qn' 'qo' 'qt' 'ri' 'rl' 'rs' 'sj' 'su' 'tj' 'u0' 'ua' 'w0' 'wh' 'wi' 'wp' 'wr' 'wx' 'wz' 'ys' 'yx' - f | '1y' '4f' 'ao' 'bv' 'co' 'cq' 'dd' 'df' 'dy' 'eq' 'eu' 'ex' 'gg' 'gm' 'gr' 'hm' 'iw' 'j9' 'jb' 'jg' 'jo' 'ju' 'k0' 'km' 'lf' 'ng' 'np' 'nw' 'nz' 'od' 'oj' 'or' 'pp' 'pr' 'pu' 'q5' 'q7' 'q9' 'qc' 'qd' 'qn' 'qx' 'r9' 'rd' 'rk' 'ro' 'sb' 'ta' 'th' 'tv' 'ty' 'tz' 'uq' 'vb' 'wb' 'wj' 'xo' - f | '4m' 'at' 'be' 'bo' 'bz' 'cv' 'd0' 'd8' 'ea' 'ec' 'ed' 'ex' 'f9' 'ff' 'fk' 'fv' 'gc' 'ge' 'i8' 'ij' 'is' 'k4' 'k6' 'mw' 'ob' 'ov' 'oy' 'pv' 'q1' 'q2' 'q9' 'qc' 'ql' 'r4' 'rn' 'ru' 'rx' 'ry' 'sj' 'tg' 'ts' 'tv' 'uj' 'vf' 'vh' 'w4' 'wc' 'ww' 'x5' 'xf' 'xx' 'y5' 'yg' 'yy' 'zg' 'zn' 'zw' - f | '4p' 'am' 'bj' 'bq' 'cg' 'cm' 'cz' 'dm' 'ds' 'du' 'e6' 'eg' 'en' 'eo' 'eq' 'et' 'ev' 'ex' 'fw' 'go' 'hb' 'hq' 'i0' 'ij' 'k6' 'kg' 'l6' 'lm' 'lq' 'mr' 'ms' 'oa' 'p6' 'q7' 'qa' 'qi' 'qk' 'r8' 'ra' 're' 'rr' 's8' 'sj' 't6' 't9' 'tc' 'v1' 'vd' 'w1' 'wf' 'wj' 'xy' 'yd' 'yn' 'yo' 'yt' 'yu' - f | 'a6' 'aa' 'ae' 'bn' 'c0' 'c7' 'ck' 'df' 'dn' 'ds' 'dt' 'e0' 'ea' 'eo' 'eu' 'f0' 'f6' 'g1' 'gd' 'i4' 'j6' 'ja' 'je' 'jt' 'mv' 'no' 'of' 'os' 'ou' 'ox' 'pi' 'pq' 'pr' 'q6' 'qk' 'ql' 'qv' 'qw' 'rh' 'rm' 'rv' 'sv' 't6' 'u2' 'un' 'ux' 'wc' 'wl' 'wo' 'wp' 'xg' 'y7' 'y8' 'ya' 'yn' 'yx' 'zl' - f | 'ab' 'am' 'aq' 'b9' 'bj' 'ck' 'cw' 'e2' 'e8' 'ek' 'eo' 'ew' 'f2' 'ff' 'fh' 'ft' 'gl' 'gs' 'h7' 'i0' 'ik' 'jl' 'jo' 'jq' 'jr' 'kc' 'kg' 'kw' 'mm' 'mo' 'pb' 'pn' 'qe' 'qg' 'qk' 'qr' 'qu' 'r2' 's8' 'sd' 'sq' 'su' 'td' 'tr' 'uh' 'uu' 'v2' 'vl' 'w8' 'wg' 'wm' 'wt' 'xl' 'xq' 'yi' 'yo' 'yq' - f | 'ah' 'ar' 'bk' 'db' 'dd' 'dq' 'eo' 'fe' 'fj' 'fv' 'g2' 'gc' 'hp' 'i7' 'j2' 'k9' 'kt' 'l9' 'ld' 'lu' 'lz' 'me' 'mr' 'mu' 'mx' 'nr' 'o6' 'ol' 'or' 'pa' 'ph' 'pi' 'q4' 'qn' 'qp' 'r5' 'rb' 'rn' 's8' 'sf' 't6' 'tl' 'tq' 'tx' 'u6' 'uc' 'uv' 'wf' 'x1' 'x7' 'xa' 'xl' 'xm' 'yb' 'yh' 'yw' 'zw' - f | '1m' '2u' 'ai' 'ap' 'bb' 'bd' 'cb' 'dn' 'fl' 'fm' 'fo' 'fu' 'gi' 'gl' 'gs' 'hg' 'hi' 'hm' 'hp' 'i0' 'if' 'ij' 'jn' 'jt' 'ju' 'k8' 'ld' 'lm' 'oe' 'ox' 'pa' 'q7' 'qe' 'qi' 'qo' 'qw' 'rf' 'rg' 'ry' 'sq' 't7' 'tk' 'tn' 'tz' 'u5' 'uc' 'v4' 'vr' 'w4' 'w6' 'wd' 'wo' 'ws' 'wt' 'wu' 'wv' 'x3' 'yl' - f | '1x' '2h' '6b' 'ah' 'av' 'ay' 'b9' 'bu' 'bv' 'c6' 'dv' 'e2' 'ea' 'ec' 'fh' 'gk' 'ia' 'j2' 'ji' 'k2' 'kc' 'kg' 'kq' 'kz' 'la' 'mq' 'nb' 'od' 'oe' 'ot' 'p8' 'pm' 'pp' 'qc' 'qi' 'qr' 'rg' 'rp' 'rw' 'tl' 'u5' 'ub' 'un' 'uv' 'vc' 'vi' 'wc' 'wi' 'wp' 'wv' 'xg' 'xq' 'yq' 'z3' 'zd' 'zv' 'zx' 'zy' - f | 'as' 'bb' 'bn' 'ce' 'd4' 'dp' 'ez' 'gu' 'gw' 'hd' 'hp' 'ia' 'ih' 'ij' 'j6' 'jm' 'jz' 'ku' 'kw' 'lh' 'mu' 'nl' 'nu' 'oc' 'pa' 'pm' 'q8' 'qd' 'qg' 'qi' 'qm' 'qn' 'qr' 'qs' 'qw' 'r9' 're' 'ro' 'ru' 'rv' 'rw' 't1' 't3' 'tm' 'u4' 'uf' 'uw' 'wa' 'wf' 'wk' 'xa' 'xs' 'yg' 'yp' 'ys' 'yu' 'yz' 'z4' - f | '3q' '5p' 'a2' 'be' 'bv' 'bw' 'cv' 'd8' 'df' 'dh' 'dk' 'e2' 'ec' 'eo' 'fc' 'g8' 'gq' 'h8' 'ig' 'io' 'ir' 'iv' 'je' 'kq' 'kx' 'lc' 'lr' 'lt' 'ok' 'pk' 'q1' 'q4' 'qe' 'qh' 'qw' 'qz' 'rj' 'rv' 't8' 'tf' 'tl' 'tq' 'uh' 'uz' 'wk' 'wp' 'ws' 'wt' 'wv' 'x3' 'yb' 'ye' 'yo' 'yp' 'yy' 'z6' 'z8' 'zb' 'zm' - f | 'a1' 'ag' 'ai' 'ap' 'bp' 'by' 'dc' 'dy' 'e5' 'eg' 'ei' 'el' 'em' 'fc' 'gl' 'h8' 'hr' 'i1' 'ib' 'ie' 'if' 'ij' 'j5' 'jj' 'kf' 'kx' 'lc' 'm3' 'mg' 'nb' 'ol' 'ph' 'py' 'q1' 'q5' 'qi' 'qk' 'ql' 'qp' 'qs' 'sv' 'td' 'te' 'tn' 'vd' 'vm' 'vq' 'wv' 'xp' 'xu' 'yb' 'yo' 'yq' 'yx' 'yy' 'zp' 'zr' 'zx' 'zz' - f | 'af' 'ag' 'ak' 'bb' 'cr' 'cy' 'd7' 'db' 'df' 'di' 'eh' 'ew' 'fl' 'fr' 'gd' 'gp' 'hf' 'hk' 'hu' 'ib' 'ik' 'io' 'ix' 'jg' 'k9' 'kc' 'ke' 'm4' 'ma' 'mx' 'mz' 'ob' 'oi' 'oo' 'or' 'ox' 'pg' 'pl' 'pu' 'qc' 'qe' 'qg' 'rl' 'ro' 'rv' 'rz' 'sj' 'sq' 'u4' 've' 'we' 'wi' 'wk' 'wy' 'xd' 'yh' 'yq' 'yv' 'yx' - f | 'ah' 'c3' 'd1' 'dh' 'dz' 'ei' 'em' 'ex' 'fe' 'fk' 'g7' 'gz' 'hi' 'hx' 'i3' 'iu' 'j1' 'jm' 'k8' 'kb' 'ku' 'lf' 'lv' 'lz' 'm3' 'mn' 'my' 'nc' 'oj' 'pk' 'qh' 'qn' 'qo' 'qq' 'rf' 'ru' 'ry' 's9' 'sa' 'sc' 'sd' 'se' 'sz' 'td' 'tg' 'tq' 'tx' 'tz' 'vr' 'vu' 'w3' 'wd' 'wl' 'wp' 'x9' 'yb' 'yd' 'yr' 'yt' - f | '0z' 'ae' 'br' 'cr' 'dd' 'di' 'du' 'e7' 'ea' 'eb' 'ee' 'ej' 'em' 'eo' 'fo' 'fs' 'gc' 'gf' 'gj' 'hu' 'iv' 'kc' 'lb' 'nw' 'o9' 'ov' 'oz' 'pq' 'qq' 'ro' 'rr' 'rt' 'ru' 'rw' 's4' 'te' 'tm' 'tq' 'um' 'v7' 'va' 'vi' 'vr' 'wb' 'wg' 'wo' 'wp' 'wr' 'ww' 'wy' 'xv' 'y1' 'y9' 'ya' 'ym' 'yt' 'yu' 'z5' 'zl' 'zs' - f | '3i' 'a3' 'af' 'cc' 'cs' 'cv' 'd8' 'ei' 'ej' 'ep' 'et' 'ex' 'fb' 'fe' 'fx' 'hc' 'ij' 'jf' 'jp' 'jr' 'js' 'kc' 'km' 'kz' 'lc' 'm7' 'nv' 'ob' 'oc' 'pt' 'q5' 'q7' 'q8' 'qc' 'qm' 'qn' 'qs' 'qy' 'ra' 're' 'rk' 'rx' 's0' 's4' 'se' 'sh' 'sm' 't7' 'tl' 'to' 'tz' 'ue' 'um' 'w3' 'wk' 'wp' 'ya' 'yl' 'yn' 'ys' - f | 'a5' 'al' 'b6' 'bu' 'co' 'd0' 'd4' 'db' 'e0' 'e2' 'eb' 'ei' 'ej' 'el' 'et' 'ez' 'g7' 'gu' 'j6' 'jl' 'k3' 'kf' 'km' 'kw' 'kx' 'lp' 'me' 'mw' 'my' 'n2' 'n5' 'n7' 'nr' 'ny' 'oj' 'q8' 'qb' 'qg' 'qn' 'qq' 'qz' 'ra' 'rk' 'rl' 'rn' 's3' 's6' 't2' 't8' 'tk' 'tm' 'ul' 'uq' 'wg' 'y1' 'yc' 'yn' 'yq' 'yy' 'zr' - f | 'af' 'am' 'az' 'ce' 'ci' 'dd' 'e5' 'eb' 'ee' 'eh' 'fe' 'fj' 'fv' 'ge' 'gl' 'hp' 'hx' 'i5' 'i8' 'ia' 'ig' 'jb' 'jm' 'jv' 'k6' 'kg' 'kv' 'li' 'ls' 'lx' 'm2' 'md' 'mz' 'o0' 'oe' 'oy' 'pv' 'q5' 'qh' 'qm' 'qo' 'qy' 'rf' 'ri' 'rp' 'rz' 'su' 'tj' 'tz' 'u2' 'uh' 'ur' 'vg' 'we' 'wi' 'xj' 'yb' 'yx' 'yz' 'z0' - f | 'b9' 'c9' 'cc' 'cx' 'di' 'ec' 'eo' 'f5' 'fn' 'gd' 'ge' 'gh' 'hc' 'hy' 'i8' 'if' 'iq' 'ir' 'ix' 'ja' 'jl' 'jx' 'km' 'l5' 'l8' 'le' 'mm' 'oc' 'pr' 'qg' 'qo' 'rm' 'ro' 'rv' 'sm' 'td' 'th' 'tm' 'tx' 'u7' 'uc' 'ue' 'uk' 'ux' 'vw' 'wa' 'wb' 'wd' 'wh' 'wr' 'ww' 'xa' 'xj' 'xz' 'y4' 'yd' 'yl' 'ym' 'zf' 'zv' - f | '1a' 'a0' 'ab' 'ah' 'ay' 'cd' 'db' 'di' 'eh' 'eo' 'eq' 'f4' 'f8' 'fs' 'gv' 'h7' 'he' 'hf' 'i0' 'id' 'il' 'io' 'ip' 'kc' 'l1' 'l7' 'lw' 'mp' 'na' 'nb' 'nj' 'oj' 'ou' 'pa' 'pn' 'ps' 'pu' 'qb' 'qf' 'ql' 'qn' 'qr' 'rn' 'rw' 'so' 'tf' 'tg' 'th' 'tr' 'us' 'w3' 'w6' 'wb' 'wc' 'wf' 'wg' 'wp' 'wr' 'yv' 'yw' 'zh' - f | '1n' 'am' 'b9' 'bc' 'bi' 'dl' 'dv' 'ea' 'eq' 'ey' 'fj' 'g1' 'g3' 'g8' 'ge' 'it' 'iu' 'jh' 'jz' 'kk' 'ln' 'mc' 'mw' 'nf' 'nj' 'o4' 'o9' 'ob' 'ox' 'pq' 'q4' 'qb' 'qk' 'qn' 'qs' 'qt' 'r4' 'ra' 'rg' 'rp' 'rx' 'sj' 'sq' 'ss' 't0' 'tg' 'ty' 'ue' 'ui' 'uj' 'uv' 'v6' 'vr' 'w1' 'w9' 'wj' 'wy' 'xt' 'ym' 'ys' 'yz' - f | '1v' '20' 'au' 'bo' 'br' 'c9' 'cv' 'dp' 'ep' 'fo' 'gj' 'hb' 'hx' 'iv' 'iw' 'jc' 'jm' 'jw' 'k8' 'kg' 'km' 'kn' 'nd' 'o8' 'oh' 'ok' 'op' 'oy' 'px' 'pz' 'qh' 'qo' 'qp' 'qs' 'qw' 're' 'rr' 'ry' 's3' 'sj' 't7' 't9' 'tt' 'tw' 'u9' 'ue' 'us' 'uu' 'v2' 'w3' 'wa' 'wf' 'wg' 'wt' 'wv' 'xc' 'yi' 'yu' 'yv' 'z4' 'zj' - f | '6g' 'ar' 'ce' 'cn' 'ds' 'dt' 'e9' 'eh' 'el' 'ew' 'gt' 'gw' 'gx' 'hb' 'ho' 'i8' 'j8' 'jp' 'jr' 'kb' 'la' 'lr' 'mg' 'mi' 'ml' 'nd' 'ng' 'oj' 'op' 'pb' 'pd' 'pn' 'py' 'qc' 'qe' 'qk' 'qr' 'qt' 'qv' 'qz' 'sm' 'tc' 'tg' 'ti' 'tp' 'ty' 'u8' 'uq' 'uw' 'uy' 've' 'w0' 'wb' 'wd' 'wp' 'x4' 'xm' 'ym' 'z2' 'zh' 'zz' - f | 'af' 'as' 'b2' 'bl' 'bz' 'ca' 'cd' 'co' 'de' 'dp' 'e4' 'ed' 'en' 'eo' 'eu' 'ft' 'g0' 'gj' 'ha' 'hh' 'hn' 'hy' 'ij' 'jb' 'jj' 'jn' 'l7' 'll' 'lp' 'oh' 'ot' 'pb' 'ph' 'pi' 'pn' 'qc' 'qx' 'r5' 'rb' 'ri' 'sb' 'sd' 'sn' 'sv' 'sw' 't7' 'tb' 'ti' 'ty' 'u7' 'un' 'uo' 'wb' 'we' 'wf' 'wh' 'wo' 'wp' 'yb' 'yc' 'ys' - f | '4a' 'a9' 'au' 'bn' 'bs' 'cg' 'd4' 'dx' 'fb' 'fk' 'gc' 'hh' 'ht' 'hy' 'iy' 'je' 'jg' 'jj' 'jp' 'ju' 'kz' 'la' 'm8' 'na' 'oa' 'oj' 'ol' 'or' 'ov' 'ox' 'p6' 'po' 'q0' 'q9' 'qc' 'qe' 'qi' 'qq' 'qt' 'qx' 'si' 'sn' 'su' 'sw' 't8' 'ta' 'tv' 'tz' 'uq' 'ut' 'w6' 'w7' 'we' 'wk' 'wl' 'wq' 'wy' 'y4' 'ya' 'ze' 'zq' 'zy' - f | 'b0' 'bg' 'bh' 'cu' 'd8' 'dv' 'er' 'fd' 'fm' 'fo' 'gg' 'ij' 'ir' 'iu' 'jc' 'jl' 'jn' 'jo' 'k4' 'kb' 'ku' 'lq' 'ly' 'mw' 'of' 'op' 'ph' 'pk' 'ps' 'px' 'q0' 'qd' 'qg' 'qz' 'rb' 'rp' 'rs' 'rv' 'su' 't9' 'tm' 'tp' 'tx' 'ty' 'ug' 'ul' 'uo' 'up' 'uv' 'v5' 'wh' 'wr' 'ww' 'xw' 'y1' 'yd' 'yf' 'yn' 'z4' 'z8' 'zf' 'zn' - f | '1i' 'ap' 'bh' 'ce' 'cp' 'di' 'dm' 'dt' 'es' 'eu' 'f2' 'f3' 'fw' 'fx' 'g5' 'gf' 'gy' 'h2' 'hl' 'i4' 'ii' 'iy' 'kp' 'lh' 'lr' 'me' 'of' 'og' 'ok' 'on' 'oo' 'pb' 'pd' 'pt' 'qf' 'qk' 'qz' 'r4' 'r7' 're' 'ri' 'rj' 'ro' 'rx' 's2' 'tk' 'tl' 'tq' 'tx' 'ub' 'ui' 'us' 'vm' 'w1' 'w7' 'wi' 'wj' 'wq' 'wx' 'xp' 'yg' 'yr' 'yx' - f | 'a2' 'a7' 'aq' 'bv' 'cy' 'd0' 'df' 'dg' 'do' 'ei' 'ek' 'ev' 'ey' 'fg' 'ge' 'gg' 'h4' 'hv' 'im' 'iq' 'ix' 'j4' 'j5' 'jt' 'kv' 'nc' 'o2' 'ou' 'ow' 'ph' 'pz' 'qf' 'qj' 'qm' 'qo' 'qq' 'qs' 'ra' 'rl' 'rz' 'te' 'tj' 'tp' 'tr' 'ts' 'tx' 'u5' 'ue' 'uj' 'uk' 'uo' 'us' 'v7' 'w2' 'wa' 'wu' 'wv' 'ww' 'wz' 'y2' 'y8' 'yy' 'zg' - f | 'a6' 'ao' 'az' 'by' 'db' 'dl' 'eg' 'ei' 'el' 'eo' 'fh' 'fv' 'gl' 'h0' 'hl' 'hx' 'i9' 'iq' 'j7' 'jx' 'kg' 'kh' 'l9' 'nz' 'o9' 'oi' 'om' 'on' 'oq' 'or' 'pf' 'pz' 'qf' 'qj' 'r5' 'rq' 'rw' 'sr' 'sx' 'tc' 'tg' 'tj' 'tl' 'to' 'uj' 'un' 'vw' 'w1' 'w5' 'w8' 'wn' 'ws' 'wv' 'x2' 'x3' 'xi' 'y4' 'yd' 'ym' 'ys' 'z7' 'zq' 'zv' - f | 'am' 'as' 'ay' 'bl' 'e0' 'e1' 'ea' 'eg' 'f6' 'fc' 'fx' 'ia' 'io' 'jq' 'jx' 'kd' 'ky' 'lb' 'll' 'lr' 'lv' 'me' 'nv' 'o4' 'o7' 'oe' 'ok' 'oq' 'pe' 'pr' 'py' 'q7' 'qg' 'qh' 'qj' 'qk' 'ql' 'qq' 'qr' 'qs' 'r1' 'rg' 'rl' 'rm' 'ro' 'sq' 'sr' 'sz' 'ti' 'ts' 'ue' 'ui' 'uu' 'uz' 'vk' 'y0' 'y2' 'y9' 'ym' 'yt' 'yx' 'z8' 'zd' - f | '1o' 'ah' 'ap' 'ba' 'bn' 'cs' 'cu' 'd2' 'd8' 'dg' 'dr' 'ed' 'ee' 'er' 'et' 'eu' 'ey' 'ff' 'g1' 'hj' 'i3' 'i5' 'il' 'in' 'jd' 'kd' 'ku' 'lm' 'lv' 'mu' 'nu' 'ok' 'ol' 'oo' 'ov' 'oy' 'p8' 'pu' 'qb' 'qq' 'qx' 'r5' 'rm' 'rp' 'rv' 'sh' 'sk' 'sl' 'sr' 'tx' 'uy' 'v3' 'vu' 'w3' 'wa' 'wb' 'wv' 'xh' 'xq' 'ys' 'yt' 'yx' 'z2' 'zi' - f | '2e' 'ak' 'bv' 'cb' 'ch' 'cy' 'd9' 'dq' 'dr' 'e8' 'ec' 'ef' 'ek' 'es' 'f1' 'fw' 'gj' 'gw' 'hh' 'hr' 'i3' 'i6' 'ic' 'ji' 'jr' 'jy' 'kr' 'md' 'mu' 'od' 'og' 'oo' 'p9' 'pd' 'q1' 'q3' 'qq' 'qw' 'qy' 'ra' 'rj' 'rl' 'rn' 'sf' 't4' 't5' 't7' 'tl' 'tm' 'ts' 'u4' 'ug' 'ui' 'uw' 'vo' 'ws' 'ww' 'wy' 'yd' 'yo' 'yq' 'zm' 'zp' 'zw' - f | '5y' 'af' 'av' 'az' 'cj' 'cp' 'cq' 'dk' 'e6' 'e7' 'ea' 'ec' 'ee' 'ei' 'ek' 'em' 'ga' 'gk' 'gw' 'hc' 'id' 'ie' 'kp' 'kv' 'lw' 'm0' 'mf' 'mn' 'mw' 'ny' 'ob' 'ol' 'p9' 'ph' 'pk' 'pp' 'pw' 'q0' 'qg' 'qi' 'ql' 'qv' 'qx' 'r3' 'rg' 'ry' 'se' 'sh' 'sq' 'sz' 't1' 'tu' 'u2' 'uh' 'uj' 'uk' 'ut' 'w2' 'wb' 'ww' 'wx' 'yf' 'yo' 'zr' - f | 'ab' 'an' 'bm' 'bn' 'bp' 'ca' 'd9' 'dc' 'e0' 'e4' 'e5' 'eh' 'er' 'fe' 'fk' 'fv' 'ga' 'ge' 'hy' 'ic' 'ie' 'io' 'ja' 'jb' 'je' 'kp' 'ks' 'ln' 'md' 'ng' 'nr' 'oj' 'oy' 'p5' 'p6' 'p7' 'pe' 'pg' 'pu' 'qa' 'qq' 'qv' 'qw' 'qy' 'rg' 'rj' 'rk' 'sk' 'tf' 'tw' 'ui' 'um' 'uu' 'v9' 'vu' 'wo' 'wp' 'wx' 'wy' 'xf' 'y0' 'yp' 'z6' 'zi' - f | 'ae' 'cn' 'ct' 'dz' 'eb' 'ee' 'ff' 'fi' 'fk' 'fo' 'ft' 'gj' 'gr' 'ie' 'il' 'iv' 'iw' 'iy' 'jb' 'jf' 'ji' 'ke' 'ku' 'kx' 'l3' 'la' 'of' 'ol' 'ox' 'pb' 'pi' 'pq' 'q8' 'qi' 'qj' 'qp' 'qq' 'qw' 'r0' 'r5' 'rk' 'rq' 'ru' 'rz' 'su' 't8' 'tb' 'u0' 'ue' 'um' 'w2' 'wc' 'wm' 'ws' 'wt' 'x4' 'xc' 'xd' 'xm' 'xn' 'y0' 'yb' 'zi' 'zp' - f | '3r' '6o' 'ab' 'ay' 'b3' 'bc' 'bh' 'd8' 'dd' 'df' 'eb' 'ee' 'eh' 'el' 'eu' 'ex' 'fn' 'g3' 'ge' 'gr' 'gz' 'hd' 'ib' 'ie' 'ih' 'il' 'it' 'iu' 'jd' 'jq' 'jt' 'jv' 'li' 'pc' 'pp' 'qc' 'ql' 'qp' 'qu' 'qx' 'qz' 'ro' 'rq' 'sj' 'sz' 'te' 'tt' 'tu' 'uh' 'uo' 'up' 'us' 'uu' 'ux' 'v7' 'w3' 'wl' 'wn' 'xf' 'xu' 'ya' 'yh' 'yk' 'za' 'zt' - f | '5o' 'ad' 'aw' 'az' 'bi' 'bo' 'd1' 'db' 'di' 'do' 'e7' 'eb' 'ei' 'em' 'ep' 'eq' 'eu' 'fo' 'gg' 'gw' 'i0' 'ig' 'ih' 'iu' 'j3' 'j4' 'jo' 'js' 'kq' 'lc' 'lo' 'lu' 'm9' 'mi' 'mk' 'mt' 'n4' 'ni' 'o7' 'od' 'ot' 'pc' 'pg' 'pp' 'qc' 'qr' 'qw' 'rd' 'rx' 'se' 'sq' 'sy' 't5' 'ts' 'ub' 'vz' 'wb' 'wl' 'wr' 'wt' 'xe' 'xt' 'yg' 'yr' 'zw' - f | '5t' 'ad' 'am' 'ed' 'ei' 'en' 'eo' 'ey' 'f0' 'fp' 'fr' 'gc' 'hp' 'hz' 'ic' 'ix' 'jt' 'kn' 'kr' 'lk' 'ls' 'm1' 'mt' 'nk' 'od' 'p3' 'pa' 'pe' 'pi' 'q4' 'qa' 'qi' 'qk' 'qq' 'qt' 'qv' 'rb' 'rr' 'rv' 's3' 'se' 'sr' 't0' 'tj' 'tk' 'tp' 'tu' 'u1' 'ud' 'uf' 'uv' 'ux' 'vd' 'vu' 'wh' 'wi' 'wp' 'wu' 'x9' 'xa' 'ye' 'yn' 'yw' 'zj' 'zs' - f | '6h' '7f' 'ab' 'aq' 'ax' 'bc' 'bl' 'ce' 'cm' 'dn' 'do' 'e1' 'ec' 'ed' 'en' 'fc' 'fl' 'g1' 'ga' 'ge' 'gn' 'h8' 'i0' 'id' 'iq' 'iw' 'jc' 'ji' 'lt' 'mj' 'nm' 'o0' 'o6' 'oo' 'pa' 'pb' 'pt' 'q6' 'qf' 'qg' 'qz' 'r1' 'r2' 'r8' 'rd' 'rf' 's9' 'se' 't8' 'tc' 'te' 'u3' 'u5' 'uc' 'ug' 'ul' 'us' 'uu' 'v2' 'vz' 'wp' 'x8' 'ye' 'yq' 'yw' - f | 'a5' 'ae' 'ah' 'as' 'b9' 'bh' 'c5' 'd0' 'dc' 'dd' 'do' 'dp' 'dz' 'eg' 'eh' 'fg' 'fu' 'g3' 'gh' 'gu' 'ha' 'hb' 'hd' 'hg' 'hw' 'i3' 'ib' 'jb' 'je' 'jm' 'js' 'm8' 'ma' 'n4' 'nu' 'nz' 'ob' 'oy' 'qu' 'qz' 'r5' 'rd' 're' 'rr' 'ru' 's5' 'sf' 'sp' 'st' 'tn' 'uz' 'vi' 'vo' 'w9' 'wj' 'wm' 'ws' 'wu' 'wv' 'wy' 'xa' 'xc' 'yl' 'zd' 'zp' - f | 'a6' 'ad' 'ak' 'ap' 'd1' 'de' 'dj' 'eb' 'eg' 'fi' 'fq' 'ft' 'gx' 'gy' 'hn' 'hs' 'i4' 'ia' 'im' 'is' 'kf' 'ko' 'kt' 'l0' 'lr' 'm8' 'mv' 'oi' 'on' 'pc' 'pl' 'pn' 'pp' 'q2' 'q4' 'qj' 'ql' 'qv' 'qy' 'r2' 'rc' 'rd' 're' 'rm' 'rt' 'sj' 'tb' 'tc' 'tf' 'tk' 'uf' 'uk' 'ul' 'um' 'uq' 'uy' 'vj' 'wl' 'wv' 'ww' 'x7' 'xj' 'y7' 'yx' 'zb' - f | 'ab' 'ct' 'cv' 'cw' 'dg' 'ds' 'e0' 'e5' 'f3' 'fj' 'fl' 'fn' 'gy' 'ha' 'ie' 'if' 'jf' 'ju' 'jx' 'kg' 'ki' 'kw' 'l0' 'ls' 'm9' 'mj' 'nj' 'of' 'om' 'oq' 'os' 'ov' 'pv' 'pz' 'q0' 'q5' 'q8' 'qf' 'qg' 'qi' 'qj' 'qm' 'qn' 'qw' 'rk' 'ru' 'sc' 'sn' 'so' 't1' 'tn' 'tz' 'un' 'uv' 'uw' 'vn' 'wh' 'wn' 'ww' 'xy' 'yd' 'yr' 'yv' 'zh' 'zy' - f | 'am' 'db' 'dj' 'dn' 'ec' 'eo' 'ev' 'ex' 'fn' 'fw' 'fz' 'gi' 'gn' 'gq' 'ha' 'ho' 'in' 'is' 'ja' 'jj' 'jk' 'js' 'kv' 'lc' 'lz' 'mj' 'o2' 'oa' 'of' 'oi' 'pe' 'pn' 'ps' 'pv' 'py' 'q3' 'qa' 'qb' 'qi' 'qn' 'qo' 'qq' 'qr' 'qv' 'rk' 'rl' 't8' 'tk' 'tv' 'tw' 'ug' 'uk' 'ux' 'uz' 'v7' 'vr' 'wd' 'wf' 'wk' 'wv' 'ya' 'yq' 'yu' 'zp' 'zx' - f | 'ao' 'as' 'bz' 'dc' 'dl' 'dn' 'du' 'dy' 'e3' 'ed' 'ee' 'ej' 'em' 'eo' 'ep' 'eq' 'er' 'fc' 'ft' 'g4' 'gc' 'gm' 'gq' 'gr' 'gy' 'hq' 'jg' 'k6' 'k9' 'ks' 'kz' 'l7' 'lo' 'lu' 'lw' 'ly' 'nb' 'oc' 'oo' 'q9' 'qc' 'qd' 'qk' 'qs' 'qv' 'ro' 'rx' 'ry' 'sk' 'sv' 'sy' 'ti' 'tk' 'ui' 'un' 'uq' 'vl' 'w4' 'wl' 'wm' 'yj' 'yo' 'yu' 'yw' 'zg' - f | '1i' '3z' 'b6' 'bh' 'bn' 'bu' 'cm' 'do' 'dt' 'ef' 'eg' 'ek' 'eo' 'et' 'ex' 'f9' 'g3' 'gj' 'gl' 'h7' 'hp' 'i8' 'ih' 'im' 'in' 'jn' 'k1' 'k7' 'kd' 'lq' 'ms' 'mv' 'n7' 'nj' 'nt' 'o4' 'o9' 'ov' 'q2' 'qa' 'qh' 'qk' 'ql' 'qp' 'qu' 'qv' 'rj' 's8' 'sf' 'sh' 't7' 't8' 'um' 'un' 'uo' 'ut' 'uu' 'w1' 'wa' 'wf' 'wi' 'wx' 'xj' 'yc' 'yo' 'yt' - f | '1w' '5i' 'a4' 'a6' 'bz' 'de' 'dg' 'dt' 'ed' 'ei' 'ej' 'ey' 'fa' 'ff' 'gg' 'hi' 'hj' 'ho' 'i2' 'ii' 'ij' 'jq' 'jy' 'ka' 'kr' 'kw' 'lv' 'm7' 'nd' 'ot' 'ov' 'oy' 'oz' 'pe' 'pj' 'qb' 'qj' 'qq' 'qt' 'qv' 'r0' 'r3' 'r4' 'ri' 'rj' 'si' 'ta' 'tc' 'ts' 'tx' 'u3' 'uj' 'ur' 'v6' 'vh' 'w3' 'w6' 'wc' 'wg' 'wq' 'x5' 'xv' 'yb' 'yt' 'yx' 'zl' - f | '10' '2p' '6r' 'a2' 'bb' 'c9' 'dq' 'dy' 'ec' 'eh' 'ej' 'ek' 'es' 'fl' 'g1' 'g9' 'gd' 'h1' 'h2' 'hj' 'hn' 'hw' 'ip' 'iz' 'jg' 'jl' 'l5' 'le' 'lk' 'me' 'mi' 'o0' 'o6' 'oa' 'oe' 'ox' 'pd' 'pm' 'pr' 'py' 'q1' 'qo' 'qq' 'qz' 'r3' 'rs' 'rw' 's7' 'sl' 'sw' 't2' 't4' 'td' 'tj' 'u3' 'ud' 'ur' 'uw' 'vy' 'wi' 'wj' 'wl' 'wq' 'ys' 'yu' 'z7' 'zp' - f | '19' 'a4' 'a7' 'ae' 'ar' 'as' 'bj' 'bp' 'bv' 'cp' 'd8' 'dp' 'e0' 'e8' 'ea' 'ee' 'ek' 'ep' 'es' 'et' 'ez' 'fp' 'gu' 'h2' 'h4' 'he' 'hp' 'ii' 'im' 'in' 'lc' 'lk' 'lu' 'lv' 'ma' 'od' 'oe' 'p7' 'pm' 'q7' 'qa' 'qe' 'qf' 'qi' 'qr' 'qu' 'rc' 'rd' 're' 'rj' 'rl' 'ro' 'tj' 'tw' 'v4' 'vb' 'w7' 'wc' 'wg' 'wi' 'wm' 'wu' 'y4' 'yj' 'yv' 'z1' 'zt' 'zu' - f | '2f' 'af' 'al' 'at' 'bc' 'bo' 'cd' 'd6' 'dk' 'dp' 'dy' 'ea' 'eu' 'ev' 'ey' 'fe' 'ft' 'gd' 'gi' 'ha' 'hm' 'hr' 'i6' 'im' 'is' 'iz' 'jb' 'jt' 'ju' 'li' 'm4' 'my' 'na' 'nm' 'nu' 'oh' 'p4' 'pa' 'pn' 'po' 'qk' 'qz' 'r4' 're' 'rf' 'rs' 'sm' 'tb' 'ti' 'to' 'tw' 'u7' 'ut' 'vz' 'w3' 'w5' 'w9' 'wc' 'wj' 'wq' 'wr' 'x7' 'xg' 'xz' 'ye' 'yx' 'ze' 'zo' - f | '2o' '4i' '93' 'a2' 'az' 'b5' 'bk' 'cd' 'do' 'ea' 'ed' 'ej' 'el' 'es' 'ez' 'fd' 'fk' 'fu' 'h7' 'hg' 'hr' 'hs' 'hv' 'iz' 'j0' 'j9' 'jb' 'jc' 'jt' 'ln' 'lu' 'lw' 'lz' 'mz' 'ns' 'nv' 'o8' 'oi' 'oq' 'ov' 'oz' 'pr' 'qi' 'qw' 'rb' 'rg' 'ro' 'rq' 'sb' 'sc' 't8' 't9' 'te' 'ti' 'tm' 'uo' 'uz' 'wd' 'wi' 'wn' 'wp' 'ww' 'wx' 'xt' 'xv' 'yb' 'yr' 'zs' - f | '2p' '3w' 'ah' 'at' 'bg' 'bv' 'bz' 'cu' 'd8' 'dw' 'eb' 'ec' 'ei' 'er' 'es' 'fn' 'gl' 'hd' 'hx' 'ij' 'ip' 'ki' 'lh' 'mx' 'my' 'nq' 'o3' 'o7' 'ok' 'or' 'os' 'pd' 'pu' 'q5' 'qa' 'qc' 'qd' 'qg' 'qh' 'ql' 'qs' 'qt' 'qu' 'qx' 'qy' 'rb' 'rd' 'rk' 'rs' 'ru' 'rv' 'rw' 'sn' 'tm' 'vw' 'w9' 'wa' 'wb' 'wm' 'ws' 'xe' 'xo' 'xu' 'y6' 'yh' 'yj' 'z4' 'zg' - f | '45' '85' 'an' 'bi' 'ca' 'cb' 'co' 'cq' 'cu' 'dt' 'e4' 'eb' 'ed' 'ef' 'em' 'eq' 'er' 'et' 'ex' 'fe' 'fk' 'fv' 'fw' 'gm' 'gq' 'hu' 'it' 'iw' 'j5' 'jj' 'jm' 'jr' 'lq' 'lz' 'mc' 'n0' 'nq' 'o7' 'ok' 'pd' 'ph' 'q6' 'qe' 'qh' 'qm' 'qy' 'qz' 'r2' 'r5' 'ra' 're' 's6' 'sb' 'sl' 'tb' 'tg' 'ur' 'vl' 'wa' 'wf' 'wt' 'wv' 'ww' 'xr' 'xz' 'yb' 'yf' 'zx' - f | 'a2' 'af' 'an' 'aq' 'ar' 'bj' 'bz' 'cx' 'dy' 'eh' 'ek' 'en' 'ex' 'fy' 'gg' 'gr' 'hh' 'if' 'ip' 'it' 'ix' 'j5' 'jf' 'jn' 'k0' 'kk' 'lq' 'nc' 'oc' 'od' 'og' 'oh' 'oo' 'or' 'pm' 'po' 'qa' 'qd' 'qe' 'qh' 'ql' 'qo' 'qt' 'qx' 'qz' 'rq' 'rr' 'ru' 'sj' 'sk' 'tl' 'tr' 'ug' 'wa' 'wb' 'wd' 'we' 'wk' 'wn' 'wq' 'wr' 'xl' 'xo' 'xw' 'yf' 'yq' 'ys' 'yw' - f | 'a6' 'ao' 'aq' 'ba' 'be' 'bg' 'cb' 'ci' 'cj' 'cq' 'cs' 'dc' 'e6' 'ea' 'ei' 'ex' 'fw' 'fy' 'g7' 'gg' 'gq' 'hg' 'ib' 'ig' 'im' 'jz' 'kv' 'me' 'mi' 'mj' 'mp' 'o6' 'oq' 'os' 'ov' 'pd' 'pl' 'pr' 'px' 'qf' 'qg' 'ql' 'qr' 'qz' 'r5' 'rl' 'ro' 'ru' 'rz' 'sy' 'sz' 't7' 'tt' 'u7' 'uf' 'ug' 'uo' 'vt' 'w3' 'w8' 'wb' 'wp' 'ww' 'wy' 'xl' 'xv' 'yc' 'z3' - f | 'a7' 'ab' 'ba' 'bv' 'e7' 'ea' 'eh' 'ep' 'er' 'es' 'f4' 'gq' 'hg' 'hi' 'hs' 'ip' 'iw' 'je' 'k0' 'kh' 'kn' 'ko' 'lo' 'ly' 'm6' 'n2' 'o1' 'oh' 'on' 'op' 'os' 'pi' 'q6' 'q9' 'qh' 'qj' 'qm' 'qn' 'qp' 'qs' 's7' 'sa' 'sb' 'sc' 'tn' 'tq' 'tt' 'u4' 'un' 'ux' 've' 'w1' 'wd' 'wf' 'wj' 'wl' 'wr' 'wu' 'wy' 'xf' 'xx' 'y3' 'y9' 'yk' 'zh' 'zn' 'zo' 'zr' - f | '1l' '2u' 'ao' 'av' 'bb' 'c7' 'c8' 'ca' 'cm' 'cp' 'dc' 'di' 'e2' 'e4' 'eo' 'er' 'et' 'ex' 'g1' 'gb' 'hi' 'hr' 'ht' 'i9' 'id' 'ie' 'jk' 'jq' 'ju' 'k3' 'k5' 'lb' 'lm' 'lv' 'mg' 'no' 'o8' 'oo' 'ot' 'p2' 'pn' 'pw' 'py' 'q6' 'qg' 'ql' 'qo' 'r2' 'rf' 'rk' 'rq' 'ry' 't2' 'tb' 'tn' 'tr' 'tw' 'ub' 'w1' 'wa' 'wb' 'we' 'wi' 'ww' 'xo' 'yk' 'ys' 'yx' 'zg' - f | '26' '4v' 'a1' 'ac' 'af' 'ah' 'au' 'cn' 'd3' 'dn' 'e4' 'ec' 'eg' 'ep' 'er' 'fa' 'fb' 'g6' 'gr' 'h3' 'hb' 'hf' 'hj' 'iu' 'iv' 'j1' 'jk' 'jy' 'k2' 'k8' 'kd' 'lc' 'lf' 'mb' 'nn' 'ot' 'p5' 'pb' 'pl' 'pw' 'py' 'q4' 'q9' 'qf' 'r6' 'ra' 'rl' 'rr' 'ru' 'rz' 'ta' 'te' 'ti' 'tp' 'u1' 'uy' 'w6' 'wb' 'wc' 'wd' 'wh' 'wv' 'wy' 'xo' 'xu' 'yd' 'yh' 'yo' 'yt' - f | 'a3' 'ac' 'af' 'ar' 'b4' 'cu' 'd6' 'dx' 'dz' 'eb' 'ej' 'ek' 'eo' 'es' 'eu' 'ff' 'gj' 'go' 'h6' 'ho' 'hx' 'i6' 'ih' 'ir' 'iy' 'jd' 'jg' 'jh' 'k5' 'n0' 'nn' 'oe' 'og' 'pj' 'pk' 'pq' 'px' 'q5' 'qc' 'qf' 'qg' 'qi' 'qm' 'qx' 'rb' 'rd' 're' 'rx' 'sn' 'sq' 'tp' 'uh' 'ul' 'um' 'uo' 'up' 'uq' 'uz' 'vt' 'wb' 'wj' 'wp' 'ws' 'wv' 'xk' 'xo' 'yv' 'yw' 'yx' - f | 'a9' 'ag' 'bh' 'dk' 'dq' 'e3' 'e7' 'ea' 'ee' 'ej' 'et' 'ez' 'f1' 'gh' 'gn' 'gr' 'hb' 'hf' 'hh' 'hz' 'ip' 'iw' 'j7' 'jh' 'ki' 'kp' 'kx' 'ml' 'nu' 'ov' 'ow' 'pg' 'ph' 'pn' 'po' 'pu' 'qd' 'qe' 'qr' 'qs' 'qv' 'r2' 'r3' 'rb' 'rd' 'ry' 's3' 'sd' 'sr' 'su' 'sz' 'tp' 'ty' 'uh' 'un' 'v0' 'v4' 'w0' 'wa' 'wk' 'wq' 'wx' 'xe' 'y6' 'yd' 'yf' 'yj' 'yl' 'zt' - f | 'ah' 'ai' 'aj' 'cb' 'cg' 'dd' 'dl' 'dp' 'eb' 'ee' 'ei' 'es' 'ey' 'f5' 'fm' 'fr' 'g0' 'gj' 'gp' 'gq' 'gw' 'hc' 'hf' 'ig' 'ij' 'iu' 'kg' 'ld' 'mr' 'nh' 'o2' 'pp' 'pz' 'qk' 'qr' 'qv' 'qz' 'r4' 'rb' 'rd' 'ro' 'rt' 'rw' 'rz' 'sr' 'sw' 'sy' 't6' 't9' 'tb' 'ts' 'tt' 'uo' 'vg' 'wd' 'wj' 'wl' 'wo' 'wp' 'wq' 'wt' 'ww' 'x8' 'yw' 'yx' 'z3' 'z9' 'zi' 'zz' - f | 'al' 'ar' 'ax' 'be' 'bu' 'dl' 'do' 'du' 'e5' 'es' 'ey' 'fb' 'fn' 'ga' 'he' 'hh' 'ho' 'i8' 'ic' 'in' 'iy' 'j1' 'j6' 'j9' 'jk' 'jm' 'ko' 'kv' 'kz' 'lk' 'ls' 'nb' 'oe' 'ou' 'pf' 'pj' 'pm' 'pu' 'py' 'q4' 'q7' 'qe' 'ql' 'qo' 'qr' 'qt' 'rh' 'rs' 'ry' 's3' 's6' 'sl' 'sq' 'tm' 'ud' 'uf' 'ur' 'vy' 'w5' 'w8' 'wp' 'wt' 'wx' 'xq' 'yc' 'yd' 'yi' 'yl' 'ym' - f | 'b0' 'b7' 'bh' 'bt' 'c0' 'cd' 'dv' 'e5' 'ed' 'ef' 'eq' 'ff' 'fj' 'fk' 'ga' 'gt' 'gx' 'hm' 'hx' 'i1' 'ih' 'il' 'j5' 'ja' 'jf' 'jp' 'jt' 'm2' 'nf' 'nt' 'of' 'oo' 'ou' 'p7' 'pa' 'pl' 'po' 'qa' 'qd' 'qr' 'qs' 'qt' 'qx' 'rb' 'rd' 're' 'rf' 'rv' 's0' 'sj' 'sw' 'tf' 'tu' 'tz' 'u2' 'ue' 'uq' 'ur' 'us' 'vw' 'w8' 'wc' 'wo' 'wt' 'x8' 'xo' 'yd' 'zi' 'zq' - f | '1i' '6c' 'ah' 'aq' 'ax' 'b4' 'cg' 'd5' 'da' 'do' 'eu' 'ex' 'ez' 'fa' 'go' 'gt' 'h5' 'h8' 'hi' 'i2' 'ib' 'il' 'in' 'is' 'iz' 'j0' 'j7' 'jn' 'jq' 'ku' 'kw' 'li' 'ls' 'n3' 'nc' 'nt' 'o6' 'oc' 'p4' 'p7' 'pc' 'qc' 'qi' 'qp' 'qs' 'qw' 'r3' 'rl' 'rq' 'ry' 's5' 'si' 'sm' 'ta' 'tr' 'ud' 'ut' 'uw' 'v0' 'w7' 'wg' 'wm' 'xi' 'yi' 'yl' 'yn' 'z1' 'z5' 'zk' 'zp' - f | '3y' 'a9' 'au' 'bp' 'ca' 'cf' 'cn' 'de' 'dh' 'dk' 'dw' 'ec' 'er' 'ey' 'fn' 'g1' 'hb' 'hd' 'hg' 'hi' 'hj' 'ib' 'if' 'iu' 'j7' 'jd' 'jg' 'jj' 'jx' 'k2' 'km' 'kv' 'ld' 'lo' 'mz' 'o3' 'ok' 'ot' 'pl' 'pn' 'pr' 'q0' 'q2' 'q7' 'qa' 'qc' 'qg' 'qm' 'qp' 'qx' 'rc' 'rq' 'rt' 'si' 'so' 'tp' 'tu' 'uj' 'vi' 'vx' 'w7' 'wb' 'wd' 'wm' 'ws' 'wt' 'wv' 'y9' 'yf' 'yq' - f | '7u' 'a4' 'ah' 'aj' 'al' 'cu' 'e1' 'fi' 'fq' 'fr' 'he' 'hi' 'hk' 'hn' 'hp' 'hs' 'ih' 'iw' 'je' 'k4' 'k6' 'kx' 'l1' 'lf' 'li' 'lj' 'lm' 'lp' 'ls' 'lu' 'ma' 'nr' 'o2' 'o4' 'oe' 'oh' 'os' 'ow' 'p7' 'pl' 'q3' 'qj' 'qt' 'qy' 'qz' 'r3' 'rc' 'rm' 'ro' 's0' 's8' 'so' 't7' 'th' 'ti' 'tm' 'tt' 'ut' 'v5' 'va' 've' 'w7' 'wc' 'wk' 'wl' 'wm' 'wq' 'wu' 'x0' 'zh' - f | '8l' 'au' 'ay' 'bn' 'bs' 'ch' 'co' 'cu' 'dq' 'dv' 'eq' 'ev' 'ff' 'fp' 'fr' 'fz' 'gb' 'gu' 'hb' 'he' 'hj' 'id' 'ih' 'jb' 'kg' 'lm' 'ls' 'nm' 'oe' 'ot' 'p6' 'pb' 'ps' 'q0' 'q8' 'qg' 'qh' 'qj' 'qo' 'qp' 'qr' 'qt' 'qz' 'r1' 're' 'rh' 'rp' 'ry' 's1' 's4' 'sc' 'sd' 'sn' 'tf' 'ty' 'u4' 'us' 'vt' 'vx' 'wc' 'wd' 'ws' 'wv' 'xp' 'y2' 'y5' 'y6' 'yy' 'zh' 'zx' - f | 'a2' 'al' 'av' 'b9' 'bd' 'bo' 'c8' 'cq' 'dz' 'ej' 'el' 'ep' 'ew' 'f3' 'f5' 'fr' 'ft' 'h2' 'i3' 'ic' 'id' 'iy' 'jd' 'ke' 'kf' 'kr' 'l3' 'lf' 'mg' 'mu' 'nj' 'o3' 'ob' 'ol' 'ov' 'pb' 'pj' 'pl' 'pp' 'qn' 'qz' 're' 'rg' 'rk' 'ru' 's3' 'sa' 'sh' 'sj' 'su' 'sw' 'tb' 'tk' 'tv' 'ue' 'uh' 'ul' 'un' 'vk' 'vu' 'w0' 'wa' 'wm' 'wo' 'wq' 'xa' 'xl' 'ya' 'yn' 'yo' - f | 'a3' 'a4' 'a5' 'bd' 'co' 'cq' 'd8' 'da' 'dn' 'do' 'du' 'dv' 'ed' 'ei' 'en' 'eo' 'es' 'et' 'f1' 'fa' 'ha' 'hr' 'i6' 'id' 'ie' 'if' 'ir' 'jx' 'kd' 'kh' 'mb' 'mq' 'mv' 'o6' 'oa' 'or' 'oy' 'pl' 'pr' 'q3' 'q8' 'qa' 'qb' 'qc' 'qd' 'qk' 'qs' 'ra' 'rc' 'rp' 'rz' 'se' 'tg' 'tj' 'u2' 'uq' 'uv' 'uy' 'vm' 'vz' 'wh' 'wu' 'xf' 'xs' 'yc' 'ye' 'yl' 'yn' 'z1' 'zf' - f | '18' 'at' 'c6' 'ca' 'dj' 'dl' 'dx' 'ec' 'ek' 'f4' 'fo' 'fs' 'fz' 'h8' 'hn' 'ik' 'il' 'j4' 'jn' 'ld' 'ln' 'ls' 'lx' 'nu' 'o6' 'os' 'ot' 'ox' 'pa' 'pe' 'pp' 'pw' 'q1' 'qc' 'qd' 'qh' 'qk' 'qq' 'qr' 'rq' 'rr' 'rt' 'rv' 'rz' 's5' 'sd' 'sh' 'sy' 't3' 'tu' 'ty' 'uj' 'uo' 'up' 'uu' 've' 'vl' 'vu' 'wa' 'wd' 'wo' 'wr' 'ws' 'ww' 'xk' 'y0' 'y6' 'yi' 'yq' 'yy' 'z3' - f | '1t' 'a7' 'aj' 'au' 'ca' 'cn' 'cw' 'dg' 'dp' 'ec' 'ei' 'en' 'ew' 'ez' 'f3' 'f8' 'hp' 'ht' 'hw' 'is' 'iv' 'jd' 'ji' 'kn' 'l4' 'lq' 'lz' 'm7' 'o6' 'oj' 'oz' 'p4' 'p7' 'po' 'pp' 'q2' 'qa' 'qj' 'qo' 'qt' 'qv' 'qw' 'qy' 'ra' 'rd' 'ri' 'rn' 'rr' 'ry' 's7' 'st' 'sz' 'tg' 'to' 'tu' 'tx' 'w2' 'w4' 'w5' 'wd' 'we' 'wg' 'wo' 'ws' 'xm' 'y5' 'yf' 'yl' 'ym' 'yr' 'yz' - f | '2e' '3u' 'aa' 'an' 'ap' 'bc' 'br' 'bx' 'cc' 'dt' 'ea' 'ei' 'ej' 'em' 'eo' 'ep' 'eq' 'es' 'et' 'ey' 'fb' 'fw' 'gh' 'gs' 'gv' 'hd' 'hg' 'hn' 'i7' 'it' 'jf' 'ka' 'kl' 'l8' 'la' 'm0' 'nu' 'o2' 'oc' 'oy' 'ph' 'po' 'pz' 'q2' 'qg' 'qh' 'qn' 'qp' 'qt' 'r7' 'r8' 'rd' 'rf' 'rj' 'rk' 'sm' 't1' 'tb' 'th' 'ub' 'ui' 'ul' 'um' 'uu' 've' 'w8' 'wj' 'wx' 'y8' 'zf' 'zn' - f | '2n' 'ar' 'b3' 'bc' 'bd' 'c0' 'c9' 'cb' 'cp' 'do' 'ee' 'eh' 'ep' 'f3' 'fb' 'fj' 'gg' 'gk' 'gm' 'h9' 'hk' 'hp' 'ht' 'i7' 'ij' 'iw' 'kx' 'm2' 'mt' 'mv' 'nj' 'ns' 'o7' 'oa' 'od' 'of' 'om' 'or' 'pg' 'pm' 'pu' 'qq' 'qw' 'qx' 'r0' 'rb' 'rg' 'rk' 'rv' 'se' 'sj' 'sp' 'su' 'tf' 'uu' 'v9' 've' 'vg' 'vm' 'w5' 'w6' 'w8' 'wa' 'wp' 'y5' 'y9' 'yk' 'yy' 'z2' 'zd' 'zw' - f | '4z' 'a8' 'ds' 'dv' 'dx' 'eq' 'f1' 'fs' 'gs' 'h9' 'hm' 'hp' 'hu' 'hz' 'i6' 'ip' 'is' 'j4' 'jf' 'k3' 'ku' 'lc' 'le' 'lj' 'nb' 'o6' 'ob' 'of' 'p6' 'pj' 'pm' 'ps' 'qc' 'qf' 'qh' 'ql' 'qr' 'qu' 'qx' 'r3' 'rf' 'ri' 's1' 's4' 'sr' 'st' 'sw' 'tb' 'tc' 'tg' 'tr' 'tz' 'ug' 'uq' 'uu' 'uv' 'uw' 'uz' 'vl' 'w2' 'w4' 'wq' 'wx' 'xe' 'xv' 'xx' 'y6' 'yg' 'yn' 'yp' 'yr' - f | '6x' 'a4' 'ae' 'as' 'd5' 'df' 'e9' 'ek' 'ew' 'ex' 'ez' 'fa' 'fe' 'fr' 'gk' 'hb' 'hg' 'hl' 'hp' 'id' 'it' 'ix' 'jc' 'jg' 'jk' 'jm' 'js' 'ju' 'ln' 'lq' 'mo' 'ms' 'o7' 'od' 'oi' 'oo' 'pn' 'qd' 'qt' 'qv' 'r0' 'rc' 'rj' 'rk' 'rn' 'rp' 'rx' 'rz' 'sc' 'sg' 'sk' 'so' 'ti' 'tn' 'to' 'tr' 'tz' 'uk' 'um' 'ut' 'v1' 'vr' 'wo' 'wr' 'wu' 'wz' 'xn' 'xu' 'y4' 'yg' 'z2' - f | '6y' 'aw' 'az' 'cn' 'cs' 'cx' 'd4' 'di' 'dl' 'e7' 'eg' 'eh' 'eo' 'ep' 'et' 'ew' 'ex' 'ey' 'fv' 'fz' 'gu' 'gv' 'hg' 'hl' 'hm' 'iv' 'jc' 'lb' 'me' 'nc' 'nz' 'o2' 'ox' 'pn' 'po' 'pw' 'q4' 'q5' 'qg' 'qq' 'qv' 'qz' 'r7' 're' 'rl' 'rq' 'rs' 'rx' 'sa' 'se' 'ti' 'tj' 'tn' 'tz' 'u6' 'uq' 'ur' 'us' 'v9' 'we' 'wn' 'wp' 'wr' 'ws' 'xo' 'ya' 'ye' 'yg' 'yx' 'zj' 'zl' - f | '8j' 'ao' 'bc' 'bh' 'co' 'cz' 'di' 'dq' 'dr' 'e1' 'ec' 'ei' 'el' 'et' 'eu' 'ex' 'f8' 'g7' 'gl' 'hq' 'hw' 'ib' 'kh' 'kn' 'kt' 'lc' 'md' 'n7' 'oe' 'p0' 'pg' 'pl' 'pm' 'po' 'pr' 'q0' 'q6' 'ql' 'qv' 'qy' 'r7' 'ra' 're' 'rq' 'ru' 'ry' 'sm' 'sn' 't4' 't5' 'tw' 'ty' 'u7' 'ud' 'uh' 'um' 'us' 'ux' 'v0' 'v7' 'vn' 'w6' 'we' 'wi' 'wm' 'x1' 'xo' 'xz' 'yn' 'yo' 'yz' - f | 'a9' 'b8' 'de' 'dl' 'eg' 'fb' 'fi' 'fm' 'gj' 'hp' 'hx' 'if' 'ig' 'ii' 'ik' 'io' 'iw' 'ix' 'j1' 'j3' 'j8' 'ji' 'ki' 'kt' 'ld' 'm8' 'mi' 'mj' 'mx' 'mz' 'n2' 'nk' 'oe' 'ok' 'oq' 'os' 'ot' 'pj' 'q4' 'qe' 'ql' 'qp' 'qq' 'r4' 'rl' 'rq' 'rx' 'sf' 'ss' 'td' 'ti' 'tl' 'u4' 'u5' 'uq' 'ux' 'v4' 'vl' 'vs' 'wb' 'wc' 'wg' 'wj' 'wn' 'ww' 'x9' 'xa' 'ye' 'yg' 'yl' 'yn' - f | 'ab' 'ap' 'ay' 'b6' 'bg' 'bm' 'bq' 'br' 'ce' 'co' 'dt' 'e4' 'ef' 'ej' 'ek' 'ep' 'fj' 'fq' 'fu' 'g3' 'gg' 'gp' 'hk' 'i3' 'i9' 'je' 'k7' 'ku' 'li' 'lq' 'lt' 'lx' 'mc' 'mz' 'n7' 'nl' 'o1' 'of' 'os' 'pm' 'qc' 'ql' 'qs' 'qw' 'r6' 'rf' 'rj' 'rp' 'ru' 'rx' 'rz' 'tl' 'tw' 'ty' 'u1' 'u3' 'ud' 'un' 'uw' 'vu' 'w6' 'wf' 'wl' 'wq' 'xh' 'xm' 'yc' 'yh' 'yr' 'yw' 'z9' - f | 'ad' 'af' 'ar' 'b0' 'bw' 'c4' 'cn' 'do' 'e0' 'e4' 'em' 'eo' 'fn' 'ga' 'gg' 'gy' 'hf' 'ht' 'id' 'ig' 'ik' 'iv' 'j6' 'mi' 'n5' 'ng' 'ob' 'og' 'ou' 'oy' 'p4' 'pd' 'pg' 'pk' 'po' 'pt' 'pw' 'q3' 'q5' 'qb' 'qr' 'qt' 'r3' 'rc' 'rd' 'rl' 's0' 'sy' 't9' 'tc' 'tm' 'tq' 'ub' 'up' 'uv' 'ux' 'vm' 'vv' 'vw' 'wc' 'wg' 'wk' 'wm' 'x0' 'xp' 'xr' 'xv' 'xx' 'yg' 'ym' 'yw' - f | 'at' 'bm' 'd0' 'd5' 'eg' 'ei' 'ek' 'eq' 'er' 'eu' 'ez' 'fs' 'gh' 'hc' 'ho' 'ix' 'ko' 'kq' 'l1' 'll' 'mz' 'ob' 'ou' 'p2' 'pi' 'pl' 'ps' 'pt' 'q0' 'q5' 'qa' 'qg' 'qj' 'qm' 'qq' 'qt' 'rf' 'ri' 'rm' 'ru' 'rv' 'rx' 'ry' 's4' 't4' 'td' 'tn' 'tq' 'tx' 'u3' 'up' 'uq' 'vn' 'wb' 'wf' 'wk' 'wr' 'wv' 'wy' 'xh' 'xx' 'y3' 'yc' 'ye' 'ys' 'za' 'zd' 'zo' 'zs' 'zv' 'zy' - f | '0p' 'ab' 'as' 'az' 'b5' 'bl' 'cn' 'cr' 'ct' 'cu' 'dq' 'dr' 'em' 'eq' 'fe' 'fj' 'fk' 'fy' 'gn' 'gx' 'h9' 'hg' 'hk' 'hr' 'io' 'iy' 'jh' 'la' 'lb' 'll' 'ln' 'lq' 'me' 'mp' 'mr' 'mv' 'ns' 'od' 'op' 'os' 'ov' 'po' 'pv' 'qa' 'qn' 'qp' 'qs' 'qt' 'r0' 'rc' 'rd' 'rl' 's5' 'si' 't7' 'tc' 'th' 'u0' 'um' 'uz' 'vg' 'vo' 'wa' 'wp' 'wu' 'xk' 'yg' 'yh' 'yi' 'yo' 'yy' 'zy' - f | '10' 'ac' 'as' 'av' 'b6' 'bb' 'cj' 'd3' 'd8' 'dt' 'eb' 'ey' 'ez' 'fm' 'g8' 'gf' 'gs' 'hn' 'hq' 'ib' 'ii' 'j0' 'jb' 'k4' 'l5' 'ld' 'mg' 'mu' 'nz' 'oa' 'ou' 'pn' 'q1' 'qa' 'qg' 'qj' 'qm' 'qn' 'qs' 'qx' 'qy' 'rj' 'rm' 'rr' 'ry' 's0' 's7' 't4' 't6' 'tb' 'td' 'u0' 'ua' 'ut' 'ux' 'uz' 'v1' 'v9' 'vd' 'vz' 'wc' 'wj' 'wo' 'ws' 'ww' 'x9' 'xu' 'y7' 'yq' 'yv' 'zw' 'zy' - f | '11' '3o' '7a' 'au' 'bc' 'bx' 'cg' 'dz' 'e5' 'ea' 'eb' 'ee' 'eg' 'eo' 'er' 'eu' 'g2' 'gq' 'gx' 'i4' 'in' 'ip' 'iv' 'k4' 'kd' 'kw' 'lf' 'ls' 'oq' 'ot' 'oz' 'p7' 'ph' 'pk' 'ps' 'qa' 'qd' 'qf' 'qi' 'qj' 'qr' 'qt' 'r4' 'rb' 'rd' 'rl' 'th' 'tl' 'tx' 'uc' 'uj' 'v6' 'vj' 'vz' 'we' 'wj' 'wl' 'wm' 'wp' 'wy' 'wz' 'xj' 'xk' 'xl' 'ya' 'yj' 'yl' 'yo' 'yw' 'zs' 'zv' 'zw' - f | '5h' 'ak' 'al' 'aq' 'cw' 'd7' 'dm' 'ec' 'ee' 'ef' 'el' 'f4' 'f5' 'f8' 'fh' 'fr' 'fu' 'gh' 'hk' 'ir' 'ix' 'k0' 'k2' 'ka' 'ku' 'lg' 'lr' 'm8' 'mf' 'nq' 'of' 'ph' 'pr' 'px' 'q2' 'qf' 'qh' 'qk' 'qn' 'qq' 'qs' 'qv' 'qx' 'qy' 'qz' 're' 'rh' 'ro' 'rx' 'sa' 'sh' 'sk' 'tb' 'u1' 'ud' 'uo' 'us' 'uw' 'vg' 'vp' 'w3' 'wb' 'wc' 'wf' 'wg' 'wi' 'wk' 'yr' 'ys' 'yw' 'zw' 'zz' - f | 'ae' 'ak' 'ax' 'cm' 'cw' 'cx' 'df' 'dm' 'dw' 'eb' 'ef' 'ei' 'el' 'f1' 'fn' 'gu' 'hf' 'hk' 'ik' 'iw' 'jd' 'ju' 'jz' 'k7' 'ku' 'lq' 'mf' 'mw' 'oa' 'od' 'oe' 'ou' 'ow' 'pc' 'ph' 'pw' 'q0' 'q2' 'qd' 'qq' 'qw' 'qx' 'rj' 'rk' 'rn' 'rx' 's2' 'sm' 'sx' 't9' 'ti' 'tu' 'tw' 'u3' 'ud' 'wf' 'wm' 'wt' 'xl' 'xo' 'xs' 'yh' 'yi' 'ym' 'yr' 'yu' 'yw' 'z2' 'za' 'zf' 'zl' 'zz' - f | 'ae' 'ap' 'bi' 'd7' 'ee' 'ej' 'ek' 'ev' 'ew' 'fb' 'gm' 'hf' 'ho' 'hz' 'i6' 'ib' 'id' 'il' 'im' 'io' 'ip' 'ir' 'iv' 'j2' 'jf' 'jl' 'jo' 'kh' 'kk' 'kt' 'lr' 'm4' 'mi' 'nm' 'ns' 'og' 'pv' 'pw' 'q4' 'qd' 'qi' 'r6' 'r8' 'rn' 'rp' 'si' 't7' 'ta' 'td' 'te' 'ti' 'tm' 'u6' 'ub' 'vu' 'w6' 'w8' 'wa' 'wb' 'wc' 'wd' 'wf' 'wh' 'wn' 'wt' 'wz' 'y5' 'ya' 'yg' 'yk' 'yn' 'z3' - f | 'as' 'b1' 'bc' 'br' 'cd' 'dw' 'e2' 'ec' 'ew' 'f1' 'f2' 'f6' 'g0' 'gi' 'gy' 'hf' 'hk' 'if' 'ii' 'is' 'iy' 'j2' 'jp' 'kl' 'ku' 'l4' 'lg' 'lp' 'lv' 'lw' 'nd' 'ng' 'oo' 'q4' 'q6' 'q9' 'qc' 'qh' 'ql' 'qm' 'qo' 'qx' 'qz' 're' 'ro' 'sm' 't5' 'th' 'tt' 'ul' 'um' 'v0' 'vb' 'vf' 'vx' 'w9' 'wc' 'we' 'wp' 'xi' 'xl' 'xt' 'xv' 'y3' 'y6' 'ye' 'yg' 'yq' 'yw' 'yx' 'zs' 'zu' - f | '3y' 'a1' 'ay' 'cn' 'd3' 'dc' 'dh' 'e9' 'ef' 'ew' 'f9' 'fd' 'fg' 'ft' 'h6' 'hs' 'hu' 'hz' 'i0' 'ib' 'ip' 'iu' 'jw' 'kf' 'kp' 'kw' 'li' 'lp' 'mg' 'mj' 'mz' 'ng' 'oc' 'od' 'om' 'ou' 'ov' 'p8' 'pi' 'q8' 'qe' 'qf' 'qh' 'qj' 'qo' 'qt' 'qv' 'qz' 'r2' 'r3' 'rc' 'rm' 't4' 'ty' 'tz' 'u6' 'ua' 'ut' 'v3' 'vj' 'w0' 'w6' 'wi' 'wj' 'wq' 'wt' 'wu' 'xe' 'xv' 'xy' 'y3' 'yv' 'zw' - f | '4a' '56' 'ae' 'ao' 'ay' 'bn' 'bx' 'c9' 'cl' 'd0' 'd7' 'en' 'et' 'fj' 'fk' 'fu' 'g4' 'ha' 'ht' 'hv' 'ih' 'ij' 'ir' 'iw' 'k1' 'k7' 'kq' 'kw' 'lg' 'm5' 'me' 'nw' 'oj' 'ou' 'p6' 'pl' 'pq' 'pt' 'qc' 'qg' 'qk' 'ql' 'qm' 'qo' 'qq' 'rj' 's1' 'sd' 'sq' 'sz' 'tb' 'td' 'tr' 'tw' 'ug' 'uj' 'uo' 'ut' 'uy' 'wj' 'wk' 'ws' 'wz' 'x2' 'xx' 'xy' 'y1' 'y4' 'yc' 'yk' 'yt' 'yx' 'zp' - f | 'a2' 'a4' 'ac' 'ag' 'ao' 'ay' 'ce' 'di' 'dm' 'dr' 'dz' 'e9' 'eg' 'ej' 'el' 'em' 'eq' 'et' 'ey' 'fa' 'fw' 'fz' 'gg' 'gh' 'gp' 'gr' 'h6' 'hj' 'hk' 'i7' 'id' 'ik' 'iu' 'j2' 'kc' 'l1' 'lh' 'm5' 'm9' 'mj' 'mr' 'mx' 'ok' 'ot' 'ov' 'oz' 'p5' 'q0' 'q1' 'qe' 'qg' 'r2' 'rj' 'rk' 'rw' 'sa' 't3' 't6' 't9' 'te' 'tf' 'tj' 'tk' 'u4' 'wa' 'wf' 'wu' 'wy' 'yg' 'yj' 'ys' 'zk' 'zy' - f | '2b' '2e' 'ao' 'ap' 'au' 'bo' 'by' 'cr' 'do' 'du' 'dy' 'e7' 'eb' 'eh' 'em' 'ey' 'f6' 'ff' 'fu' 'gc' 'gi' 'gk' 'h1' 'hp' 'hs' 'hy' 'ia' 'ic' 'ig' 'ik' 'im' 'je' 'jf' 'ji' 'jr' 'l5' 'lf' 'lp' 'mv' 'ne' 'nt' 'oa' 'os' 'pj' 'po' 'q9' 'qd' 'qe' 'qf' 'qw' 'qy' 'ra' 'rg' 'rk' 'rp' 'rq' 'rx' 'sb' 'si' 'sn' 'ta' 'ux' 'v1' 'vd' 'wa' 'wk' 'wr' 'wu' 'xo' 'yg' 'yl' 'yz' 'zi' 'zt' - f | '2l' 'a5' 'ah' 'ak' 'ar' 'be' 'e6' 'eh' 'ge' 'gj' 'gn' 'gt' 'gy' 'ia' 'ii' 'iw' 'ix' 'jb' 'k9' 'kd' 'ke' 'kh' 'kv' 'lm' 'ly' 'me' 'mt' 'nb' 'np' 'o5' 'oe' 'of' 'og' 'pd' 'pf' 'pm' 'pt' 'pw' 'q3' 'qc' 'qe' 'qg' 'qh' 'qk' 'ql' 'qs' 'qt' 'qx' 'qz' 'rr' 'rw' 'rx' 'ry' 'sk' 'sr' 'su' 'sv' 'sz' 'ug' 'uk' 'uq' 'vg' 'w7' 'wb' 'wn' 'wr' 'ws' 'wz' 'x6' 'yg' 'yj' 'yo' 'zb' 'zx' - f | '3h' '4h' '93' 'ak' 'ao' 'bq' 'cw' 'db' 'dx' 'eb' 'ef' 'el' 'eu' 'ex' 'fc' 'fg' 'fo' 'fr' 'fs' 'fx' 'g5' 'gl' 'go' 'hs' 'i0' 'ii' 'ix' 'j3' 'jc' 'ke' 'l2' 'lf' 'lo' 'm6' 'ms' 'ne' 'oi' 'ox' 'p6' 'pb' 'pr' 'q9' 'qa' 'qi' 'qj' 'qq' 'qs' 'qu' 'qw' 're' 'rq' 's0' 'sa' 'se' 'sk' 'sx' 'tj' 'tk' 'u2' 'ua' 'uh' 'un' 'vy' 'wf' 'wh' 'wj' 'wq' 'wv' 'xl' 'xq' 'yf' 'yi' 'yw' 'z3' - f | 'a6' 'ac' 'ao' 'au' 'cf' 'co' 'cr' 'd9' 'da' 'ds' 'du' 'e4' 'ea' 'ew' 'f1' 'fx' 'hr' 'hu' 'ic' 'id' 'ii' 'in' 'iq' 'iy' 'j3' 'jb' 'ji' 'jl' 'kf' 'mi' 'mp' 'o3' 'oe' 'oj' 'on' 'p5' 'ph' 'pk' 'q1' 'q9' 'qa' 'qs' 'qu' 'qv' 'r9' 'rc' 'rj' 'rv' 's1' 'se' 'sl' 'su' 't2' 't5' 'tk' 'u8' 'uv' 'v9' 'vd' 'vi' 'w0' 'w3' 'we' 'wh' 'wm' 'wp' 'xm' 'y0' 'y1' 'y8' 'yf' 'yo' 'yw' 'zv' - f | 'a0' 'ad' 'ak' 'ao' 'au' 'b0' 'dg' 'dt' 'e9' 'em' 'fg' 'fk' 'fn' 'gq' 'h7' 'hf' 'hm' 'ia' 'id' 'ig' 'iv' 'kv' 'l1' 'lm' 'lz' 'm1' 'mw' 'mz' 'na' 'nh' 'nl' 'nn' 'o1' 'o3' 'om' 'p2' 'p9' 'pj' 'pw' 'pz' 'qk' 'qm' 'qy' 'rs' 'ru' 'ry' 'sf' 'su' 'sw' 'sx' 'td' 'tl' 'tp' 'tw' 'u1' 'ug' 'un' 'uo' 'uq' 'ur' 'v5' 'vk' 'wd' 'wf' 'wj' 'wq' 'wy' 'xg' 'xk' 'xn' 'xq' 'yb' 'z0' 'zt' 'zu' - f | 'a2' 'ag' 'cz' 'd0' 'd3' 'da' 'di' 'ds' 'e5' 'e6' 'e7' 'ej' 'em' 'ex' 'ff' 'fq' 'gm' 'go' 'gt' 'gv' 'gx' 'ho' 'hv' 'i8' 'ic' 'ii' 'il' 'im' 'it' 'ix' 'j1' 'ja' 'jd' 'js' 'kw' 'l6' 'le' 'lk' 'ln' 'mt' 'my' 'ns' 'ol' 'op' 'os' 'p0' 'pu' 'pv' 'pz' 'q4' 'q5' 'qe' 'qh' 'ql' 'qx' 'qy' 'r1' 'rk' 'rr' 'sb' 'sn' 'te' 'tt' 'ui' 'v6' 'w6' 'we' 'ws' 'wy' 'xt' 'xu' 'yf' 'yq' 'yz' 'zy' - f | '1l' '2t' '2z' 'af' 'az' 'be' 'br' 'cm' 'cu' 'cy' 'd2' 'd4' 'dd' 'dx' 'ec' 'ed' 'g2' 'gs' 'gu' 'id' 'ig' 'in' 'it' 'jh' 'jl' 'l6' 'ld' 'lg' 'lr' 'ly' 'nf' 'nj' 'ob' 'ot' 'p4' 'p8' 'pe' 'ph' 'q5' 'qa' 'qd' 'qe' 'qi' 'qo' 'qv' 'qz' 'ra' 'ri' 'rl' 'rr' 's7' 'sj' 'sx' 't0' 'tj' 'to' 'u2' 'ui' 'uq' 'ut' 'uw' 'vg' 'vs' 'w2' 'wa' 'wi' 'wl' 'wr' 'wu' 'ww' 'x7' 'xm' 'ym' 'yn' 'yp' 'zh' - f | 'a0' 'a7' 'ad' 'ae' 'ah' 'ay' 'b8' 'bh' 'dc' 'di' 'do' 'e5' 'ea' 'eb' 'ep' 'ev' 'fn' 'gq' 'gr' 'gs' 'h8' 'hc' 'hj' 'i3' 'ih' 'it' 'iw' 'ja' 'jf' 'kc' 'l8' 'lp' 'lx' 'ly' 'm1' 'og' 'oy' 'p7' 'pp' 'q7' 'qa' 'qm' 'qq' 'qr' 'qu' 'r3' 'rd' 'rg' 'rj' 'rw' 'si' 'sq' 'ss' 't0' 'tb' 'tg' 'tl' 'to' 'tp' 'tx' 'u9' 'ub' 'um' 'up' 'ux' 'vc' 'w0' 'w2' 'w5' 'wl' 'wy' 'xg' 'yb' 'yg' 'yh' 'za' - f | 'af' 'as' 'at' 'aw' 'ax' 'ay' 'az' 'b2' 'bf' 'bl' 'ck' 'cs' 'df' 'dn' 'dv' 'ei' 'ek' 'ev' 'fg' 'fm' 'h0' 'hb' 'hk' 'i0' 'ib' 'if' 'ir' 'lk' 'm5' 'mr' 'np' 'om' 'p5' 'p7' 'pl' 'pq' 'q3' 'qh' 'qi' 'qj' 'qo' 'qp' 'qs' 'qt' 'qu' 'qv' 'qy' 'ra' 'sh' 'sm' 'so' 't2' 'ta' 'ti' 'tv' 'tz' 'u1' 'ug' 'um' 'v5' 'vd' 'w1' 'wn' 'wp' 'wr' 'ws' 'ww' 'xk' 'xx' 'y7' 'yf' 'yh' 'yk' 'yt' 'zr' 'zv' - f | 'ai' 'ak' 'as' 'ax' 'b8' 'bd' 'bq' 'bv' 'd1' 'd8' 'dl' 'e9' 'ew' 'ez' 'fz' 'gj' 'gu' 'hk' 'hq' 'i2' 'ie' 'ip' 'iq' 'is' 'iz' 'jf' 'k0' 'kk' 'l6' 'md' 'mx' 'na' 'nf' 'o5' 'ol' 'p3' 'pe' 'q2' 'qf' 'qj' 'qm' 'qn' 'qs' 'qv' 'r2' 'ra' 'rv' 'ry' 's1' 's5' 'si' 't0' 'tc' 'te' 'tn' 'tq' 'tz' 'u1' 'uj' 'ut' 'w3' 'wa' 'wd' 'wh' 'wj' 'wo' 'wu' 'wv' 'ww' 'xk' 'xo' 'yi' 'yk' 'yl' 'zb' 'zw' - f | '3b' '4p' 'ae' 'aj' 'ap' 'b3' 'bb' 'c0' 'cc' 'cf' 'cn' 'dz' 'e1' 'e9' 'eo' 'ew' 'ey' 'fk' 'fo' 'gd' 'gp' 'hb' 'i8' 'ia' 'ib' 'ie' 'ij' 'iq' 'ir' 'ix' 'jo' 'kp' 'lc' 'ld' 'lp' 'ml' 'on' 'p8' 'pd' 'pf' 'pi' 'pn' 'pr' 'pt' 'pw' 'qe' 'qi' 'qj' 'qp' 'qr' 'qw' 'qz' 'r0' 'rj' 'rn' 'ro' 'rq' 'rr' 's3' 'sg' 'sz' 'tn' 'tw' 'ty' 'ui' 'uj' 'vj' 'vr' 'w5' 'w9' 'wa' 'wp' 'xt' 'ya' 'ym' 'z0' 'zr' - f | 'a7' 'ag' 'aq' 'aw' 'az' 'bf' 'bg' 'ce' 'cp' 'cx' 'dc' 'ek' 'en' 'es' 'fj' 'gb' 'hr' 'hs' 'ie' 'ik' 'is' 'jl' 'jr' 'kc' 'ku' 'l5' 'lj' 'lq' 'lr' 'nu' 'og' 'oj' 'os' 'oy' 'p1' 'p6' 'pc' 'pk' 'pr' 'pu' 'px' 'py' 'q1' 'q4' 'qb' 'qc' 'ql' 'qo' 'qv' 'qz' 'r9' 'ra' 'rg' 'rp' 'sq' 'ts' 'tx' 'u8' 'ue' 'ui' 'uj' 'uk' 'up' 'v4' 'vg' 'vn' 'w2' 'w9' 'wa' 'wk' 'wt' 'y0' 'y9' 'yc' 'ym' 'ys' 'zh' - f | 'aq' 'bo' 'cx' 'dl' 'dm' 'ed' 'ee' 'ei' 'er' 'eu' 'f4' 'fe' 'fw' 'g2' 'gj' 'h7' 'hv' 'im' 'j8' 'jf' 'kl' 'la' 'lj' 'mo' 'oc' 'oh' 'oi' 'ol' 'pa' 'pb' 'pf' 'pk' 'po' 'q9' 'qf' 'qj' 'qq' 'qv' 'r8' 're' 'rg' 'rh' 'rj' 'rr' 'ru' 'ry' 's8' 'sd' 'sg' 'sj' 'sk' 'so' 'sv' 't3' 'tz' 'u5' 'ua' 'uk' 'um' 'vi' 'vt' 'wb' 'we' 'wk' 'wn' 'wt' 'wu' 'wz' 'xk' 'xz' 'yi' 'yl' 'yz' 'z9' 'zd' 'zu' 'zw' - f | '3w' 'aa' 'ad' 'cj' 'cu' 'cv' 'dv' 'ei' 'ej' 'ep' 'er' 'fc' 'fd' 'g5' 'ga' 'go' 'gr' 'gs' 'hq' 'hx' 'hy' 'i1' 'i5' 'ie' 'ip' 'jh' 'jn' 'jq' 'k7' 'ks' 'l0' 'l6' 'l9' 'lm' 'm5' 'mj' 'ng' 'nh' 'o8' 'od' 'on' 'pq' 'q3' 'q8' 'qb' 'qc' 'qf' 'qp' 'qs' 'r1' 'ra' 'rj' 'rw' 'si' 'sp' 't6' 'ta' 'tc' 'tk' 'tm' 'ty' 'ua' 'ud' 'ug' 'v7' 'vg' 'vm' 'w3' 'wc' 'wi' 'wp' 'wy' 'xk' 'xv' 'yi' 'ym' 'ys' 'zl' - f | '5c' 'ag' 'ak' 'ao' 'ap' 'at' 'bo' 'br' 'bs' 'bw' 'cv' 'e9' 'et' 'ex' 'ez' 'fj' 'fu' 'fz' 'gm' 'gq' 'gu' 'h8' 'hl' 'i7' 'ic' 'ik' 'il' 'is' 'j1' 'jc' 'k2' 'k3' 'k6' 'k7' 'kf' 'km' 'kr' 'lj' 'mb' 'md' 'nf' 'om' 'ow' 'p4' 'pk' 'pw' 'q2' 'qe' 'qh' 'qn' 'qq' 'qu' 'qz' 'r0' 'ra' 'rp' 'ru' 'rw' 'rz' 'sv' 'ta' 'ti' 'tq' 'ua' 'up' 'us' 'ut' 'uz' 'vd' 'wl' 'wq' 'xg' 'xk' 'y0' 'yd' 'yw' 'zd' 'zq' - f | 'a0' 'a8' 'b4' 'b9' 'bb' 'cb' 'cn' 'cq' 'ds' 'e2' 'eb' 'ex' 'ez' 'fa' 'fu' 'fz' 'ge' 'gy' 'h0' 'hj' 'ht' 'hu' 'ig' 'in' 'iq' 'is' 'iu' 'jg' 'le' 'ln' 'lr' 'nj' 'nu' 'oe' 'oo' 'ow' 'oz' 'pf' 'ph' 'pj' 'ps' 'pu' 'pv' 'q0' 'q4' 'qf' 'qm' 'qn' 'qo' 'qq' 'qs' 'qw' 'r2' 'r3' 'ra' 'rk' 's7' 'sa' 'sp' 'tl' 'ue' 'ui' 'vo' 'vs' 'wj' 'wk' 'wn' 'wv' 'ww' 'x0' 'xg' 'xh' 'xx' 'y0' 'yd' 'yr' 'zg' 'zn' - f | '18' '5j' '6y' 'ax' 'bw' 'c5' 'de' 'dh' 'dl' 'do' 'dx' 'ek' 'el' 'en' 'et' 'ez' 'f1' 'fe' 'fw' 'ge' 'h1' 'h9' 'ha' 'hb' 'hl' 'it' 'jy' 'kl' 'ky' 'lo' 'nr' 'o3' 'o9' 'p0' 'pe' 'pg' 'ph' 'q1' 'q5' 'qd' 'qg' 'qh' 'qi' 'qj' 'qp' 'qq' 'qr' 'qv' 'qx' 'r0' 'rd' 're' 'rj' 'rm' 'rn' 'rq' 'rv' 'se' 'sv' 'to' 'tz' 'uo' 'v1' 'vm' 'vr' 'wa' 'we' 'wf' 'wh' 'wp' 'wt' 'wy' 'xe' 'xh' 'xz' 'y6' 'yn' 'zm' 'zy' - f | '1s' '95' 'ac' 'aq' 'cb' 'ch' 'cq' 'ct' 'cz' 'dg' 'dj' 'eb' 'ed' 'ee' 'eg' 'en' 'eo' 'ep' 'es' 'fc' 'fl' 'fp' 'gu' 'gw' 'hx' 'i0' 'i2' 'ig' 'ih' 'il' 'in' 'iu' 'iy' 'iz' 'jc' 'k7' 'ka' 'kl' 'km' 'kn' 'lz' 'mx' 'n6' 'nc' 'og' 'pt' 'pw' 'q0' 'qa' 'qe' 'qf' 'ql' 'qm' 'qp' 'qt' 'r7' 'rp' 'ru' 'rw' 'sb' 'sd' 't0' 't7' 'ta' 'tl' 'tn' 'tv' 'ty' 'ui' 'vx' 'wa' 'wc' 'wm' 'wp' 'wy' 'xi' 'xj' 'xk' 'yz' - f | '2o' '2w' 'ac' 'an' 'bb' 'be' 'c3' 'db' 'dj' 'dm' 'dw' 'eb' 'ed' 'eh' 'ew' 'ex' 'fb' 'fi' 'g2' 'gh' 'gv' 'h6' 'hw' 'id' 'ie' 'iz' 'j8' 'jk' 'jv' 'kd' 'lk' 'm8' 'ml' 'mr' 'mx' 'np' 'oe' 'on' 'oz' 'pj' 'pk' 'pm' 'pu' 'qb' 'ql' 'qv' 'qx' 'r0' 're' 'rj' 'rq' 'rw' 't3' 'ta' 'tc' 'td' 'th' 'to' 'tu' 'tx' 'ty' 'tz' 'u3' 'uf' 'vp' 'vz' 'wd' 'wh' 'wk' 'wl' 'wx' 'wy' 'wz' 'xe' 'xm' 'xs' 'ye' 'zd' 'zk' - f | '2v' '7g' 'a1' 'ar' 'bw' 'cs' 'dt' 'ee' 'eg' 'f4' 'fd' 'fl' 'g0' 'ge' 'gx' 'hf' 'hk' 'ht' 'i7' 'il' 'jy' 'ka' 'kl' 'lb' 'lm' 'lv' 'mu' 'nk' 'ns' 'nw' 'oe' 'og' 'oj' 'ol' 'ou' 'pk' 'q2' 'q3' 'qi' 'qn' 'qo' 'qs' 'qy' 'r5' 'rh' 'rj' 'rm' 'rq' 'rr' 'rs' 'rt' 'ru' 'rv' 'ss' 'th' 'tj' 'uj' 'ul' 'um' 'un' 'uo' 'uv' 'w2' 'w6' 'wf' 'wh' 'wk' 'wo' 'wx' 'y4' 'y5' 'yb' 'yn' 'yo' 'ys' 'yz' 'zb' 'zt' 'zw' - f | 'ab' 'ah' 'ax' 'bd' 'ca' 'dh' 'e3' 'ea' 'ed' 'ef' 'en' 'eo' 'er' 'ev' 'ex' 'ez' 'f5' 'fh' 'fo' 'fv' 'ga' 'gb' 'gk' 'id' 'ik' 'in' 'ir' 'jp' 'js' 'jv' 'kf' 'kr' 'lx' 'mu' 'mz' 'n5' 'od' 'on' 'pr' 'pt' 'pv' 'qb' 'qe' 'qj' 'ql' 'qq' 'qu' 'qw' 'rd' 'rl' 'rz' 's4' 'sv' 'ta' 'tj' 'tk' 'u8' 'ui' 'uj' 'uk' 'um' 'ux' 'uz' 'v9' 'vh' 'vl' 'w7' 'wb' 'wi' 'wm' 'ws' 'xi' 'ye' 'yg' 'yr' 'yw' 'zj' 'zn' 'zs' - f | '17' 'bv' 'c5' 'cn' 'd0' 'di' 'dm' 'dz' 'ec' 'ef' 'et' 'ev' 'ew' 'fa' 'fb' 'fh' 'fw' 'fy' 'g8' 'he' 'hf' 'hp' 'hu' 'hz' 'i2' 'ig' 'ix' 'jy' 'lt' 'mi' 'nm' 'nq' 'o5' 'oa' 'od' 'oi' 'oo' 'ox' 'pb' 'po' 'qb' 'qc' 'qf' 'qr' 'qs' 'qt' 'qw' 'qx' 'r0' 'rb' 'rf' 'rh' 'rk' 'ry' 'sb' 'sf' 'sh' 'st' 'tb' 'tj' 'tn' 'tq' 'tu' 'ty' 'u9' 'ud' 'ut' 'uw' 'ux' 'w4' 'wi' 'xm' 'ya' 'yl' 'ys' 'yt' 'yv' 'yy' 'zb' 'zv' - f | '19' '1t' 'aa' 'ah' 'ak' 'an' 'aq' 'bx' 'cc' 'dd' 'ed' 'ee' 'ei' 'ej' 'en' 'er' 'es' 'eu' 'fj' 'fs' 'fy' 'gd' 'gs' 'hw' 'ie' 'ig' 'ix' 'jc' 'jj' 'jk' 'jq' 'kb' 'kv' 'li' 'mr' 'no' 'nq' 'o9' 'oo' 'os' 'oy' 'p2' 'pd' 'ph' 'pn' 'pp' 'ql' 'qr' 'qy' 'r1' 'rb' 'rd' 'rn' 'ru' 's9' 'sf' 'tk' 'to' 'u1' 'u5' 'ub' 'ud' 'uf' 'vh' 'vx' 'w1' 'w8' 'wg' 'wn' 'wq' 'wu' 'wy' 'xc' 'xm' 'xw' 'y8' 'yk' 'yr' 'z3' 'zm' - f | '1n' '2h' 'at' 'ax' 'c0' 'cn' 'dc' 'df' 'dg' 'e1' 'e5' 'ea' 'eq' 'ex' 'ez' 'f7' 'f8' 'fa' 'gt' 'gv' 'gy' 'hb' 'i3' 'i8' 'id' 'iw' 'ix' 'jc' 'jg' 'k0' 'kh' 'ky' 'kz' 'lj' 'lm' 'lq' 'mj' 'o8' 'og' 'op' 'ow' 'p5' 'p9' 'qc' 'qe' 'qf' 'qg' 'qo' 'qq' 'qv' 'qz' 'rq' 'rt' 'ru' 's2' 's5' 'sf' 'sl' 'sm' 'st' 'sw' 'tc' 'tl' 'ts' 'tz' 'ud' 'ur' 've' 'wd' 'wj' 'wn' 'wq' 'xa' 'xb' 'yf' 'yj' 'yn' 'yt' 'yv' 'zf' - f | '5h' 'a2' 'ad' 'ag' 'ar' 'cf' 'ch' 'dd' 'e6' 'ei' 'el' 'em' 'f3' 'fx' 'ga' 'gm' 'h8' 'hi' 'hl' 'hm' 'ix' 'jb' 'l5' 'm2' 'mh' 'mm' 'mu' 'n9' 'nf' 'nq' 'nw' 'o4' 'oe' 'pa' 'pl' 'q1' 'qb' 'qd' 'qi' 'qj' 'qk' 'qu' 'qy' 'r0' 'r3' 'r8' 'rg' 'rj' 'rt' 's1' 'sa' 'ss' 'sx' 'tb' 'tc' 'tj' 'uf' 'uh' 'um' 'uo' 'ut' 'vf' 'vo' 'w4' 'w7' 'wf' 'wk' 'wp' 'wq' 'wr' 'wt' 'wy' 'wz' 'xk' 'xt' 'ye' 'yv' 'zd' 'zp' 'zw' - f | 'a9' 'ad' 'az' 'bo' 'bq' 'by' 'c8' 'cb' 'cp' 'd9' 'de' 'dj' 'dp' 'e4' 'ef' 'ey' 'fn' 'gc' 'ge' 'gp' 'gs' 'gu' 'gx' 'hf' 'ic' 'ie' 'ir' 'kc' 'kp' 'ku' 'l2' 'lv' 'mx' 'n7' 'o3' 'o7' 'oe' 'pb' 'ps' 'q3' 'q4' 'qb' 'qn' 'qo' 'qq' 'qu' 'qw' 'r3' 'ra' 'rk' 'sg' 'si' 'sv' 'sw' 'tb' 'tg' 'tw' 'u2' 'uc' 'uf' 'uj' 'us' 'ut' 'uu' 'v3' 'va' 'wa' 'wd' 'ww' 'wy' 'wz' 'xk' 'y6' 'y8' 'ya' 'yc' 'ye' 'yo' 'yx' 'z3' - f | '2b' 'a4' 'aa' 'ay' 'br' 'cf' 'd3' 'dh' 'dw' 'e1' 'e8' 'eg' 'en' 'ep' 'ff' 'fj' 'g2' 'go' 'gq' 'i4' 'i8' 'if' 'jc' 'jj' 'jo' 'k1' 'kk' 'ks' 'la' 'ld' 'lm' 'm4' 'ml' 'o4' 'oe' 'og' 'os' 'oy' 'pc' 'pd' 'pr' 'px' 'q5' 'q6' 'qd' 'qh' 'qj' 'ql' 'qo' 'qt' 'qu' 'qw' 'rb' 're' 'rk' 'rn' 'rr' 'rw' 'sl' 'sm' 'te' 'tf' 'tk' 'u6' 'ui' 'ul' 'uo' 'ur' 'us' 'va' 'vt' 'w7' 'wf' 'wg' 'wm' 'wn' 'ws' 'xo' 'y8' 'yj' 'zh' - f | '3q' 'am' 'ar' 'az' 'bo' 'cb' 'cv' 'dd' 'ds' 'e1' 'e7' 'en' 'eq' 'er' 'eu' 'ew' 'fq' 'fx' 'g8' 'gm' 'h5' 'ic' 'if' 'jp' 'kw' 'kz' 'l4' 'lf' 'li' 'lm' 'lq' 'lu' 'lz' 'ml' 'nz' 'o4' 'o6' 'op' 'ow' 'po' 'py' 'qj' 'qk' 'qn' 'qp' 'qw' 'qz' 'r6' 'r8' 'rc' 'rl' 'rt' 's6' 'sh' 'sx' 'sz' 'tc' 'ti' 'tt' 'ty' 'uf' 'uh' 'uj' 'um' 'ut' 'uu' 'uv' 'vn' 'vt' 'we' 'wh' 'wi' 'wq' 'wu' 'wz' 'xn' 'y4' 'yu' 'yz' 'z9' 'ze' - f | 'a5' 'ao' 'av' 'b0' 'bh' 'bk' 'cj' 'd0' 'do' 'ds' 'e1' 'ee' 'ep' 'fa' 'fh' 'fv' 'hc' 'hh' 'hq' 'i1' 'if' 'iv' 'ix' 'iy' 'jq' 'jw' 'jz' 'ko' 'll' 'lr' 'lx' 'ns' 'nz' 'og' 'or' 'os' 'oz' 'p0' 'p8' 'pr' 'pv' 'q6' 'q9' 'qb' 'qd' 'qe' 'qh' 'qi' 'qq' 'qt' 'qu' 'qy' 'rd' 'ri' 'rq' 'rr' 'ry' 'sf' 'sg' 'sh' 'si' 'sw' 'sx' 'tc' 'tq' 'u3' 'uh' 'ul' 'us' 'wg' 'wn' 'wx' 'x3' 'xb' 'ya' 'yg' 'yn' 'yo' 'yp' 'yr' 'yw' - f | 'a7' 'ad' 'aq' 'ck' 'cu' 'd8' 'db' 'do' 'dt' 'e7' 'eg' 'em' 'eq' 'ex' 'ez' 'fl' 'gr' 'h4' 'hd' 'he' 'hg' 'id' 'ie' 'if' 'in' 'io' 'j4' 'j7' 'je' 'jm' 'jo' 'k5' 'kg' 'kj' 'kn' 'l9' 'lb' 'lc' 'ld' 'lf' 'li' 'lj' 'na' 'nz' 'oa' 'ok' 'oo' 'pj' 'px' 'q3' 'q4' 'q6' 'qd' 'qi' 'qm' 'qq' 'qt' 'qx' 'r1' 'rg' 'ri' 'rn' 'sm' 'so' 'su' 'sy' 'tq' 'u5' 'uc' 'ue' 'us' 'w3' 'wc' 'we' 'wj' 'wk' 'wt' 'x8' 'xj' 'yc' 'yy' - f | '2l' 'a4' 'ae' 'ag' 'cb' 'cs' 'cu' 'cy' 'dd' 'dj' 'ed' 'ee' 'ef' 'es' 'ex' 'ey' 'ft' 'fy' 'gs' 'h4' 'ho' 'ht' 'i2' 'i7' 'ia' 'iq' 'iz' 'jz' 'ku' 'kv' 'lq' 'mv' 'of' 'oi' 'op' 'pb' 'pd' 'pl' 'pn' 'pq' 'q0' 'q3' 'qe' 'qi' 'qp' 'qs' 'qy' 'qz' 'r3' 'ra' 'rh' 'rk' 'ry' 'sj' 'st' 'ta' 'tc' 'te' 'tf' 'th' 'ti' 'to' 'tt' 'u2' 'u5' 'vf' 'vx' 'w4' 'w7' 'wm' 'wt' 'ww' 'wx' 'x2' 'xi' 'y1' 'ye' 'yq' 'yv' 'za' 'zo' 'zy' - f | '4a' 'aj' 'an' 'aq' 'ax' 'bb' 'bh' 'ci' 'cn' 'd4' 'df' 'ea' 'ef' 'ek' 'en' 'eo' 'ex' 'fc' 'fj' 'fz' 'gb' 'gl' 'h7' 'hg' 'i1' 'i4' 'ia' 'im' 'j9' 'jg' 'ji' 'jj' 'kl' 'l2' 'lb' 'ld' 'lh' 'm1' 'mf' 'n1' 'nd' 'ob' 'od' 'ot' 'ph' 'pk' 'po' 'pv' 'q1' 'q2' 'q4' 'qb' 'qu' 'qz' 'ra' 'rh' 'ro' 'rp' 's1' 'si' 'sl' 'sn' 't7' 't8' 'ta' 'tc' 'ub' 'ue' 'ul' 'v9' 'w7' 'wd' 'xq' 'y5' 'yj' 'yl' 'yp' 'yw' 'z9' 'zb' 'zu' 'zy' - f | '8w' 'ao' 'aw' 'bk' 'by' 'c5' 'ch' 'cw' 'd0' 'dh' 'e2' 'eq' 'ev' 'ha' 'hk' 'ii' 'ir' 'iv' 'j8' 'ki' 'kp' 'lm' 'lx' 'nh' 'ns' 'nt' 'o3' 'on' 'ov' 'ow' 'ox' 'pl' 'q3' 'qc' 'qd' 'qe' 'qh' 'qm' 'qo' 'qr' 'qs' 'qt' 'qx' 'r2' 'r8' 'rc' 'rd' 'rh' 'ri' 'rk' 'rq' 'rt' 's3' 'sm' 'sq' 'sz' 't1' 't6' 'tx' 'u2' 'ue' 'up' 'uq' 'ut' 'ux' 'vg' 'vp' 'w3' 'wb' 'wc' 'wd' 'wr' 'xf' 'xp' 'y1' 'ya' 'yf' 'yg' 'yi' 'yn' 'yq' 'zn' - f | '5i' 'a4' 'a8' 'ae' 'af' 'am' 'ap' 'ax' 'bf' 'cr' 'cw' 'dk' 'dz' 'ea' 'eb' 'ec' 'ef' 'ep' 'er' 'et' 'ex' 'fp' 'fv' 'gh' 'hy' 'hz' 'i8' 'ig' 'io' 'iq' 'ji' 'jw' 'kt' 'le' 'lo' 'mi' 'nb' 'nq' 'o1' 'ol' 'oo' 'oq' 'oy' 'q3' 'q8' 'qi' 'qk' 'qo' 'qq' 'qs' 'r0' 'rg' 'rl' 'rt' 's6' 'sb' 'sk' 'sq' 't9' 'ta' 'tb' 'tg' 'tj' 'tu' 'ty' 'un' 'ur' 'uw' 'vi' 'wf' 'wk' 'wo' 'wt' 'wy' 'xj' 'xq' 'y0' 'y1' 'y9' 'yc' 'yn' 'yu' 'zs' - f | '6q' 'ac' 'ag' 'av' 'aw' 'az' 'bj' 'bq' 'bx' 'cj' 'dc' 'dd' 'dq' 'eg' 'ei' 'ek' 'em' 'f7' 'fg' 'fq' 'fx' 'gb' 'gp' 'hb' 'hk' 'i9' 'ii' 'im' 'ip' 'ir' 'iz' 'jx' 'le' 'lf' 'll' 'lx' 'mp' 'mv' 'of' 'oq' 'pb' 'pg' 'pj' 'py' 'pz' 'qi' 'qk' 'ql' 'qs' 'qu' 'qw' 'r8' 'ra' 'rc' 'rj' 'sj' 'tg' 'tk' 'tl' 'tm' 'tp' 'tw' 'u4' 'uf' 'ui' 'um' 'vu' 'w7' 'wd' 'wn' 'wt' 'wy' 'wz' 'y1' 'y3' 'y9' 'yl' 'yo' 'yq' 'yu' 'zb' 'zu' 'zv' - f | 'a1' 'a6' 'aq' 'bk' 'bu' 'ch' 'dd' 'dj' 'dx' 'e7' 'ef' 'ei' 'ek' 'el' 'ez' 'fp' 'ft' 'gb' 'go' 'hd' 'hh' 'hj' 'ho' 'hs' 'ic' 'il' 'ip' 'jh' 'jn' 'jq' 'jx' 'kq' 'kv' 'la' 'm8' 'mv' 'ng' 'oh' 'or' 'p7' 'p9' 'pe' 'pk' 'px' 'q7' 'qc' 'qd' 'qh' 'qj' 'qk' 'qr' 'qx' 'rd' 'rl' 'rq' 'sl' 'sq' 'st' 'sy' 'th' 'tn' 'u9' 'uc' 'ud' 'ug' 'uy' 'uz' 'vl' 'vn' 'wa' 'wb' 'wd' 'wt' 'xi' 'xs' 'xz' 'y5' 'yi' 'yk' 'yt' 'yw' 'z7' 'zg' - f | 'a2' 'a8' 'ae' 'b3' 'b5' 'bn' 'bq' 'd4' 'd7' 'dh' 'dl' 'dt' 'e6' 'es' 'fa' 'fb' 'ff' 'gd' 'ge' 'h8' 'hj' 'hm' 'hq' 'hw' 'hy' 'im' 'ip' 'kb' 'kh' 'ku' 'lm' 'ly' 'mf' 'nn' 'o0' 'oi' 'ok' 'or' 'pn' 'po' 'pq' 'pt' 'pv' 'q4' 'qc' 'qd' 'qu' 'qv' 'rd' 'rh' 'ri' 'rk' 'rp' 'rr' 'ru' 'si' 'sk' 'sq' 'ss' 'sx' 'sy' 'ta' 'tm' 'tq' 'tx' 'ul' 'um' 'uv' 'vj' 'vt' 'w7' 'wh' 'wi' 'wl' 'wn' 'wp' 'wq' 'ww' 'xb' 'y6' 'yq' 'z2' 'zn' - f | 'a3' 'b3' 'bv' 'ck' 'cl' 'cs' 'cv' 'cy' 'dc' 'dl' 'dq' 'dr' 'du' 'eg' 'el' 'en' 'ex' 'fv' 'fx' 'gd' 'gk' 'gu' 'h7' 'ic' 'jf' 'jz' 'lf' 'lq' 'nl' 'nm' 'o6' 'ob' 'ol' 'pe' 'po' 'q2' 'qa' 'qk' 'qo' 'qp' 'qr' 'qv' 'qz' 'r5' 're' 'rk' 'ss' 'sv' 't2' 'tf' 'ti' 'tj' 'ua' 'ud' 'v8' 'vb' 'vg' 'vh' 'vi' 'w1' 'w5' 'wb' 'wd' 'we' 'wf' 'wj' 'wr' 'ws' 'wv' 'xq' 'xs' 'y1' 'ye' 'yi' 'yp' 'yr' 'ys' 'yu' 'yw' 'z7' 'z8' 'zf' 'zq' - f | 'a7' 'a9' 'al' 'aq' 'ar' 'as' 'b2' 'cj' 'cl' 'co' 'cu' 'cv' 'dc' 'e3' 'e5' 'e7' 'ea' 'ee' 'eh' 'en' 'es' 'ev' 'ez' 'fl' 'fp' 'gm' 'hm' 'it' 'iu' 'iv' 'j8' 'jd' 'jo' 'kn' 'ko' 'ky' 'ln' 'mr' 'nc' 'o2' 'ok' 'ot' 'pc' 'pe' 'pz' 'q2' 'qe' 'qi' 'qq' 'qr' 'qs' 'r8' 'rm' 'rn' 'ro' 'rx' 'sz' 'te' 'tn' 'tq' 'tx' 'u1' 'ub' 'ul' 'up' 'uq' 'wf' 'wh' 'wk' 'wl' 'wo' 'wq' 'wt' 'wu' 'wv' 'xh' 'xz' 'y5' 'y9' 'yv' 'yx' 'yz' 'zj' - f | 'a8' 'am' 'as' 'bb' 'cn' 'co' 'dd' 'di' 'dx' 'dz' 'e3' 'e5' 'ea' 'ef' 'eh' 'el' 'en' 'eo' 'ew' 'fe' 'fg' 'gc' 'gi' 'gj' 'gp' 'ha' 'hl' 'hy' 'i3' 'id' 'ij' 'ir' 'is' 'k9' 'kc' 'kf' 'l9' 'ln' 'ls' 'lw' 'm0' 'm8' 'my' 'nn' 'on' 'p9' 'ph' 'pm' 'pw' 'q1' 'q3' 'qc' 'qm' 'qq' 'qv' 'rg' 'ro' 'se' 'sw' 'sx' 'ta' 'tl' 'tz' 'u4' 'uo' 'vg' 'vl' 'vm' 'vn' 'w3' 'wd' 'wm' 'wo' 'wp' 'wt' 'yg' 'yj' 'yv' 'yx' 'yz' 'zu' 'zx' 'zy' - f | 'ag' 'ao' 'ay' 'bk' 'bl' 'bw' 'cq' 'dp' 'dv' 'e5' 'ee' 'eu' 'fv' 'fx' 'gg' 'h8' 'ha' 'hk' 'ii' 'im' 'iq' 'it' 'je' 'jf' 'jr' 'js' 'km' 'ks' 'lo' 'lz' 'o8' 'ot' 'pg' 'pl' 'q3' 'q9' 'qa' 'qd' 'qe' 'qf' 'qh' 'qn' 'qq' 'qy' 'qz' 'r7' 're' 'ri' 'rl' 'ro' 'rr' 'rx' 'ry' 'sh' 'sl' 'ss' 'sw' 't5' 'ta' 'te' 'th' 'tr' 'tt' 'tu' 'tv' 'u7' 'uk' 'um' 'up' 'uu' 'vl' 'vw' 'w0' 'wl' 'ws' 'wt' 'ww' 'xe' 'ya' 'yb' 'ye' 'yf' 'za' - f | '2i' 'ap' 'aw' 'bd' 'bs' 'c9' 'db' 'di' 'dl' 'dq' 'dr' 'e5' 'ej' 'el' 'ez' 'f9' 'ga' 'gp' 'hm' 'if' 'im' 'j9' 'k9' 'ka' 'kd' 'kj' 'lc' 'lf' 'lk' 'mf' 'mq' 'mr' 'o3' 'oe' 'oj' 'op' 'p5' 'p7' 'p8' 'pb' 'pv' 'py' 'qc' 'qd' 'qf' 'qi' 'qk' 'qq' 'qu' 'qw' 'qz' 'r2' 'r8' 'r9' 'rj' 'rk' 'ry' 'rz' 'sd' 'sk' 'sm' 't2' 't6' 't7' 'tp' 'tq' 'tt' 'tw' 'u7' 'uh' 'uv' 'vh' 'w0' 'wc' 'we' 'wh' 'wk' 'wl' 'yf' 'yk' 'ys' 'zi' 'zq' 'zs' - f | '39' 'a9' 'ag' 'ap' 'ax' 'b9' 'c3' 'dn' 'e4' 'ea' 'f7' 'f9' 'g2' 'gi' 'h4' 'ht' 'ie' 'ij' 'ir' 'is' 'it' 'jd' 'jj' 'jv' 'kp' 'kt' 'kv' 'lh' 'ln' 'ls' 'mg' 'mh' 'nc' 'nh' 'o0' 'o2' 'od' 'ox' 'pb' 'pn' 'po' 'pz' 'q1' 'q8' 'qc' 'qd' 'qf' 'qh' 'qi' 'qk' 'qz' 'rd' 'rg' 'ro' 'rz' 's7' 'sa' 'sf' 'sn' 'sw' 'tk' 'tu' 'tw' 'u7' 'uu' 'vh' 'wa' 'wd' 'wk' 'wr' 'wu' 'wy' 'wz' 'xh' 'xm' 'xt' 'yb' 'yi' 'yj' 'yn' 'yr' 'z1' 'z8' 'zn' - f | '14' '1t' 'aa' 'ab' 'ag' 'aq' 'ax' 'bc' 'cb' 'cm' 'd1' 'db' 'ea' 'ek' 'en' 'ey' 'f0' 'g6' 'g7' 'gx' 'ha' 'ho' 'i1' 'i6' 'i8' 'it' 'iw' 'jl' 'jp' 'ko' 'kt' 'la' 'le' 'mb' 'mk' 'mt' 'n6' 'na' 'ob' 'oc' 'od' 'on' 'op' 'or' 'ot' 'q6' 'qd' 'qf' 'qk' 'qp' 'qq' 'qr' 'qw' 'qy' 'r5' 'rf' 'rk' 'se' 'sp' 'sx' 't0' 'tj' 'tk' 'tq' 'tv' 'tx' 'u0' 'ub' 'ue' 'uf' 'um' 'us' 'uu' 'v9' 'vi' 'wb' 'wo' 'wu' 'wy' 'x9' 'xs' 'xu' 'y0' 'yo' 'zi' - f | '1a' '3r' 'ai' 'b8' 'bp' 'd6' 'do' 'dp' 'du' 'e0' 'e1' 'ek' 'es' 'eu' 'ey' 'ez' 'fq' 'fs' 'fy' 'gd' 'gm' 'h1' 'hu' 'i0' 'is' 'iv' 'iz' 'jd' 'jn' 'jz' 'kc' 'lf' 'lp' 'lt' 'ly' 'm2' 'mb' 'mt' 'n1' 'nr' 'of' 'oi' 'oq' 'ow' 'pe' 'pm' 'pn' 'q0' 'qb' 'qg' 'qk' 'qo' 'qz' 'r4' 'r8' 'ra' 'rn' 'rs' 's7' 'sa' 'sj' 'so' 'su' 'sv' 'sx' 'ti' 'tl' 'ts' 'ty' 'u4' 'up' 'vr' 'w2' 'wg' 'wi' 'wk' 'wl' 'ws' 'xl' 'xx' 'yf' 'yk' 'yn' 'yq' 'yv' - f | '1r' '2z' '3d' '6d' 'ay' 'bb' 'bh' 'bq' 'bv' 'cn' 'cr' 'cs' 'd5' 'd9' 'dh' 'dq' 'dw' 'dz' 'e5' 'ea' 'eb' 'eh' 'er' 'fu' 'g0' 'ga' 'gc' 'h4' 'hf' 'hh' 'hx' 'ih' 'io' 'jz' 'kk' 'ko' 'li' 'lz' 'n1' 'n4' 'nn' 'nt' 'o8' 'oa' 'oe' 'oh' 'ov' 'oz' 'p2' 'p5' 'pg' 'pt' 'px' 'qa' 'qe' 'qg' 'qm' 'qt' 'qv' 'qx' 'ro' 'rp' 'rv' 's6' 'sh' 'sn' 'sx' 'sz' 't6' 'th' 'tj' 'tu' 'u5' 'uk' 'uq' 'wg' 'wp' 'wv' 'xe' 'xv' 'y4' 'yf' 'yo' 'yx' 'zr' - f | '27' '4g' 'a2' 'al' 'bp' 'ca' 'cp' 'da' 'dt' 'e8' 'ee' 'ef' 'eg' 'ej' 'eq' 'eu' 'ev' 'fe' 'gn' 'gq' 'gy' 'i5' 'ic' 'il' 'io' 'ir' 'iw' 'iz' 'j2' 'kz' 'l2' 'l8' 'lh' 'ln' 'lt' 'np' 'ns' 'oi' 'oj' 'or' 'ph' 'pr' 'pt' 'qa' 'qh' 'qi' 'qs' 'qt' 'qu' 'r1' 'rd' 'ry' 's8' 'sj' 'sk' 'sl' 'sq' 'td' 'te' 'tg' 'tj' 'tq' 'tx' 'u0' 'ub' 'ul' 'uu' 'ux' 'uy' 'v0' 've' 'vg' 'vk' 'wa' 'wj' 'ws' 'wu' 'yb' 'yd' 'yi' 'yk' 'yo' 'yr' 'ys' 'zp' - f | '2a' '3o' '6w' '9h' 'ag' 'ak' 'cu' 'db' 'e3' 'ei' 'ep' 'eq' 'f6' 'fm' 'fq' 'fz' 'g5' 'gi' 'gk' 'gn' 'gp' 'gx' 'hb' 'hw' 'ia' 'ic' 'id' 'ig' 'ih' 'im' 'jh' 'jy' 'k3' 'kh' 'kv' 'l4' 'lf' 'lh' 'lu' 'ni' 'o7' 'oe' 'oi' 'op' 'os' 'ow' 'p0' 'p1' 'pj' 'pp' 'qa' 'qe' 'qi' 'qm' 'qz' 'rb' 're' 'rh' 'sa' 'sn' 'te' 'th' 'u3' 'ui' 'uj' 'us' 'vh' 'w1' 'w7' 'wc' 'wg' 'wh' 'wo' 'wx' 'xr' 'xw' 'xz' 'y5' 'y6' 'y9' 'ye' 'yo' 'ze' 'zg' 'zp' - f | '2i' 'am' 'ar' 'bl' 'by' 'ci' 'da' 'db' 'dc' 'dg' 'ea' 'ed' 'ei' 'ek' 'em' 'en' 'er' 'f2' 'f7' 'fi' 'fj' 'fu' 'fy' 'g0' 'g8' 'gn' 'gr' 'hl' 'hr' 'i2' 'it' 'ix' 'jg' 'jr' 'ju' 'jx' 'lh' 'lj' 'm8' 'n7' 'o1' 'o5' 'ob' 'og' 'ok' 'oz' 'pg' 'pl' 'pz' 'q4' 'q9' 'qf' 'qh' 'qn' 'qt' 'qu' 'qw' 're' 'ri' 'rn' 'ro' 'rp' 'rz' 's7' 'st' 'sz' 't3' 'tk' 'tw' 'u1' 'u5' 'uh' 'uy' 've' 'vh' 'w1' 'x2' 'xi' 'y4' 'ya' 'yi' 'yk' 'yn' 'zb' 'zo' - f | '2v' '7j' '7r' 'a7' 'aj' 'av' 'br' 'cg' 'd8' 'ei' 'en' 'ez' 'f5' 'fb' 'go' 'h0' 'hf' 'hk' 'hm' 'i9' 'ia' 'ig' 'ik' 'im' 'iq' 'it' 'iv' 'ja' 'je' 'ji' 'ks' 'lo' 'mo' 'mq' 'nj' 'o5' 'of' 'on' 'ot' 'ox' 'p1' 'p6' 'pb' 'ph' 'pj' 'pn' 'pp' 'pq' 'pr' 'qa' 'qh' 'qj' 'qm' 'qn' 'qo' 'qu' 'r2' 'ro' 's1' 'sa' 'se' 'sp' 't6' 'ti' 'tu' 'u2' 'ug' 'uh' 'ui' 'ur' 'uy' 'v9' 'wl' 'wn' 'wt' 'wv' 'wy' 'wz' 'xm' 'xs' 'ya' 'yi' 'yu' 'z8' 'zl' - f | '68' 'af' 'as' 'at' 'cf' 'ci' 'cx' 'dl' 'dt' 'dv' 'e4' 'ea' 'eg' 'es' 'fd' 'fi' 'fo' 'g1' 'gd' 'gy' 'he' 'hu' 'hw' 'hz' 'i0' 'ia' 'if' 'io' 'it' 'iv' 'iz' 'j0' 'jc' 'jj' 'jm' 'kb' 'ki' 'kk' 'ku' 'ky' 'lr' 'nv' 'oi' 'om' 'p3' 'pa' 'pt' 'qe' 'qh' 'qj' 'qm' 'qq' 'qs' 'qt' 'qu' 'r3' 'rp' 'rr' 'rw' 'rx' 's2' 'sc' 'sp' 'ti' 'tn' 'tv' 'ty' 'ue' 'uk' 'ul' 'up' 'ut' 'uu' 'uw' 'vf' 'vp' 'w2' 'w7' 'wb' 'wl' 'wq' 'xg' 'yc' 'yh' 'z3' - f | 'a8' 'ad' 'ai' 'al' 'as' 'av' 'ay' 'az' 'bf' 'bg' 'c6' 'dc' 'dd' 'dz' 'ef' 'eh' 'eu' 'ft' 'g0' 'g4' 'gf' 'gm' 'hq' 'im' 'iw' 'ix' 'j5' 'jj' 'jn' 'jv' 'k1' 'ko' 'ma' 'n0' 'nw' 'o9' 'om' 'op' 'ox' 'p4' 'pa' 'pe' 'pm' 'ps' 'q2' 'qb' 'qc' 'qf' 'qi' 'ql' 'qu' 'qw' 'qy' 'qz' 'rl' 'rn' 'rs' 'ru' 'se' 'sf' 'sm' 't8' 'tc' 'tg' 'uq' 'uz' 'va' 'vj' 'wa' 'we' 'wg' 'wk' 'wq' 'wu' 'wy' 'xb' 'xo' 'y3' 'ye' 'ym' 'ys' 'yt' 'z8' 'zd' 'zo' - f | 'aa' 'af' 'bp' 'c2' 'cd' 'ci' 'dl' 'du' 'dz' 'es' 'fn' 'fo' 'g7' 'ga' 'gc' 'gj' 'gx' 'hc' 'ht' 'iu' 'jl' 'jm' 'ke' 'kk' 'kr' 'lc' 'le' 'lr' 'lw' 'ms' 'n2' 'nq' 'nw' 'nz' 'o1' 'op' 'oq' 'ot' 'ow' 'p0' 'pb' 'pp' 'pr' 'qc' 'qh' 'qp' 'qr' 'qu' 'qy' 'r3' 'r9' 'rc' 'rk' 'rr' 'ry' 'sb' 'sf' 'sk' 'sl' 't1' 'ta' 'tb' 'tk' 'tz' 'ud' 'ur' 've' 'vn' 'vt' 'vw' 'w6' 'w9' 'wg' 'wk' 'xg' 'yg' 'yh' 'yj' 'ys' 'yt' 'yv' 'yy' 'z3' 'zo' 'zq' - f | '1o' '1v' '4v' 'ab' 'at' 'b5' 'bw' 'bz' 'cm' 'dc' 'dh' 'dj' 'dk' 'dq' 'e1' 'e5' 'ee' 'el' 'em' 'eo' 'ep' 'ew' 'fi' 'fy' 'gd' 'gh' 'gw' 'h7' 'i6' 'ia' 'ii' 'ip' 'is' 'iu' 'jk' 'k5' 'kb' 'mf' 'mp' 'mw' 'mx' 'n1' 'na' 'nm' 'nq' 'ob' 'of' 'pa' 'pb' 'pd' 'pu' 'q4' 'qb' 'qe' 'qf' 'qj' 'qo' 'qr' 're' 'rj' 'ro' 'rt' 'rv' 'sb' 'sp' 'st' 'ub' 'ue' 'um' 'vz' 'w3' 'w7' 'w8' 'wh' 'wo' 'wt' 'x5' 'x9' 'xd' 'xj' 'xm' 'ya' 'yb' 'yg' 'zr' 'zv' - f | '2d' '2w' 'ag' 'aj' 'am' 'ce' 'd2' 'd4' 'dd' 'dn' 'e0' 'e6' 'ef' 'ej' 'ek' 'er' 'f9' 'fd' 'fe' 'fq' 'ft' 'fv' 'gq' 'gy' 'he' 'ij' 'iq' 'it' 'iy' 'jf' 'jl' 'jr' 'ju' 'kq' 'ku' 'lm' 'nj' 'nw' 'nx' 'oa' 'oe' 'oj' 'p2' 'pa' 'pm' 'pq' 'pv' 'q5' 'q9' 'qb' 'qd' 'qg' 'qj' 'qk' 'ql' 'qm' 'qu' 'qx' 'qz' 'rp' 'rq' 'rs' 's5' 'sk' 'ss' 'sy' 't2' 'ta' 'ts' 'tw' 'tx' 'u5' 'w3' 'w7' 'wa' 'wp' 'ws' 'wu' 'wy' 'wz' 'xz' 'y4' 'yn' 'yy' 'zt' 'zw' - f | '3h' 'aw' 'az' 'b2' 'bl' 'bq' 'bt' 'c0' 'cd' 'ck' 'cm' 'cw' 'cy' 'd3' 'e0' 'ef' 'eo' 'ew' 'fj' 'fk' 'gb' 'gc' 'gm' 'gt' 'gz' 'hv' 'in' 'ix' 'jb' 'jg' 'jl' 'jn' 'js' 'k7' 'kg' 'ku' 'kx' 'kz' 'ld' 'nb' 'ni' 'o3' 'o6' 'od' 'om' 'p3' 'p4' 'pn' 'pu' 'q7' 'qd' 'qh' 'qp' 'qq' 'qr' 'qv' 'qy' 'qz' 'rq' 'sb' 'se' 'sf' 'sh' 'sm' 't0' 'tg' 'tj' 'tl' 'tq' 'ua' 'uk' 'us' 'wd' 'we' 'wj' 'wn' 'wq' 'wv' 'wz' 'yd' 'yg' 'ym' 'yn' 'yr' 'zc' 'zz' - f | 'ac' 'al' 'am' 'ap' 'ar' 'av' 'c1' 'cl' 'cz' 'dm' 'dp' 'dt' 'du' 'e0' 'es' 'eu' 'ev' 'ez' 'fa' 'fc' 'fj' 'fw' 'fz' 'gm' 'gx' 'ha' 'i6' 'i7' 'if' 'iz' 'j0' 'jd' 'jx' 'k0' 'kr' 'mj' 'mk' 'mp' 'n4' 'nm' 'om' 'ot' 'pl' 'px' 'qa' 'qb' 'qg' 'qk' 'qn' 'qo' 'qr' 'qu' 'r7' 'rc' 'rk' 's3' 's4' 'se' 'ss' 'su' 't8' 'ta' 'te' 'tj' 'tk' 'ts' 'tz' 'ub' 'v2' 'wb' 'wd' 'we' 'wf' 'wj' 'wq' 'wr' 'ws' 'wv' 'wx' 'y6' 'ym' 'yn' 'yq' 'ys' 'yv' 'zt' - f | '20' 'ag' 'az' 'b9' 'bg' 'bh' 'c2' 'dd' 'di' 'do' 'dw' 'e1' 'e3' 'e8' 'eb' 'ev' 'fc' 'fj' 'fu' 'fw' 'fx' 'gb' 'gf' 'hu' 'hw' 'hz' 'ih' 'ik' 'iq' 'jc' 'jq' 'ka' 'kf' 'kh' 'kw' 'ky' 'ly' 'lz' 'mc' 'n8' 'o3' 'o7' 'oa' 'os' 'ou' 'pf' 'q5' 'q6' 'qg' 'ql' 'qo' 'qy' 'r2' 'rf' 'rp' 'ru' 'ry' 's1' 's5' 'se' 'si' 'sn' 'su' 'ta' 'tn' 'tu' 'tv' 'tw' 'u9' 'ua' 'uh' 'uu' 'vo' 'w8' 'wa' 'wc' 'we' 'wi' 'wu' 'ww' 'wy' 'x3' 'xl' 'yb' 'yg' 'yk' 'yv' - f | '5f' 'ac' 'ad' 'aj' 'av' 'bg' 'ch' 'ci' 'dg' 'di' 'dl' 'dx' 'ep' 'ew' 'f8' 'fp' 'fr' 'fs' 'g4' 'g6' 'gb' 'h5' 'he' 'hk' 'hx' 'i1' 'ic' 'im' 'ji' 'jm' 'js' 'k9' 'kq' 'lf' 'ln' 'mj' 'n2' 'ni' 'no' 'o5' 'oi' 'ot' 'oy' 'pa' 'pj' 'qk' 'qq' 'qt' 'qu' 'r4' 'r8' 'rb' 'rh' 'rj' 'rm' 'ro' 'rp' 'rq' 'rt' 'ru' 'sb' 'th' 'tu' 'tz' 'u9' 'un' 'uo' 'us' 've' 'vi' 'vq' 'vx' 'wf' 'wg' 'wi' 'wq' 'wr' 'wt' 'x5' 'xz' 'y6' 'yd' 'yo' 'yq' 'yx' 'z3' 'zl' - f | 'a5' 'av' 'dc' 'dm' 'dt' 'dw' 'e2' 'ea' 'en' 'et' 'ez' 'f2' 'fg' 'fn' 'fv' 'gn' 'gu' 'h5' 'ht' 'i2' 'ie' 'ik' 'il' 'in' 'it' 'iv' 'iy' 'ji' 'jj' 'ju' 'kf' 'kn' 'ko' 'ku' 'l6' 'ls' 'nz' 'o2' 'o4' 'of' 'on' 'op' 'p7' 'q0' 'q2' 'qa' 'qt' 'r1' 'r9' 'ri' 'rq' 'rv' 'se' 'sp' 'sx' 't0' 't2' 't9' 'te' 'tq' 'tu' 'u8' 'uc' 'ue' 'ug' 'ui' 'ut' 'uu' 'v0' 'v7' 'vs' 'w0' 'wc' 'wh' 'wn' 'wo' 'ws' 'y1' 'y3' 'y5' 'ya' 'yc' 'yi' 'ys' 'yt' 'zf' 'zq' - f | 'ab' 'ag' 'aw' 'b4' 'd4' 'ds' 'e1' 'ea' 'eb' 'ee' 'ef' 'eh' 'ek' 'el' 'en' 'eu' 'ex' 'fv' 'fx' 'gi' 'gp' 'gt' 'i3' 'ie' 'jd' 'jk' 'jn' 'kv' 'l7' 'n8' 'ns' 'nx' 'o2' 'o3' 'oh' 'os' 'ow' 'p2' 'pr' 'pu' 'q3' 'qd' 'qe' 'qk' 'qq' 'qz' 'r9' 'rb' 'ri' 'rm' 'rr' 'rw' 'ry' 'sc' 'sj' 'sl' 'sx' 'tk' 'tr' 'tw' 'u4' 'ub' 'ud' 'uf' 'uh' 'un' 'uv' 'v6' 'vc' 'vf' 'vi' 'wg' 'wr' 'ws' 'wv' 'x7' 'xf' 'xo' 'y0' 'yi' 'yo' 'yt' 'yy' 'za' 'zm' 'zu' 'zx' - f | 'ad' 'af' 'aj' 'am' 'aq' 'ba' 'bo' 'c0' 'c5' 'cx' 'da' 'dc' 'dk' 'ed' 'ek' 'en' 'eo' 'ep' 'ew' 'fb' 'g4' 'gd' 'h2' 'hm' 'ho' 'hy' 'ib' 'if' 'io' 'ir' 'iv' 'iz' 'jc' 'jp' 'jt' 'kr' 'lo' 'me' 'na' 'nc' 'nh' 'o6' 'ok' 'oq' 'or' 'ow' 'pn' 'ps' 'q6' 'qa' 'qj' 'qo' 'qx' 'r2' 'r6' 'rd' 're' 'rg' 'rw' 'ta' 'tk' 'uj' 'uo' 'vn' 'wc' 'wh' 'wj' 'wl' 'wu' 'wv' 'ww' 'wx' 'xe' 'xl' 'xn' 'xo' 'xs' 'yb' 'yl' 'yp' 'yw' 'zg' 'zq' 'zr' 'zu' 'zx' 'zy' - f | 'am' 'as' 'b1' 'd3' 'db' 'dl' 'ea' 'ed' 'ee' 'el' 'em' 'ep' 'er' 'ew' 'ez' 'f4' 'fm' 'go' 'h9' 'he' 'hl' 'i0' 'ie' 'ii' 'iz' 'ji' 'kl' 'kn' 'lc' 'lr' 'm7' 'mb' 'mt' 'my' 'no' 'nu' 'oo' 'or' 'ov' 'ox' 'pe' 'pq' 'q7' 'qc' 'qd' 'qi' 'qj' 'qo' 'qr' 'qs' 'qv' 'qw' 'r0' 'r8' 're' 'rm' 'rn' 'rp' 's0' 'so' 'sp' 'sw' 'sy' 'te' 'tg' 'tl' 'tr' 'uq' 'uz' 'vk' 'wk' 'wl' 'wp' 'wq' 'wr' 'wu' 'ww' 'xs' 'y0' 'yb' 'yn' 'yr' 'ys' 'yt' 'yy' 'zh' 'zr' - f | '1v' '44' 'aj' 'al' 'am' 'ba' 'bi' 'br' 'bt' 'c0' 'd6' 'dg' 'di' 'e1' 'e4' 'ei' 'ek' 'er' 'et' 'eu' 'ev' 'ex' 'fa' 'fc' 'fe' 'fh' 'fr' 'ga' 'gh' 'gu' 'hp' 'hq' 'ib' 'if' 'io' 'ix' 'jo' 'jv' 'kd' 'l1' 'me' 'mh' 'mn' 'mo' 'mt' 'n7' 'nf' 'nk' 'oe' 'oy' 'pa' 'pg' 'pj' 'pm' 'pn' 'pq' 'py' 'q1' 'qd' 'qe' 'qi' 'qj' 'qr' 'qs' 'qz' 'rb' 'rs' 'su' 'sw' 't5' 'ti' 'tx' 'uc' 'ug' 'ui' 'uq' 'uz' 'v6' 'vp' 'we' 'wf' 'wj' 'wm' 'wx' 'xe' 'y7' 'yt' 'yy' - f | '1x' 'a2' 'ah' 'az' 'bb' 'be' 'bz' 'd1' 'dd' 'du' 'ee' 'eh' 'en' 'eu' 'ey' 'ez' 'fl' 'fs' 'g4' 'gm' 'gr' 'ha' 'hf' 'hg' 'hr' 'hw' 'i6' 'id' 'ij' 'j2' 'jv' 'k9' 'lg' 'm9' 'md' 'me' 'mg' 'mp' 'nd' 'nf' 'ng' 'nj' 'ny' 'nz' 'oj' 'os' 'pi' 'pj' 'pt' 'q9' 'qa' 'qf' 'qh' 'qk' 'qn' 'qq' 'qs' 'rz' 'sb' 't7' 't8' 'tw' 'ty' 'u2' 'u3' 'ua' 'ub' 'uy' 'uz' 'w1' 'w3' 'w5' 'wa' 'wc' 'we' 'wh' 'wp' 'wr' 'xd' 'xs' 'y3' 'yd' 'yk' 'yl' 'yo' 'ze' 'zh' 'zo' - f | '4n' '6a' 'a3' 'a5' 'aa' 'ae' 'ag' 'b9' 'ca' 'cf' 'd1' 'da' 'dr' 'dz' 'ee' 'el' 'et' 'ey' 'fj' 'fs' 'gl' 'hk' 'hl' 'hn' 'ie' 'ih' 'im' 'ix' 'j1' 'jr' 'kf' 'kk' 'lc' 'lk' 'lp' 'lx' 'mh' 'mt' 'mx' 'my' 'nr' 'nu' 'o6' 'og' 'oo' 'p4' 'p7' 'pj' 'pr' 'q3' 'qc' 'qd' 'qj' 'qk' 'ql' 'qp' 'qt' 'qv' 'qx' 'r8' 're' 'rm' 'rs' 'ru' 'rv' 'sp' 'sw' 'td' 'tk' 'to' 'tw' 'tz' 'u8' 'uf' 'vf' 'vw' 'w7' 'wq' 'wr' 'xe' 'ym' 'yo' 'yr' 'ys' 'yz' 'zc' 'zn' 'zs' - f | 'a3' 'ag' 'ar' 'au' 'ax' 'be' 'cy' 'd7' 'dd' 'do' 'e3' 'eb' 'eo' 'er' 'ev' 'ey' 'fp' 'gj' 'hk' 'hw' 'hy' 'if' 'ig' 'ii' 'in' 'io' 'iq' 'is' 'kh' 'll' 'n8' 'nd' 'np' 'nz' 'og' 'ot' 'ox' 'oy' 'pe' 'px' 'qa' 'qb' 'qe' 'qf' 'qh' 'qj' 'qm' 'qn' 'qt' 'qu' 'qw' 'qx' 'r6' 'r9' 'rb' 'rc' 'rd' 'rn' 'rq' 'se' 't7' 'tb' 'ti' 'tq' 'u0' 'u6' 'ub' 'ud' 'vd' 'vj' 'vl' 'vo' 'wd' 'wg' 'wh' 'wi' 'wk' 'wo' 'wv' 'wz' 'xy' 'y9' 'yl' 'ym' 'yo' 'yp' 'ys' 'yu' - f | 'ad' 'av' 'be' 'bi' 'bk' 'bu' 'ce' 'cv' 'd8' 'df' 'dh' 'du' 'e1' 'em' 'ep' 'ex' 'ey' 'f1' 'fd' 'fh' 'fp' 'gn' 'gu' 'h8' 'i8' 'ii' 'il' 'iz' 'j0' 'ji' 'jv' 'jz' 'k2' 'k9' 'ky' 'l9' 'lo' 'm6' 'mc' 'md' 'ng' 'nz' 'ot' 'p2' 'pa' 'pj' 'ps' 'pv' 'q5' 'qd' 'qf' 'qm' 'qq' 'qr' 'qt' 'qv' 'qx' 'rc' 'rh' 'rp' 'rq' 'rs' 'sb' 'sc' 'te' 'to' 'ts' 'tw' 'up' 'vn' 'vw' 'wa' 'wg' 'wh' 'wn' 'wp' 'wz' 'xc' 'xf' 'xt' 'y5' 'yh' 'yn' 'yy' 'yz' 'za' 'zh' 'zy' - f | 'aj' 'ak' 'aq' 'ba' 'bd' 'bj' 'bk' 'bx' 'cr' 'cx' 'd0' 'dj' 'dl' 'ef' 'ei' 'ej' 'ek' 'em' 'eq' 'er' 'f3' 'f7' 'fi' 'fp' 'gv' 'h7' 'ha' 'hc' 'hh' 'hk' 'hn' 'ia' 'id' 'im' 'iq' 'is' 'iw' 'ji' 'jk' 'js' 'ju' 'jx' 'l0' 'm1' 'mq' 'ng' 'o8' 'ot' 'ov' 'oy' 'p1' 'pu' 'pv' 'pz' 'qc' 'qh' 'qm' 'qq' 'qs' 'qt' 'qw' 'rc' 'rq' 'rt' 'sd' 't5' 'ta' 'tb' 'tj' 'tp' 'uk' 'un' 'v9' 'vt' 'wa' 'wd' 'wf' 'wm' 'wn' 'wv' 'wz' 'xp' 'ya' 'yg' 'ym' 'yn' 'yr' 'zi' - f | 'aj' 'as' 'aw' 'bc' 'bj' 'bm' 'cd' 'd7' 'd9' 'di' 'ef' 'er' 'es' 'ew' 'fi' 'ft' 'g2' 'gq' 'gx' 'h9' 'he' 'hg' 'hu' 'i8' 'ie' 'ik' 'im' 'iq' 'ix' 'iy' 'jf' 'ji' 'ka' 'kl' 'kt' 'la' 'lf' 'm8' 'm9' 'nh' 'nw' 'o7' 'oi' 'pi' 'ps' 'q6' 'qa' 'qb' 'qf' 'qm' 'qo' 'qq' 'qs' 'r9' 'rb' 'rf' 'rk' 'rx' 'sa' 'sb' 'sp' 't1' 'tb' 'tl' 'tm' 'u6' 'uc' 'um' 'un' 'ur' 'w3' 'wj' 'wu' 'xm' 'xr' 'xy' 'ya' 'yd' 'ye' 'yf' 'yj' 'yo' 'ys' 'zc' 'zg' 'zl' 'zv' 'zz' - f | '1g' '3e' 'a5' 'ae' 'ar' 'bt' 'cb' 'ck' 'co' 'd5' 'dh' 'dm' 'dr' 'dt' 'e0' 'e9' 'eh' 'el' 'em' 'f2' 'f3' 'fb' 'ff' 'fg' 'gu' 'hb' 'he' 'hr' 'hx' 'i7' 'ib' 'if' 'ig' 'io' 'iu' 'jc' 'jo' 'jv' 'kh' 'kx' 'lb' 'lh' 'lw' 'lx' 'ly' 'mp' 'oa' 'oc' 'oz' 'pa' 'q9' 'qi' 'qp' 'qq' 'qw' 'r3' 'rb' 'rd' 're' 'rn' 'ry' 'rz' 's5' 'su' 'sz' 'td' 'tf' 'tt' 'u9' 'uh' 'um' 'un' 'va' 'vh' 'vt' 'w7' 'wb' 'wc' 'wg' 'wh' 'wx' 'xd' 'xz' 'y8' 'yf' 'yg' 'yh' 'ym' 'zy' - f | '1t' 'am' 'aw' 'b4' 'bi' 'bm' 'bx' 'cd' 'cj' 'cv' 'e5' 'ek' 'eo' 'er' 'ex' 'f5' 'f8' 'fp' 'fr' 'fw' 'ga' 'gy' 'hm' 'hs' 'ip' 'ir' 'is' 'it' 'iv' 'iz' 'jf' 'jm' 'jp' 'ju' 'k6' 'kk' 'kr' 'lb' 'le' 'lt' 'mj' 'nk' 'np' 'nr' 'oa' 'oj' 'pm' 'pn' 'q1' 'q3' 'qb' 'qc' 'qf' 'qh' 'qi' 'qj' 'qq' 'qt' 'qw' 'qy' 'r2' 'r8' 'rf' 'rh' 'rw' 's9' 't3' 'te' 'ti' 'tn' 'tq' 'uu' 'uv' 'uw' 'w0' 'w1' 'w3' 'w7' 'wc' 'wi' 'wk' 'wn' 'wr' 'wt' 'xm' 'y5' 'ya' 'z7' 'zu' - f | '1f' '2j' '2q' '5c' 'am' 'ap' 'aq' 'az' 'bu' 'bx' 'cr' 'e0' 'e7' 'ea' 'eb' 'ec' 'eg' 'ew' 'ey' 'fh' 'fk' 'g2' 'go' 'h4' 'h8' 'ha' 'hb' 'hm' 'ia' 'ic' 'ie' 'ik' 'iu' 'iv' 'j7' 'ja' 'k9' 'kw' 'lp' 'ly' 'mb' 'ns' 'ny' 'o0' 'of' 'ok' 'oz' 'pe' 'pr' 'qb' 'qd' 'qg' 'qt' 'qx' 'qy' 'rg' 'ro' 'rq' 'rz' 'sh' 'si' 'sl' 'sm' 'td' 'tg' 'tx' 'uh' 'um' 'up' 'uw' 've' 'vl' 'vp' 'vw' 'w2' 'w5' 'wc' 'wf' 'wi' 'wn' 'wq' 'ws' 'wv' 'wy' 'wz' 'x1' 'xr' 'y1' 'zs' 'zz' - f | '1q' '2n' '4p' '8n' 'a3' 'aa' 'bn' 'bq' 'bu' 'cg' 'db' 'dl' 'dx' 'dy' 'e2' 'e3' 'e4' 'ea' 'eq' 'ff' 'fk' 'fn' 'fv' 'gf' 'gh' 'h1' 'hs' 'hu' 'ij' 'in' 'iv' 'kc' 'ke' 'ln' 'm2' 'mc' 'mf' 'o2' 'og' 'oo' 'ov' 'p8' 'pl' 'pm' 'ps' 'pv' 'pz' 'qa' 'qc' 'qe' 'qh' 'qi' 'qj' 'qm' 'qo' 'qq' 'qv' 'qw' 'qy' 're' 'rz' 'sd' 'si' 'sj' 'sp' 'su' 'tc' 'tk' 'tn' 'tq' 'u7' 'ut' 'uv' 'v5' 'vr' 'wb' 'we' 'wm' 'ws' 'wv' 'xc' 'xy' 'y9' 'yj' 'yq' 'yv' 'yw' 'zb' 'zw' 'zz' - f | '2t' '4d' 'bb' 'bd' 'cg' 'co' 'd6' 'db' 'dg' 'dn' 'do' 'dy' 'e1' 'e3' 'e6' 'ec' 'em' 'en' 'ep' 'es' 'ev' 'ew' 'fu' 'fz' 'gc' 'gt' 'hg' 'hl' 'hy' 'id' 'in' 'io' 'ir' 'iu' 'iw' 'jl' 'jo' 'jp' 'jw' 'k4' 'ke' 'ku' 'ld' 'lg' 'li' 'lj' 'lr' 'ls' 'm7' 'n8' 'o6' 'oi' 'op' 'oy' 'pi' 'pq' 'qk' 'qm' 'qn' 'qp' 'qr' 'qu' 'qv' 'qz' 'r3' 're' 'rj' 'rk' 'rq' 'ru' 'rw' 'rz' 'sh' 'si' 'ta' 'to' 'uq' 'ux' 'vc' 'vr' 'wb' 'wh' 'wn' 'wo' 'wy' 'xg' 'y7' 'yb' 'yd' 'yo' - f | '4w' 'ad' 'ak' 'as' 'b3' 'bd' 'cy' 'dt' 'du' 'dy' 'e2' 'e4' 'en' 'es' 'et' 'ew' 'ex' 'f2' 'fl' 'fr' 'ft' 'gv' 'h1' 'ha' 'hd' 'hh' 'hs' 'hv' 'ic' 'io' 'ja' 'jm' 'k2' 'ku' 'kw' 'ld' 'ls' 'lx' 'ma' 'nu' 'ny' 'ov' 'pm' 'ps' 'q6' 'qa' 'qd' 'qo' 'qs' 'qv' 'qy' 'ra' 'rs' 'rt' 'rv' 'rw' 's8' 'sa' 'sw' 'tj' 'ug' 'ui' 'ur' 'ut' 'ux' 'v6' 'vt' 'w4' 'w7' 'wa' 'wb' 'wc' 'wg' 'wh' 'wi' 'wt' 'wu' 'wy' 'wz' 'x2' 'xb' 'xm' 'yg' 'yh' 'yi' 'yu' 'yy' 'z5' 'zg' 'zo' - f | '97' 'ac' 'ah' 'bn' 'c8' 'dl' 'ds' 'dw' 'e2' 'eb' 'eg' 'ej' 'ep' 'eu' 'f1' 'fe' 'g9' 'gs' 'h0' 'ha' 'he' 'hh' 'hy' 'i0' 'i8' 'i9' 'ia' 'ij' 'ip' 'jr' 'jt' 'jw' 'kq' 'lo' 'lq' 'm7' 'me' 'na' 'ns' 'o1' 'od' 'oe' 'oi' 'om' 'oz' 'pc' 'pj' 'pk' 'qh' 'qi' 'qj' 'ql' 'qo' 'qq' 'qr' 'qs' 'qt' 'qw' 'qz' 'rc' 're' 'rh' 'ri' 'rr' 'sz' 'tc' 'tf' 'tk' 'tl' 'ue' 'uh' 'uj' 'un' 'uz' 'vq' 'vr' 'vv' 'w3' 'w4' 'we' 'wk' 'wt' 'wx' 'xa' 'xh' 'xl' 'yg' 'yi' 'yj' 'ym' - f | 'ae' 'av' 'bj' 'bk' 'c3' 'd3' 'dj' 'eb' 'ed' 'ef' 'ei' 'ej' 'en' 'ep' 'fl' 'fp' 'fr' 'fu' 'fz' 'g8' 'gd' 'h6' 'ht' 'hu' 'i7' 'if' 'im' 'j0' 'j1' 'je' 'jx' 'ku' 'l6' 'l7' 'll' 'lp' 'mc' 'ns' 'o4' 'oi' 'op' 'oz' 'q4' 'qa' 'qf' 'qh' 'ql' 'qn' 'qv' 'r0' 'r1' 'r4' 'rh' 'rj' 'rw' 'sd' 'sn' 'so' 't6' 'ti' 'tw' 'uc' 'uf' 'uk' 'ul' 'uw' 'v3' 'v8' 'vd' 'vr' 'w7' 'w9' 'wa' 'wb' 'wg' 'wh' 'wk' 'wl' 'wn' 'wo' 'wz' 'xf' 'xg' 'xo' 'xv' 'y2' 'yl' 'zr' 'zs' 'zw' - f | '4k' 'af' 'ah' 'b7' 'bj' 'by' 'ci' 'df' 'ds' 'eh' 'em' 'eq' 'ey' 'ez' 'fu' 'fv' 'g1' 'ga' 'go' 'gs' 'gy' 'gz' 'ho' 'i0' 'ie' 'ir' 'iv' 'k5' 'kj' 'ks' 'kw' 'l4' 'li' 'm0' 'mo' 'o8' 'ol' 'oq' 'os' 'ox' 'pk' 'pl' 'pq' 'q3' 'q6' 'qa' 'qg' 'qj' 'qn' 'qq' 'qr' 'qu' 'r5' 'r9' 'rb' 'ri' 'rl' 'rs' 'ry' 's7' 's9' 'sa' 'sd' 'sx' 'ta' 'tc' 'td' 'tf' 'tg' 'th' 'ti' 'to' 'tq' 'tv' 'uo' 'ut' 've' 'vt' 'w8' 'wg' 'wm' 'ws' 'wy' 'xy' 'y8' 'ya' 'yc' 'yf' 'ys' 'yu' 'zy' - f | 'aa' 'av' 'ay' 'bn' 'by' 'cm' 'da' 'dd' 'dj' 'dk' 'dr' 'dz' 'eu' 'ev' 'ez' 'f1' 'f8' 'gr' 'he' 'hp' 'hs' 'hw' 'iz' 'k1' 'kc' 'km' 'ko' 'kt' 'ln' 'ls' 'mx' 'n3' 'nl' 'oe' 'oj' 'om' 'os' 'oy' 'pg' 'pr' 'pt' 'pv' 'qa' 'qc' 'qe' 'qi' 'qj' 'qm' 'qn' 'qq' 'qu' 'qx' 'qz' 'r0' 'r1' 'rf' 'rs' 'rx' 'ry' 's4' 'sg' 't2' 'tq' 'tt' 'tv' 'ty' 'ua' 'uj' 'uq' 'vi' 'vk' 'vx' 'wc' 'we' 'wi' 'wl' 'wo' 'wp' 'wv' 'ww' 'xi' 'xr' 'xs' 'y9' 'yg' 'ym' 'yr' 'yt' 'yz' 'zo' 'zw' - f | 'ag' 'bm' 'co' 'cr' 'd7' 'dk' 'do' 'e6' 'e9' 'eb' 'ed' 'ef' 'eh' 'ep' 'eq' 'ew' 'f4' 'fd' 'fr' 'gc' 'gg' 'gq' 'gt' 'gv' 'h0' 'h4' 'hi' 'hm' 'i4' 'ib' 'im' 'is' 'j0' 'jc' 'jd' 'jo' 'ka' 'kk' 'kx' 'm3' 'nj' 'nr' 'nu' 'o2' 'ob' 'oe' 'oi' 'om' 'ow' 'oz' 'p3' 'pd' 'pf' 'ph' 'qi' 'qj' 'qu' 'qw' 'qy' 'r2' 'ra' 'rg' 'rj' 'rm' 'rn' 'ro' 'rp' 'rr' 'ry' 'rz' 's3' 'sa' 'so' 't6' 't7' 'ta' 'tc' 'u3' 'ub' 'vj' 'vm' 'vn' 'w7' 'wb' 'wf' 'wk' 'wn' 'x2' 'yi' 'zf' 'zh' - f | 'av' 'aw' 'bl' 'bt' 'cj' 'cx' 'df' 'ea' 'ed' 'ee' 'ef' 'ew' 'f5' 'fq' 'ft' 'fu' 'g6' 'gl' 'gs' 'ha' 'hj' 'hr' 'i8' 'ia' 'ic' 'ir' 'iz' 'j0' 'j5' 'jp' 'ju' 'k6' 'ki' 'lk' 'lz' 'mh' 'nl' 'o4' 'ob' 'oo' 'op' 'pf' 'po' 'pq' 'pr' 'q2' 'q3' 'q6' 'qd' 'qj' 'qk' 'qn' 'qt' 'qw' 'ro' 'rr' 'ru' 'rv' 'sc' 'sw' 'sy' 'sz' 't2' 'tb' 'tg' 'tl' 'tp' 'tv' 'u4' 'u9' 'um' 'uu' 'vj' 'vt' 'vu' 'wg' 'wh' 'wj' 'wo' 'wq' 'wt' 'xd' 'y1' 'yc' 'ye' 'yl' 'yw' 'zb' 'zd' 'zm' 'zr' - f | '41' '7r' 'a2' 'ad' 'aj' 'ak' 'ao' 'as' 'b1' 'b2' 'cn' 'db' 'eb' 'ec' 'ee' 'em' 'ev' 'ex' 'f8' 'fa' 'fo' 'fs' 'fu' 'gh' 'gk' 'gr' 'gx' 'iq' 'iv' 'j9' 'jb' 'jc' 'jr' 'kf' 'lg' 'm1' 'mq' 'mt' 'ne' 'nv' 'o5' 'od' 'os' 'ox' 'pc' 'pj' 'pq' 'q1' 'q6' 'q7' 'q8' 'qk' 'qm' 'qn' 'qp' 'qs' 'rj' 'rk' 's5' 'sb' 'sk' 'sn' 'sv' 't4' 'tb' 'th' 'ti' 'uc' 'uj' 'ut' 'vh' 'vp' 'wg' 'wi' 'wk' 'wp' 'ws' 'wv' 'wx' 'wy' 'x6' 'xa' 'xg' 'xr' 'xy' 'y6' 'yj' 'yo' 'yv' 'yw' 'zt' 'zw' - f | '5z' 'ad' 'al' 'ax' 'b4' 'bx' 'd7' 'da' 'dp' 'dz' 'e3' 'ef' 'en' 'es' 'et' 'ev' 'ex' 'f6' 'fj' 'fr' 'ft' 'fv' 'h9' 'hn' 'hx' 'i0' 'ii' 'it' 'jf' 'jk' 'jl' 'ka' 'kr' 'l9' 'lf' 'm3' 'mm' 'nc' 'nd' 'nr' 'o8' 'oe' 'ok' 'om' 'pd' 'ph' 'pz' 'qd' 'qf' 'qo' 'qp' 'qw' 'qx' 'qy' 'r6' 'rc' 'rg' 'rw' 'rx' 's8' 'se' 't3' 'tb' 'tc' 'ti' 'tl' 'to' 'tw' 'ub' 'ue' 'uf' 'uy' 'vf' 'vh' 'vr' 'w0' 'w9' 'wd' 'wf' 'wh' 'wp' 'wu' 'x0' 'x8' 'y2' 'ye' 'yq' 'ys' 'yt' 'z6' 'zf' 'zp' - f | '7u' '7z' 'a0' 'a5' 'c6' 'cj' 'cx' 'do' 'e4' 'eg' 'eh' 'eq' 'ey' 'ez' 'fe' 'fu' 'gk' 'gy' 'h9' 'ih' 'ik' 'il' 'j0' 'jt' 'jy' 'k1' 'k5' 'ke' 'lp' 'lx' 'ly' 'ma' 'mj' 'mm' 'mt' 'nb' 'nm' 'o3' 'o5' 'of' 'oi' 'ov' 'p2' 'pf' 'ph' 'pt' 'q8' 'qj' 'qk' 'qp' 'qs' 'r1' 'r8' 'rl' 'rp' 'rt' 's5' 'sb' 'sh' 'sk' 'so' 'sr' 'su' 'tc' 'tn' 'tr' 'tx' 'u4' 'uh' 'ul' 'up' 'ur' 'uw' 'uy' 'vm' 'w5' 'wb' 'wj' 'wm' 'wq' 'wy' 'x6' 'xq' 'xs' 'y8' 'y9' 'yi' 'yj' 'yv' 'yx' 'zi' 'zy' - f | 'a7' 'a9' 'ab' 'af' 'ah' 'bc' 'bg' 'bi' 'c2' 'cj' 'cl' 'dj' 'dn' 'do' 'dv' 'dw' 'e5' 'ec' 'es' 'f4' 'fa' 'fk' 'fv' 'h9' 'hi' 'hu' 'i5' 'ic' 'ig' 'im' 'ir' 'ji' 'jt' 'k2' 'kc' 'kd' 'ki' 'km' 'kz' 'l3' 'lh' 'li' 'lv' 'ml' 'ne' 'ni' 'nt' 'nx' 'o9' 'pm' 'pn' 'pr' 'px' 'q0' 'qa' 'qe' 'qj' 'r6' 'ra' 'rj' 'rn' 'rq' 'ru' 'ry' 's0' 'sq' 'tb' 'to' 'tw' 'ua' 'ub' 'uf' 'ui' 'un' 'uu' 'vi' 'vo' 'vv' 'w5' 'w7' 'w8' 'wo' 'wv' 'wy' 'xy' 'y2' 'ya' 'yh' 'yj' 'yo' 'za' 'zh' - f | '0h' '42' 'ak' 'al' 'bf' 'bz' 'co' 'd3' 'dc' 'dt' 'dy' 'ed' 'ee' 'eg' 'ev' 'ew' 'fg' 'fr' 'fz' 'gi' 'gy' 'ha' 'hb' 'hd' 'hl' 'hn' 'hs' 'ht' 'ij' 'io' 'iv' 'jh' 'jl' 'jr' 'kc' 'kj' 'kt' 'ku' 'ky' 'ln' 'ml' 'mu' 'ng' 'nm' 'o0' 'o1' 'of' 'oh' 'or' 'p8' 'q8' 'qb' 'qp' 'qr' 'qt' 'qu' 'qv' 'r8' 'rj' 'rl' 'rp' 'rw' 'sk' 'ss' 'sw' 't9' 'th' 'tl' 'u5' 'uj' 'us' 'vg' 'vt' 'vz' 'w4' 'wb' 'we' 'wl' 'ww' 'wx' 'wy' 'wz' 'xj' 'yb' 'yg' 'yn' 'yp' 'ys' 'yw' 'yz' 'zk' 'zp' 'zu' - f | '2s' 'a7' 'ae' 'am' 'bx' 'd5' 'de' 'do' 'ds' 'dt' 'e1' 'e3' 'e7' 'ed' 'ee' 'eg' 'ek' 'em' 'es' 'ev' 'fi' 'fw' 'g2' 'gf' 'gs' 'gu' 'gy' 'h7' 'hh' 'hi' 'hm' 'hu' 'ih' 'in' 'io' 'jr' 'js' 'jw' 'k0' 'k5' 'kb' 'l4' 'lu' 'm6' 'm8' 'mc' 'nb' 'od' 'ox' 'pc' 'pg' 'pv' 'py' 'q5' 'qc' 'qe' 'qf' 'qh' 'ql' 'qm' 'qo' 'qq' 'ra' 'rk' 'ro' 'rp' 'rr' 'ru' 'rw' 'se' 'sh' 't9' 'to' 'tq' 'tt' 'tw' 'u1' 'u4' 'ui' 'un' 'uq' 'vr' 'vy' 'vz' 'wh' 'wp' 'ws' 'wt' 'wy' 'xt' 'y8' 'yd' 'zp' - f | '3g' 'af' 'aj' 'ak' 'ap' 'as' 'au' 'cp' 'cx' 'dh' 'dn' 'dr' 'ds' 'ej' 'en' 'eo' 'et' 'eu' 'ex' 'fq' 'fs' 'ft' 'gw' 'h5' 'i5' 'ix' 'j6' 'jc' 'jg' 'k1' 'kb' 'kh' 'kn' 'kv' 'lj' 'lu' 'mc' 'mi' 'na' 'nq' 'ns' 'o4' 'o7' 'of' 'q5' 'q7' 'qe' 'qh' 'qi' 'qk' 'qr' 'qu' 'qw' 'r9' 'rb' 'ri' 'rx' 's3' 'sf' 'sm' 'so' 'sq' 'ss' 'su' 'sy' 't7' 'ta' 'ti' 'tn' 'tr' 'tx' 'u1' 'u3' 'ue' 'uh' 'up' 'uw' 'uy' 'va' 'w6' 'wg' 'wm' 'wp' 'ws' 'wy' 'x1' 'y4' 'y6' 'yk' 'ys' 'yy' 'zf' 'zv' - f | 'a4' 'ae' 'ax' 'bb' 'bg' 'ca' 'ch' 'cq' 'cv' 'dm' 'dn' 'en' 'ep' 'eu' 'ev' 'f0' 'g3' 'gk' 'gm' 'hd' 'ho' 'hp' 'hy' 'ij' 'im' 'iy' 'jl' 'jr' 'jy' 'kj' 'kt' 'ku' 'lp' 'mo' 'mr' 'mz' 'n4' 'nk' 'oc' 'ol' 'oo' 'os' 'oy' 'oz' 'p8' 'p9' 'ps' 'qb' 'qd' 'qg' 'qi' 'qv' 'qx' 'r1' 'ra' 'rf' 'rg' 'rm' 'ro' 'rr' 'rv' 'rz' 's7' 'sm' 'ss' 'tl' 'tr' 'tu' 'ty' 'u5' 'ui' 'un' 'uq' 'uv' 'vn' 'w1' 'w2' 'w6' 'wd' 'we' 'wg' 'wn' 'wp' 'wy' 'y2' 'y6' 'yc' 'yd' 'yt' 'yw' 'z8' 'ze' 'zs' - f | 'ai' 'cg' 'cs' 'dc' 'dg' 'di' 'dj' 'dk' 'dp' 'du' 'eb' 'ec' 'ee' 'ei' 'ek' 'eu' 'f2' 'fh' 'fm' 'fy' 'hc' 'hm' 'i2' 'ia' 'jj' 'ke' 'kl' 'lb' 'lc' 'ln' 'me' 'nc' 'nf' 'o6' 'oc' 'ok' 'os' 'pc' 'po' 'px' 'q2' 'q8' 'qa' 'qb' 'qd' 'qe' 'qg' 'qi' 'qo' 'qs' 'qt' 'qv' 'qx' 'r4' 'r9' 'ri' 'rn' 'rq' 'rr' 'rz' 's9' 'sf' 'sr' 'su' 'sw' 'sy' 'sz' 't2' 'ti' 'tv' 'ud' 'uv' 'wa' 'wc' 'wi' 'wk' 'wm' 'ws' 'wu' 'wv' 'wz' 'x2' 'xa' 'xf' 'y8' 'yb' 'yd' 'yl' 'yp' 'yr' 'ys' 'z5' 'zh' - f | '1a' 'a0' 'ae' 'av' 'be' 'bj' 'bv' 'bx' 'bz' 'ck' 'd4' 'do' 'ds' 'du' 'ee' 'eu' 'fg' 'fm' 'fy' 'g1' 'gh' 'h8' 'h9' 'ha' 'he' 'hl' 'hy' 'i6' 'ic' 'io' 'jb' 'jk' 'jq' 'k1' 'kf' 'km' 'kv' 'l8' 'n0' 'n3' 'oh' 'oz' 'po' 'q9' 'qg' 'qh' 'qi' 'ql' 'qp' 'qr' 'qu' 'qw' 'qx' 'rb' 're' 'rf' 'ri' 'rj' 'rv' 'sj' 'sm' 'sz' 't1' 'tl' 'tm' 'tw' 'tz' 'ua' 'uc' 'ug' 'ut' 'uv' 'ux' 'v7' 'wa' 'wc' 'wf' 'wi' 'wl' 'wp' 'wq' 'wr' 'x3' 'x6' 'xk' 'xx' 'y4' 'yp' 'yr' 'yw' 'yy' 'ze' 'zk' 'zy' - f | '1z' 'aq' 'cf' 'cl' 'e1' 'e5' 'ee' 'eg' 'eo' 'er' 'ff' 'fh' 'fx' 'g5' 'ga' 'gd' 'gm' 'gn' 'hc' 'hf' 'hi' 'hk' 'ho' 'ib' 'ik' 'in' 'iw' 'ix' 'jd' 'kl' 'ky' 'lp' 'm3' 'm5' 'n4' 'ng' 'o0' 'oc' 'oj' 'ot' 'ou' 'p2' 'pa' 'pd' 'pg' 'ps' 'pw' 'q1' 'q8' 'qb' 'qc' 'qd' 'qi' 'qr' 'qs' 'qu' 'qv' 'ra' 'rg' 'rt' 'rz' 's7' 'sm' 'sn' 't0' 'tb' 'th' 'tx' 'tz' 'ub' 'ud' 'ue' 'un' 'ur' 'ut' 'vb' 'vj' 'wg' 'wh' 'wj' 'wl' 'wn' 'ww' 'wz' 'x0' 'xc' 'xq' 'xr' 'xw' 'y8' 'y9' 'yr' 'z7' 'zt' - f | '27' '3p' '6r' 'a8' 'an' 'ap' 'au' 'ay' 'bi' 'c6' 'cy' 'dh' 'ds' 'eg' 'eh' 'ej' 'es' 'eu' 'fm' 'fp' 'fq' 'fx' 'g2' 'gg' 'gv' 'hz' 'ij' 'il' 'iq' 'iw' 'j8' 'j9' 'k3' 'kr' 'lq' 'm7' 'm9' 'mj' 'n0' 'n6' 'nr' 'nx' 'ox' 'pq' 'qc' 'qd' 'qe' 'qg' 'qh' 'qj' 'qk' 'qm' 'qn' 'qr' 'qu' 'qv' 'qw' 'qx' 're' 'rg' 'rr' 'rz' 's1' 'sj' 'sl' 'sy' 't4' 't7' 'tb' 'tr' 'uc' 'un' 'uq' 'ut' 'vf' 'w7' 'w9' 'wg' 'wh' 'wp' 'wq' 'ws' 'wx' 'x8' 'xp' 'xy' 'y0' 'y1' 'ya' 'yu' 'yz' 'z2' 'z5' 'zz' - f | '3k' 'ag' 'ak' 'bi' 'bl' 'bw' 'by' 'ch' 'cm' 'dw' 'e1' 'e2' 'ed' 'ej' 'ek' 'er' 'eu' 'ez' 'f8' 'fd' 'fi' 'fl' 'gi' 'gm' 'gx' 'gy' 'h2' 'h8' 'hl' 'hn' 'ij' 'ip' 'iq' 'it' 'jb' 'jl' 'jn' 'k7' 'kh' 'kl' 'kn' 'kt' 'nh' 'nk' 'pa' 'pe' 'pg' 'pp' 'q5' 'q7' 'qi' 'qk' 'ql' 'qs' 'r8' 'ri' 'rj' 'rl' 'rw' 'rx' 'so' 'tb' 'tj' 'tm' 'to' 'tu' 'tv' 'tz' 'u4' 'ue' 'ul' 'uv' 'v1' 'vj' 'vy' 'wc' 'wr' 'wt' 'wx' 'wz' 'xh' 'xj' 'xp' 'xt' 'y4' 'yb' 'yf' 'ym' 'yo' 'yq' 'yy' 'zp' 'zq' 'zt' - f | '9a' 'aa' 'bo' 'br' 'bv' 'bz' 'c1' 'c7' 'cz' 'db' 'dh' 'dj' 'e4' 'e8' 'eh' 'ew' 'gd' 'gg' 'gp' 'gu' 'gx' 'hc' 'ho' 'ht' 'hv' 'ia' 'ii' 'ij' 'ir' 'iw' 'j5' 'jk' 'jv' 'la' 'lc' 'lp' 'lt' 'lw' 'na' 'nt' 'nv' 'nw' 'o8' 'o9' 'ou' 'oy' 'p0' 'pc' 'pq' 'py' 'q4' 'q8' 'qi' 'qk' 'qn' 'qr' 'qt' 'qw' 'r5' 'r8' 're' 'rj' 'rt' 'rv' 'ry' 'rz' 's8' 'sg' 'sv' 'ta' 'td' 'tl' 'tm' 'ul' 'up' 'ut' 'uz' 'vf' 'w1' 'wb' 'wc' 'wd' 'wf' 'wi' 'wl' 'wu' 'x8' 'xg' 'xm' 'y1' 'yl' 'yr' 'ys' 'z8' - f | 'a3' 'a5' 'ai' 'av' 'ay' 'c3' 'cp' 'd3' 'dn' 'du' 'dz' 'e9' 'ed' 'ei' 'ek' 'em' 'eo' 'ev' 'f2' 'gh' 'gl' 'gy' 'h6' 'h8' 'hb' 'he' 'hg' 'hm' 'hp' 'hy' 'hz' 'id' 'ie' 'it' 'ix' 'j2' 'ja' 'jc' 'jf' 'k5' 'ki' 'kk' 'kv' 'l4' 'l6' 'ls' 'm9' 'mg' 'mt' 'nb' 'ng' 'oh' 'ot' 'oz' 'pw' 'py' 'pz' 'q0' 'q3' 'q8' 'qb' 'qh' 'qj' 'qk' 'ql' 'qp' 'qq' 'qr' 'qt' 'qw' 'rb' 'rz' 's3' 'sl' 'tb' 'u0' 'u5' 'uo' 'us' 'vc' 'wg' 'wh' 'wj' 'wr' 'wt' 'wu' 'wy' 'x1' 'y1' 'yh' 'z7' 'zf' 'zs' 'zu' - f | 'af' 'aq' 'at' 'bh' 'cc' 'd3' 'dd' 'dg' 'dq' 'e1' 'e6' 'ec' 'ee' 'ex' 'ey' 'go' 'gt' 'h3' 'h8' 'hd' 'he' 'hm' 'i7' 'ii' 'ik' 'io' 'ip' 'it' 'iz' 'jg' 'k0' 'k5' 'k7' 'kb' 'ks' 'kw' 'ln' 'lp' 'lx' 'm3' 'nf' 'nh' 'of' 'og' 'oj' 'ok' 'ow' 'p7' 'pd' 'pj' 'pm' 'q1' 'q7' 'qg' 'qh' 'qk' 'qq' 'qr' 'qt' 'qw' 'rr' 'rs' 'ry' 's4' 's7' 'sd' 'si' 'sl' 'sp' 'sw' 'sz' 't6' 't8' 'tc' 'u2' 'um' 'uo' 'ux' 'vb' 'w2' 'wb' 'wi' 'wj' 'ws' 'wt' 'wx' 'x3' 'xf' 'xt' 'yb' 'ym' 'zb' 'ze' 'zm' - f | '1s' 'a0' 'ac' 'aj' 'am' 'ao' 'aw' 'bi' 'bm' 'by' 'ca' 'cu' 'dc' 'di' 'dp' 'e0' 'e9' 'eg' 'eq' 'fh' 'fi' 'gc' 'gi' 'gq' 'gx' 'h4' 'he' 'hk' 'i2' 'ix' 'jc' 'jl' 'jm' 'k1' 'kf' 'kg' 'kn' 'kq' 'la' 'nj' 'nw' 'o9' 'og' 'p0' 'p6' 'pj' 'pn' 'pp' 'q4' 'qf' 'ql' 'qm' 'qs' 'qt' 'qu' 'qz' 'r5' 'r6' 'ri' 'ro' 'rw' 'sv' 'sx' 't5' 'th' 'tl' 'tp' 'u0' 'ue' 'uf' 'ug' 'un' 'uq' 'vx' 'wd' 'we' 'wg' 'wj' 'wk' 'wp' 'wr' 'ws' 'wt' 'x2' 'xq' 'xt' 'xw' 'xx' 'ya' 'yk' 'yl' 'yr' 'yt' 'za' 'zh' - f | '2o' 'af' 'ah' 'aj' 'al' 'ap' 'as' 'bc' 'd2' 'di' 'dm' 'do' 'ds' 'dt' 'dy' 'eb' 'ed' 'ee' 'ej' 'es' 'ev' 'ex' 'ey' 'gs' 'hk' 'hr' 'hw' 'hz' 'ig' 'is' 'jp' 'jv' 'jx' 'k4' 'kh' 'kv' 'lb' 'll' 'lw' 'na' 'no' 'o1' 'o8' 'oh' 'oo' 'op' 'pb' 'po' 'q2' 'q4' 'q7' 'qa' 'qc' 'qe' 'qg' 'qm' 'qo' 'qt' 'qw' 'qy' 'qz' 'r4' 'r7' 'r8' 'rp' 'rs' 's2' 'sb' 'sg' 'sl' 't1' 'tj' 'to' 'ts' 'tx' 'u5' 'uz' 'v6' 'w4' 'w5' 'we' 'wh' 'wn' 'wr' 'wu' 'wv' 'x1' 'y1' 'y8' 'ya' 'yk' 'ym' 'yw' 'z5' 'zx' - f | '2s' '4q' '8x' 'aa' 'ac' 'ah' 'av' 'aw' 'c4' 'c5' 'ce' 'd1' 'dj' 'dq' 'e7' 'e9' 'ea' 'ef' 'fc' 'fs' 'fz' 'gb' 'ge' 'gx' 'hf' 'hl' 'hm' 'i5' 'i7' 'ie' 'ii' 'iw' 'kf' 'ko' 'kx' 'lb' 'm6' 'mh' 'ob' 'od' 'oe' 'og' 'oq' 'or' 'pn' 'ps' 'pu' 'q4' 'qf' 'qg' 'qj' 'ql' 'qn' 'qt' 'qy' 'r3' 'ra' 'ri' 'rj' 'rv' 'sr' 'tg' 'th' 'tl' 'tx' 'u7' 'ub' 'uh' 'uq' 'uu' 'uy' 'vn' 'vt' 'wa' 'we' 'wh' 'wn' 'wp' 'wr' 'ww' 'xg' 'xk' 'xs' 'ya' 'yc' 'yd' 'yf' 'yl' 'ym' 'ys' 'z6' 'zj' 'zm' 'zv' 'zw' - f | '4d' '5a' 'ab' 'ae' 'ah' 'ai' 'be' 'cb' 'd6' 'dw' 'ea' 'eu' 'fb' 'fd' 'fe' 'fj' 'fk' 'g0' 'g9' 'gf' 'hc' 'hh' 'hk' 'i4' 'ik' 'iv' 'iz' 'j1' 'j3' 'j5' 'ju' 'jz' 'kc' 'md' 'ng' 'nk' 'o1' 'o9' 'oi' 'om' 'p4' 'pl' 'pn' 'ps' 'q8' 'qa' 'qb' 'qh' 'qj' 'qm' 'qn' 'qo' 'qq' 'qr' 'qt' 'qu' 'qy' 'qz' 'ro' 'rs' 's2' 'sl' 'sm' 'sp' 'ti' 'tn' 'tu' 'tw' 'u7' 'ug' 'ui' 'ul' 'us' 'ut' 'vc' 've' 'vj' 'vk' 'w0' 'w2' 'w8' 'wi' 'wk' 'wl' 'wm' 'wt' 'x6' 'xh' 'xy' 'yr' 'ys' 'yt' 'yy' 'yz' 'zr' - f | 'a1' 'ab' 'ad' 'ag' 'bd' 'bf' 'bo' 'bt' 'cl' 'cn' 'cq' 'cz' 'd4' 'db' 'dc' 'dk' 'dn' 'dq' 'ea' 'el' 'em' 'ev' 'ew' 'ex' 'fr' 'ft' 'ga' 'gc' 'gd' 'gg' 'gl' 'gp' 'hp' 'hu' 'id' 'ij' 'jn' 'kl' 'kt' 'lh' 'lk' 'lm' 'mi' 'mw' 'n4' 'nk' 'oj' 'ok' 'on' 'ph' 'pk' 'pt' 'q0' 'q8' 'qa' 'qe' 'qf' 'qg' 'qh' 'qi' 'qr' 'qt' 'qv' 'qw' 'qx' 'rd' 're' 'rg' 'rq' 'rr' 'sb' 'sg' 'te' 'tj' 'us' 'w5' 'w9' 'wb' 'wf' 'wh' 'wp' 'wq' 'ws' 'wt' 'wu' 'wv' 'y7' 'yb' 'ye' 'yf' 'yq' 'yu' 'yy' 'yz' 'zx' - f | 'ah' 'ao' 'aq' 'aw' 'az' 'b5' 'bd' 'bg' 'bj' 'bq' 'bu' 'cc' 'cv' 'dt' 'e0' 'e2' 'e4' 'ec' 'eu' 'fd' 'fi' 'gg' 'gu' 'ha' 'hb' 'he' 'hj' 'hy' 'ih' 'in' 'iv' 'ji' 'js' 'ks' 'l1' 'la' 'lg' 'mh' 'mi' 'mk' 'o7' 'oh' 'oj' 'om' 'on' 'op' 'oq' 'or' 'os' 'p1' 'pe' 'pq' 'pv' 'q8' 'qe' 'qg' 'qj' 'ql' 'qm' 'qn' 'qo' 'qr' 'qs' 'qt' 'qv' 'qy' 'r1' 'r9' 'rb' 'rl' 'ro' 'ru' 'rw' 'ry' 's0' 's6' 'sn' 't0' 't1' 'tr' 'ty' 'u0' 'u8' 'up' 'uv' 'vn' 'wb' 'wg' 'ww' 'x1' 'yb' 'yj' 'ym' 'yx' 'zz' - f | '2k' '2v' '4h' '7c' 'a3' 'aa' 'al' 'an' 'aq' 'aw' 'b7' 'bb' 'bl' 'bn' 'by' 'ca' 'cf' 'cu' 'dd' 'dn' 'e6' 'ej' 'en' 'et' 'eu' 'ey' 'g8' 'gg' 'go' 'gw' 'hj' 'ho' 'i3' 'ii' 'iy' 'ke' 'kh' 'le' 'lf' 'lu' 'ml' 'ny' 'oa' 'oc' 'og' 'oh' 'os' 'pf' 'pl' 'qb' 'qd' 'qf' 'qj' 'qn' 'qu' 'qw' 'r8' 'rd' 're' 'rh' 'sc' 'sn' 'ss' 't0' 't7' 'tl' 'tt' 'tx' 'tz' 'u9' 'uc' 'ud' 'ul' 'us' 'ux' 'vb' 'vf' 'vg' 'vp' 'w5' 'w6' 'wk' 'wt' 'wy' 'xd' 'xk' 'xm' 'xn' 'xr' 'xx' 'y0' 'yf' 'yl' 'yo' 'yw' 'zv' - f | '3k' '3v' 'a0' 'ah' 'am' 'an' 'ay' 'bd' 'cs' 'cu' 'cw' 'db' 'dp' 'e3' 'et' 'ey' 'fi' 'gl' 'gq' 'hh' 'hk' 'hx' 'i8' 'ih' 'ii' 'iv' 'iz' 'j5' 'jf' 'jo' 'jr' 'kd' 'km' 'lg' 'li' 'lt' 'm4' 'mo' 'mv' 'mw' 'n3' 'nh' 'nn' 'o9' 'od' 'of' 'oh' 'ok' 'or' 'pc' 'pg' 'pm' 'pn' 'q6' 'qj' 'qn' 'qr' 'qs' 'qw' 'r7' 'r9' 're' 'rm' 'rr' 'rt' 'rz' 'sb' 'sf' 'td' 'tu' 'uc' 'ug' 'uj' 'uu' 'w3' 'wa' 'wc' 'wg' 'wi' 'wm' 'wp' 'ww' 'wx' 'wz' 'xb' 'xi' 'y0' 'y9' 'ya' 'yd' 'yg' 'yh' 'yv' 'yy' 'zo' 'zz' - f | '3z' 'ah' 'ai' 'aq' 'bn' 'ck' 'cm' 'cq' 'ct' 'cy' 'd1' 'db' 'dn' 'e2' 'e7' 'ed' 'ee' 'eh' 'eq' 'er' 'ew' 'fc' 'fn' 'fp' 'fs' 'fw' 'fx' 'gl' 'gp' 'gw' 'ha' 'hq' 'hw' 'hx' 'i2' 'ir' 'j9' 'jb' 'jn' 'jr' 'jx' 'jy' 'kk' 'kl' 'kx' 'kz' 'li' 'm2' 'me' 'mj' 'mr' 'na' 'nu' 'oe' 'ol' 'on' 'po' 'pv' 'q7' 'qa' 'qb' 'qd' 'qh' 'qp' 'qv' 'rh' 'rz' 'sf' 'sp' 't6' 'tl' 'to' 'tx' 'ty' 'ue' 'uf' 'un' 'uu' 'uw' 'vb' 'vq' 'wg' 'wp' 'wt' 'wu' 'xb' 'xp' 'y1' 'y9' 'yi' 'yj' 'ys' 'yu' 'z7' 'zf' 'zt' - f | '4d' '9f' 'aa' 'ad' 'av' 'b1' 'bb' 'bd' 'bk' 'd2' 'd6' 'd7' 'de' 'do' 'ds' 'dx' 'e4' 'eh' 'en' 'ep' 'ew' 'fd' 'fg' 'fo' 'fp' 'ga' 'gw' 'gz' 'h2' 'h4' 'hc' 'hk' 'ic' 'ih' 'io' 'iv' 'iy' 'jj' 'jt' 'jz' 'kb' 'ke' 'kl' 'lh' 'lm' 'nc' 'ni' 'nl' 'o9' 'oi' 'oo' 'pd' 'pj' 'po' 'pr' 'qa' 'qf' 'qi' 'qm' 'qn' 'qp' 'qt' 'qu' 'rb' 're' 'rf' 'rn' 'rz' 'sz' 't0' 'tg' 'ti' 'ut' 'v9' 'vg' 'wb' 'wc' 'wf' 'wx' 'x0' 'xb' 'xv' 'y6' 'yb' 'yg' 'yh' 'yk' 'ym' 'yn' 'yw' 'yx' 'yz' 'z8' 'zp' 'zs' 'zw' - f | '5t' 'al' 'av' 'ax' 'by' 'cr' 'cx' 'df' 'dh' 'dp' 'dq' 'e4' 'e7' 'en' 'ff' 'fg' 'fo' 'g4' 'ge' 'gg' 'gm' 'gr' 'gt' 'hc' 'hn' 'hq' 'hs' 'i0' 'i1' 'id' 'ig' 'im' 'jb' 'jv' 'k9' 'kl' 'km' 'lh' 'lp' 'lq' 'mm' 'ne' 'nk' 'nu' 'od' 'os' 'pj' 'pw' 'pz' 'q0' 'q1' 'qa' 'qd' 'qe' 'qf' 'qk' 'ql' 'qn' 'qp' 'qt' 'r0' 'rx' 'sx' 'sy' 't0' 't1' 't8' 'th' 'ti' 'tp' 'tv' 'tz' 'ub' 'uc' 'ul' 'v5' 'vp' 'vu' 'w8' 'wd' 'we' 'wf' 'wh' 'wm' 'wq' 'wr' 'ws' 'wy' 'wz' 'xg' 'y9' 'yg' 'yh' 'yq' 'yz' 'z0' - f | '6w' 'a1' 'aa' 'ab' 'af' 'ao' 'aq' 'ay' 'cd' 'cg' 'dn' 'dp' 'dy' 'e0' 'e6' 'eb' 'ej' 'fh' 'fi' 'fs' 'g0' 'gc' 'gi' 'gs' 'gv' 'gw' 'hd' 'ho' 'i8' 'i9' 'ia' 'ic' 'ie' 'ik' 'ja' 'jd' 'ji' 'jm' 'k0' 'kj' 'lj' 'm8' 'n0' 'nh' 'nl' 'nn' 'o8' 'of' 'ou' 'oz' 'ph' 'po' 'q0' 'q3' 'q6' 'q7' 'qa' 'qc' 'qd' 'qn' 'qp' 'qq' 'qu' 'qw' 'qz' 'rd' 'rm' 'rn' 'rp' 'rs' 'sj' 'tg' 'tj' 'tr' 'tv' 'u0' 'uf' 'uh' 'uk' 'vv' 'wa' 'wc' 'wf' 'wg' 'wn' 'wo' 'wp' 'wr' 'ww' 'xt' 'y3' 'y7' 'yc' 'yf' 'yp' 'yy' - f | '9y' 'a0' 'a6' 'ad' 'aj' 'az' 'bw' 'by' 'cg' 'ci' 'dc' 'dk' 'dm' 'dw' 'e0' 'e4' 'ee' 'ef' 'eg' 'en' 'eu' 'fc' 'fg' 'fm' 'fx' 'g7' 'gm' 'go' 'hw' 'hy' 'i4' 'i7' 'ip' 'iq' 'ir' 'jr' 'ju' 'jx' 'kr' 'ky' 'la' 'lk' 'lq' 'm9' 'mg' 'mp' 'my' 'n6' 'nv' 'nz' 'o1' 'o3' 'oe' 'oy' 'pj' 'pv' 'pz' 'qg' 'ql' 'qp' 'qt' 'qy' 'r4' 'rf' 'rg' 'rk' 'ro' 'rw' 'ry' 's3' 'sd' 'sf' 'sm' 'tf' 'tg' 'tq' 'tu' 'ty' 'tz' 'ub' 'uc' 'uf' 'um' 'vi' 'vn' 'wa' 'wc' 'we' 'xo' 'xr' 'xs' 'yb' 'yi' 'yw' 'zn' 'zo' - f | 'ag' 'ai' 'az' 'br' 'c2' 'cd' 'ck' 'db' 'du' 'e0' 'e7' 'eb' 'ee' 'em' 'ep' 'eq' 'es' 'eu' 'ex' 'fh' 'fi' 'ga' 'gm' 'gn' 'hj' 'hq' 'if' 'ig' 'ii' 'ix' 'jk' 'kd' 'kg' 'kk' 'kr' 'kt' 'ku' 'lx' 'mp' 'mq' 'nx' 'o9' 'oa' 'om' 'oq' 'p5' 'pd' 'pr' 'pu' 'pw' 'q3' 'qa' 'qd' 'qe' 'qf' 'qi' 'qj' 'ql' 'qn' 'qo' 'qr' 'qt' 'qu' 'qv' 'qx' 'rc' 'rf' 'rl' 'rm' 'rn' 'rp' 'rr' 's7' 'se' 'st' 'sx' 't8' 'to' 'u1' 'ua' 'uq' 'ux' 'w6' 'w8' 'wf' 'wt' 'xb' 'xf' 'xk' 'xn' 'y3' 'ym' 'yp' 'yz' 'zi' 'zk' - f | '1k' '1p' 'a6' 'ah' 'ap' 'ay' 'az' 'bi' 'bj' 'cv' 'd1' 'd2' 'd5' 'db' 'dm' 'dn' 'e7' 'eg' 'eh' 'el' 'en' 'eo' 'er' 'f4' 'fy' 'gf' 'gp' 'hd' 'hj' 'hp' 'i2' 'i7' 'ij' 'ik' 'jq' 'jy' 'kg' 'kh' 'kw' 'lr' 'ls' 'n5' 'ne' 'o7' 'od' 'oi' 'oo' 'os' 'oy' 'pd' 'ph' 'pm' 'pw' 'px' 'qa' 'qb' 'qc' 'qk' 'ql' 'qm' 'qn' 'qr' 'qs' 'qu' 'qx' 'r5' 'rh' 'ri' 'rk' 'rn' 'ry' 's0' 'sa' 'si' 'sr' 'st' 'ub' 'uc' 'uf' 'uq' 'ur' 'v9' 'vn' 'w9' 'wd' 'we' 'wk' 'wt' 'wv' 'ww' 'xj' 'xk' 'yh' 'yk' 'zd' 'zw' 'zy' - f | '2u' '3q' '7p' 'b4' 'b8' 'bb' 'bq' 'bu' 'ce' 'dk' 'dn' 'e0' 'ea' 'ed' 'ef' 'eg' 'eh' 'ej' 'ek' 'em' 'en' 'eo' 'et' 'fh' 'fr' 'fw' 'gj' 'gx' 'ho' 'i7' 'i9' 'if' 'ii' 'il' 'in' 'iq' 'iw' 'jy' 'k0' 'k1' 'km' 'kr' 'lh' 'lq' 'lw' 'ma' 'mb' 'md' 'mj' 'nh' 'ni' 'nj' 'ob' 'p5' 'p7' 'pc' 'pk' 'pl' 'pu' 'q2' 'qa' 'qc' 'qe' 'qi' 'qm' 'qn' 'qt' 'rb' 'rp' 'rx' 'sd' 't6' 'tb' 'tl' 'tw' 'u5' 'u7' 'ug' 'uh' 'ut' 'vs' 'wi' 'wj' 'wp' 'ws' 'wu' 'wv' 'xm' 'xs' 'y2' 'y6' 'yd' 'yh' 'yl' 'yr' 'yx' 'yz' - f | '2u' 'au' 'bz' 'cd' 'cj' 'cm' 'cq' 'ct' 'cw' 'd1' 'ds' 'dw' 'dz' 'ec' 'ei' 'eo' 'fk' 'fq' 'fx' 'g6' 'gl' 'gs' 'i5' 'if' 'im' 'iq' 'jd' 'k0' 'k5' 'kr' 'lv' 'lx' 'n5' 'na' 'ny' 'ob' 'ot' 'ox' 'pa' 'pi' 'ps' 'qa' 'qc' 'qg' 'qh' 'qj' 'ql' 'qm' 'qo' 'qr' 'qs' 'r2' 'rh' 'rl' 'rr' 'rw' 'rx' 's0' 's8' 'sb' 'sc' 'sg' 'si' 'sl' 'so' 'sv' 't7' 't9' 'tc' 'te' 'tl' 'tn' 'to' 'tr' 'tx' 'ty' 'un' 'uz' 'vg' 'vw' 'ws' 'wt' 'wu' 'xj' 'xy' 'y0' 'y1' 'ya' 'ye' 'yl' 'yn' 'yr' 'ys' 'yt' 'z3' 'z6' 'zw' - f | '3a' '5k' 'a2' 'ah' 'au' 'ba' 'bb' 'bh' 'bu' 'cb' 'cj' 'cv' 'cx' 'df' 'dy' 'e4' 'e6' 'ed' 'ep' 'et' 'ev' 'ez' 'f1' 'fb' 'fl' 'fx' 'fz' 'gg' 'h5' 'hj' 'io' 'iz' 'ja' 'k7' 'kf' 'lu' 'lv' 'md' 'ne' 'nh' 'oh' 'on' 'ow' 'p0' 'p8' 'pc' 'px' 'q3' 'qa' 'qf' 'qg' 'qj' 'qr' 'qs' 'qt' 'qv' 'qx' 'qz' 'r3' 'rm' 'rq' 'rt' 'rv' 'sa' 'sf' 'so' 't1' 't7' 'tb' 'tn' 'tq' 'tr' 'ts' 'tu' 'tz' 'u7' 'uf' 'uk' 'um' 'ut' 'va' 'vj' 'vm' 'vx' 'vz' 'wa' 'we' 'wf' 'xe' 'xg' 'ya' 'yb' 'yu' 'zg' 'zo' 'zt' 'zz' - f | '3t' '5u' 'aa' 'ab' 'b9' 'd8' 'di' 'dj' 'dq' 'dx' 'dy' 'ea' 'ek' 'el' 'es' 'ev' 'ey' 'ez' 'fk' 'fp' 'g3' 'gb' 'gd' 'gg' 'gi' 'go' 'hj' 'hz' 'i3' 'i5' 'in' 'io' 'is' 'jv' 'kb' 'kr' 'lu' 'md' 'nd' 'ny' 'o6' 'oe' 'ok' 'ow' 'ox' 'p4' 'p7' 'p8' 'pb' 'pu' 'q1' 'q3' 'q4' 'qd' 'qj' 'qk' 'qq' 'qx' 'qy' 'r6' 'ro' 'rw' 'rx' 'ry' 'se' 'sp' 't0' 't1' 'th' 'tl' 'tn' 'tp' 'tr' 'tx' 'u5' 'uf' 'uo' 'ux' 'vc' 'vz' 'w2' 'w5' 'wa' 'wb' 'we' 'wi' 'wj' 'ws' 'wt' 'xh' 'y1' 'yb' 'yg' 'yi' 'zd' 'zm' 'zt' - f | '4g' '8w' 'ab' 'aq' 'at' 'bc' 'bi' 'c7' 'cb' 'cj' 'cs' 'd2' 'd3' 'di' 'dm' 'dx' 'dz' 'ed' 'ex' 'fj' 'gs' 'h1' 'h6' 'he' 'hj' 'hr' 'i1' 'ia' 'ie' 'il' 'ix' 'iy' 'j2' 'jd' 'jo' 'jy' 'kx' 'la' 'lv' 'ma' 'mh' 'mp' 'mt' 'n9' 'na' 'nf' 'ng' 'np' 'o7' 'ob' 'on' 'ou' 'ov' 'p9' 'pg' 'po' 'pq' 'q0' 'q4' 'q5' 'qc' 'qj' 'qp' 'qq' 'qt' 'ra' 'rb' 'rq' 'ru' 'sl' 'sp' 't8' 'ta' 'te' 'tl' 'tz' 'u1' 'ud' 'ui' 'uv' 'uw' 'vf' 'vt' 'vu' 'vz' 'w0' 'w7' 'wc' 'wg' 'wh' 'wq' 'wr' 'wz' 'y6' 'y7' 'ye' 'yh' - f | '6u' 'a5' 'ai' 'au' 'b1' 'be' 'bg' 'by' 'ce' 'co' 'cw' 'db' 'dw' 'e8' 'ec' 'em' 'en' 'er' 'f7' 'fh' 'fu' 'gb' 'gq' 'h4' 'hf' 'hy' 'hz' 'ig' 'iq' 'kj' 'kk' 'kp' 'ky' 'ld' 'lr' 'na' 'nd' 'ny' 'o8' 'o9' 'og' 'oh' 'ok' 'ot' 'ou' 'ow' 'ox' 'p8' 'pl' 'pp' 'ps' 'px' 'q0' 'q1' 'qa' 'qm' 'qr' 'qs' 'qx' 'r9' 'rd' 're' 'rf' 'rl' 'rn' 'rr' 'ru' 'rz' 'sc' 'sn' 'so' 't1' 'tl' 'tr' 'ts' 'tt' 'tx' 'ub' 'um' 'un' 'uo' 'ut' 'vr' 'wa' 'wb' 'wc' 'wj' 'wl' 'wp' 'wq' 'ws' 'wx' 'wz' 'yp' 'yt' 'yy' 'zc' - f | '7h' '7k' 'bi' 'c8' 'cc' 'cj' 'cs' 'd7' 'dh' 'dl' 'dp' 'dt' 'e9' 'ea' 'eh' 'ei' 'ej' 'el' 'ew' 'fo' 'fp' 'ge' 'gg' 'gi' 'gk' 'h2' 'h7' 'hk' 'hs' 'hy' 'ii' 'j4' 'kd' 'kh' 'kp' 'ks' 'm2' 'n1' 'n3' 'nk' 'nr' 'od' 'ok' 'om' 'oy' 'pb' 'ph' 'pm' 'pp' 'pt' 'q2' 'q8' 'q9' 'qc' 'qf' 'qg' 'qh' 'qj' 'qk' 'qn' 'qo' 'qq' 'qv' 'qx' 'r4' 'rc' 'rg' 'rj' 's9' 'sb' 'sg' 'sj' 'ss' 't6' 'ta' 'tc' 'tm' 'tv' 'us' 'uu' 'uy' 'w7' 'wf' 'wh' 'wk' 'wz' 'x2' 'xr' 'ya' 'yc' 'yk' 'yp' 'ys' 'yz' 'z4' 'zg' 'zn' - f | '15' '4o' '7h' 'aa' 'av' 'b0' 'cb' 'da' 'dh' 'di' 'dr' 'e0' 'ee' 'eo' 'ep' 'ey' 'fi' 'fo' 'fq' 'fx' 'fz' 'g9' 'ge' 'hd' 'hh' 'hs' 'i4' 'i9' 'iq' 'it' 'iw' 'ix' 'iy' 'j8' 'jr' 'ld' 'm1' 'mo' 'nx' 'ob' 'ol' 'ot' 'pj' 'qf' 'qj' 'qk' 'qp' 'qv' 'qy' 'r4' 'r5' 'rl' 'rm' 'rq' 'rz' 's8' 'sc' 'sd' 'sf' 'sh' 'sn' 'ss' 't5' 't7' 'tj' 'to' 'tw' 'u6' 'uc' 'ud' 'ug' 'ui' 'uk' 'ut' 'uy' 'uz' 'vi' 'w0' 'w9' 'wb' 'wg' 'wl' 'ww' 'wx' 'wy' 'xa' 'xb' 'xo' 'ya' 'yi' 'yu' 'yv' 'yz' 'zm' 'zv' 'zx' 'zy' 'zz' - f | '1g' '6b' 'az' 'be' 'c5' 'dp' 'dt' 'e6' 'eg' 'en' 'es' 'et' 'f1' 'fi' 'fn' 'ft' 'fz' 'g0' 'gj' 'gv' 'h8' 'hp' 'hs' 'hu' 'hw' 'hz' 'ia' 'im' 'is' 'iv' 'iw' 'iy' 'jd' 'kw' 'ky' 'l4' 'l7' 'mn' 'nn' 'nr' 'ny' 'ot' 'p2' 'p8' 'pt' 'q8' 'qa' 'qb' 'qf' 'qh' 'qi' 'qm' 'qt' 'qu' 'qv' 'qy' 'ra' 'rl' 'ro' 'rw' 'rx' 's8' 's9' 'sk' 'sn' 'st' 'sv' 'th' 'tl' 'to' 'tp' 'tu' 'ua' 'uk' 'un' 'uv' 'v8' 've' 'vt' 'vu' 'vv' 'w4' 'wc' 'wh' 'wo' 'wq' 'wy' 'xe' 'xm' 'xp' 'xu' 'yc' 'yn' 'yq' 'zf' 'zj' 'zs' 'zt' - f | '1t' 'a0' 'ah' 'ar' 'at' 'be' 'bs' 'bt' 'co' 'd9' 'e9' 'ea' 'ec' 'ei' 'eo' 'er' 'ez' 'fa' 'fz' 'gl' 'gt' 'h8' 'h9' 'hb' 'hv' 'ia' 'ic' 'if' 'im' 'io' 'iq' 'ix' 'k9' 'kq' 'lo' 'm4' 'md' 'mo' 'mz' 'ni' 'nr' 'nz' 'o8' 'ox' 'pk' 'pr' 'q1' 'q3' 'q8' 'qa' 'qb' 'qf' 'qg' 'qi' 'ql' 'qr' 'r3' 'rc' 'rf' 'rg' 'rr' 's0' 'sf' 'tg' 'tw' 'u2' 'uh' 'un' 'ur' 'ux' 'vb' 'vr' 'w7' 'w9' 'wd' 'wh' 'wm' 'wo' 'wr' 'ws' 'wv' 'x4' 'xj' 'xx' 'y3' 'y8' 'yd' 'yl' 'yo' 'yq' 'yr' 'yw' 'z8' 'za' 'zb' 'zg' 'zo' 'zs' - f | '2n' 'aa' 'ab' 'ae' 'ah' 'aj' 'as' 'av' 'ax' 'bc' 'be' 'bi' 'by' 'cg' 'ck' 'cm' 'cx' 'dq' 'dr' 'e5' 'ed' 'ef' 'ei' 'em' 'eu' 'fd' 'fq' 'fu' 'gd' 'gl' 'gs' 'he' 'ia' 'iz' 'jc' 'je' 'jt' 'k7' 'km' 'ko' 'l4' 'lh' 'mk' 'nl' 'ny' 'oa' 'oh' 'op' 'p1' 'pj' 'pm' 'ps' 'q1' 'q7' 'qc' 'qg' 'qj' 'qo' 'qr' 'qt' 'qu' 'qv' 'qw' 'rf' 'ri' 'rl' 'rw' 'sf' 'su' 't4' 'tc' 'tn' 'to' 'tq' 'tr' 'tt' 'tv' 'tz' 'u0' 'u6' 'ub' 'ug' 'up' 'uv' 'ux' 'uy' 'vv' 'w2' 'wx' 'xi' 'xu' 'xw' 'yj' 'yt' 'yw' 'z3' 'z7' 'zb' - f | '2y' '4v' 'ac' 'at' 'av' 'bd' 'bs' 'c3' 'ca' 'cf' 'cg' 'cq' 'cw' 'cz' 'db' 'dx' 'e7' 'eg' 'ei' 'el' 'et' 'ey' 'fo' 'fq' 'fw' 'fx' 'gd' 'gl' 'gw' 'h6' 'hd' 'hp' 'hy' 'il' 'ir' 'is' 'j5' 'k4' 'k8' 'kc' 'kp' 'kz' 'l0' 'l1' 'm4' 'm6' 'mn' 'mr' 'nx' 'ov' 'ox' 'pn' 'pq' 'q3' 'qd' 'qj' 'qk' 'qo' 'qw' 'qx' 'rd' 're' 'rg' 'rq' 'rr' 'rx' 'rz' 'sb' 't0' 't3' 't4' 'tc' 'tk' 'tp' 'tr' 'tv' 'u7' 'uf' 'um' 'uo' 'uq' 'va' 'vc' 'vi' 'vy' 'vz' 'w1' 'w6' 'wo' 'wq' 'xh' 'xn' 'y5' 'yo' 'ys' 'yt' 'zs' 'zu' - f | '4y' '6h' 'a0' 'a6' 'bo' 'bq' 'co' 'cv' 'dv' 'ec' 'ee' 'eh' 'ei' 'en' 'er' 'ew' 'f1' 'fk' 'fq' 'fy' 'ga' 'gj' 'gp' 'gv' 'gx' 'hv' 'i0' 'i5' 'ij' 'ik' 'in' 'kb' 'ks' 'kw' 'kz' 'la' 'lh' 'lq' 'ls' 'mu' 'nl' 'og' 'oi' 'om' 'pf' 'pu' 'pv' 'q6' 'q9' 'qe' 'qk' 'ql' 'qo' 'qq' 'qu' 'qw' 'r5' 'rb' 'rn' 'rr' 'rt' 's4' 'sc' 'sg' 'si' 'sm' 'sn' 'sp' 'ss' 'tb' 'tc' 'tg' 'tk' 'tm' 'tr' 'tt' 'tu' 'tv' 'u0' 'u3' 'uj' 'un' 'uz' 'vp' 'w6' 'w8' 'wc' 'we' 'wi' 'wt' 'wx' 'xp' 'xz' 'y0' 'y4' 'z0' 'zj' 'zy' - f | '5e' 'a1' 'ak' 'at' 'av' 'bi' 'bj' 'c7' 'dl' 'do' 'e0' 'ee' 'el' 'em' 'es' 'fl' 'fy' 'g2' 'g6' 'g7' 'gr' 'gv' 'h3' 'hg' 'hr' 'ht' 'ii' 'il' 'is' 'j8' 'jm' 'jq' 'jv' 'k0' 'ka' 'kk' 'l3' 'la' 'lh' 'ms' 'mz' 'no' 'oa' 'os' 'ox' 'pa' 'pc' 'pl' 'pq' 'px' 'q1' 'q3' 'q5' 'q8' 'qb' 'qi' 'ql' 'qw' 'qy' 'qz' 're' 'ri' 'rp' 'rx' 's1' 's3' 'sh' 'sq' 't1' 't5' 'ta' 'td' 'tj' 'to' 'tt' 'ub' 'uc' 'ur' 'uv' 'v1' 'vs' 'w7' 'wb' 'wd' 'wj' 'wl' 'wm' 'wq' 'xr' 'xt' 'ym' 'yu' 'yx' 'yz' 'z8' 'zp' 'zr' 'zs' - f | '5o' 'ar' 'at' 'c1' 'ch' 'cr' 'dp' 'dy' 'ed' 'el' 'er' 'f0' 'f7' 'fc' 'fh' 'fm' 'fx' 'g8' 'gd' 'gf' 'gh' 'gl' 'gp' 'gv' 'gy' 'hs' 'ht' 'hv' 'hy' 'i2' 'ia' 'lc' 'lu' 'lv' 'ly' 'mb' 'mq' 'nn' 'nz' 'oa' 'oq' 'ot' 'ox' 'oz' 'p8' 'pc' 'pn' 'py' 'q2' 'q7' 'qe' 'qj' 'qn' 'qs' 'qv' 'qw' 'r1' 'r4' 'rh' 'rn' 'rq' 'rr' 's1' 'sa' 'sd' 'se' 'sr' 't6' 'ta' 'tf' 'ts' 'tw' 'tx' 'uk' 'ul' 'uw' 'v7' 'vu' 'vz' 'w7' 'wc' 'wd' 'wk' 'wl' 'wo' 'wr' 'wv' 'x9' 'xe' 'xj' 'xl' 'xu' 'yi' 'ym' 'yq' 'z1' 'z5' 'zm' - f | '8d' 'a2' 'af' 'ap' 'as' 'cg' 'cr' 'd4' 'dg' 'du' 'dv' 'eg' 'ei' 'el' 'em' 'fg' 'ft' 'g1' 'gt' 'h1' 'hg' 'i5' 'i8' 'ih' 'im' 'ir' 'iw' 'jd' 'jf' 'js' 'jw' 'jz' 'k8' 'ko' 'ky' 'la' 'lr' 'mq' 'no' 'ox' 'p8' 'p9' 'pb' 'pv' 'pw' 'px' 'q0' 'q1' 'q5' 'q7' 'qd' 'qh' 'qj' 'ql' 'qn' 'qo' 'qq' 'qx' 'qy' 'rc' 'rd' 'rf' 'rh' 'rr' 'rs' 'sd' 'sw' 'sz' 't3' 'tf' 'tn' 'to' 'tv' 'tx' 'ty' 'ua' 'ub' 'ud' 'uk' 'ul' 'us' 'uu' 'ux' 'vl' 'vs' 'wf' 'wg' 'wl' 'wq' 'wt' 'wu' 'wx' 'x6' 'xi' 'yg' 'yh' 'z7' 'ze' - f | 'a3' 'ai' 'ap' 'ar' 'at' 'be' 'br' 'bz' 'cw' 'd2' 'd6' 'df' 'dk' 'do' 'dp' 'dr' 'dw' 'e2' 'ef' 'eg' 'ej' 'eq' 'fb' 'fd' 'fq' 'ge' 'gr' 'h0' 'hp' 'i8' 'ih' 'ir' 'jb' 'jd' 'kj' 'kz' 'l4' 'lm' 'lt' 'mj' 'mz' 'nc' 'ni' 'nw' 'of' 'oh' 'ol' 'ot' 'ov' 'p5' 'pf' 'pn' 'pp' 'pv' 'py' 'q2' 'q3' 'q7' 'qb' 'qc' 'qj' 'qo' 'qp' 'qr' 'qs' 'r6' 'ra' 'rc' 'rd' 'rx' 'se' 'sn' 'sr' 'sx' 't2' 't9' 'tm' 'tr' 'uj' 'vh' 'vn' 'vs' 'w0' 'w2' 'w3' 'wd' 'we' 'wj' 'wq' 'wr' 'wv' 'xe' 'xo' 'ya' 'ye' 'yq' 'yt' 'yw' - f | '1h' '4r' 'a9' 'ag' 'ah' 'ak' 'as' 'at' 'b2' 'be' 'cs' 'd6' 'd7' 'da' 'dg' 'dl' 'dr' 'dy' 'dz' 'e8' 'e9' 'eg' 'eh' 'ek' 'eq' 'es' 'f3' 'f9' 'fs' 'gh' 'go' 'gq' 'gz' 'ha' 'hc' 'hi' 'hp' 'if' 'ig' 'im' 'iv' 'jd' 'jt' 'kp' 'kt' 'l6' 'le' 'lg' 'lo' 'mz' 'ns' 'oc' 'ok' 'p6' 'pa' 'pt' 'pv' 'px' 'q1' 'q5' 'qa' 'qe' 'qg' 'qh' 'qi' 'qn' 'qp' 'qs' 'qu' 'qv' 'qy' 'rh' 'rl' 'rp' 'rv' 'rx' 'rz' 's8' 'sc' 'sr' 'su' 't6' 'tb' 'tq' 'tw' 'u9' 'vr' 'w7' 'wb' 'wj' 'wq' 'ww' 'xr' 'xx' 'yb' 'yl' 'yn' 'yr' 'zw' - f | '1r' '2q' 'aa' 'ae' 'ah' 'aq' 'at' 'bx' 'c6' 'ca' 'cl' 'da' 'dp' 'ds' 'e0' 'e6' 'ea' 'eb' 'eg' 'er' 'ev' 'ew' 'fs' 'fw' 'gm' 'gx' 'hl' 'hv' 'hw' 'ib' 'iw' 'jc' 'jn' 'jr' 'ju' 'k0' 'k5' 'k8' 'kf' 'kn' 'l1' 'ln' 'lp' 'ly' 'mz' 'n3' 'nc' 'ng' 'nk' 'o8' 'oe' 'oi' 'ol' 'ot' 'pb' 'pe' 'pk' 'q0' 'qa' 'qe' 'qg' 'qi' 'ql' 'qo' 'qv' 'r0' 'r2' 'r4' 'rg' 'rj' 'rm' 'sv' 'tk' 'to' 'tt' 'u7' 'ud' 'uo' 'ur' 'us' 'uv' 'vf' 'vm' 'vn' 'w6' 'w7' 'wa' 'wb' 'wf' 'wg' 'wi' 'wv' 'ww' 'wx' 'xk' 'xy' 'yg' 'yh' 'ym' - f | '6i' 'a3' 'ac' 'ax' 'br' 'ck' 'dc' 'dj' 'dn' 'dq' 'dt' 'dv' 'eb' 'ed' 'ei' 'en' 'eq' 'ey' 'f4' 'fk' 'fm' 'fp' 'fw' 'gn' 'gw' 'he' 'ho' 'io' 'ja' 'jj' 'jn' 'ju' 'jy' 'kc' 'kk' 'l1' 'l4' 'lt' 'lx' 'm1' 'm7' 'mc' 'o6' 'ol' 'p3' 'pc' 'pe' 'pp' 'pv' 'px' 'q0' 'qb' 'qe' 'qr' 'qs' 'qt' 'qu' 'qv' 'qy' 'rg' 'rq' 's3' 'sm' 'so' 't5' 't9' 'tb' 'tp' 'ty' 'tz' 'u6' 'ua' 'ui' 'uv' 'vm' 'vw' 'w2' 'wa' 'wf' 'wi' 'wj' 'wt' 'ww' 'wz' 'xg' 'y1' 'ya' 'yb' 'yd' 'yg' 'yh' 'yi' 'yk' 'yp' 'yz' 'zb' 'zk' 'zm' 'zo' - f | '0e' 'a0' 'a1' 'a5' 'ad' 'ao' 'bp' 'bw' 'c0' 'ca' 'cw' 'cx' 'cy' 'dg' 'e8' 'eb' 'ef' 'ek' 'fd' 'fk' 'fv' 'g3' 'gj' 'gn' 'gw' 'hr' 'if' 'ij' 'is' 'ix' 'j7' 'km' 'kp' 'kq' 'ks' 'kt' 'kv' 'lk' 'mj' 'mn' 'mv' 'mz' 'nn' 'o5' 'oa' 'ol' 'on' 'os' 'pa' 'pb' 'pf' 'po' 'pq' 'pr' 'pv' 'py' 'q1' 'q6' 'qf' 'qh' 'qj' 'qn' 'qt' 'qu' 'qw' 'rf' 'rk' 'rt' 's5' 'sq' 'ss' 't5' 'ta' 'th' 'tj' 'to' 'ts' 'tx' 'tz' 'u0' 'ua' 'ug' 'ui' 'uw' 'ux' 'vh' 'vp' 'we' 'wg' 'wi' 'wj' 'wp' 'wq' 'xm' 'y6' 'yr' 'yt' 'yu' 'z8' 'zs' - f | '1l' '24' '2o' 'ad' 'am' 'ao' 'bf' 'bl' 'ca' 'ce' 'cn' 'cw' 'cy' 'd4' 'dm' 'e9' 'ed' 'ee' 'ei' 'ep' 'f0' 'fk' 'fu' 'fw' 'g3' 'g4' 'gm' 'gn' 'gu' 'ha' 'hh' 'id' 'iq' 'it' 'iw' 'ix' 'iy' 'j4' 'j6' 'jo' 'jp' 'js' 'jv' 'k6' 'kb' 'ke' 'kh' 'l9' 'lh' 'lr' 'mr' 'n7' 'n8' 'nd' 'o6' 'of' 'oi' 'op' 'ox' 'p6' 'q2' 'qb' 'qd' 'qi' 'qk' 'qn' 'qq' 'qt' 'qy' 'r2' 're' 'rf' 'rk' 's4' 'sb' 'se' 'sh' 't4' 'tb' 'tn' 'tq' 'ty' 'u4' 'u5' 'ub' 'ue' 'uf' 'uk' 'uo' 'vc' 'vs' 'vu' 'w7' 'wd' 'wh' 'xm' 'yk' 'yv' 'zl' 'zz' - f | '1m' '37' 'a8' 'ab' 'at' 'ce' 'cx' 'd6' 'dk' 'e9' 'eh' 'ei' 'el' 'eo' 'eu' 'ew' 'ex' 'f4' 'fh' 'fr' 'fs' 'fy' 'g6' 'gf' 'gl' 'gm' 'gp' 'h8' 'ha' 'hf' 'hi' 'hl' 'hr' 'ie' 'ir' 'j6' 'jd' 'jr' 'jv' 'kz' 'l1' 'lg' 'ln' 'lp' 'ls' 'm0' 'ma' 'mi' 'mo' 'mu' 'na' 'np' 'o8' 'oc' 'oe' 'oh' 'ot' 'oy' 'ps' 'pw' 'q5' 'qa' 'qe' 'qg' 'qh' 'qi' 'qq' 'qz' 'rd' 'rg' 'ru' 'rv' 's9' 'sr' 'sv' 'sy' 'sz' 'ts' 'ty' 'uc' 'uh' 'uj' 'uk' 'up' 'v9' 'vc' 'vg' 'vt' 'vy' 'wl' 'wm' 'wv' 'wy' 'wz' 'xg' 'yg' 'yv' 'yw' 'yy' 'z0' - f | '2s' '3k' '4b' '6s' 'a1' 'b2' 'b9' 'bi' 'bz' 'c0' 'c3' 'ck' 'cl' 'cu' 'de' 'dl' 'do' 'dp' 'e8' 'ec' 'ee' 'ei' 'ek' 'eu' 'ez' 'fi' 'fo' 'fr' 'fu' 'fx' 'gh' 'gk' 'go' 'gq' 'h4' 'hd' 'hm' 'hz' 'ia' 'id' 'iv' 'jb' 'jh' 'km' 'kp' 'l4' 'l7' 'l8' 'ld' 'lj' 'nb' 'o7' 'oc' 'oh' 'os' 'p4' 'p9' 'pd' 'q6' 'q7' 'q8' 'qc' 'qh' 'qj' 'qk' 'qw' 'rg' 'rj' 'rm' 'rn' 'rp' 'ru' 'rz' 's5' 'si' 'sv' 'sy' 'tc' 'tg' 'tk' 'tv' 'ue' 'uf' 'ux' 'uz' 'vl' 'w2' 'w6' 'wf' 'wm' 'wu' 'wx' 'xh' 'ya' 'yc' 'ye' 'z7' 'zi' 'zo' 'zq' - f | '3s' '6q' 'ad' 'am' 'as' 'ay' 'cd' 'ci' 'cl' 'dw' 'eb' 'en' 'ep' 'ew' 'ey' 'fh' 'g6' 'gb' 'gk' 'h5' 'hk' 'hp' 'hy' 'i8' 'ia' 'ic' 'ih' 'in' 'is' 'iw' 'jd' 'je' 'kh' 'kp' 'ks' 'lg' 'lz' 'mx' 'my' 'nf' 'nh' 'no' 'nv' 'o4' 'oa' 'oc' 'op' 'p2' 'p5' 'pa' 'pb' 'pt' 'pw' 'q2' 'q4' 'qb' 'qg' 'qh' 'qq' 'qs' 'qt' 'qx' 'qy' 'qz' 'r7' 'rc' 'rk' 'rt' 'sr' 'sx' 'sy' 'tc' 'tn' 'tr' 'tv' 'u1' 'u7' 'ua' 'uh' 'uz' 'vi' 'wd' 'wr' 'x3' 'x5' 'xa' 'xe' 'y5' 'y7' 'yb' 'yg' 'yk' 'ym' 'yr' 'yt' 'z7' 'zc' 'zf' 'zj' 'zp' - f | '5k' 'a1' 'ab' 'an' 'ar' 'as' 'av' 'ax' 'br' 'c2' 'c7' 'd5' 'da' 'dl' 'dr' 'dz' 'e3' 'ec' 'ed' 'ek' 'el' 'em' 'eu' 'ew' 'fd' 'ff' 'fw' 'g8' 'gb' 'gl' 'hh' 'hs' 'hz' 'i6' 'ia' 'ig' 'ii' 'ik' 'iq' 'ix' 'j0' 'j9' 'jf' 'jl' 'jo' 'jw' 'ko' 'kt' 'lm' 'nl' 'nm' 'ov' 'p3' 'p6' 'p7' 'pg' 'pl' 'pn' 'pp' 'qd' 'qe' 'qf' 'qn' 'qo' 'qq' 'qs' 'qt' 'qv' 'r9' 'rf' 'rj' 'rt' 'rw' 'sa' 'sl' 't2' 'tg' 'tk' 'tq' 'ty' 'ua' 'ud' 'vi' 'vm' 'w2' 'w6' 'wb' 'wd' 'wf' 'wi' 'wl' 'wq' 'wr' 'wy' 'x5' 'x8' 'y0' 'y1' 'ys' 'yx' - f | '6f' 'a1' 'ag' 'ak' 'ap' 'au' 'b1' 'b5' 'bi' 'c1' 'cu' 'd5' 'dc' 'dr' 'dv' 'eg' 'ej' 'ek' 'em' 'et' 'fe' 'fr' 'fz' 'ga' 'gb' 'gk' 'gu' 'gv' 'h5' 'hh' 'ho' 'hy' 'ii' 'ik' 'ip' 'iv' 'ja' 'jg' 'jz' 'k0' 'kt' 'm6' 'mj' 'nd' 'o0' 'o1' 'oj' 'or' 'ot' 'ov' 'oz' 'ph' 'pm' 'pv' 'qa' 'qe' 'qf' 'qg' 'qh' 'qj' 'qn' 'qt' 'qy' 'r5' 'rd' 'rg' 'rs' 'ru' 'rz' 's7' 'sf' 'si' 'sl' 'sw' 'sy' 't0' 't3' 'th' 'tn' 'tq' 'tu' 'ub' 'us' 'ux' 'vb' 'vo' 'wa' 'wd' 'wn' 'wq' 'wt' 'ww' 'wx' 'y1' 'yh' 'yo' 'yq' 'yz' 'z7' 'ze' - f | 'an' 'as' 'cm' 'dh' 'dk' 'do' 'ds' 'dv' 'e1' 'eh' 'ek' 'el' 'er' 'ew' 'ff' 'fo' 'fq' 'g2' 'gz' 'h0' 'hi' 'hk' 'hm' 'hs' 'i4' 'ij' 'iy' 'j5' 'jj' 'jk' 'jw' 'kq' 'kx' 'ky' 'lo' 'lp' 'lw' 'ly' 'm2' 'm6' 'md' 'mv' 'ng' 'of' 'om' 'oy' 'p7' 'pr' 'q7' 'qc' 'ql' 'qq' 'qs' 'qy' 'qz' 'r2' 'r9' 'ra' 'rn' 'ro' 'rq' 'rv' 'rx' 'ry' 'rz' 's0' 'si' 'sy' 'sz' 't6' 't7' 'to' 'tt' 'tv' 'tx' 'ub' 'ur' 'uv' 've' 'vj' 'w3' 'w8' 'w9' 'wa' 'wb' 'wc' 'we' 'wm' 'wt' 'wu' 'wx' 'wy' 'y9' 'yd' 'yg' 'ym' 'yp' 'yt' 'yu' 'yy' - f | '1h' '1j' '1o' '2d' 'ac' 'ak' 'b6' 'cx' 'd3' 'ds' 'dv' 'e7' 'es' 'ev' 'ez' 'fs' 'fx' 'fy' 'gc' 'gh' 'gs' 'gt' 'hk' 'ho' 'i8' 'id' 'ig' 'ii' 'iq' 'j5' 'jd' 'je' 'jg' 'jk' 'jl' 'jo' 'jr' 'kf' 'kl' 'kq' 'lf' 'lh' 'lz' 'mb' 'mu' 'ni' 'nz' 'o4' 'ol' 'p7' 'pl' 'pm' 'pv' 'q5' 'qa' 'qe' 'qh' 'qi' 'ql' 'qq' 'qw' 'ra' 'rb' 'rn' 's5' 'sf' 'sh' 'so' 'sp' 'sq' 'sw' 'tj' 'tu' 'ty' 'u3' 'ue' 'uo' 'ux' 'v8' 'vo' 'vr' 'vz' 'w4' 'w8' 'wa' 'wk' 'wl' 'wo' 'wr' 'wu' 'ww' 'xs' 'xx' 'yb' 'yf' 'yn' 'yt' 'yw' 'zf' 'zg' 'zs' - f | '1l' '2l' 'ag' 'aj' 'al' 'an' 'ay' 'bo' 'cg' 'cs' 'cw' 'cx' 'db' 'dd' 'e1' 'e8' 'ee' 'ef' 'ej' 'eu' 'ew' 'fc' 'ff' 'fh' 'fo' 'fv' 'fz' 'g1' 'gj' 'gk' 'gw' 'ha' 'i3' 'ib' 'iu' 'jv' 'k9' 'kq' 'ku' 'kw' 'lv' 'lw' 'ly' 'mp' 'my' 'nf' 'ng' 'nw' 'o6' 'oe' 'or' 'ou' 'oz' 'p4' 'pc' 'ph' 'pn' 'py' 'q0' 'qd' 'qe' 'qp' 'qs' 'qu' 'qv' 'qz' 'r6' 'rg' 'rj' 'rt' 'ru' 'rv' 'rw' 'sx' 't2' 'tl' 'tn' 'tq' 'tx' 'tz' 'ue' 'ui' 'ul' 'uo' 'uu' 'w0' 'w1' 'w4' 'w6' 'w8' 'w9' 'wh' 'wi' 'wp' 'ws' 'wt' 'wx' 'yd' 'zd' 'zf' 'zx' - f | '1x' '3m' 'a1' 'a5' 'a7' 'al' 'aq' 'ar' 'c8' 'cc' 'cd' 'd0' 'dt' 'e6' 'ei' 'em' 'ez' 'f6' 'fc' 'ff' 'fg' 'fp' 'gm' 'h3' 'ha' 'hg' 'ho' 'hs' 'ib' 'ie' 'il' 'it' 'ix' 'j3' 'jl' 'jt' 'jv' 'jw' 'kd' 'ki' 'l1' 'lk' 'lr' 'lv' 'm0' 'mx' 'my' 'n6' 'ni' 'o2' 'o4' 'o9' 'om' 'p0' 'p6' 'pi' 'pj' 'q5' 'qa' 'qc' 'qg' 'qp' 'qs' 'qu' 'qw' 'rg' 'rt' 'rz' 'se' 'sj' 'sp' 'su' 'sw' 'te' 'tp' 'tq' 'tt' 'tu' 'tx' 'u3' 'un' 'uo' 'up' 'uq' 'vc' 'vj' 'vu' 'w2' 'wh' 'wj' 'ws' 'wv' 'ww' 'xg' 'yc' 'yf' 'yv' 'z2' 'zl' 'zw' 'zy' - f | '5q' '7c' 'a1' 'ac' 'ao' 'az' 'bw' 'cz' 'd8' 'dr' 'dy' 'ec' 'eg' 'ej' 'el' 'es' 'et' 'fa' 'fh' 'g2' 'gm' 'go' 'gp' 'gy' 'h6' 'hd' 'he' 'ho' 'ij' 'ik' 'it' 'jc' 'ji' 'k8' 'ku' 'l8' 'lf' 'mw' 'n8' 'nb' 'nd' 'o3' 'ok' 'ol' 'ot' 'p8' 'pi' 'pl' 'pn' 'ps' 'qd' 'qe' 'qk' 'ql' 'qt' 'qz' 'r0' 'r7' 'rd' 'rg' 'rh' 'rm' 'ro' 'rp' 'ru' 'rz' 'sj' 'sm' 'sr' 'su' 'sv' 'sx' 't1' 'tc' 'tl' 'tq' 'uj' 'um' 'uv' 'uw' 'wa' 'wb' 'wi' 'wo' 'wr' 'ws' 'wv' 'wz' 'xb' 'xg' 'ya' 'yb' 'yh' 'yi' 'ym' 'yn' 'yr' 'yv' 'z6' 'zf' 'zz' - f | 'ab' 'ad' 'al' 'cm' 'cw' 'dh' 'dn' 'dy' 'e8' 'ea' 'ed' 'ej' 'eq' 'er' 'eu' 'ev' 'f5' 'fb' 'fj' 'fr' 'gc' 'gp' 'h2' 'h6' 'hc' 'hw' 'i3' 'i5' 'id' 'ir' 'ix' 'ka' 'kj' 'ko' 'lv' 'lx' 'mj' 'mv' 'nq' 'ns' 'oa' 'od' 'ok' 'om' 'os' 'p1' 'p5' 'pb' 'pg' 'pj' 'pl' 'ps' 'pu' 'q1' 'qb' 'qd' 'qe' 'qj' 'qk' 'ql' 'qr' 'qs' 'rd' 'rl' 'ro' 'rs' 'rx' 's1' 'sf' 'sg' 't0' 'tb' 'te' 'tf' 'tg' 'tl' 'tp' 'tq' 'u3' 'uf' 'ug' 'up' 'uq' 'vx' 'vy' 'w5' 'wa' 'wc' 'wr' 'ws' 'wu' 'wx' 'wy' 'wz' 'yf' 'ym' 'yz' 'ze' 'zs' 'zy' 'zz' - f | '13' '6k' 'ae' 'ah' 'an' 'at' 'aw' 'cc' 'cn' 'du' 'dx' 'ek' 'el' 'em' 'en' 'es' 'ey' 'f2' 'fe' 'fh' 'fk' 'fv' 'g6' 'g9' 'gw' 'hj' 'hn' 'hs' 'hu' 'hv' 'ia' 'im' 'iq' 'ix' 'jh' 'jm' 'ki' 'lj' 'lo' 'lt' 'lw' 'm1' 'nv' 'nw' 'ny' 'oj' 'oo' 'ot' 'ov' 'ox' 'pd' 'pv' 'q1' 'q7' 'qc' 'qd' 'qj' 'ql' 'qp' 'qq' 'qv' 'qw' 'qx' 'r7' 'ry' 'so' 'st' 'su' 'sx' 't9' 'tf' 'th' 'ty' 'uh' 'uo' 'ut' 'uv' 'uy' 'vj' 'vp' 'vu' 'vw' 'w3' 'w5' 'w8' 'wc' 'we' 'wg' 'wi' 'wl' 'wv' 'xh' 'xx' 'yb' 'yd' 'yj' 'yn' 'yo' 'yr' 'yy' 'zk' 'zx' - f | '4m' 'a3' 'ac' 'ag' 'ai' 'au' 'av' 'bd' 'bu' 'cl' 'd0' 'dc' 'dy' 'ej' 'el' 'eo' 'es' 'eu' 'ex' 'fr' 'fx' 'g6' 'g9' 'gk' 'gq' 'gr' 'gw' 'gx' 'gz' 'hv' 'ii' 'ix' 'jj' 'jy' 'k0' 'kl' 'kn' 'kw' 'l5' 'lo' 'lz' 'm1' 'n5' 'n9' 'nf' 'nm' 'nz' 'ol' 'os' 'ot' 'oy' 'p1' 'p9' 'pa' 'pj' 'pl' 'pp' 'qc' 'qh' 'qj' 'qr' 'qs' 'qw' 'qz' 'rg' 'ri' 'rl' 'rn' 'rp' 'rt' 'rx' 'rz' 'se' 'sl' 'sv' 'sy' 't8' 'td' 'th' 'tr' 'tx' 'u3' 'u7' 'uc' 'ud' 'ul' 'un' 'up' 'uv' 've' 'vz' 'w7' 'wr' 'y0' 'y6' 'yh' 'yo' 'yu' 'yy' 'yz' 'z6' 'zm' - f | '6p' 'an' 'aq' 'au' 'br' 'bz' 'c3' 'ca' 'cg' 'cn' 'db' 'dk' 'dq' 'e0' 'e8' 'ek' 'eo' 'er' 'ez' 'fd' 'ft' 'g0' 'gd' 'gh' 'gk' 'gn' 'gr' 'gv' 'gy' 'hb' 'hc' 'he' 'ht' 'ii' 'ip' 'iu' 'j9' 'jn' 'jo' 'jq' 'jz' 'kh' 'kn' 'ko' 'l3' 'ls' 'lz' 'nh' 'nk' 'ok' 'oy' 'p7' 'pd' 'ph' 'pu' 'pw' 'py' 'q7' 'qa' 'qi' 'qk' 'ql' 'qn' 'qq' 'qr' 'qs' 'qv' 'qx' 'qy' 'rg' 'rs' 'ru' 'sg' 'sl' 'sq' 'sr' 'su' 'ts' 'tt' 'ui' 'um' 'ut' 'uu' 'v0' 'v5' 'w7' 'wb' 'wc' 'wf' 'wi' 'wm' 'xd' 'xs' 'xz' 'y3' 'y4' 'y9' 'yi' 'yp' 'yx' 'z0' 'zf' - f | '1f' '3z' '43' 'am' 'b9' 'bg' 'cc' 'ct' 'cx' 'd0' 'de' 'dm' 'dr' 'e0' 'e8' 'ef' 'en' 'ev' 'fd' 'ff' 'fx' 'ga' 'gm' 'gp' 'gr' 'gs' 'gy' 'h1' 'hi' 'hl' 'hs' 'i4' 'i5' 'ic' 'jb' 'jj' 'kj' 'lh' 'ng' 'ni' 'nn' 'ns' 'nx' 'o1' 'oa' 'oe' 'og' 'p4' 'pd' 'ph' 'pj' 'pl' 'pw' 'q9' 'qa' 'qd' 'qh' 'ql' 'qr' 'qs' 'qw' 'qx' 'ro' 'rs' 'rw' 'ry' 'rz' 'sb' 'sj' 'sm' 'so' 't0' 't6' 'tc' 'ti' 'tk' 'tn' 'uk' 'um' 'uw' 'uy' 'v9' 'vd' 'vg' 'w3' 'wf' 'wg' 'wi' 'wk' 'wm' 'wx' 'xe' 'xm' 'xn' 'y5' 'ye' 'yk' 'yq' 'z3' 'zj' 'zk' 'zt' 'zz' - f | '1t' 'ai' 'cu' 'cw' 'cx' 'dd' 'de' 'ds' 'e0' 'e2' 'e6' 'eb' 'ei' 'eq' 'eu' 'ev' 'ez' 'f5' 'f8' 'fc' 'g2' 'gd' 'gs' 'gv' 'hu' 'hy' 'id' 'ig' 'ij' 'ir' 'iv' 'ju' 'ka' 'kj' 'kl' 'ks' 'kv' 'kw' 'kx' 'la' 'lh' 'lm' 'ls' 'lv' 'lz' 'mg' 'mh' 'mp' 'ns' 'nt' 'nu' 'nx' 'o6' 'o9' 'oc' 'oj' 'pa' 'pj' 'pl' 'pv' 'q3' 'q5' 'qb' 'qh' 'ql' 'qn' 'qr' 'qs' 'qz' 'rb' 'rf' 'rh' 'rm' 'sm' 'sn' 'sw' 't5' 't9' 'tq' 'ty' 'uj' 'v2' 'vz' 'w6' 'wc' 'wg' 'wi' 'wj' 'wk' 'wm' 'wn' 'wt' 'wv' 'ww' 'wz' 'x0' 'xb' 'xc' 'yb' 'yt' 'yv' 'zp' 'zy' - f | 'a3' 'ad' 'ar' 'at' 'bb' 'bf' 'bt' 'cg' 'cx' 'd6' 'de' 'df' 'e4' 'e6' 'eg' 'et' 'ex' 'f0' 'fe' 'fg' 'fj' 'fo' 'gh' 'hb' 'hj' 'hq' 'hr' 'hv' 'ia' 'id' 'is' 'it' 'iy' 'ja' 'jj' 'jq' 'jw' 'l2' 'l7' 'lc' 'lu' 'nc' 'no' 'np' 'nt' 'ob' 'od' 'og' 'oo' 'os' 'pe' 'pj' 'pl' 'po' 'pq' 'pu' 'q2' 'qa' 'qc' 'qf' 'qr' 'qt' 'qx' 're' 'rk' 'rn' 'ro' 'ru' 'rw' 'rx' 's7' 's8' 'sy' 'tc' 'tf' 'tg' 'th' 'tm' 'to' 'tv' 'tx' 'tz' 'va' 'vp' 'w0' 'w1' 'wh' 'wl' 'wq' 'wr' 'wt' 'ww' 'wy' 'x7' 'xl' 'xy' 'y0' 'ye' 'yr' 'yy' 'z3' 'zf' 'zo' - f | 'ad' 'ar' 'az' 'b7' 'cf' 'cm' 'ct' 'cw' 'cy' 'dh' 'dn' 'ds' 'ef' 'en' 'eo' 'er' 'fh' 'fi' 'fq' 'fr' 'fx' 'gp' 'gq' 'gu' 'gx' 'h8' 'hf' 'hj' 'hk' 'ho' 'hw' 'hy' 'i3' 'i4' 'ik' 'iu' 'iy' 'jj' 'kn' 'l8' 'lb' 'lg' 'lo' 'm7' 'mx' 'nj' 'nt' 'o6' 'ob' 'oh' 'ok' 'ot' 'pr' 'pz' 'q7' 'q8' 'qa' 'qd' 'qg' 'qh' 'qi' 'qn' 'qz' 'r7' 're' 'ri' 'rk' 'ry' 's4' 'sa' 'sd' 'sm' 'sn' 'sp' 'sw' 'sy' 'tf' 'th' 'to' 'tr' 'tv' 'tz' 'u0' 'u1' 'u5' 'ue' 'uk' 'uq' 'vb' 'vp' 'vr' 'vu' 'wa' 'wb' 'wg' 'wi' 'wk' 'wm' 'wt' 'ya' 'yj' 'yl' 'z3' - f | '12' 'ac' 'ay' 'bc' 'bj' 'bm' 'bo' 'ce' 'cf' 'ck' 'cr' 'db' 'do' 'du' 'dy' 'ea' 'ej' 'ek' 'eo' 'ep' 'et' 'f2' 'fc' 'fl' 'fs' 'fy' 'g9' 'gf' 'hj' 'hk' 'hp' 'i1' 'i5' 'ih' 'ii' 'im' 'jp' 'jx' 'k9' 'kf' 'ky' 'mb' 'mj' 'mk' 'nb' 'nc' 'o0' 'o9' 'oc' 'oj' 'oq' 'pm' 'ps' 'pt' 'pu' 'pv' 'py' 'qd' 'qi' 'qj' 'qk' 'ql' 'qm' 'qs' 'qz' 'ra' 'rc' 'rd' 'rh' 'ri' 'ro' 's0' 's3' 's7' 's9' 'sv' 't2' 't7' 'tm' 'tp' 'tw' 'tx' 'ty' 'u3' 'u6' 'uf' 'ug' 'ui' 'uk' 'ut' 'ux' 'v5' 'w3' 'wk' 'wt' 'xo' 'xq' 'xr' 'xu' 'ye' 'yl' 'yn' 'yt' 'yv' 'zm' - f | '1o' '7o' 'a1' 'aq' 'b4' 'bi' 'bp' 'c9' 'cb' 'cd' 'cf' 'co' 'cp' 'd3' 'd8' 'ds' 'dv' 'dy' 'eb' 'ee' 'ef' 'eh' 'em' 'eo' 'es' 'ez' 'fi' 'fk' 'fl' 'gj' 'gn' 'gt' 'h1' 'hn' 'ho' 'hp' 'hu' 'ic' 'ik' 'il' 'im' 'ir' 'iw' 'ji' 'jp' 'k4' 'ks' 'm0' 'ml' 'n0' 'ns' 'oj' 'on' 'or' 'os' 'ov' 'pr' 'pv' 'px' 'py' 'q6' 'qa' 'qc' 'qg' 'qj' 'qm' 'qn' 'qp' 'qq' 'qs' 'qv' 'qz' 'r0' 'ra' 's0' 'te' 'tg' 'tm' 'to' 'tw' 'u1' 'u7' 'ug' 'um' 'uw' 'ux' 'uz' 'vm' 'vu' 'w4' 'wg' 'wh' 'wj' 'wk' 'wm' 'wo' 'wq' 'ws' 'wt' 'wz' 'x0' 'x5' 'yg' 'yn' 'z6' - f | '2l' '9f' 'a2' 'ak' 'as' 'av' 'bi' 'c1' 'cf' 'ct' 'cu' 'db' 'dp' 'du' 'dv' 'e3' 'e9' 'ee' 'eg' 'el' 'fm' 'gu' 'hc' 'he' 'i3' 'i9' 'ib' 'ik' 'im' 'ir' 'j7' 'jp' 'jv' 'ki' 'kl' 'l6' 'lj' 'lo' 'lw' 'md' 'mj' 'mk' 'ms' 'mw' 'na' 'nq' 'o2' 'od' 'ox' 'p3' 'pj' 'pp' 'q6' 'q7' 'q9' 'qb' 'qg' 'qr' 'qs' 'r7' 'r8' 'rd' 're' 'rf' 'rg' 'rp' 'rv' 'rw' 's6' 'sc' 'sq' 't6' 'tb' 'tc' 'te' 'tj' 'tn' 'tx' 'tz' 'uh' 'uq' 'uu' 'v4' 'vw' 'w3' 'w8' 'wa' 'wj' 'wk' 'wp' 'x3' 'xg' 'xy' 'y1' 'y6' 'yc' 'yi' 'yn' 'yo' 'yw' 'yz' 'z6' 'z8' 'zk' 'zz' - f | '80' 'ak' 'al' 'an' 'aq' 'as' 'at' 'bg' 'bn' 'bq' 'bx' 'd4' 'db' 'dg' 'dq' 'dv' 'ef' 'ej' 'en' 'eu' 'ex' 'f9' 'fu' 'g1' 'g2' 'g3' 'gj' 'gn' 'gs' 'gt' 'gx' 'h4' 'h8' 'hb' 'hn' 'hx' 'hy' 'i5' 'im' 'ji' 'jo' 'jy' 'k1' 'kb' 'kh' 'kp' 'kv' 'l4' 'lf' 'lt' 'ml' 'mv' 'na' 'ny' 'o2' 'oj' 'oq' 'or' 'os' 'pp' 'q3' 'q8' 'qc' 'qf' 'qp' 'qt' 'qv' 'rc' 'rg' 'rm' 's4' 'sa' 'sk' 'sp' 'su' 'sv' 'sy' 't5' 'te' 'tm' 'tn' 'to' 'tt' 'tw' 'tx' 'tz' 'ue' 'ug' 'ul' 'un' 'uq' 'us' 'v3' 'wd' 'wr' 'wu' 'wy' 'x3' 'x8' 'xk' 'yf' 'yj' 'yk' 'yp' 'z0' - f | '1b' '42' 'a7' 'ab' 'ak' 'ap' 'at' 'av' 'ay' 'b0' 'b9' 'bb' 'bp' 'bu' 'bz' 'cq' 'da' 'de' 'dn' 'e0' 'eb' 'ef' 'eg' 'ek' 'eq' 'er' 'eu' 'ey' 'fn' 'ft' 'gg' 'h4' 'hk' 'hl' 'i7' 'ig' 'ik' 'ip' 'ir' 'iu' 'iw' 'jr' 'jw' 'jx' 'kg' 'lc' 'lg' 'm0' 'na' 'np' 'om' 'on' 'oz' 'pg' 'pn' 'ps' 'pt' 'pz' 'q3' 'q6' 'qa' 'qb' 'ql' 'qq' 'qt' 'qv' 'qw' 'qy' 'r8' 'rf' 'ri' 'rk' 'rl' 'rw' 'sg' 'si' 'sp' 'sw' 'ta' 'th' 'ua' 'uj' 'uu' 'uv' 'uz' 'vj' 'vk' 'vm' 'wc' 'wf' 'wh' 'wn' 'wo' 'ww' 'xb' 'xk' 'xt' 'xw' 'y7' 'ye' 'yl' 'yt' 'yw' 'z4' 'z7' 'zc' 'zw' - f | '1h' '3s' 'ab' 'ae' 'ax' 'b1' 'bz' 'cy' 'dk' 'dq' 'ds' 'du' 'e8' 'ef' 'ej' 'ek' 'ex' 'f1' 'fe' 'ff' 'fn' 'fo' 'ft' 'fx' 'ge' 'go' 'gz' 'h6' 'hz' 'i2' 'iv' 'iy' 'j5' 'j6' 'ke' 'kf' 'lh' 'lr' 'mc' 'mj' 'na' 'ng' 'oh' 'om' 'oy' 'p2' 'pi' 'pk' 'py' 'q3' 'qb' 'qc' 'qg' 'qn' 'qo' 'qq' 'qu' 'qw' 'qx' 'qy' 'qz' 'r1' 'rk' 'rl' 'rq' 'rs' 'rt' 'ry' 'rz' 'sk' 'sl' 'so' 't9' 'td' 'te' 'tn' 'tw' 'tz' 'ud' 'uk' 'uo' 'uq' 'uw' 'ux' 'uy' 'v1' 'vg' 'vq' 'w4' 'w9' 'wa' 'wg' 'wj' 'wm' 'wo' 'wr' 'ww' 'wy' 'xf' 'xg' 'y9' 'yh' 'yi' 'yk' 'ym' 'yq' 'yv' 'zm' - t | -(514 rows) - -drop index wowidx; -create index wowidx on test_tsvector using gin (a); -set enable_seqscan=off; -SELECT count(*) FROM test_tsvector WHERE a @@ 'wr|qh'; - count -------- - 158 -(1 row) - -SELECT count(*) FROM test_tsvector WHERE a @@ 'wr&qh'; - count -------- - 17 -(1 row) - -SELECT count(*) FROM test_tsvector WHERE a @@ 'eq&yt'; - count -------- - 6 -(1 row) - -SELECT count(*) FROM test_tsvector WHERE a @@ 'eq|yt'; - count -------- - 98 -(1 row) - -SELECT count(*) FROM test_tsvector WHERE a @@ '(eq&yt)|(wr&qh)'; - count -------- - 23 -(1 row) - -SELECT count(*) FROM test_tsvector WHERE a @@ '(eq|yt)&(wr|qh)'; - count -------- - 39 -(1 row) - diff --git a/contrib/tsearch2/sql/tsearch2.sql b/contrib/tsearch2/sql/tsearch2.sql deleted file mode 100644 index 2c37c4a1dd..0000000000 --- a/contrib/tsearch2/sql/tsearch2.sql +++ /dev/null @@ -1,339 +0,0 @@ -CREATE EXTENSION tsearch2; - --- Check whether any of our opclasses fail amvalidate -SELECT amname, opcname -FROM pg_opclass opc LEFT JOIN pg_am am ON am.oid = opcmethod -WHERE opc.oid >= 16384 AND NOT amvalidate(opc.oid); - ---tsvector -SELECT '1'::tsvector; -SELECT '1 '::tsvector; -SELECT ' 1'::tsvector; -SELECT ' 1 '::tsvector; -SELECT '1 2'::tsvector; -SELECT '''1 2'''::tsvector; -SELECT E'''1 \\''2'''::tsvector; -SELECT E'''1 \\''2''3'::tsvector; -SELECT E'''1 \\''2'' 3'::tsvector; -SELECT E'''1 \\''2'' '' 3'' 4 '::tsvector; -select '''w'':4A,3B,2C,1D,5 a:8'; -select 'a:3A b:2a'::tsvector || 'ba:1234 a:1B'; -select setweight('w:12B w:13* w:12,5,6 a:1,3* a:3 w asd:1dc asd zxc:81,567,222A'::tsvector, 'c'); -select strip('w:12B w:13* w:12,5,6 a:1,3* a:3 w asd:1dc asd'::tsvector); - - ---tsquery -SELECT '1'::tsquery; -SELECT '1 '::tsquery; -SELECT ' 1'::tsquery; -SELECT ' 1 '::tsquery; -SELECT '''1 2'''::tsquery; -SELECT E'''1 \\''2'''::tsquery; -SELECT '!1'::tsquery; -SELECT '1|2'::tsquery; -SELECT '1|!2'::tsquery; -SELECT '!1|2'::tsquery; -SELECT '!1|!2'::tsquery; -SELECT '!(!1|!2)'::tsquery; -SELECT '!(!1|2)'::tsquery; -SELECT '!(1|!2)'::tsquery; -SELECT '!(1|2)'::tsquery; -SELECT '1&2'::tsquery; -SELECT '!1&2'::tsquery; -SELECT '1&!2'::tsquery; -SELECT '!1&!2'::tsquery; -SELECT '(1&2)'::tsquery; -SELECT '1&(2)'::tsquery; -SELECT '!(1)&2'::tsquery; -SELECT '!(1&2)'::tsquery; -SELECT '1|2&3'::tsquery; -SELECT '1|(2&3)'::tsquery; -SELECT '(1|2)&3'::tsquery; -SELECT '1|2&!3'::tsquery; -SELECT '1|!2&3'::tsquery; -SELECT '!1|2&3'::tsquery; -SELECT '!1|(2&3)'::tsquery; -SELECT '!(1|2)&3'::tsquery; -SELECT '(!1|2)&3'::tsquery; -SELECT '1|(2|(4|(5|6)))'::tsquery; -SELECT '1|2|4|5|6'::tsquery; -SELECT '1&(2&(4&(5&6)))'::tsquery; -SELECT '1&2&4&5&6'::tsquery; -SELECT '1&(2&(4&(5|6)))'::tsquery; -SELECT '1&(2&(4&(5|!6)))'::tsquery; -SELECT E'1&(''2''&('' 4''&(\\|5 | ''6 \\'' !|&'')))'::tsquery; -SELECT '''the wether'':dc & '' sKies '':BC & a:d b:a'; - -select 'a' < 'b & c'::tsquery; -select 'a' > 'b & c'::tsquery; -select 'a | f' < 'b & c'::tsquery; -select 'a | ff' < 'b & c'::tsquery; -select 'a | f | g' < 'b & c'::tsquery; - -select numnode( 'new'::tsquery ); -select numnode( 'new & york'::tsquery ); -select numnode( 'new & york | qwery'::tsquery ); - -create table test_tsquery (txtkeyword text, txtsample text); -\set ECHO none -\copy test_tsquery from stdin -'New York' new & york | big & apple | nyc -Moscow moskva | moscow -'Sanct Peter' Peterburg | peter | 'Sanct Peterburg' -'foo bar qq' foo & (bar | qq) & city -\. -\set ECHO all - -alter table test_tsquery add column keyword tsquery; -update test_tsquery set keyword = to_tsquery('english', txtkeyword); -alter table test_tsquery add column sample tsquery; -update test_tsquery set sample = to_tsquery('english', txtsample::text); - -create unique index bt_tsq on test_tsquery (keyword); - -select count(*) from test_tsquery where keyword < 'new & york'; -select count(*) from test_tsquery where keyword <= 'new & york'; -select count(*) from test_tsquery where keyword = 'new & york'; -select count(*) from test_tsquery where keyword >= 'new & york'; -select count(*) from test_tsquery where keyword > 'new & york'; - -set enable_seqscan=off; - -select count(*) from test_tsquery where keyword < 'new & york'; -select count(*) from test_tsquery where keyword <= 'new & york'; -select count(*) from test_tsquery where keyword = 'new & york'; -select count(*) from test_tsquery where keyword >= 'new & york'; -select count(*) from test_tsquery where keyword > 'new & york'; - -set enable_seqscan=on; - -select rewrite('foo & bar & qq & new & york', 'new & york'::tsquery, 'big & apple | nyc | new & york & city'); - -select rewrite('moscow', 'select keyword, sample from test_tsquery'::text ); -select rewrite('moscow & hotel', 'select keyword, sample from test_tsquery'::text ); -select rewrite('bar & new & qq & foo & york', 'select keyword, sample from test_tsquery'::text ); - -select rewrite( ARRAY['moscow', keyword, sample] ) from test_tsquery; -select rewrite( ARRAY['moscow & hotel', keyword, sample] ) from test_tsquery; -select rewrite( ARRAY['bar & new & qq & foo & york', keyword, sample] ) from test_tsquery; - - -select keyword from test_tsquery where keyword @> 'new'; -select keyword from test_tsquery where keyword @> 'moscow'; -select keyword from test_tsquery where keyword <@ 'new'; -select keyword from test_tsquery where keyword <@ 'moscow'; -select rewrite( ARRAY[query, keyword, sample] ) from test_tsquery, to_tsquery('english', 'moscow') as query where keyword <@ query; -select rewrite( ARRAY[query, keyword, sample] ) from test_tsquery, to_tsquery('english', 'moscow & hotel') as query where keyword <@ query; -select rewrite( ARRAY[query, keyword, sample] ) from test_tsquery, to_tsquery('english', 'bar & new & qq & foo & york') as query where keyword <@ query; -select rewrite( ARRAY[query, keyword, sample] ) from test_tsquery, to_tsquery('english', 'moscow') as query where query @> keyword; -select rewrite( ARRAY[query, keyword, sample] ) from test_tsquery, to_tsquery('english', 'moscow & hotel') as query where query @> keyword; -select rewrite( ARRAY[query, keyword, sample] ) from test_tsquery, to_tsquery('english', 'bar & new & qq & foo & york') as query where query @> keyword; - -create index qq on test_tsquery using gist (keyword gist_tp_tsquery_ops); -set enable_seqscan='off'; - -select keyword from test_tsquery where keyword @> 'new'; -select keyword from test_tsquery where keyword @> 'moscow'; -select keyword from test_tsquery where keyword <@ 'new'; -select keyword from test_tsquery where keyword <@ 'moscow'; -select rewrite( ARRAY[query, keyword, sample] ) from test_tsquery, to_tsquery('english', 'moscow') as query where keyword <@ query; -select rewrite( ARRAY[query, keyword, sample] ) from test_tsquery, to_tsquery('english', 'moscow & hotel') as query where keyword <@ query; -select rewrite( ARRAY[query, keyword, sample] ) from test_tsquery, to_tsquery('english', 'bar & new & qq & foo & york') as query where keyword <@ query; -select rewrite( ARRAY[query, keyword, sample] ) from test_tsquery, to_tsquery('english', 'moscow') as query where query @> keyword; -select rewrite( ARRAY[query, keyword, sample] ) from test_tsquery, to_tsquery('english', 'moscow & hotel') as query where query @> keyword; -select rewrite( ARRAY[query, keyword, sample] ) from test_tsquery, to_tsquery('english', 'bar & new & qq & foo & york') as query where query @> keyword; -set enable_seqscan='on'; - - - -select lexize('simple', 'ASD56 hsdkf'); -select lexize('english_stem', 'SKIES Problems identity'); - -select * from token_type('default'); -select * from parse('default', '345 qwe@efd.r '' http://www.com/ http://aew.werc.ewr/?ad=qwe&dw 1aew.werc.ewr/?ad=qwe&dw 2aew.werc.ewr http://3aew.werc.ewr/?ad=qwe&dw http://4aew.werc.ewr http://5aew.werc.ewr:8100/? ad=qwe&dw 6aew.werc.ewr:8100/?ad=qwe&dw 7aew.werc.ewr:8100/?ad=qwe&dw=%20%32 +4.0e-10 qwe qwe qwqwe 234.435 455 5.005 teodor@stack.net qwe-wer asdf qwer jf sdjk ewr1> ewri2 -/usr/local/fff /awdf/dwqe/4325 rewt/ewr wefjn /wqe-324/ewr gist.h gist.h.c gist.c. readline 4.2 4.2. 4.2, readline-4.2 readline-4.2. 234 - wow < jqw <> qwerty'); - -SELECT to_tsvector('english', '345 qwe@efd.r '' http://www.com/ http://aew.werc.ewr/?ad=qwe&dw 1aew.werc.ewr/?ad=qwe&dw 2aew.werc.ewr http://3aew.werc.ewr/?ad=qwe&dw http://4aew.werc.ewr http://5aew.werc.ewr:8100/? ad=qwe&dw 6aew.werc.ewr:8100/?ad=qwe&dw 7aew.werc.ewr:8100/?ad=qwe&dw=%20%32 +4.0e-10 qwe qwe qwqwe 234.435 455 5.005 teodor@stack.net qwe-wer asdf qwer jf sdjk ewr1> ewri2 -/usr/local/fff /awdf/dwqe/4325 rewt/ewr wefjn /wqe-324/ewr gist.h gist.h.c gist.c. readline 4.2 4.2. 4.2, readline-4.2 readline-4.2. 234 - wow < jqw <> qwerty'); - -SELECT length(to_tsvector('english', '345 qw')); - -SELECT length(to_tsvector('english', '345 qwe@efd.r '' http://www.com/ http://aew.werc.ewr/?ad=qwe&dw 1aew.werc.ewr/?ad=qwe&dw 2aew.werc.ewr http://3aew.werc.ewr/?ad=qwe&dw http://4aew.werc.ewr http://5aew.werc.ewr:8100/? ad=qwe&dw 6aew.werc.ewr:8100/?ad=qwe&dw 7aew.werc.ewr:8100/?ad=qwe&dw=%20%32 +4.0e-10 qwe qwe qwqwe 234.435 455 5.005 teodor@stack.net qwe-wer asdf qwer jf sdjk ewr1> ewri2 -/usr/local/fff /awdf/dwqe/4325 rewt/ewr wefjn /wqe-324/ewr gist.h gist.h.c gist.c. readline 4.2 4.2. 4.2, readline-4.2 readline-4.2. 234 - wow < jqw <> qwerty')); - - -select to_tsquery('english', 'qwe & sKies '); -select to_tsquery('simple', 'qwe & sKies '); -select to_tsquery('english', '''the wether'':dc & '' sKies '':BC '); -select to_tsquery('english', 'asd&(and|fghj)'); -select to_tsquery('english', '(asd&and)|fghj'); -select to_tsquery('english', '(asd&!and)|fghj'); -select to_tsquery('english', '(the|and&(i&1))&fghj'); - -select plainto_tsquery('english', 'the and z 1))& fghj'); -select plainto_tsquery('english', 'foo bar') && plainto_tsquery('english', 'asd'); -select plainto_tsquery('english', 'foo bar') || plainto_tsquery('english', 'asd fg'); -select plainto_tsquery('english', 'foo bar') || !!plainto_tsquery('english', 'asd fg'); -select plainto_tsquery('english', 'foo bar') && 'asd | fg'; - -select 'a b:89 ca:23A,64b d:34c'::tsvector @@ 'd:AC & ca'; -select 'a b:89 ca:23A,64b d:34c'::tsvector @@ 'd:AC & ca:B'; -select 'a b:89 ca:23A,64b d:34c'::tsvector @@ 'd:AC & ca:A'; -select 'a b:89 ca:23A,64b d:34c'::tsvector @@ 'd:AC & ca:C'; -select 'a b:89 ca:23A,64b d:34c'::tsvector @@ 'd:AC & ca:CB'; - -CREATE TABLE test_tsvector( t text, a tsvector ); - -\copy test_tsvector from 'data/test_tsearch.data' - -SELECT count(*) FROM test_tsvector WHERE a @@ 'wr|qh'; -SELECT count(*) FROM test_tsvector WHERE a @@ 'wr&qh'; -SELECT count(*) FROM test_tsvector WHERE a @@ 'eq&yt'; -SELECT count(*) FROM test_tsvector WHERE a @@ 'eq|yt'; -SELECT count(*) FROM test_tsvector WHERE a @@ '(eq&yt)|(wr&qh)'; -SELECT count(*) FROM test_tsvector WHERE a @@ '(eq|yt)&(wr|qh)'; - -create index wowidx on test_tsvector using gist (a); -set enable_seqscan=off; - -SELECT count(*) FROM test_tsvector WHERE a @@ 'wr|qh'; -SELECT count(*) FROM test_tsvector WHERE a @@ 'wr&qh'; -SELECT count(*) FROM test_tsvector WHERE a @@ 'eq&yt'; -SELECT count(*) FROM test_tsvector WHERE a @@ 'eq|yt'; -SELECT count(*) FROM test_tsvector WHERE a @@ '(eq&yt)|(wr&qh)'; -SELECT count(*) FROM test_tsvector WHERE a @@ '(eq|yt)&(wr|qh)'; - -select set_curcfg('english'); - -CREATE TRIGGER tsvectorupdate -BEFORE UPDATE OR INSERT ON test_tsvector -FOR EACH ROW EXECUTE PROCEDURE tsearch2(a, t); - -SELECT count(*) FROM test_tsvector WHERE a @@ to_tsquery('345&qwerty'); - -INSERT INTO test_tsvector (t) VALUES ('345 qwerty'); - -SELECT count(*) FROM test_tsvector WHERE a @@ to_tsquery('345&qwerty'); - -UPDATE test_tsvector SET t = null WHERE t = '345 qwerty'; - -SELECT count(*) FROM test_tsvector WHERE a @@ to_tsquery('345&qwerty'); - -insert into test_tsvector (t) values ('345 qwerty copyright'); -select count(*) FROM test_tsvector WHERE a @@ to_tsquery('345&qwerty'); -select count(*) FROM test_tsvector WHERE a @@ to_tsquery('copyright'); - -select rank(' a:1 s:2C d g'::tsvector, 'a | s'); -select rank(' a:1 s:2B d g'::tsvector, 'a | s'); -select rank(' a:1 s:2 d g'::tsvector, 'a | s'); -select rank(' a:1 s:2C d g'::tsvector, 'a & s'); -select rank(' a:1 s:2B d g'::tsvector, 'a & s'); -select rank(' a:1 s:2 d g'::tsvector, 'a & s'); - -insert into test_tsvector (t) values ('foo bar foo the over foo qq bar'); -drop trigger tsvectorupdate on test_tsvector; -select * from stat('select a from test_tsvector') order by ndoc desc, nentry desc, word collate "C"; -insert into test_tsvector values ('1', 'a:1a,2,3b b:5a,6a,7c,8'); -insert into test_tsvector values ('1', 'a:1a,2,3c b:5a,6b,7c,8b'); -select * from stat('select a from test_tsvector','a') order by ndoc desc, nentry desc, word collate "C"; -select * from stat('select a from test_tsvector','b') order by ndoc desc, nentry desc, word collate "C"; -select * from stat('select a from test_tsvector','c') order by ndoc desc, nentry desc, word collate "C"; -select * from stat('select a from test_tsvector','d') order by ndoc desc, nentry desc, word collate "C"; -select * from stat('select a from test_tsvector','ad') order by ndoc desc, nentry desc, word collate "C"; - -select to_tsquery('english', 'skies & books'); - -select rank_cd(to_tsvector('Erosion It took the sea a thousand years, -A thousand years to trace -The granite features of this cliff -In crag and scarp and base. -It took the sea an hour one night -An hour of storm to place -The sculpture of these granite seams, -Upon a woman s face. E. J. Pratt (1882 1964) -'), to_tsquery('sea&thousand&years')); - -select rank_cd(to_tsvector('Erosion It took the sea a thousand years, -A thousand years to trace -The granite features of this cliff -In crag and scarp and base. -It took the sea an hour one night -An hour of storm to place -The sculpture of these granite seams, -Upon a woman s face. E. J. Pratt (1882 1964) -'), to_tsquery('granite&sea')); - -select rank_cd(to_tsvector('Erosion It took the sea a thousand years, -A thousand years to trace -The granite features of this cliff -In crag and scarp and base. -It took the sea an hour one night -An hour of storm to place -The sculpture of these granite seams, -Upon a woman s face. E. J. Pratt (1882 1964) -'), to_tsquery('sea')); - -select headline('Erosion It took the sea a thousand years, -A thousand years to trace -The granite features of this cliff -In crag and scarp and base. -It took the sea an hour one night -An hour of storm to place -The sculpture of these granite seams, -Upon a woman s face. E. J. Pratt (1882 1964) -', to_tsquery('sea&thousand&years')); - -select headline('Erosion It took the sea a thousand years, -A thousand years to trace -The granite features of this cliff -In crag and scarp and base. -It took the sea an hour one night -An hour of storm to place -The sculpture of these granite seams, -Upon a woman s face. E. J. Pratt (1882 1964) -', to_tsquery('granite&sea')); - -select headline('Erosion It took the sea a thousand years, -A thousand years to trace -The granite features of this cliff -In crag and scarp and base. -It took the sea an hour one night -An hour of storm to place -The sculpture of these granite seams, -Upon a woman s face. E. J. Pratt (1882 1964) -', to_tsquery('sea')); - - -select headline(' - - - -Sea view wow foo bar qq -YES   -ff-bg - - -', -to_tsquery('sea&foo'), 'HighlightAll=true'); ---check debug -select * from public.ts_debug('Tsearch module for PostgreSQL 7.3.3'); - ---check ordering -insert into test_tsvector values (null, null); -select a is null, a from test_tsvector order by a; - -drop index wowidx; -create index wowidx on test_tsvector using gin (a); -set enable_seqscan=off; - -SELECT count(*) FROM test_tsvector WHERE a @@ 'wr|qh'; -SELECT count(*) FROM test_tsvector WHERE a @@ 'wr&qh'; -SELECT count(*) FROM test_tsvector WHERE a @@ 'eq&yt'; -SELECT count(*) FROM test_tsvector WHERE a @@ 'eq|yt'; -SELECT count(*) FROM test_tsvector WHERE a @@ '(eq&yt)|(wr&qh)'; -SELECT count(*) FROM test_tsvector WHERE a @@ '(eq|yt)&(wr|qh)'; diff --git a/contrib/tsearch2/tsearch2--1.0.sql b/contrib/tsearch2/tsearch2--1.0.sql deleted file mode 100644 index 68bb43fd7c..0000000000 --- a/contrib/tsearch2/tsearch2--1.0.sql +++ /dev/null @@ -1,576 +0,0 @@ -/* contrib/tsearch2/tsearch2--1.0.sql */ - --- complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use "CREATE EXTENSION tsearch2" to load this file. \quit - --- These domains are just to catch schema-qualified references to the --- old data types. -CREATE DOMAIN tsvector AS pg_catalog.tsvector; -CREATE DOMAIN tsquery AS pg_catalog.tsquery; -CREATE DOMAIN gtsvector AS pg_catalog.gtsvector; -CREATE DOMAIN gtsq AS pg_catalog.text; - ---dict interface -CREATE FUNCTION lexize(oid, text) - RETURNS _text - as 'ts_lexize' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT; - -CREATE FUNCTION lexize(text, text) - RETURNS _text - as 'MODULE_PATHNAME', 'tsa_lexize_byname' - LANGUAGE C - RETURNS NULL ON NULL INPUT; - -CREATE FUNCTION lexize(text) - RETURNS _text - as 'MODULE_PATHNAME', 'tsa_lexize_bycurrent' - LANGUAGE C - RETURNS NULL ON NULL INPUT; - -CREATE FUNCTION set_curdict(int) - RETURNS void - as 'MODULE_PATHNAME', 'tsa_set_curdict' - LANGUAGE C - RETURNS NULL ON NULL INPUT; - -CREATE FUNCTION set_curdict(text) - RETURNS void - as 'MODULE_PATHNAME', 'tsa_set_curdict_byname' - LANGUAGE C - RETURNS NULL ON NULL INPUT; - ---built-in dictionaries -CREATE FUNCTION dex_init(internal) - RETURNS internal - as 'MODULE_PATHNAME', 'tsa_dex_init' - LANGUAGE C; - -CREATE FUNCTION dex_lexize(internal,internal,int4) - RETURNS internal - as 'MODULE_PATHNAME', 'tsa_dex_lexize' - LANGUAGE C - RETURNS NULL ON NULL INPUT; - -CREATE FUNCTION snb_en_init(internal) - RETURNS internal - as 'MODULE_PATHNAME', 'tsa_snb_en_init' - LANGUAGE C; - -CREATE FUNCTION snb_lexize(internal,internal,int4) - RETURNS internal - as 'MODULE_PATHNAME', 'tsa_snb_lexize' - LANGUAGE C - RETURNS NULL ON NULL INPUT; - -CREATE FUNCTION snb_ru_init_koi8(internal) - RETURNS internal - as 'MODULE_PATHNAME', 'tsa_snb_ru_init_koi8' - LANGUAGE C; - -CREATE FUNCTION snb_ru_init_utf8(internal) - RETURNS internal - as 'MODULE_PATHNAME', 'tsa_snb_ru_init_utf8' - LANGUAGE C; - -CREATE FUNCTION snb_ru_init(internal) - RETURNS internal - as 'MODULE_PATHNAME', 'tsa_snb_ru_init' - LANGUAGE C; - -CREATE FUNCTION spell_init(internal) - RETURNS internal - as 'MODULE_PATHNAME', 'tsa_spell_init' - LANGUAGE C; - -CREATE FUNCTION spell_lexize(internal,internal,int4) - RETURNS internal - as 'MODULE_PATHNAME', 'tsa_spell_lexize' - LANGUAGE C - RETURNS NULL ON NULL INPUT; - -CREATE FUNCTION syn_init(internal) - RETURNS internal - as 'MODULE_PATHNAME', 'tsa_syn_init' - LANGUAGE C; - -CREATE FUNCTION syn_lexize(internal,internal,int4) - RETURNS internal - as 'MODULE_PATHNAME', 'tsa_syn_lexize' - LANGUAGE C - RETURNS NULL ON NULL INPUT; - -CREATE FUNCTION thesaurus_init(internal) - RETURNS internal - as 'MODULE_PATHNAME', 'tsa_thesaurus_init' - LANGUAGE C; - -CREATE FUNCTION thesaurus_lexize(internal,internal,int4,internal) - RETURNS internal - as 'MODULE_PATHNAME', 'tsa_thesaurus_lexize' - LANGUAGE C - RETURNS NULL ON NULL INPUT; - ---sql-level interface -CREATE TYPE tokentype - as (tokid int4, alias text, descr text); - -CREATE FUNCTION token_type(int4) - RETURNS setof tokentype - as 'ts_token_type_byid' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT - ROWS 16; - -CREATE FUNCTION token_type(text) - RETURNS setof tokentype - as 'ts_token_type_byname' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT - ROWS 16; - -CREATE FUNCTION token_type() - RETURNS setof tokentype - as 'MODULE_PATHNAME', 'tsa_token_type_current' - LANGUAGE C - RETURNS NULL ON NULL INPUT - ROWS 16; - -CREATE FUNCTION set_curprs(int) - RETURNS void - as 'MODULE_PATHNAME', 'tsa_set_curprs' - LANGUAGE C - RETURNS NULL ON NULL INPUT; - -CREATE FUNCTION set_curprs(text) - RETURNS void - as 'MODULE_PATHNAME', 'tsa_set_curprs_byname' - LANGUAGE C - RETURNS NULL ON NULL INPUT; - -CREATE TYPE tokenout - as (tokid int4, token text); - -CREATE FUNCTION parse(oid,text) - RETURNS setof tokenout - as 'ts_parse_byid' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT; - -CREATE FUNCTION parse(text,text) - RETURNS setof tokenout - as 'ts_parse_byname' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT; - -CREATE FUNCTION parse(text) - RETURNS setof tokenout - as 'MODULE_PATHNAME', 'tsa_parse_current' - LANGUAGE C - RETURNS NULL ON NULL INPUT; - ---default parser -CREATE FUNCTION prsd_start(internal,int4) - RETURNS internal - as 'MODULE_PATHNAME', 'tsa_prsd_start' - LANGUAGE C; - -CREATE FUNCTION prsd_getlexeme(internal,internal,internal) - RETURNS int4 - as 'MODULE_PATHNAME', 'tsa_prsd_getlexeme' - LANGUAGE C; - -CREATE FUNCTION prsd_end(internal) - RETURNS void - as 'MODULE_PATHNAME', 'tsa_prsd_end' - LANGUAGE C; - -CREATE FUNCTION prsd_lextype(internal) - RETURNS internal - as 'MODULE_PATHNAME', 'tsa_prsd_lextype' - LANGUAGE C; - -CREATE FUNCTION prsd_headline(internal,internal,internal) - RETURNS internal - as 'MODULE_PATHNAME', 'tsa_prsd_headline' - LANGUAGE C; - ---tsearch config -CREATE FUNCTION set_curcfg(int) - RETURNS void - as 'MODULE_PATHNAME', 'tsa_set_curcfg' - LANGUAGE C - RETURNS NULL ON NULL INPUT; - -CREATE FUNCTION set_curcfg(text) - RETURNS void - as 'MODULE_PATHNAME', 'tsa_set_curcfg_byname' - LANGUAGE C - RETURNS NULL ON NULL INPUT; - -CREATE FUNCTION show_curcfg() - RETURNS oid - AS 'get_current_ts_config' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT STABLE; - -CREATE FUNCTION length(tsvector) - RETURNS int4 - AS 'tsvector_length' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION to_tsvector(oid, text) - RETURNS tsvector - AS 'to_tsvector_byid' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION to_tsvector(text, text) - RETURNS tsvector - AS 'MODULE_PATHNAME', 'tsa_to_tsvector_name' - LANGUAGE C RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION to_tsvector(text) - RETURNS tsvector - AS 'to_tsvector' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION strip(tsvector) - RETURNS tsvector - AS 'tsvector_strip' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION setweight(tsvector,"char") - RETURNS tsvector - AS 'tsvector_setweight' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION concat(tsvector,tsvector) - RETURNS tsvector - AS 'tsvector_concat' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION querytree(tsquery) - RETURNS text - AS 'tsquerytree' - LANGUAGE INTERNAL RETURNS NULL ON NULL INPUT; - -CREATE FUNCTION to_tsquery(oid, text) - RETURNS tsquery - AS 'to_tsquery_byid' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION to_tsquery(text, text) - RETURNS tsquery - AS 'MODULE_PATHNAME','tsa_to_tsquery_name' - LANGUAGE C RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION to_tsquery(text) - RETURNS tsquery - AS 'to_tsquery' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION plainto_tsquery(oid, text) - RETURNS tsquery - AS 'plainto_tsquery_byid' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION plainto_tsquery(text, text) - RETURNS tsquery - AS 'MODULE_PATHNAME','tsa_plainto_tsquery_name' - LANGUAGE C RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION plainto_tsquery(text) - RETURNS tsquery - AS 'plainto_tsquery' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT IMMUTABLE; - ---Trigger -CREATE FUNCTION tsearch2() - RETURNS trigger - AS 'MODULE_PATHNAME', 'tsa_tsearch2' - LANGUAGE C; - ---Relevation -CREATE FUNCTION rank(float4[], tsvector, tsquery) - RETURNS float4 - AS 'ts_rank_wtt' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION rank(float4[], tsvector, tsquery, int4) - RETURNS float4 - AS 'ts_rank_wttf' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION rank(tsvector, tsquery) - RETURNS float4 - AS 'ts_rank_tt' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION rank(tsvector, tsquery, int4) - RETURNS float4 - AS 'ts_rank_ttf' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION rank_cd(float4[], tsvector, tsquery) - RETURNS float4 - AS 'ts_rankcd_wtt' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION rank_cd(float4[], tsvector, tsquery, int4) - RETURNS float4 - AS 'ts_rankcd_wttf' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION rank_cd(tsvector, tsquery) - RETURNS float4 - AS 'ts_rankcd_tt' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION rank_cd(tsvector, tsquery, int4) - RETURNS float4 - AS 'ts_rankcd_ttf' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION headline(oid, text, tsquery, text) - RETURNS text - AS 'ts_headline_byid_opt' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION headline(oid, text, tsquery) - RETURNS text - AS 'ts_headline_byid' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION headline(text, text, tsquery, text) - RETURNS text - AS 'MODULE_PATHNAME', 'tsa_headline_byname' - LANGUAGE C RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION headline(text, text, tsquery) - RETURNS text - AS 'MODULE_PATHNAME', 'tsa_headline_byname' - LANGUAGE C RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION headline(text, tsquery, text) - RETURNS text - AS 'ts_headline_opt' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION headline(text, tsquery) - RETURNS text - AS 'ts_headline' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT IMMUTABLE; - --- CREATE the OPERATOR class -CREATE OPERATOR CLASS gist_tsvector_ops -FOR TYPE tsvector USING gist -AS - OPERATOR 1 @@ (tsvector, tsquery), - FUNCTION 1 gtsvector_consistent (internal, tsvector, smallint, oid, internal), - FUNCTION 2 gtsvector_union (internal, internal), - FUNCTION 3 gtsvector_compress (internal), - FUNCTION 4 gtsvector_decompress (internal), - FUNCTION 5 gtsvector_penalty (internal, internal, internal), - FUNCTION 6 gtsvector_picksplit (internal, internal), - FUNCTION 7 gtsvector_same (gtsvector, gtsvector, internal), - STORAGE gtsvector; - ---stat info -CREATE TYPE statinfo - as (word text, ndoc int4, nentry int4); - -CREATE FUNCTION stat(text) - RETURNS setof statinfo - as 'ts_stat1' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT; - -CREATE FUNCTION stat(text,text) - RETURNS setof statinfo - as 'ts_stat2' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT; - ---reset - just for debugging -CREATE FUNCTION reset_tsearch() - RETURNS void - as 'MODULE_PATHNAME', 'tsa_reset_tsearch' - LANGUAGE C - RETURNS NULL ON NULL INPUT; - ---get cover (debug for rank_cd) -CREATE FUNCTION get_covers(tsvector,tsquery) - RETURNS text - as 'MODULE_PATHNAME', 'tsa_get_covers' - LANGUAGE C - RETURNS NULL ON NULL INPUT; - ---debug function -create type tsdebug as ( - ts_name text, - tok_type text, - description text, - token text, - dict_name text[], - "tsvector" tsvector -); - -CREATE FUNCTION _get_parser_from_curcfg() -RETURNS text as -$$select prsname::text from pg_catalog.pg_ts_parser p join pg_ts_config c on cfgparser = p.oid where c.oid = show_curcfg();$$ -LANGUAGE SQL RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION ts_debug(text) -RETURNS setof tsdebug as $$ -select - (select c.cfgname::text from pg_catalog.pg_ts_config as c - where c.oid = show_curcfg()), - t.alias as tok_type, - t.descr as description, - p.token, - ARRAY ( SELECT m.mapdict::pg_catalog.regdictionary::pg_catalog.text - FROM pg_catalog.pg_ts_config_map AS m - WHERE m.mapcfg = show_curcfg() AND m.maptokentype = p.tokid - ORDER BY m.mapseqno ) - AS dict_name, - strip(to_tsvector(p.token)) as tsvector -from - parse( _get_parser_from_curcfg(), $1 ) as p, - token_type() as t -where - t.tokid = p.tokid -$$ LANGUAGE SQL RETURNS NULL ON NULL INPUT; - -CREATE FUNCTION numnode(tsquery) - RETURNS int4 - as 'tsquery_numnode' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION tsquery_and(tsquery,tsquery) - RETURNS tsquery - as 'tsquery_and' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION tsquery_or(tsquery,tsquery) - RETURNS tsquery - as 'tsquery_or' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION tsquery_not(tsquery) - RETURNS tsquery - as 'tsquery_not' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT IMMUTABLE; - ---------------rewrite subsystem - -CREATE FUNCTION rewrite(tsquery, text) - RETURNS tsquery - as 'tsquery_rewrite_query' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION rewrite(tsquery, tsquery, tsquery) - RETURNS tsquery - as 'tsquery_rewrite' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION rewrite_accum(tsquery,tsquery[]) - RETURNS tsquery - AS 'MODULE_PATHNAME', 'tsa_rewrite_accum' - LANGUAGE C; - -CREATE FUNCTION rewrite_finish(tsquery) - RETURNS tsquery - as 'MODULE_PATHNAME', 'tsa_rewrite_finish' - LANGUAGE C; - -CREATE AGGREGATE rewrite ( - BASETYPE = tsquery[], - SFUNC = rewrite_accum, - STYPE = tsquery, - FINALFUNC = rewrite_finish -); - -CREATE FUNCTION tsq_mcontains(tsquery, tsquery) - RETURNS bool - as 'tsq_mcontains' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE FUNCTION tsq_mcontained(tsquery, tsquery) - RETURNS bool - as 'tsq_mcontained' - LANGUAGE INTERNAL - RETURNS NULL ON NULL INPUT IMMUTABLE; - -CREATE OPERATOR CLASS gist_tp_tsquery_ops -FOR TYPE tsquery USING gist -AS - OPERATOR 7 @> (tsquery, tsquery), - OPERATOR 8 <@ (tsquery, tsquery), - FUNCTION 1 gtsquery_consistent (internal, tsquery, smallint, oid, internal), - FUNCTION 2 gtsquery_union (internal, internal), - FUNCTION 3 gtsquery_compress (internal), - FUNCTION 4 gtsquery_decompress (internal), - FUNCTION 5 gtsquery_penalty (internal, internal, internal), - FUNCTION 6 gtsquery_picksplit (internal, internal), - FUNCTION 7 gtsquery_same (bigint, bigint, internal), - STORAGE bigint; - -CREATE OPERATOR CLASS gin_tsvector_ops -FOR TYPE tsvector USING gin -AS - OPERATOR 1 @@ (tsvector, tsquery), - OPERATOR 2 @@@ (tsvector, tsquery), - FUNCTION 1 bttextcmp(text, text), - FUNCTION 2 gin_extract_tsvector(tsvector,internal,internal), - FUNCTION 3 gin_extract_tsquery(tsvector,internal,smallint,internal,internal,internal,internal), - FUNCTION 4 gin_tsquery_consistent(internal,smallint,tsvector,int,internal,internal,internal,internal), - FUNCTION 5 gin_cmp_prefix(text,text,smallint,internal), - STORAGE text; - -CREATE OPERATOR CLASS tsvector_ops -FOR TYPE tsvector USING btree AS - OPERATOR 1 < , - OPERATOR 2 <= , - OPERATOR 3 = , - OPERATOR 4 >= , - OPERATOR 5 > , - FUNCTION 1 tsvector_cmp(tsvector, tsvector); - -CREATE OPERATOR CLASS tsquery_ops -FOR TYPE tsquery USING btree AS - OPERATOR 1 < , - OPERATOR 2 <= , - OPERATOR 3 = , - OPERATOR 4 >= , - OPERATOR 5 > , - FUNCTION 1 tsquery_cmp(tsquery, tsquery); diff --git a/contrib/tsearch2/tsearch2--unpackaged--1.0.sql b/contrib/tsearch2/tsearch2--unpackaged--1.0.sql deleted file mode 100644 index e123297132..0000000000 --- a/contrib/tsearch2/tsearch2--unpackaged--1.0.sql +++ /dev/null @@ -1,144 +0,0 @@ -/* contrib/tsearch2/tsearch2--unpackaged--1.0.sql */ - --- complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use "CREATE EXTENSION tsearch2 FROM unpackaged" to load this file. \quit - -ALTER EXTENSION tsearch2 ADD type @extschema@.tsvector; -ALTER EXTENSION tsearch2 ADD type @extschema@.tsquery; -ALTER EXTENSION tsearch2 ADD type @extschema@.gtsvector; -ALTER EXTENSION tsearch2 ADD type gtsq; -ALTER EXTENSION tsearch2 ADD function lexize(oid,text); -ALTER EXTENSION tsearch2 ADD function lexize(text,text); -ALTER EXTENSION tsearch2 ADD function lexize(text); -ALTER EXTENSION tsearch2 ADD function set_curdict(integer); -ALTER EXTENSION tsearch2 ADD function set_curdict(text); -ALTER EXTENSION tsearch2 ADD function dex_init(internal); -ALTER EXTENSION tsearch2 ADD function dex_lexize(internal,internal,integer); -ALTER EXTENSION tsearch2 ADD function snb_en_init(internal); -ALTER EXTENSION tsearch2 ADD function snb_lexize(internal,internal,integer); -ALTER EXTENSION tsearch2 ADD function snb_ru_init_koi8(internal); -ALTER EXTENSION tsearch2 ADD function snb_ru_init_utf8(internal); -ALTER EXTENSION tsearch2 ADD function snb_ru_init(internal); -ALTER EXTENSION tsearch2 ADD function spell_init(internal); -ALTER EXTENSION tsearch2 ADD function spell_lexize(internal,internal,integer); -ALTER EXTENSION tsearch2 ADD function syn_init(internal); -ALTER EXTENSION tsearch2 ADD function syn_lexize(internal,internal,integer); -ALTER EXTENSION tsearch2 ADD function @extschema@.thesaurus_init(internal); -ALTER EXTENSION tsearch2 ADD function thesaurus_lexize(internal,internal,integer,internal); -ALTER EXTENSION tsearch2 ADD type tokentype; -ALTER EXTENSION tsearch2 ADD function token_type(integer); -ALTER EXTENSION tsearch2 ADD function token_type(text); -ALTER EXTENSION tsearch2 ADD function token_type(); -ALTER EXTENSION tsearch2 ADD function set_curprs(integer); -ALTER EXTENSION tsearch2 ADD function set_curprs(text); -ALTER EXTENSION tsearch2 ADD type tokenout; -ALTER EXTENSION tsearch2 ADD function parse(oid,text); -ALTER EXTENSION tsearch2 ADD function parse(text,text); -ALTER EXTENSION tsearch2 ADD function parse(text); -ALTER EXTENSION tsearch2 ADD function @extschema@.prsd_start(internal,integer); -ALTER EXTENSION tsearch2 ADD function prsd_getlexeme(internal,internal,internal); -ALTER EXTENSION tsearch2 ADD function @extschema@.prsd_end(internal); -ALTER EXTENSION tsearch2 ADD function @extschema@.prsd_lextype(internal); -ALTER EXTENSION tsearch2 ADD function prsd_headline(internal,internal,internal); -ALTER EXTENSION tsearch2 ADD function set_curcfg(integer); -ALTER EXTENSION tsearch2 ADD function set_curcfg(text); -ALTER EXTENSION tsearch2 ADD function show_curcfg(); -ALTER EXTENSION tsearch2 ADD function @extschema@.length(tsvector); -ALTER EXTENSION tsearch2 ADD function to_tsvector(oid,text); -ALTER EXTENSION tsearch2 ADD function to_tsvector(text,text); -ALTER EXTENSION tsearch2 ADD function @extschema@.to_tsvector(text); -ALTER EXTENSION tsearch2 ADD function @extschema@.strip(tsvector); -ALTER EXTENSION tsearch2 ADD function @extschema@.setweight(tsvector,"char"); -ALTER EXTENSION tsearch2 ADD function concat(tsvector,tsvector); -ALTER EXTENSION tsearch2 ADD function @extschema@.querytree(tsquery); -ALTER EXTENSION tsearch2 ADD function to_tsquery(oid,text); -ALTER EXTENSION tsearch2 ADD function to_tsquery(text,text); -ALTER EXTENSION tsearch2 ADD function @extschema@.to_tsquery(text); -ALTER EXTENSION tsearch2 ADD function plainto_tsquery(oid,text); -ALTER EXTENSION tsearch2 ADD function plainto_tsquery(text,text); -ALTER EXTENSION tsearch2 ADD function @extschema@.plainto_tsquery(text); -ALTER EXTENSION tsearch2 ADD function tsearch2(); -ALTER EXTENSION tsearch2 ADD function rank(real[],tsvector,tsquery); -ALTER EXTENSION tsearch2 ADD function rank(real[],tsvector,tsquery,integer); -ALTER EXTENSION tsearch2 ADD function rank(tsvector,tsquery); -ALTER EXTENSION tsearch2 ADD function rank(tsvector,tsquery,integer); -ALTER EXTENSION tsearch2 ADD function rank_cd(real[],tsvector,tsquery); -ALTER EXTENSION tsearch2 ADD function rank_cd(real[],tsvector,tsquery,integer); -ALTER EXTENSION tsearch2 ADD function rank_cd(tsvector,tsquery); -ALTER EXTENSION tsearch2 ADD function rank_cd(tsvector,tsquery,integer); -ALTER EXTENSION tsearch2 ADD function headline(oid,text,tsquery,text); -ALTER EXTENSION tsearch2 ADD function headline(oid,text,tsquery); -ALTER EXTENSION tsearch2 ADD function headline(text,text,tsquery,text); -ALTER EXTENSION tsearch2 ADD function headline(text,text,tsquery); -ALTER EXTENSION tsearch2 ADD function headline(text,tsquery,text); -ALTER EXTENSION tsearch2 ADD function headline(text,tsquery); -ALTER EXTENSION tsearch2 ADD operator family gist_tsvector_ops using gist; -ALTER EXTENSION tsearch2 ADD operator class gist_tsvector_ops using gist; -ALTER EXTENSION tsearch2 ADD type statinfo; -ALTER EXTENSION tsearch2 ADD function stat(text); -ALTER EXTENSION tsearch2 ADD function stat(text,text); -ALTER EXTENSION tsearch2 ADD function reset_tsearch(); -ALTER EXTENSION tsearch2 ADD function get_covers(tsvector,tsquery); -ALTER EXTENSION tsearch2 ADD type tsdebug; -ALTER EXTENSION tsearch2 ADD function _get_parser_from_curcfg(); -ALTER EXTENSION tsearch2 ADD function @extschema@.ts_debug(text); -ALTER EXTENSION tsearch2 ADD function @extschema@.numnode(tsquery); -ALTER EXTENSION tsearch2 ADD function @extschema@.tsquery_and(tsquery,tsquery); -ALTER EXTENSION tsearch2 ADD function @extschema@.tsquery_or(tsquery,tsquery); -ALTER EXTENSION tsearch2 ADD function @extschema@.tsquery_not(tsquery); -ALTER EXTENSION tsearch2 ADD function rewrite(tsquery,text); -ALTER EXTENSION tsearch2 ADD function rewrite(tsquery,tsquery,tsquery); -ALTER EXTENSION tsearch2 ADD function rewrite_accum(tsquery,tsquery[]); -ALTER EXTENSION tsearch2 ADD function rewrite_finish(tsquery); -ALTER EXTENSION tsearch2 ADD function rewrite(tsquery[]); -ALTER EXTENSION tsearch2 ADD function @extschema@.tsq_mcontains(tsquery,tsquery); -ALTER EXTENSION tsearch2 ADD function @extschema@.tsq_mcontained(tsquery,tsquery); -ALTER EXTENSION tsearch2 ADD operator family gist_tp_tsquery_ops using gist; -ALTER EXTENSION tsearch2 ADD operator class gist_tp_tsquery_ops using gist; -ALTER EXTENSION tsearch2 ADD operator family gin_tsvector_ops using gin; -ALTER EXTENSION tsearch2 ADD operator class gin_tsvector_ops using gin; -ALTER EXTENSION tsearch2 ADD operator family @extschema@.tsvector_ops using btree; -ALTER EXTENSION tsearch2 ADD operator class @extschema@.tsvector_ops using btree; -ALTER EXTENSION tsearch2 ADD operator family @extschema@.tsquery_ops using btree; -ALTER EXTENSION tsearch2 ADD operator class @extschema@.tsquery_ops using btree; - --- tsearch2 relies on the core functions gin_extract_tsvector, --- gin_extract_tsquery, and gin_tsquery_consistent, which changed signature in --- 9.1. To support upgrading, pg_catalog contains entries for these functions --- with both the old and new signatures, and the former is what would have --- been added to our opclass during initial restore of a 9.0 dump script. --- Avert your eyes while we hack the pg_amproc entries to make them link to --- the new forms ... - -UPDATE pg_catalog.pg_amproc -SET amproc = 'pg_catalog.gin_extract_tsvector(pg_catalog.tsvector,internal,internal)'::pg_catalog.regprocedure -WHERE amprocfamily = - (SELECT oid FROM pg_catalog.pg_opfamily WHERE opfname = 'gin_tsvector_ops' AND - opfnamespace = (SELECT oid FROM pg_catalog.pg_namespace - WHERE nspname = '@extschema@')) - AND amproclefttype = 'pg_catalog.tsvector'::pg_catalog.regtype - AND amprocrighttype = 'pg_catalog.tsvector'::pg_catalog.regtype - AND amprocnum = 2 - AND amproc = 'pg_catalog.gin_extract_tsvector(pg_catalog.tsvector,internal)'::pg_catalog.regprocedure; - -UPDATE pg_catalog.pg_amproc -SET amproc = 'pg_catalog.gin_extract_tsquery(pg_catalog.tsquery,internal,smallint,internal,internal,internal,internal)'::pg_catalog.regprocedure -WHERE amprocfamily = - (SELECT oid FROM pg_catalog.pg_opfamily WHERE opfname = 'gin_tsvector_ops' AND - opfnamespace = (SELECT oid FROM pg_catalog.pg_namespace - WHERE nspname = '@extschema@')) - AND amproclefttype = 'pg_catalog.tsvector'::pg_catalog.regtype - AND amprocrighttype = 'pg_catalog.tsvector'::pg_catalog.regtype - AND amprocnum = 3 - AND amproc = 'pg_catalog.gin_extract_tsquery(pg_catalog.tsquery,internal,smallint,internal,internal)'::pg_catalog.regprocedure; - -UPDATE pg_catalog.pg_amproc -SET amproc = 'pg_catalog.gin_tsquery_consistent(internal,smallint,pg_catalog.tsquery,integer,internal,internal,internal,internal)'::pg_catalog.regprocedure -WHERE amprocfamily = - (SELECT oid FROM pg_catalog.pg_opfamily WHERE opfname = 'gin_tsvector_ops' AND - opfnamespace = (SELECT oid FROM pg_catalog.pg_namespace - WHERE nspname = '@extschema@')) - AND amproclefttype = 'pg_catalog.tsvector'::pg_catalog.regtype - AND amprocrighttype = 'pg_catalog.tsvector'::pg_catalog.regtype - AND amprocnum = 4 - AND amproc = 'pg_catalog.gin_tsquery_consistent(internal,smallint,pg_catalog.tsquery,integer,internal,internal)'::pg_catalog.regprocedure; diff --git a/contrib/tsearch2/tsearch2.c b/contrib/tsearch2/tsearch2.c deleted file mode 100644 index 16772a2b92..0000000000 --- a/contrib/tsearch2/tsearch2.c +++ /dev/null @@ -1,542 +0,0 @@ -/*------------------------------------------------------------------------- - * - * tsearch2.c - * Backwards-compatibility package for old contrib/tsearch2 API - * - * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group - * - * - * IDENTIFICATION - * contrib/tsearch2/tsearch2.c - * - *------------------------------------------------------------------------- - */ -#include "postgres.h" - -#include "catalog/namespace.h" -#include "catalog/pg_type.h" -#include "commands/trigger.h" -#include "tsearch/ts_utils.h" -#include "utils/builtins.h" -#include "utils/guc.h" -#include "utils/regproc.h" -#include "utils/syscache.h" - -PG_MODULE_MAGIC; - -static Oid current_dictionary_oid = InvalidOid; -static Oid current_parser_oid = InvalidOid; - -/* insert given value at argument position 0 */ -#define INSERT_ARGUMENT0(argument, isnull) \ - do { \ - int i; \ - for (i = fcinfo->nargs; i > 0; i--) \ - { \ - fcinfo->arg[i] = fcinfo->arg[i-1]; \ - fcinfo->argnull[i] = fcinfo->argnull[i-1]; \ - } \ - fcinfo->arg[0] = (argument); \ - fcinfo->argnull[0] = (isnull); \ - fcinfo->nargs++; \ - } while (0) - -#define TextGetObjectId(infunction, text) \ - DatumGetObjectId(DirectFunctionCall1(infunction, \ - CStringGetDatum(text_to_cstring(text)))) - -#define UNSUPPORTED_FUNCTION(name) \ - PG_FUNCTION_INFO_V1(name); \ - Datum \ - name(PG_FUNCTION_ARGS) \ - { \ - ereport(ERROR, \ - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),\ - errmsg("function %s is no longer supported", \ - format_procedure(fcinfo->flinfo->fn_oid)), \ - errhint("Switch to new tsearch functionality."))); \ - /* keep compiler quiet */ \ - PG_RETURN_NULL(); \ - } \ - extern int no_such_variable - -static Oid GetCurrentDict(void); -static Oid GetCurrentParser(void); - -PG_FUNCTION_INFO_V1(tsa_lexize_byname); -PG_FUNCTION_INFO_V1(tsa_lexize_bycurrent); -PG_FUNCTION_INFO_V1(tsa_set_curdict); -PG_FUNCTION_INFO_V1(tsa_set_curdict_byname); -PG_FUNCTION_INFO_V1(tsa_token_type_current); -PG_FUNCTION_INFO_V1(tsa_set_curprs); -PG_FUNCTION_INFO_V1(tsa_set_curprs_byname); -PG_FUNCTION_INFO_V1(tsa_parse_current); -PG_FUNCTION_INFO_V1(tsa_set_curcfg); -PG_FUNCTION_INFO_V1(tsa_set_curcfg_byname); -PG_FUNCTION_INFO_V1(tsa_to_tsvector_name); -PG_FUNCTION_INFO_V1(tsa_to_tsquery_name); -PG_FUNCTION_INFO_V1(tsa_plainto_tsquery_name); -PG_FUNCTION_INFO_V1(tsa_headline_byname); -PG_FUNCTION_INFO_V1(tsa_ts_stat); -PG_FUNCTION_INFO_V1(tsa_tsearch2); -PG_FUNCTION_INFO_V1(tsa_rewrite_accum); -PG_FUNCTION_INFO_V1(tsa_rewrite_finish); - - -/* - * List of unsupported functions - * - * The parser and dictionary functions are defined only so that the former - * contents of pg_ts_parser and pg_ts_dict can be loaded into the system, - * for ease of reference while creating the new tsearch configuration. - */ - -UNSUPPORTED_FUNCTION(tsa_dex_init); -UNSUPPORTED_FUNCTION(tsa_dex_lexize); - -UNSUPPORTED_FUNCTION(tsa_snb_en_init); -UNSUPPORTED_FUNCTION(tsa_snb_lexize); -UNSUPPORTED_FUNCTION(tsa_snb_ru_init_koi8); -UNSUPPORTED_FUNCTION(tsa_snb_ru_init_utf8); -UNSUPPORTED_FUNCTION(tsa_snb_ru_init); - -UNSUPPORTED_FUNCTION(tsa_spell_init); -UNSUPPORTED_FUNCTION(tsa_spell_lexize); - -UNSUPPORTED_FUNCTION(tsa_syn_init); -UNSUPPORTED_FUNCTION(tsa_syn_lexize); - -UNSUPPORTED_FUNCTION(tsa_thesaurus_init); -UNSUPPORTED_FUNCTION(tsa_thesaurus_lexize); - -UNSUPPORTED_FUNCTION(tsa_prsd_start); -UNSUPPORTED_FUNCTION(tsa_prsd_getlexeme); -UNSUPPORTED_FUNCTION(tsa_prsd_end); -UNSUPPORTED_FUNCTION(tsa_prsd_lextype); -UNSUPPORTED_FUNCTION(tsa_prsd_headline); - -UNSUPPORTED_FUNCTION(tsa_reset_tsearch); -UNSUPPORTED_FUNCTION(tsa_get_covers); - - -/* - * list of redefined functions - */ - -/* lexize(text, text) */ -Datum -tsa_lexize_byname(PG_FUNCTION_ARGS) -{ - text *dictname = PG_GETARG_TEXT_PP(0); - Datum arg1 = PG_GETARG_DATUM(1); - - return DirectFunctionCall2(ts_lexize, - ObjectIdGetDatum(TextGetObjectId(regdictionaryin, dictname)), - arg1); -} - -/* lexize(text) */ -Datum -tsa_lexize_bycurrent(PG_FUNCTION_ARGS) -{ - Datum arg0 = PG_GETARG_DATUM(0); - Oid id = GetCurrentDict(); - - return DirectFunctionCall2(ts_lexize, - ObjectIdGetDatum(id), - arg0); -} - -/* set_curdict(int) */ -Datum -tsa_set_curdict(PG_FUNCTION_ARGS) -{ - Oid dict_oid = PG_GETARG_OID(0); - - if (!SearchSysCacheExists(TSDICTOID, - ObjectIdGetDatum(dict_oid), - 0, 0, 0)) - elog(ERROR, "cache lookup failed for text search dictionary %u", - dict_oid); - - current_dictionary_oid = dict_oid; - - PG_RETURN_VOID(); -} - -/* set_curdict(text) */ -Datum -tsa_set_curdict_byname(PG_FUNCTION_ARGS) -{ - text *name = PG_GETARG_TEXT_PP(0); - Oid dict_oid; - - dict_oid = get_ts_dict_oid(stringToQualifiedNameList(text_to_cstring(name)), false); - - current_dictionary_oid = dict_oid; - - PG_RETURN_VOID(); -} - -/* token_type() */ -Datum -tsa_token_type_current(PG_FUNCTION_ARGS) -{ - INSERT_ARGUMENT0(ObjectIdGetDatum(GetCurrentParser()), false); - return ts_token_type_byid(fcinfo); -} - -/* set_curprs(int) */ -Datum -tsa_set_curprs(PG_FUNCTION_ARGS) -{ - Oid parser_oid = PG_GETARG_OID(0); - - if (!SearchSysCacheExists(TSPARSEROID, - ObjectIdGetDatum(parser_oid), - 0, 0, 0)) - elog(ERROR, "cache lookup failed for text search parser %u", - parser_oid); - - current_parser_oid = parser_oid; - - PG_RETURN_VOID(); -} - -/* set_curprs(text) */ -Datum -tsa_set_curprs_byname(PG_FUNCTION_ARGS) -{ - text *name = PG_GETARG_TEXT_PP(0); - Oid parser_oid; - - parser_oid = get_ts_parser_oid(stringToQualifiedNameList(text_to_cstring(name)), false); - - current_parser_oid = parser_oid; - - PG_RETURN_VOID(); -} - -/* parse(text) */ -Datum -tsa_parse_current(PG_FUNCTION_ARGS) -{ - INSERT_ARGUMENT0(ObjectIdGetDatum(GetCurrentParser()), false); - return ts_parse_byid(fcinfo); -} - -/* set_curcfg(int) */ -Datum -tsa_set_curcfg(PG_FUNCTION_ARGS) -{ - Oid arg0 = PG_GETARG_OID(0); - char *name; - - name = DatumGetCString(DirectFunctionCall1(regconfigout, - ObjectIdGetDatum(arg0))); - - SetConfigOption("default_text_search_config", name, - PGC_USERSET, PGC_S_SESSION); - - PG_RETURN_VOID(); -} - -/* set_curcfg(text) */ -Datum -tsa_set_curcfg_byname(PG_FUNCTION_ARGS) -{ - text *arg0 = PG_GETARG_TEXT_PP(0); - char *name; - - name = text_to_cstring(arg0); - - SetConfigOption("default_text_search_config", name, - PGC_USERSET, PGC_S_SESSION); - - PG_RETURN_VOID(); -} - -/* to_tsvector(text, text) */ -Datum -tsa_to_tsvector_name(PG_FUNCTION_ARGS) -{ - text *cfgname = PG_GETARG_TEXT_PP(0); - Datum arg1 = PG_GETARG_DATUM(1); - Oid config_oid; - - config_oid = TextGetObjectId(regconfigin, cfgname); - - return DirectFunctionCall2(to_tsvector_byid, - ObjectIdGetDatum(config_oid), arg1); -} - -/* to_tsquery(text, text) */ -Datum -tsa_to_tsquery_name(PG_FUNCTION_ARGS) -{ - text *cfgname = PG_GETARG_TEXT_PP(0); - Datum arg1 = PG_GETARG_DATUM(1); - Oid config_oid; - - config_oid = TextGetObjectId(regconfigin, cfgname); - - return DirectFunctionCall2(to_tsquery_byid, - ObjectIdGetDatum(config_oid), arg1); -} - - -/* plainto_tsquery(text, text) */ -Datum -tsa_plainto_tsquery_name(PG_FUNCTION_ARGS) -{ - text *cfgname = PG_GETARG_TEXT_PP(0); - Datum arg1 = PG_GETARG_DATUM(1); - Oid config_oid; - - config_oid = TextGetObjectId(regconfigin, cfgname); - - return DirectFunctionCall2(plainto_tsquery_byid, - ObjectIdGetDatum(config_oid), arg1); -} - -/* headline(text, text, tsquery [,text]) */ -Datum -tsa_headline_byname(PG_FUNCTION_ARGS) -{ - Datum arg0 = PG_GETARG_DATUM(0); - Datum arg1 = PG_GETARG_DATUM(1); - Datum arg2 = PG_GETARG_DATUM(2); - Datum result; - Oid config_oid; - - /* first parameter has to be converted to oid */ - config_oid = DatumGetObjectId(DirectFunctionCall1(regconfigin, - CStringGetDatum(TextDatumGetCString(arg0)))); - - if (PG_NARGS() == 3) - result = DirectFunctionCall3(ts_headline_byid, - ObjectIdGetDatum(config_oid), arg1, arg2); - else - { - Datum arg3 = PG_GETARG_DATUM(3); - - result = DirectFunctionCall4(ts_headline_byid_opt, - ObjectIdGetDatum(config_oid), - arg1, arg2, arg3); - } - - return result; -} - -/* - * tsearch2 version of update trigger - * - * We pass this on to the core trigger after inserting the default text - * search configuration name as the second argument. Note that this isn't - * a complete implementation of the original functionality; tsearch2 allowed - * transformation function names to be included in the list. However, that - * is deliberately removed as being a security risk. - */ -Datum -tsa_tsearch2(PG_FUNCTION_ARGS) -{ - TriggerData *trigdata; - Trigger *trigger; - char **tgargs, - **tgargs_old; - int i; - Datum res; - - /* Check call context */ - if (!CALLED_AS_TRIGGER(fcinfo)) /* internal error */ - elog(ERROR, "tsvector_update_trigger: not fired by trigger manager"); - - trigdata = (TriggerData *) fcinfo->context; - trigger = trigdata->tg_trigger; - - if (trigger->tgnargs < 2) - elog(ERROR, "TSearch: format tsearch2(tsvector_field, text_field1,...)"); - - /* create space for configuration name */ - tgargs = (char **) palloc((trigger->tgnargs + 1) * sizeof(char *)); - tgargs[0] = trigger->tgargs[0]; - for (i = 1; i < trigger->tgnargs; i++) - tgargs[i + 1] = trigger->tgargs[i]; - - tgargs[1] = pstrdup(GetConfigOptionByName("default_text_search_config", - NULL, false)); - tgargs_old = trigger->tgargs; - trigger->tgargs = tgargs; - trigger->tgnargs++; - - res = tsvector_update_trigger_byid(fcinfo); - - /* restore old trigger data */ - trigger->tgargs = tgargs_old; - trigger->tgnargs--; - - pfree(tgargs[1]); - pfree(tgargs); - - return res; -} - - -Datum -tsa_rewrite_accum(PG_FUNCTION_ARGS) -{ - TSQuery acc; - ArrayType *qa; - TSQuery q; - QTNode *qex = NULL, - *subs = NULL, - *acctree = NULL; - bool isfind = false; - Datum *elemsp; - int nelemsp; - MemoryContext aggcontext; - MemoryContext oldcontext; - - if (!AggCheckCallContext(fcinfo, &aggcontext)) - elog(ERROR, "tsa_rewrite_accum called in non-aggregate context"); - - if (PG_ARGISNULL(0) || PG_GETARG_POINTER(0) == NULL) - { - acc = (TSQuery) MemoryContextAlloc(aggcontext, HDRSIZETQ); - SET_VARSIZE(acc, HDRSIZETQ); - acc->size = 0; - } - else - acc = PG_GETARG_TSQUERY(0); - - if (PG_ARGISNULL(1) || PG_GETARG_POINTER(1) == NULL) - PG_RETURN_TSQUERY(acc); - else - qa = PG_GETARG_ARRAYTYPE_P_COPY(1); - - if (ARR_NDIM(qa) != 1) - elog(ERROR, "array must be one-dimensional, not %d dimensions", - ARR_NDIM(qa)); - if (ArrayGetNItems(ARR_NDIM(qa), ARR_DIMS(qa)) != 3) - elog(ERROR, "array must have three elements"); - if (ARR_ELEMTYPE(qa) != TSQUERYOID) - elog(ERROR, "array must contain tsquery elements"); - - deconstruct_array(qa, TSQUERYOID, -1, false, 'i', &elemsp, NULL, &nelemsp); - - q = DatumGetTSQuery(elemsp[0]); - if (q->size == 0) - { - pfree(elemsp); - PG_RETURN_POINTER(acc); - } - - if (!acc->size) - { - if (VARSIZE(acc) > HDRSIZETQ) - { - pfree(elemsp); - PG_RETURN_POINTER(acc); - } - else - acctree = QT2QTN(GETQUERY(q), GETOPERAND(q)); - } - else - acctree = QT2QTN(GETQUERY(acc), GETOPERAND(acc)); - - QTNTernary(acctree); - QTNSort(acctree); - - q = DatumGetTSQuery(elemsp[1]); - if (q->size == 0) - { - pfree(elemsp); - PG_RETURN_POINTER(acc); - } - qex = QT2QTN(GETQUERY(q), GETOPERAND(q)); - QTNTernary(qex); - QTNSort(qex); - - q = DatumGetTSQuery(elemsp[2]); - if (q->size) - subs = QT2QTN(GETQUERY(q), GETOPERAND(q)); - - acctree = findsubquery(acctree, qex, subs, &isfind); - - if (isfind || !acc->size) - { - /* pfree( acc ); do not pfree(p), because nodeAgg.c will */ - if (acctree) - { - QTNBinary(acctree); - oldcontext = MemoryContextSwitchTo(aggcontext); - acc = QTN2QT(acctree); - MemoryContextSwitchTo(oldcontext); - } - else - { - acc = (TSQuery) MemoryContextAlloc(aggcontext, HDRSIZETQ); - SET_VARSIZE(acc, HDRSIZETQ); - acc->size = 0; - } - } - - pfree(elemsp); - QTNFree(qex); - QTNFree(subs); - QTNFree(acctree); - - PG_RETURN_TSQUERY(acc); -} - -Datum -tsa_rewrite_finish(PG_FUNCTION_ARGS) -{ - TSQuery acc = PG_GETARG_TSQUERY(0); - TSQuery rewrited; - - if (acc == NULL || PG_ARGISNULL(0) || acc->size == 0) - { - rewrited = (TSQuery) palloc(HDRSIZETQ); - SET_VARSIZE(rewrited, HDRSIZETQ); - rewrited->size = 0; - } - else - { - rewrited = (TSQuery) palloc(VARSIZE(acc)); - memcpy(rewrited, acc, VARSIZE(acc)); - pfree(acc); - } - - PG_RETURN_POINTER(rewrited); -} - - -/* - * Get Oid of current dictionary - */ -static Oid -GetCurrentDict(void) -{ - if (current_dictionary_oid == InvalidOid) - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("no current dictionary"), - errhint("Execute SELECT set_curdict(...)."))); - - return current_dictionary_oid; -} - -/* - * Get Oid of current parser - * - * Here, it seems reasonable to select the "default" parser if none has been - * set. - */ -static Oid -GetCurrentParser(void) -{ - if (current_parser_oid == InvalidOid) - current_parser_oid = get_ts_parser_oid(stringToQualifiedNameList("pg_catalog.default"), false); - return current_parser_oid; -} diff --git a/contrib/tsearch2/tsearch2.control b/contrib/tsearch2/tsearch2.control deleted file mode 100644 index 3e11bcfbe8..0000000000 --- a/contrib/tsearch2/tsearch2.control +++ /dev/null @@ -1,7 +0,0 @@ -# tsearch2 extension -comment = 'compatibility package for pre-8.3 text search functions' -default_version = '1.0' -module_pathname = '$libdir/tsearch2' -# this is not relocatable because the tsearch2--unpackaged--1.0.sql script -# has to use @extschema@ to avoid conflict with items in pg_catalog -relocatable = false diff --git a/doc/src/sgml/contrib.sgml b/doc/src/sgml/contrib.sgml index c8708ecf8b..03e5889839 100644 --- a/doc/src/sgml/contrib.sgml +++ b/doc/src/sgml/contrib.sgml @@ -142,7 +142,6 @@ CREATE EXTENSION module_name FROM unpackaged; &tablefunc; &tcn; &test-decoding; - &tsearch2; &tsm-system-rows; &tsm-system-time; &unaccent; diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml index 2624c627dc..e7aa92f914 100644 --- a/doc/src/sgml/filelist.sgml +++ b/doc/src/sgml/filelist.sgml @@ -151,7 +151,6 @@ - diff --git a/doc/src/sgml/textsearch.sgml b/doc/src/sgml/textsearch.sgml index 67e4901c92..fe630a66b3 100644 --- a/doc/src/sgml/textsearch.sgml +++ b/doc/src/sgml/textsearch.sgml @@ -3864,87 +3864,4 @@ Parser: "pg_catalog.default" - - Migration from Pre-8.3 Text Search - - - Applications that use the - module for text searching will need some adjustments to work - with the - built-in features: - - - - - - Some functions have been renamed or had small adjustments in their - argument lists, and all of them are now in the pg_catalog - schema, whereas in a previous installation they would have been in - public or another non-system schema. There is a new - version of tsearch2 - that provides a compatibility layer to solve most problems in this - area. - - - - - - The old tsearch2 functions and other objects - must be suppressed when loading pg_dump - output from a pre-8.3 database. While many of them won't load anyway, - a few will and then cause problems. One simple way to deal with this - is to load the new tsearch2 module before restoring - the dump; then it will block the old objects from being loaded. - - - - - - Text search configuration setup is completely different now. - Instead of manually inserting rows into configuration tables, - search is configured through the specialized SQL commands shown - earlier in this chapter. There is no automated - support for converting an existing custom configuration for 8.3; - you're on your own here. - - - - - - Most types of dictionaries rely on some outside-the-database - configuration files. These are largely compatible with pre-8.3 - usage, but note the following differences: - - - - - Configuration files now must be placed in a single specified - directory ($SHAREDIR/tsearch_data), and must have - a specific extension depending on the type of file, as noted - previously in the descriptions of the various dictionary types. - This restriction was added to forestall security problems. - - - - - - Configuration files must be encoded in UTF-8 encoding, - regardless of what database encoding is used. - - - - - - In thesaurus configuration files, stop words must be marked with - ?. - - - - - - - - - - diff --git a/doc/src/sgml/tsearch2.sgml b/doc/src/sgml/tsearch2.sgml deleted file mode 100644 index 192eccd732..0000000000 --- a/doc/src/sgml/tsearch2.sgml +++ /dev/null @@ -1,203 +0,0 @@ - - - - tsearch2 - - - tsearch2 - - - - The tsearch2 module provides backwards-compatible - text search functionality for applications that used - tsearch2 before text searching was integrated - into core PostgreSQL in release 8.3. - - - - Portability Issues - - - Although the built-in text search features were based on - tsearch2 and are largely similar to it, - there are numerous small differences that will create portability - issues for existing applications: - - - - - - Some functions' names were changed, for example rank - to ts_rank. - The replacement tsearch2 module - provides aliases having the old names. - - - - - - The built-in text search data types and functions all exist within - the system schema pg_catalog. In an installation using - tsearch2, these objects would usually have been in - the public schema, though some users chose to place them - in a separate schema of their own. Explicitly schema-qualified - references to the objects will therefore fail in either case. - The replacement tsearch2 module - provides alias objects that are stored in public - (or another schema if necessary) so that such references will still work. - - - - - - There is no concept of a current parser or current - dictionary in the built-in text search features, only of a current - search configuration (set by the default_text_search_config - parameter). While the current parser and current dictionary were used - only by functions intended for debugging, this might still pose - a porting obstacle in some cases. - The replacement tsearch2 module emulates these - additional state variables and provides backwards-compatible functions - for setting and retrieving them. - - - - - - There are some issues that are not addressed by the replacement - tsearch2 module, and will therefore require - application code changes in any case: - - - - - - The old tsearch2 trigger function allowed items in its - argument list to be names of functions to be invoked on the text data - before it was converted to tsvector format. This was removed - as being a security hole, since it was not possible to guarantee that - the function invoked was the one intended. The recommended approach - if the data must be massaged before being indexed is to write a custom - trigger that does the work for itself. - - - - - - Text search configuration information has been moved into core - system catalogs that are noticeably different from the tables used - by tsearch2. Any applications that examined - or modified those tables will need adjustment. - - - - - - If an application used any custom text search configurations, - those will need to be set up in the core - catalogs using the new text search configuration SQL commands. - The replacement tsearch2 module offers a little - bit of support for this by making it possible to load an old set - of tsearch2 configuration tables into - PostgreSQL 8.3. (Without the module, - it is not possible to load the configuration data because values in the - regprocedure columns cannot be resolved to functions.) - While those configuration tables won't actually do - anything, at least their contents will be available to be consulted - while setting up an equivalent custom configuration in 8.3. - - - - - - The old reset_tsearch() and get_covers() - functions are not supported. - - - - - - The replacement tsearch2 module does not define - any alias operators, relying entirely on the built-in ones. - This would only pose an issue if an application used explicitly - schema-qualified operator names, which is very uncommon. - - - - - - - - Converting a pre-8.3 Installation - - - The recommended way to update a pre-8.3 installation that uses - tsearch2 is: - - - - - - Make a dump from the old installation in the usual way, - but be sure not to use -c (--clean) - option of pg_dump or pg_dumpall. - - - - - - In the new installation, create empty database(s) and install - the replacement tsearch2 module into each - database that will use text search. This must be done - before loading the dump data! If your old installation - had the tsearch2 objects in a schema other - than public, be sure to adjust the - CREATE EXTENSION command so that the replacement - objects are created in that same schema. - - - - - - Load the dump data. There will be quite a few errors reported - due to failure to recreate the original tsearch2 - objects. These errors can be ignored, but this means you cannot - restore the dump in a single transaction (eg, you cannot use - pg_restore's - - - - - Examine the contents of the restored tsearch2 - configuration tables (pg_ts_cfg and so on), and - create equivalent built-in text search configurations as needed. - You may drop the old configuration tables once you've extracted - all the useful information from them. - - - - - - Test your application. - - - - - - At a later time you may wish to rename application references - to the alias text search objects, so that you can eventually - uninstall the replacement tsearch2 module. - - - - - - References - - Tsearch2 Development Site - - - - - diff --git a/src/test/modules/test_parser/test_parser.c b/src/test/modules/test_parser/test_parser.c index 43b96120de..bb5305109e 100644 --- a/src/test/modules/test_parser/test_parser.c +++ b/src/test/modules/test_parser/test_parser.c @@ -28,7 +28,6 @@ typedef struct int pos; /* position of the parser */ } ParserState; -/* copy-paste from wparser.h of tsearch2 */ typedef struct { int lexid; diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index df06e73f0d..51b5d5449a 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -36,7 +36,7 @@ my @contrib_uselibpgport = ('oid2name', 'pg_standby', 'vacuumlo'); my @contrib_uselibpgcommon = ('oid2name', 'pg_standby', 'vacuumlo'); my $contrib_extralibs = undef; my $contrib_extraincludes = - { 'tsearch2' => ['contrib/tsearch2'], 'dblink' => ['src/backend'] }; + { 'dblink' => ['src/backend'] }; my $contrib_extrasource = { 'cube' => [ 'contrib/cube/cubescan.l', 'contrib/cube/cubeparse.y' ], 'seg' => [ 'contrib/seg/segscan.l', 'contrib/seg/segparse.y' ], }; -- cgit v1.2.3 From f10637ebe02074e264b17606c00bc09da986c60d Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Tue, 14 Feb 2017 02:30:46 +0900 Subject: Replace references to "xlog" with "wal" in docs. Commit f82ec32ac30ae7e3ec7c84067192535b2ff8ec0e renamed the pg_xlog directory to pg_wal. To make things consistent, we decided to eliminate "xlog" from user-visible docs. --- doc/src/sgml/datatype.sgml | 2 +- doc/src/sgml/pgstandby.sgml | 2 +- doc/src/sgml/protocol.sgml | 2 +- doc/src/sgml/storage.sgml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/datatype.sgml b/doc/src/sgml/datatype.sgml index 9ef7b4a025..387ba53ef0 100644 --- a/doc/src/sgml/datatype.sgml +++ b/doc/src/sgml/datatype.sgml @@ -4572,7 +4572,7 @@ SELECT * FROM pg_attribute The pg_lsn data type can be used to store LSN (Log Sequence - Number) data which is a pointer to a location in the XLOG. This type is a + Number) data which is a pointer to a location in the WAL. This type is a representation of XLogRecPtr and an internal system type of PostgreSQL. diff --git a/doc/src/sgml/pgstandby.sgml b/doc/src/sgml/pgstandby.sgml index 80c6f60062..bf4edea9f1 100644 --- a/doc/src/sgml/pgstandby.sgml +++ b/doc/src/sgml/pgstandby.sgml @@ -22,7 +22,7 @@ option archivelocation nextwalfile - xlogfilepath + walfilepath restartwalfile diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index 05a0d6e846..589b881ef2 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -1371,7 +1371,7 @@ The commands accepted in walsender mode are: - Current xlog flush location. Useful to get a known location in the + Current WAL flush location. Useful to get a known location in the transaction log where streaming can start. diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml index 5c52824dfc..127b759c14 100644 --- a/doc/src/sgml/storage.sgml +++ b/doc/src/sgml/storage.sgml @@ -810,7 +810,7 @@ data. Empty in ordinary tables. pd_lsn PageXLogRecPtr 8 bytes - LSN: next byte after last byte of xlog record for last change + LSN: next byte after last byte of WAL record for last change to this page -- cgit v1.2.3 From 0ede57a1a5161012f8f4a55049ef24c987d5b59c Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Tue, 14 Feb 2017 09:37:31 -0500 Subject: Corrections and improvements to generic parallel query documentation. David Rowley, reviewed by Brad DeJong, Amit Kapila, and me. Discussion: http://postgr.es/m/CAKJS1f81fob-M6RJyTVv3SCasxMuQpj37ReNOJ=tprhwd7hAVg@mail.gmail.com --- doc/src/sgml/parallel.sgml | 60 ++++++++++++++++++++++------------------------ 1 file changed, 29 insertions(+), 31 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/parallel.sgml b/doc/src/sgml/parallel.sgml index 5d4bb211c1..e8624fcab6 100644 --- a/doc/src/sgml/parallel.sgml +++ b/doc/src/sgml/parallel.sgml @@ -284,44 +284,41 @@ EXPLAIN SELECT * FROM pgbench_accounts WHERE filler LIKE '%x%'; The driving table may be joined to one or more other tables using nested - loops or hash joins. The outer side of the join may be any kind of + loops or hash joins. The inner side of the join may be any kind of non-parallel plan that is otherwise supported by the planner provided that it is safe to run within a parallel worker. For example, it may be an - index scan which looks up a value based on a column taken from the inner - table. Each worker will execute the outer side of the plan in full, which - is why merge joins are not supported here. The outer side of a merge join - will often involve sorting the entire inner table; even if it involves an - index, it is unlikely to be productive to have multiple processes each - conduct a full index scan of the inner table. + index scan which looks up a value taken from the outer side of the join. + Each worker will execute the inner side of the join in full, which for + hash join means that an identical hash table is built in each worker + process. Parallel Aggregation - It is not possible to perform the aggregation portion of a query entirely - in parallel. For example, if a query involves selecting - COUNT(*), each worker could compute a total, but those totals - would need to combined in order to produce a final answer. If the query - involved a GROUP BY clause, a separate total would need to - be computed for each group. Even though aggregation can't be done entirely - in parallel, queries involving aggregation are often excellent candidates - for parallel query, because they typically read many rows but return only - a few rows to the client. Queries that return many rows to the client - are often limited by the speed at which the client can read the data, - in which case parallel query cannot help very much. - - - - PostgreSQL supports parallel aggregation by aggregating - twice. First, each process participating in the parallel portion of the - query performs an aggregation step, producing a partial result for each - group of which that process is aware. This is reflected in the plan as - a PartialAggregate node. Second, the partial results are + PostgreSQL supports parallel aggregation by aggregating in + two stages. First, each process participating in the parallel portion of + the query performs an aggregation step, producing a partial result for + each group of which that process is aware. This is reflected in the plan + as a Partial Aggregate node. Second, the partial results are transferred to the leader via the Gather node. Finally, the leader re-aggregates the results across all workers in order to produce the final result. This is reflected in the plan as a - FinalizeAggregate node. + Finalize Aggregate node. + + + + Because the Finalize Aggregate node runs on the leader + process, queries which produce a relatively large number of groups in + comparison to the number of input rows will appear less favorable to the + query planner. For example, in the worst-case scenario the number of + groups seen by the Finalize Aggregate node could be as many as + the number of input rows which were seen by all worker processes in the + Partial Aggregate stage. For such cases, there is clearly + going to be no performance benefit to using parallel aggregation. The + query planner takes this into account during the planning process and is + unlikely to choose parallel aggregate in this scenario. @@ -330,10 +327,11 @@ EXPLAIN SELECT * FROM pgbench_accounts WHERE filler LIKE '%x%'; have a combine function. If the aggregate has a transition state of type internal, it must have serialization and deserialization functions. See for more details. - Parallel aggregation is not supported for ordered set aggregates or when - the query involves GROUPING SETS. It can only be used when - all joins involved in the query are also part of the parallel portion - of the plan. + Parallel aggregation is not supported if any aggregate function call + contains DISTINCT or ORDER BY clause and is also + not supported for ordered set aggregates or when the query involves + GROUPING SETS. It can only be used when all joins involved in + the query are also part of the parallel portion of the plan. -- cgit v1.2.3 From 569174f1be92be93f5366212cc46960d28a5c5cd Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Wed, 15 Feb 2017 07:41:14 -0500 Subject: btree: Support parallel index scans. This isn't exposed to the optimizer or the executor yet; we'll add support for those things in a separate patch. But this puts the basic mechanism in place: several processes can attach to a parallel btree index scan, and each one will get a subset of the tuples that would have been produced by a non-parallel scan. Each index page becomes the responsibility of a single worker, which then returns all of the TIDs on that page. Rahila Syed, Amit Kapila, Robert Haas, reviewed and tested by Anastasia Lubennikova, Tushar Ahuja, and Haribabu Kommi. --- doc/src/sgml/monitoring.sgml | 6 +- src/backend/access/nbtree/nbtree.c | 259 +++++++++++++++++++++++++++++- src/backend/access/nbtree/nbtsearch.c | 286 ++++++++++++++++++++++++++++------ src/backend/access/nbtree/nbtutils.c | 4 + src/backend/postmaster/pgstat.c | 3 + src/include/access/nbtree.h | 15 +- src/include/pgstat.h | 1 + src/tools/pgindent/typedefs.list | 3 + 8 files changed, 527 insertions(+), 50 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 5b67defdb8..fad5cb05b9 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -1207,7 +1207,7 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser Waiting in an extension. - IPC + IPC BgWorkerShutdown Waiting for background worker to shut down. @@ -1215,6 +1215,10 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser BgWorkerStartup Waiting for background worker to start up. + + BtreePage + Waiting for the page number needed to continue a parallel btree scan to become available. + ExecuteGather Waiting for activity from child process when executing Gather node. diff --git a/src/backend/access/nbtree/nbtree.c b/src/backend/access/nbtree/nbtree.c index 945e563fcc..cbc575d5cf 100644 --- a/src/backend/access/nbtree/nbtree.c +++ b/src/backend/access/nbtree/nbtree.c @@ -23,6 +23,8 @@ #include "access/xlog.h" #include "catalog/index.h" #include "commands/vacuum.h" +#include "pgstat.h" +#include "storage/condition_variable.h" #include "storage/indexfsm.h" #include "storage/ipc.h" #include "storage/lmgr.h" @@ -63,6 +65,45 @@ typedef struct MemoryContext pagedelcontext; } BTVacState; +/* + * BTPARALLEL_NOT_INITIALIZED indicates that the scan has not started. + * + * BTPARALLEL_ADVANCING indicates that some process is advancing the scan to + * a new page; others must wait. + * + * BTPARALLEL_IDLE indicates that no backend is currently advancing the scan + * to a new page; some process can start doing that. + * + * BTPARALLEL_DONE indicates that the scan is complete (including error exit). + * We reach this state once for every distinct combination of array keys. + */ +typedef enum +{ + BTPARALLEL_NOT_INITIALIZED, + BTPARALLEL_ADVANCING, + BTPARALLEL_IDLE, + BTPARALLEL_DONE +} BTPS_State; + +/* + * BTParallelScanDescData contains btree specific shared information required + * for parallel scan. + */ +typedef struct BTParallelScanDescData +{ + BlockNumber btps_scanPage; /* latest or next page to be scanned */ + BTPS_State btps_pageStatus;/* indicates whether next page is available + * for scan. see above for possible states of + * parallel scan. */ + int btps_arrayKeyCount; /* count indicating number of array + * scan keys processed by parallel + * scan */ + slock_t btps_mutex; /* protects above variables */ + ConditionVariable btps_cv; /* used to synchronize parallel scan */ +} BTParallelScanDescData; + +typedef struct BTParallelScanDescData *BTParallelScanDesc; + static void btbuildCallback(Relation index, HeapTuple htup, @@ -118,9 +159,9 @@ bthandler(PG_FUNCTION_ARGS) amroutine->amendscan = btendscan; amroutine->ammarkpos = btmarkpos; amroutine->amrestrpos = btrestrpos; - amroutine->amestimateparallelscan = NULL; - amroutine->aminitparallelscan = NULL; - amroutine->amparallelrescan = NULL; + amroutine->amestimateparallelscan = btestimateparallelscan; + amroutine->aminitparallelscan = btinitparallelscan; + amroutine->amparallelrescan = btparallelrescan; PG_RETURN_POINTER(amroutine); } @@ -491,6 +532,7 @@ btrescan(IndexScanDesc scan, ScanKey scankey, int nscankeys, } so->markItemIndex = -1; + so->arrayKeyCount = 0; BTScanPosUnpinIfPinned(so->markPos); BTScanPosInvalidate(so->markPos); @@ -652,6 +694,217 @@ btrestrpos(IndexScanDesc scan) } } +/* + * btestimateparallelscan -- estimate storage for BTParallelScanDescData + */ +Size +btestimateparallelscan(void) +{ + return sizeof(BTParallelScanDescData); +} + +/* + * btinitparallelscan -- initialize BTParallelScanDesc for parallel btree scan + */ +void +btinitparallelscan(void *target) +{ + BTParallelScanDesc bt_target = (BTParallelScanDesc) target; + + SpinLockInit(&bt_target->btps_mutex); + bt_target->btps_scanPage = InvalidBlockNumber; + bt_target->btps_pageStatus = BTPARALLEL_NOT_INITIALIZED; + bt_target->btps_arrayKeyCount = 0; + ConditionVariableInit(&bt_target->btps_cv); +} + +/* + * btparallelrescan() -- reset parallel scan + */ +void +btparallelrescan(IndexScanDesc scan) +{ + BTParallelScanDesc btscan; + ParallelIndexScanDesc parallel_scan = scan->parallel_scan; + + Assert(parallel_scan); + + btscan = (BTParallelScanDesc) OffsetToPointer((void *) parallel_scan, + parallel_scan->ps_offset); + + /* + * In theory, we don't need to acquire the spinlock here, because there + * shouldn't be any other workers running at this point, but we do so for + * consistency. + */ + SpinLockAcquire(&btscan->btps_mutex); + btscan->btps_scanPage = InvalidBlockNumber; + btscan->btps_pageStatus = BTPARALLEL_NOT_INITIALIZED; + btscan->btps_arrayKeyCount = 0; + SpinLockRelease(&btscan->btps_mutex); +} + +/* + * _bt_parallel_seize() -- Begin the process of advancing the scan to a new + * page. Other scans must wait until we call bt_parallel_release() or + * bt_parallel_done(). + * + * The return value is true if we successfully seized the scan and false + * if we did not. The latter case occurs if no pages remain for the current + * set of scankeys. + * + * If the return value is true, *pageno returns the next or current page + * of the scan (depending on the scan direction). An invalid block number + * means the scan hasn't yet started, and P_NONE means we've reached the end. + * The first time a participating process reaches the last page, it will return + * true and set *pageno to P_NONE; after that, further attempts to seize the + * scan will return false. + * + * Callers should ignore the value of pageno if the return value is false. + */ +bool +_bt_parallel_seize(IndexScanDesc scan, BlockNumber *pageno) +{ + BTScanOpaque so = (BTScanOpaque) scan->opaque; + BTPS_State pageStatus; + bool exit_loop = false; + bool status = true; + ParallelIndexScanDesc parallel_scan = scan->parallel_scan; + BTParallelScanDesc btscan; + + *pageno = P_NONE; + + btscan = (BTParallelScanDesc) OffsetToPointer((void *) parallel_scan, + parallel_scan->ps_offset); + + while (1) + { + SpinLockAcquire(&btscan->btps_mutex); + pageStatus = btscan->btps_pageStatus; + + if (so->arrayKeyCount < btscan->btps_arrayKeyCount) + { + /* Parallel scan has already advanced to a new set of scankeys. */ + status = false; + } + else if (pageStatus == BTPARALLEL_DONE) + { + /* + * We're done with this set of scankeys. This may be the end, or + * there could be more sets to try. + */ + status = false; + } + else if (pageStatus != BTPARALLEL_ADVANCING) + { + /* + * We have successfully seized control of the scan for the purpose + * of advancing it to a new page! + */ + btscan->btps_pageStatus = BTPARALLEL_ADVANCING; + *pageno = btscan->btps_scanPage; + exit_loop = true; + } + SpinLockRelease(&btscan->btps_mutex); + if (exit_loop || !status) + break; + ConditionVariableSleep(&btscan->btps_cv, WAIT_EVENT_BTREE_PAGE); + } + ConditionVariableCancelSleep(); + + return status; +} + +/* + * _bt_parallel_release() -- Complete the process of advancing the scan to a + * new page. We now have the new value btps_scanPage; some other backend + * can now begin advancing the scan. + */ +void +_bt_parallel_release(IndexScanDesc scan, BlockNumber scan_page) +{ + ParallelIndexScanDesc parallel_scan = scan->parallel_scan; + BTParallelScanDesc btscan; + + btscan = (BTParallelScanDesc) OffsetToPointer((void *) parallel_scan, + parallel_scan->ps_offset); + + SpinLockAcquire(&btscan->btps_mutex); + btscan->btps_scanPage = scan_page; + btscan->btps_pageStatus = BTPARALLEL_IDLE; + SpinLockRelease(&btscan->btps_mutex); + ConditionVariableSignal(&btscan->btps_cv); +} + +/* + * _bt_parallel_done() -- Mark the parallel scan as complete. + * + * When there are no pages left to scan, this function should be called to + * notify other workers. Otherwise, they might wait forever for the scan to + * advance to the next page. + */ +void +_bt_parallel_done(IndexScanDesc scan) +{ + BTScanOpaque so = (BTScanOpaque) scan->opaque; + ParallelIndexScanDesc parallel_scan = scan->parallel_scan; + BTParallelScanDesc btscan; + bool status_changed = false; + + /* Do nothing, for non-parallel scans */ + if (parallel_scan == NULL) + return; + + btscan = (BTParallelScanDesc) OffsetToPointer((void *) parallel_scan, + parallel_scan->ps_offset); + + /* + * Mark the parallel scan as done for this combination of scan keys, + * unless some other process already did so. See also + * _bt_advance_array_keys. + */ + SpinLockAcquire(&btscan->btps_mutex); + if (so->arrayKeyCount >= btscan->btps_arrayKeyCount && + btscan->btps_pageStatus != BTPARALLEL_DONE) + { + btscan->btps_pageStatus = BTPARALLEL_DONE; + status_changed = true; + } + SpinLockRelease(&btscan->btps_mutex); + + /* wake up all the workers associated with this parallel scan */ + if (status_changed) + ConditionVariableBroadcast(&btscan->btps_cv); +} + +/* + * _bt_parallel_advance_array_keys() -- Advances the parallel scan for array + * keys. + * + * Updates the count of array keys processed for both local and parallel + * scans. + */ +void +_bt_parallel_advance_array_keys(IndexScanDesc scan) +{ + BTScanOpaque so = (BTScanOpaque) scan->opaque; + ParallelIndexScanDesc parallel_scan = scan->parallel_scan; + BTParallelScanDesc btscan; + + btscan = (BTParallelScanDesc) OffsetToPointer((void *) parallel_scan, + parallel_scan->ps_offset); + + so->arrayKeyCount++; + SpinLockAcquire(&btscan->btps_mutex); + if (btscan->btps_pageStatus == BTPARALLEL_DONE) + { + btscan->btps_scanPage = InvalidBlockNumber; + btscan->btps_pageStatus = BTPARALLEL_NOT_INITIALIZED; + btscan->btps_arrayKeyCount++; + } + SpinLockRelease(&btscan->btps_mutex); +} + /* * Bulk deletion of all index entries pointing to a set of heap tuples. * The set of target tuples is specified via a callback routine that tells diff --git a/src/backend/access/nbtree/nbtsearch.c b/src/backend/access/nbtree/nbtsearch.c index b6459d2f2a..2f32b2e78d 100644 --- a/src/backend/access/nbtree/nbtsearch.c +++ b/src/backend/access/nbtree/nbtsearch.c @@ -30,9 +30,13 @@ static bool _bt_readpage(IndexScanDesc scan, ScanDirection dir, static void _bt_saveitem(BTScanOpaque so, int itemIndex, OffsetNumber offnum, IndexTuple itup); static bool _bt_steppage(IndexScanDesc scan, ScanDirection dir); +static bool _bt_readnextpage(IndexScanDesc scan, BlockNumber blkno, ScanDirection dir); +static bool _bt_parallel_readpage(IndexScanDesc scan, BlockNumber blkno, + ScanDirection dir); static Buffer _bt_walk_left(Relation rel, Buffer buf, Snapshot snapshot); static bool _bt_endpoint(IndexScanDesc scan, ScanDirection dir); static void _bt_drop_lock_and_maybe_pin(IndexScanDesc scan, BTScanPos sp); +static inline void _bt_initialize_more_data(BTScanOpaque so, ScanDirection dir); /* @@ -544,8 +548,10 @@ _bt_first(IndexScanDesc scan, ScanDirection dir) ScanKeyData notnullkeys[INDEX_MAX_KEYS]; int keysCount = 0; int i; + bool status = true; StrategyNumber strat_total; BTScanPosItem *currItem; + BlockNumber blkno; Assert(!BTScanPosIsValid(so->currPos)); @@ -564,6 +570,30 @@ _bt_first(IndexScanDesc scan, ScanDirection dir) if (!so->qual_ok) return false; + /* + * For parallel scans, get the starting page from shared state. If the + * scan has not started, proceed to find out first leaf page in the usual + * way while keeping other participating processes waiting. If the scan + * has already begun, use the page number from the shared structure. + */ + if (scan->parallel_scan != NULL) + { + status = _bt_parallel_seize(scan, &blkno); + if (!status) + return false; + else if (blkno == P_NONE) + { + _bt_parallel_done(scan); + return false; + } + else if (blkno != InvalidBlockNumber) + { + if (!_bt_parallel_readpage(scan, blkno, dir)) + return false; + goto readcomplete; + } + } + /*---------- * Examine the scan keys to discover where we need to start the scan. * @@ -743,7 +773,19 @@ _bt_first(IndexScanDesc scan, ScanDirection dir) * there. */ if (keysCount == 0) - return _bt_endpoint(scan, dir); + { + bool match; + + match = _bt_endpoint(scan, dir); + + if (!match) + { + /* No match, so mark (parallel) scan finished */ + _bt_parallel_done(scan); + } + + return match; + } /* * We want to start the scan somewhere within the index. Set up an @@ -773,7 +815,10 @@ _bt_first(IndexScanDesc scan, ScanDirection dir) Assert(subkey->sk_flags & SK_ROW_MEMBER); if (subkey->sk_flags & SK_ISNULL) + { + _bt_parallel_done(scan); return false; + } memcpy(scankeys + i, subkey, sizeof(ScanKeyData)); /* @@ -993,25 +1038,21 @@ _bt_first(IndexScanDesc scan, ScanDirection dir) * because nothing finer to lock exists. */ PredicateLockRelation(rel, scan->xs_snapshot); + + /* + * mark parallel scan as done, so that all the workers can finish + * their scan + */ + _bt_parallel_done(scan); + BTScanPosInvalidate(so->currPos); + return false; } else PredicateLockPage(rel, BufferGetBlockNumber(buf), scan->xs_snapshot); - /* initialize moreLeft/moreRight appropriately for scan direction */ - if (ScanDirectionIsForward(dir)) - { - so->currPos.moreLeft = false; - so->currPos.moreRight = true; - } - else - { - so->currPos.moreLeft = true; - so->currPos.moreRight = false; - } - so->numKilled = 0; /* just paranoia */ - Assert(so->markItemIndex == -1); + _bt_initialize_more_data(so, dir); /* position to the precise item on the page */ offnum = _bt_binsrch(rel, buf, keysCount, scankeys, nextkey); @@ -1060,6 +1101,7 @@ _bt_first(IndexScanDesc scan, ScanDirection dir) _bt_drop_lock_and_maybe_pin(scan, &so->currPos); } +readcomplete: /* OK, itemIndex says what to return */ currItem = &so->currPos.items[so->currPos.itemIndex]; scan->xs_ctup.t_self = currItem->heapTid; @@ -1132,6 +1174,10 @@ _bt_next(IndexScanDesc scan, ScanDirection dir) * moreLeft or moreRight (as appropriate) is cleared if _bt_checkkeys reports * that there can be no more matching tuples in the current scan direction. * + * In the case of a parallel scan, caller must have called _bt_parallel_seize + * prior to calling this function; this function will invoke + * _bt_parallel_release before returning. + * * Returns true if any matching items found on the page, false if none. */ static bool @@ -1154,6 +1200,16 @@ _bt_readpage(IndexScanDesc scan, ScanDirection dir, OffsetNumber offnum) page = BufferGetPage(so->currPos.buf); opaque = (BTPageOpaque) PageGetSpecialPointer(page); + + /* allow next page be processed by parallel worker */ + if (scan->parallel_scan) + { + if (ScanDirectionIsForward(dir)) + _bt_parallel_release(scan, opaque->btpo_next); + else + _bt_parallel_release(scan, BufferGetBlockNumber(so->currPos.buf)); + } + minoff = P_FIRSTDATAKEY(opaque); maxoff = PageGetMaxOffsetNumber(page); @@ -1278,21 +1334,16 @@ _bt_saveitem(BTScanOpaque so, int itemIndex, * if pinned, we'll drop the pin before moving to next page. The buffer is * not locked on entry. * - * On success exit, so->currPos is updated to contain data from the next - * interesting page. For success on a scan using a non-MVCC snapshot we hold - * a pin, but not a read lock, on that page. If we do not hold the pin, we - * set so->currPos.buf to InvalidBuffer. We return TRUE to indicate success. - * - * If there are no more matching records in the given direction, we drop all - * locks and pins, set so->currPos.buf to InvalidBuffer, and return FALSE. + * For success on a scan using a non-MVCC snapshot we hold a pin, but not a + * read lock, on that page. If we do not hold the pin, we set so->currPos.buf + * to InvalidBuffer. We return TRUE to indicate success. */ static bool _bt_steppage(IndexScanDesc scan, ScanDirection dir) { BTScanOpaque so = (BTScanOpaque) scan->opaque; - Relation rel; - Page page; - BTPageOpaque opaque; + BlockNumber blkno = InvalidBlockNumber; + bool status = true; Assert(BTScanPosIsValid(so->currPos)); @@ -1319,25 +1370,103 @@ _bt_steppage(IndexScanDesc scan, ScanDirection dir) so->markItemIndex = -1; } - rel = scan->indexRelation; - if (ScanDirectionIsForward(dir)) { /* Walk right to the next page with data */ - /* We must rely on the previously saved nextPage link! */ - BlockNumber blkno = so->currPos.nextPage; + if (scan->parallel_scan != NULL) + { + /* + * Seize the scan to get the next block number; if the scan has + * ended already, bail out. + */ + status = _bt_parallel_seize(scan, &blkno); + if (!status) + { + /* release the previous buffer, if pinned */ + BTScanPosUnpinIfPinned(so->currPos); + BTScanPosInvalidate(so->currPos); + return false; + } + } + else + { + /* Not parallel, so use the previously-saved nextPage link. */ + blkno = so->currPos.nextPage; + } /* Remember we left a page with data */ so->currPos.moreLeft = true; /* release the previous buffer, if pinned */ BTScanPosUnpinIfPinned(so->currPos); + } + else + { + /* Remember we left a page with data */ + so->currPos.moreRight = true; + + if (scan->parallel_scan != NULL) + { + /* + * Seize the scan to get the current block number; if the scan has + * ended already, bail out. + */ + status = _bt_parallel_seize(scan, &blkno); + BTScanPosUnpinIfPinned(so->currPos); + if (!status) + { + BTScanPosInvalidate(so->currPos); + return false; + } + } + else + { + /* Not parallel, so just use our own notion of the current page */ + blkno = so->currPos.currPage; + } + } + + if (!_bt_readnextpage(scan, blkno, dir)) + return false; + + /* Drop the lock, and maybe the pin, on the current page */ + _bt_drop_lock_and_maybe_pin(scan, &so->currPos); + return true; +} + +/* + * _bt_readnextpage() -- Read next page containing valid data for scan + * + * On success exit, so->currPos is updated to contain data from the next + * interesting page. Caller is responsible to release lock and pin on + * buffer on success. We return TRUE to indicate success. + * + * If there are no more matching records in the given direction, we drop all + * locks and pins, set so->currPos.buf to InvalidBuffer, and return FALSE. + */ +static bool +_bt_readnextpage(IndexScanDesc scan, BlockNumber blkno, ScanDirection dir) +{ + BTScanOpaque so = (BTScanOpaque) scan->opaque; + Relation rel; + Page page; + BTPageOpaque opaque; + bool status = true; + + rel = scan->indexRelation; + + if (ScanDirectionIsForward(dir)) + { for (;;) { - /* if we're at end of scan, give up */ + /* + * if we're at end of scan, give up and mark parallel scan as + * done, so that all the workers can finish their scan + */ if (blkno == P_NONE || !so->currPos.moreRight) { + _bt_parallel_done(scan); BTScanPosInvalidate(so->currPos); return false; } @@ -1359,14 +1488,32 @@ _bt_steppage(IndexScanDesc scan, ScanDirection dir) } /* nope, keep going */ - blkno = opaque->btpo_next; + if (scan->parallel_scan != NULL) + { + status = _bt_parallel_seize(scan, &blkno); + if (!status) + { + _bt_relbuf(rel, so->currPos.buf); + BTScanPosInvalidate(so->currPos); + return false; + } + } + else + blkno = opaque->btpo_next; _bt_relbuf(rel, so->currPos.buf); } } else { - /* Remember we left a page with data */ - so->currPos.moreRight = true; + /* + * Should only happen in parallel cases, when some other backend + * advanced the scan. + */ + if (so->currPos.currPage != blkno) + { + BTScanPosUnpinIfPinned(so->currPos); + so->currPos.currPage = blkno; + } /* * Walk left to the next page with data. This is much more complex @@ -1401,6 +1548,7 @@ _bt_steppage(IndexScanDesc scan, ScanDirection dir) if (!so->currPos.moreLeft) { _bt_relbuf(rel, so->currPos.buf); + _bt_parallel_done(scan); BTScanPosInvalidate(so->currPos); return false; } @@ -1412,6 +1560,7 @@ _bt_steppage(IndexScanDesc scan, ScanDirection dir) /* if we're physically at end of index, return failure */ if (so->currPos.buf == InvalidBuffer) { + _bt_parallel_done(scan); BTScanPosInvalidate(so->currPos); return false; } @@ -1432,9 +1581,46 @@ _bt_steppage(IndexScanDesc scan, ScanDirection dir) if (_bt_readpage(scan, dir, PageGetMaxOffsetNumber(page))) break; } + + /* + * For parallel scans, get the last page scanned as it is quite + * possible that by the time we try to seize the scan, some other + * worker has already advanced the scan to a different page. We + * must continue based on the latest page scanned by any worker. + */ + if (scan->parallel_scan != NULL) + { + _bt_relbuf(rel, so->currPos.buf); + status = _bt_parallel_seize(scan, &blkno); + if (!status) + { + BTScanPosInvalidate(so->currPos); + return false; + } + so->currPos.buf = _bt_getbuf(rel, blkno, BT_READ); + } } } + return true; +} + +/* + * _bt_parallel_readpage() -- Read current page containing valid data for scan + * + * On success, release lock and maybe pin on buffer. We return TRUE to + * indicate success. + */ +static bool +_bt_parallel_readpage(IndexScanDesc scan, BlockNumber blkno, ScanDirection dir) +{ + BTScanOpaque so = (BTScanOpaque) scan->opaque; + + _bt_initialize_more_data(so, dir); + + if (!_bt_readnextpage(scan, blkno, dir)) + return false; + /* Drop the lock, and maybe the pin, on the current page */ _bt_drop_lock_and_maybe_pin(scan, &so->currPos); @@ -1712,19 +1898,7 @@ _bt_endpoint(IndexScanDesc scan, ScanDirection dir) /* remember which buffer we have pinned */ so->currPos.buf = buf; - /* initialize moreLeft/moreRight appropriately for scan direction */ - if (ScanDirectionIsForward(dir)) - { - so->currPos.moreLeft = false; - so->currPos.moreRight = true; - } - else - { - so->currPos.moreLeft = true; - so->currPos.moreRight = false; - } - so->numKilled = 0; /* just paranoia */ - so->markItemIndex = -1; /* ditto */ + _bt_initialize_more_data(so, dir); /* * Now load data from the first page of the scan. @@ -1753,3 +1927,25 @@ _bt_endpoint(IndexScanDesc scan, ScanDirection dir) return true; } + +/* + * _bt_initialize_more_data() -- initialize moreLeft/moreRight appropriately + * for scan direction + */ +static inline void +_bt_initialize_more_data(BTScanOpaque so, ScanDirection dir) +{ + /* initialize moreLeft/moreRight appropriately for scan direction */ + if (ScanDirectionIsForward(dir)) + { + so->currPos.moreLeft = false; + so->currPos.moreRight = true; + } + else + { + so->currPos.moreLeft = true; + so->currPos.moreRight = false; + } + so->numKilled = 0; /* just paranoia */ + so->markItemIndex = -1; /* ditto */ +} diff --git a/src/backend/access/nbtree/nbtutils.c b/src/backend/access/nbtree/nbtutils.c index da0f330c96..5b259a31d9 100644 --- a/src/backend/access/nbtree/nbtutils.c +++ b/src/backend/access/nbtree/nbtutils.c @@ -590,6 +590,10 @@ _bt_advance_array_keys(IndexScanDesc scan, ScanDirection dir) break; } + /* advance parallel scan */ + if (scan->parallel_scan != NULL) + _bt_parallel_advance_array_keys(scan); + return found; } diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index 7176cf1bbe..ada374c0c4 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -3374,6 +3374,9 @@ pgstat_get_wait_ipc(WaitEventIPC w) case WAIT_EVENT_BGWORKER_STARTUP: event_name = "BgWorkerStartup"; break; + case WAIT_EVENT_BTREE_PAGE: + event_name = "BtreePage"; + break; case WAIT_EVENT_EXECUTE_GATHER: event_name = "ExecuteGather"; break; diff --git a/src/include/access/nbtree.h b/src/include/access/nbtree.h index 25a1dc818c..6289ffa9bd 100644 --- a/src/include/access/nbtree.h +++ b/src/include/access/nbtree.h @@ -383,6 +383,8 @@ typedef struct BTScanOpaqueData ScanKey arrayKeyData; /* modified copy of scan->keyData */ int numArrayKeys; /* number of equality-type array keys (-1 if * there are any unsatisfiable array keys) */ + int arrayKeyCount; /* count indicating number of array scan keys + * processed */ BTArrayKeyInfo *arrayKeys; /* info about each equality-type array key */ MemoryContext arrayContext; /* scan-lifespan context for array data */ @@ -426,7 +428,7 @@ typedef BTScanOpaqueData *BTScanOpaque; #define SK_BT_NULLS_FIRST (INDOPTION_NULLS_FIRST << SK_BT_INDOPTION_SHIFT) /* - * prototypes for functions in nbtree.c (external entry points for btree) + * external entry points for btree, in nbtree.c */ extern IndexBuildResult *btbuild(Relation heap, Relation index, struct IndexInfo *indexInfo); @@ -436,10 +438,13 @@ extern bool btinsert(Relation rel, Datum *values, bool *isnull, IndexUniqueCheck checkUnique, struct IndexInfo *indexInfo); extern IndexScanDesc btbeginscan(Relation rel, int nkeys, int norderbys); +extern Size btestimateparallelscan(void); +extern void btinitparallelscan(void *target); extern bool btgettuple(IndexScanDesc scan, ScanDirection dir); extern int64 btgetbitmap(IndexScanDesc scan, TIDBitmap *tbm); extern void btrescan(IndexScanDesc scan, ScanKey scankey, int nscankeys, ScanKey orderbys, int norderbys); +extern void btparallelrescan(IndexScanDesc scan); extern void btendscan(IndexScanDesc scan); extern void btmarkpos(IndexScanDesc scan); extern void btrestrpos(IndexScanDesc scan); @@ -451,6 +456,14 @@ extern IndexBulkDeleteResult *btvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats); extern bool btcanreturn(Relation index, int attno); +/* + * prototypes for internal functions in nbtree.c + */ +extern bool _bt_parallel_seize(IndexScanDesc scan, BlockNumber *pageno); +extern void _bt_parallel_release(IndexScanDesc scan, BlockNumber scan_page); +extern void _bt_parallel_done(IndexScanDesc scan); +extern void _bt_parallel_advance_array_keys(IndexScanDesc scan); + /* * prototypes for functions in nbtinsert.c */ diff --git a/src/include/pgstat.h b/src/include/pgstat.h index de8225b989..8b710ecb24 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -780,6 +780,7 @@ typedef enum { WAIT_EVENT_BGWORKER_SHUTDOWN = PG_WAIT_IPC, WAIT_EVENT_BGWORKER_STARTUP, + WAIT_EVENT_BTREE_PAGE, WAIT_EVENT_EXECUTE_GATHER, WAIT_EVENT_MQ_INTERNAL, WAIT_EVENT_MQ_PUT_MESSAGE, diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index c4235ae63a..9f876ae264 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -161,6 +161,9 @@ BTPageOpaque BTPageOpaqueData BTPageStat BTPageState +BTParallelScanDesc +BTParallelScanDescData +BTPS_State BTScanOpaque BTScanOpaqueData BTScanPos -- cgit v1.2.3 From e403732ef66d368f0a9a154d8f756f5d28615b8a Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Wed, 15 Feb 2017 07:53:38 -0500 Subject: Fix some nonstandard capitalization. Ashutosh Bapat --- doc/src/sgml/ddl.sgml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ddl.sgml b/doc/src/sgml/ddl.sgml index 39e44461e2..f909242e4c 100644 --- a/doc/src/sgml/ddl.sgml +++ b/doc/src/sgml/ddl.sgml @@ -2907,8 +2907,8 @@ VALUES ('Albany', NULL, NULL, 'NY'); - Since Primary Keys are not supprtable on partitioned tables - Foreign keys referencing partitioned tables are not supported, nor + Since primary keys are not supported on partitioned tables + foreign keys referencing partitioned tables are not supported, nor are foreign key references from a partitioned table to some other table. -- cgit v1.2.3 From 6d16ecc646d21b39092970c591fd0f73b4cfc26b Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 8 Feb 2017 22:51:09 -0500 Subject: Add CREATE COLLATION IF NOT EXISTS clause The core of the functionality was already implemented when pg_import_system_collations was added. This just exposes it as an option in the SQL command. --- doc/src/sgml/ref/create_collation.sgml | 15 +++++++++++++-- src/backend/commands/collationcmds.c | 4 ++-- src/backend/nodes/copyfuncs.c | 1 + src/backend/nodes/equalfuncs.c | 1 + src/backend/parser/gram.y | 20 ++++++++++++++++++++ src/backend/tcop/utility.c | 3 ++- src/include/commands/collationcmds.h | 2 +- src/include/nodes/parsenodes.h | 1 + src/test/regress/expected/collate.linux.utf8.out | 4 ++++ src/test/regress/sql/collate.linux.utf8.sql | 2 ++ 10 files changed, 47 insertions(+), 6 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/create_collation.sgml b/doc/src/sgml/ref/create_collation.sgml index d757cdfb43..c09e5bd6d4 100644 --- a/doc/src/sgml/ref/create_collation.sgml +++ b/doc/src/sgml/ref/create_collation.sgml @@ -18,12 +18,12 @@ -CREATE COLLATION name ( +CREATE COLLATION [ IF NOT EXISTS ] name ( [ LOCALE = locale, ] [ LC_COLLATE = lc_collate, ] [ LC_CTYPE = lc_ctype ] ) -CREATE COLLATION name FROM existing_collation +CREATE COLLATION [ IF NOT EXISTS ] name FROM existing_collation @@ -47,6 +47,17 @@ CREATE COLLATION name FROM existing_coll Parameters + + IF NOT EXISTS + + + Do not throw an error if a collation with the same name already exists. + A notice is issued in this case. Note that there is no guarantee that + the existing collation is anything like the one that would have been created. + + + + name diff --git a/src/backend/commands/collationcmds.c b/src/backend/commands/collationcmds.c index e165d4b2a6..919cfc6a06 100644 --- a/src/backend/commands/collationcmds.c +++ b/src/backend/commands/collationcmds.c @@ -37,7 +37,7 @@ * CREATE COLLATION */ ObjectAddress -DefineCollation(ParseState *pstate, List *names, List *parameters) +DefineCollation(ParseState *pstate, List *names, List *parameters, bool if_not_exists) { char *collName; Oid collNamespace; @@ -137,7 +137,7 @@ DefineCollation(ParseState *pstate, List *names, List *parameters) GetDatabaseEncoding(), collcollate, collctype, - false); + if_not_exists); if (!OidIsValid(newoid)) return InvalidObjectAddress; diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index 12324ab63f..05d8538717 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -3105,6 +3105,7 @@ _copyDefineStmt(const DefineStmt *from) COPY_NODE_FIELD(defnames); COPY_NODE_FIELD(args); COPY_NODE_FIELD(definition); + COPY_SCALAR_FIELD(if_not_exists); return newnode; } diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index 6d1dabe17e..d595cd7481 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -1211,6 +1211,7 @@ _equalDefineStmt(const DefineStmt *a, const DefineStmt *b) COMPARE_NODE_FIELD(defnames); COMPARE_NODE_FIELD(args); COMPARE_NODE_FIELD(definition); + COMPARE_SCALAR_FIELD(if_not_exists); return true; } diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 174773bdf3..5cb82977d5 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -5610,6 +5610,16 @@ DefineStmt: n->definition = $4; $$ = (Node *)n; } + | CREATE COLLATION IF_P NOT EXISTS any_name definition + { + DefineStmt *n = makeNode(DefineStmt); + n->kind = OBJECT_COLLATION; + n->args = NIL; + n->defnames = $6; + n->definition = $7; + n->if_not_exists = true; + $$ = (Node *)n; + } | CREATE COLLATION any_name FROM any_name { DefineStmt *n = makeNode(DefineStmt); @@ -5619,6 +5629,16 @@ DefineStmt: n->definition = list_make1(makeDefElem("from", (Node *) $5, @5)); $$ = (Node *)n; } + | CREATE COLLATION IF_P NOT EXISTS any_name FROM any_name + { + DefineStmt *n = makeNode(DefineStmt); + n->kind = OBJECT_COLLATION; + n->args = NIL; + n->defnames = $6; + n->definition = list_make1(makeDefElem("from", (Node *) $8, @8)); + n->if_not_exists = true; + $$ = (Node *)n; + } ; definition: '(' def_list ')' { $$ = $2; } diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index 5d3be38bf5..3bc0ae5e7e 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -1271,7 +1271,8 @@ ProcessUtilitySlow(ParseState *pstate, Assert(stmt->args == NIL); address = DefineCollation(pstate, stmt->defnames, - stmt->definition); + stmt->definition, + stmt->if_not_exists); break; default: elog(ERROR, "unrecognized define stmt type: %d", diff --git a/src/include/commands/collationcmds.h b/src/include/commands/collationcmds.h index 699ce2f9ee..3b2fcb8271 100644 --- a/src/include/commands/collationcmds.h +++ b/src/include/commands/collationcmds.h @@ -18,7 +18,7 @@ #include "catalog/objectaddress.h" #include "nodes/parsenodes.h" -extern ObjectAddress DefineCollation(ParseState *pstate, List *names, List *parameters); +extern ObjectAddress DefineCollation(ParseState *pstate, List *names, List *parameters, bool if_not_exists); extern void IsThereCollationInNamespace(const char *collname, Oid nspOid); #endif /* COLLATIONCMDS_H */ diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 07a8436143..5afc3ebea0 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -2380,6 +2380,7 @@ typedef struct DefineStmt List *defnames; /* qualified name (list of Value strings) */ List *args; /* a list of TypeName (if needed) */ List *definition; /* a list of DefElem */ + bool if_not_exists; /* just do nothing if it already exists? */ } DefineStmt; /* ---------------------- diff --git a/src/test/regress/expected/collate.linux.utf8.out b/src/test/regress/expected/collate.linux.utf8.out index 286c972fbb..293e78641e 100644 --- a/src/test/regress/expected/collate.linux.utf8.out +++ b/src/test/regress/expected/collate.linux.utf8.out @@ -963,6 +963,10 @@ END $$; CREATE COLLATION test0 FROM "C"; -- fail, duplicate name ERROR: collation "test0" for encoding "UTF8" already exists +CREATE COLLATION IF NOT EXISTS test0 FROM "C"; -- ok, skipped +NOTICE: collation "test0" for encoding "UTF8" already exists, skipping +CREATE COLLATION IF NOT EXISTS test0 (locale = 'foo'); -- ok, skipped +NOTICE: collation "test0" for encoding "UTF8" already exists, skipping do $$ BEGIN EXECUTE 'CREATE COLLATION test1 (lc_collate = ' || diff --git a/src/test/regress/sql/collate.linux.utf8.sql b/src/test/regress/sql/collate.linux.utf8.sql index 3b7cc6cf2b..c349cbde2b 100644 --- a/src/test/regress/sql/collate.linux.utf8.sql +++ b/src/test/regress/sql/collate.linux.utf8.sql @@ -325,6 +325,8 @@ BEGIN END $$; CREATE COLLATION test0 FROM "C"; -- fail, duplicate name +CREATE COLLATION IF NOT EXISTS test0 FROM "C"; -- ok, skipped +CREATE COLLATION IF NOT EXISTS test0 (locale = 'foo'); -- ok, skipped do $$ BEGIN EXECUTE 'CREATE COLLATION test1 (lc_collate = ' || -- cgit v1.2.3 From fbe7a3fa45f360e73ce141e51005a3e86cd1926c Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 15 Feb 2017 10:44:07 -0500 Subject: doc: Add advice about systemd RemoveIPC Reviewed-by: Magnus Hagander --- doc/src/sgml/runtime.sgml | 79 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) (limited to 'doc/src') diff --git a/doc/src/sgml/runtime.sgml b/doc/src/sgml/runtime.sgml index 130c386462..5e3d783c6a 100644 --- a/doc/src/sgml/runtime.sgml +++ b/doc/src/sgml/runtime.sgml @@ -1165,6 +1165,85 @@ project.max-msg-ids=(priv,4096,deny) + + systemd RemoveIPC + + + systemd + RemoveIPC + + + + If systemd is in use, some care must be taken + that IPC resources (shared memory and semaphores) are not prematurely + removed by the operating system. This is especially of concern when + installing PostgreSQL from source. Users of distribution packages of + PostgreSQL are less likely to be affected, as + the postgres user is then normally created as a system + user. + + + + The setting RemoveIPC + in logind.conf controls whether IPC objects are + removed when a user fully logs out. System users are exempt. This + setting defaults to on in stock systemd, but + some operating system distributions default it to off. + + + + A typical observed effect when this setting is on is that the semaphore + objects used by a PostgreSQL server are removed at apparently random + times, leading to the server crashing with log messages like + +LOG: semctl(1234567890, 0, IPC_RMID, ...) failed: Invalid argument + + Different types of IPC objects (shared memory vs. semaphores, System V + vs. POSIX) are treated slightly differently + by systemd, so one might observe that some IPC + resources are not removed in the same way as others. But it is not + advisable to rely on these subtle differences. + + + + A user logging out might happen as part of a maintenance + job or manually when an administrator logs in as + the postgres user or something similar, so it is hard + to prevent in general. + + + + What is a system user is determined + at systemd compile time from + the SYS_UID_MAX setting + in /etc/login.defs. + + + + Packaging and deployment scripts should be careful to create + the postgres user as a system user by + using useradd -r, adduser --system, + or equivalent. + + + + Alternatively, if the user account was created incorrectly or cannot be + changed, it is recommended to set + +RemoveIPC=no + + in /etc/systemd/logind.conf or another appropriate + configuration file. + + + + + At least one of these two things has to be ensured, or the PostgreSQL + server will be very unreliable. + + + + Resource Limits -- cgit v1.2.3 From 1330a7d7265577412cae44653a5368172af396c7 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Wed, 15 Feb 2017 11:03:41 -0500 Subject: Document new libpq connection statuses for target_session_attrs. I didn't realize these would ever be visible to clients, but Michael figured out that it can happen when using asynchronous interfaces such as PQconnectPoll. Michael Paquier --- doc/src/sgml/libpq.sgml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'doc/src') diff --git a/doc/src/sgml/libpq.sgml b/doc/src/sgml/libpq.sgml index ea7e7da9d4..4bc5bf3192 100644 --- a/doc/src/sgml/libpq.sgml +++ b/doc/src/sgml/libpq.sgml @@ -410,6 +410,24 @@ PostgresPollingStatusType PQconnectPoll(PGconn *conn); + + + CONNECTION_CHECK_WRITABLE + + + Checking if connection is able to handle write transactions. + + + + + + CONNECTION_CONSUME + + + Consuming any remaining response messages on connection. + + + Note that, although these constants will remain (in order to maintain -- cgit v1.2.3 From 51ee6f3160d2e1515ed6197594bda67eb99dc2cc Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Wed, 15 Feb 2017 13:37:24 -0500 Subject: Replace min_parallel_relation_size with two new GUCs. When min_parallel_relation_size was added, the only supported type of parallel scan was a parallel sequential scan, but there are pending patches for parallel index scan, parallel index-only scan, and parallel bitmap heap scan. Those patches introduce two new types of complications: first, what's relevant is not really the total size of the relation but the portion of it that we will scan; and second, index pages and heap pages shouldn't necessarily be treated in exactly the same way. Typically, the number of index pages will be quite small, but that doesn't necessarily mean that a parallel index scan can't pay off. Therefore, we introduce min_parallel_table_scan_size, which works out a degree of parallelism for scans based on the number of table pages that will be scanned (and which is therefore equivalent to min_parallel_relation_size for parallel sequential scans) and also min_parallel_index_scan_size which can be used to work out a degree of parallelism based on the number of index pages that will be scanned. Amit Kapila and Robert Haas Discussion: http://postgr.es/m/CAA4eK1KowGSYYVpd2qPpaPPA5R90r++QwDFbrRECTE9H_HvpOg@mail.gmail.com Discussion: http://postgr.es/m/CAA4eK1+TnM4pXQbvn7OXqam+k_HZqb0ROZUMxOiL6DWJYCyYow@mail.gmail.com --- doc/src/sgml/config.sgml | 31 +++++++++-- doc/src/sgml/release-9.6.sgml | 4 +- src/backend/optimizer/path/allpaths.c | 79 +++++++++++++++++++-------- src/backend/utils/misc/guc.c | 19 +++++-- src/backend/utils/misc/postgresql.conf.sample | 3 +- src/include/optimizer/paths.h | 3 +- src/test/regress/expected/select_parallel.out | 2 +- src/test/regress/sql/select_parallel.sql | 2 +- 8 files changed, 105 insertions(+), 38 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index dc63d7d5e4..95afc2c483 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3835,16 +3835,37 @@ ANY num_sync ( - min_parallel_relation_size (integer) + + min_parallel_table_scan_size (integer) - min_parallel_relation_size configuration parameter + min_parallel_table_scan_size configuration parameter - Sets the minimum size of relations to be considered for parallel scan. - The default is 8 megabytes (8MB). + Sets the minimum amount of table data that must be scanned in order + for a parallel scan to be considered. For a parallel sequential scan, + the amount of table data scanned is always equal to the size of the + table, but when indexes are used the amount of table data + scanned will normally be less. The default is 8 + megabytes (8MB). + + + + + + min_parallel_index_scan_size (integer) + + min_parallel_index_scan_size configuration parameter + + + + + Sets the minimum amount of index data that must be scanned in order + for a parallel scan to be considered. Note that a parallel index scan + typically won't touch the entire index; it is the number of pages + which the planner believes will actually be touched by the scan which + is relevant. The default is 512 kilobytes (512kB). diff --git a/doc/src/sgml/release-9.6.sgml b/doc/src/sgml/release-9.6.sgml index bffcaac46e..02cc8c9003 100644 --- a/doc/src/sgml/release-9.6.sgml +++ b/doc/src/sgml/release-9.6.sgml @@ -2407,8 +2407,8 @@ and many others in the same vein is available through other new configuration parameters , , , and . + linkend="guc-parallel-tuple-cost">, and + min_parallel_relation_size. diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index 5c189874ef..85505c57d3 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -57,7 +57,8 @@ typedef struct pushdown_safety_info /* These parameters are set by GUC */ bool enable_geqo = false; /* just in case GUC doesn't set it */ int geqo_threshold; -int min_parallel_relation_size; +int min_parallel_table_scan_size; +int min_parallel_index_scan_size; /* Hook for plugins to get control in set_rel_pathlist() */ set_rel_pathlist_hook_type set_rel_pathlist_hook = NULL; @@ -126,7 +127,8 @@ static void subquery_push_qual(Query *subquery, static void recurse_push_qual(Node *setOp, Query *topquery, RangeTblEntry *rte, Index rti, Node *qual); static void remove_unused_subquery_outputs(Query *subquery, RelOptInfo *rel); -static int compute_parallel_worker(RelOptInfo *rel, BlockNumber pages); +static int compute_parallel_worker(RelOptInfo *rel, BlockNumber heap_pages, + BlockNumber index_pages); /* @@ -679,7 +681,7 @@ create_plain_partial_paths(PlannerInfo *root, RelOptInfo *rel) { int parallel_workers; - parallel_workers = compute_parallel_worker(rel, rel->pages); + parallel_workers = compute_parallel_worker(rel, rel->pages, 0); /* If any limit was set to zero, the user doesn't want a parallel scan. */ if (parallel_workers <= 0) @@ -2876,13 +2878,20 @@ remove_unused_subquery_outputs(Query *subquery, RelOptInfo *rel) /* * Compute the number of parallel workers that should be used to scan a - * relation. "pages" is the number of pages from the relation that we - * expect to scan. + * relation. We compute the parallel workers based on the size of the heap to + * be scanned and the size of the index to be scanned, then choose a minimum + * of those. + * + * "heap_pages" is the number of pages from the table that we expect to scan. + * "index_pages" is the number of pages from the index that we expect to scan. */ static int -compute_parallel_worker(RelOptInfo *rel, BlockNumber pages) +compute_parallel_worker(RelOptInfo *rel, BlockNumber heap_pages, + BlockNumber index_pages) { - int parallel_workers; + int parallel_workers = 0; + int heap_parallel_workers = 1; + int index_parallel_workers = 1; /* * If the user has set the parallel_workers reloption, use that; otherwise @@ -2892,7 +2901,8 @@ compute_parallel_worker(RelOptInfo *rel, BlockNumber pages) parallel_workers = rel->rel_parallel_workers; else { - int parallel_threshold; + int heap_parallel_threshold; + int index_parallel_threshold; /* * If this relation is too small to be worth a parallel scan, just @@ -2901,25 +2911,48 @@ compute_parallel_worker(RelOptInfo *rel, BlockNumber pages) * might not be worthwhile just for this relation, but when combined * with all of its inheritance siblings it may well pay off. */ - if (pages < (BlockNumber) min_parallel_relation_size && + if (heap_pages < (BlockNumber) min_parallel_table_scan_size && + index_pages < (BlockNumber) min_parallel_index_scan_size && rel->reloptkind == RELOPT_BASEREL) return 0; - /* - * Select the number of workers based on the log of the size of the - * relation. This probably needs to be a good deal more - * sophisticated, but we need something here for now. Note that the - * upper limit of the min_parallel_relation_size GUC is chosen to - * prevent overflow here. - */ - parallel_workers = 1; - parallel_threshold = Max(min_parallel_relation_size, 1); - while (pages >= (BlockNumber) (parallel_threshold * 3)) + if (heap_pages > 0) + { + /* + * Select the number of workers based on the log of the size of + * the relation. This probably needs to be a good deal more + * sophisticated, but we need something here for now. Note that + * the upper limit of the min_parallel_table_scan_size GUC is + * chosen to prevent overflow here. + */ + heap_parallel_threshold = Max(min_parallel_table_scan_size, 1); + while (heap_pages >= (BlockNumber) (heap_parallel_threshold * 3)) + { + heap_parallel_workers++; + heap_parallel_threshold *= 3; + if (heap_parallel_threshold > INT_MAX / 3) + break; /* avoid overflow */ + } + + parallel_workers = heap_parallel_workers; + } + + if (index_pages > 0) { - parallel_workers++; - parallel_threshold *= 3; - if (parallel_threshold > INT_MAX / 3) - break; /* avoid overflow */ + /* same calculation as for heap_pages above */ + index_parallel_threshold = Max(min_parallel_index_scan_size, 1); + while (index_pages >= (BlockNumber) (index_parallel_threshold * 3)) + { + index_parallel_workers++; + index_parallel_threshold *= 3; + if (index_parallel_threshold > INT_MAX / 3) + break; /* avoid overflow */ + } + + if (parallel_workers > 0) + parallel_workers = Min(parallel_workers, index_parallel_workers); + else + parallel_workers = index_parallel_workers; } } diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 0249721204..5d8fb2edb8 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -2776,16 +2776,27 @@ static struct config_int ConfigureNamesInt[] = }, { - {"min_parallel_relation_size", PGC_USERSET, QUERY_TUNING_COST, - gettext_noop("Sets the minimum size of relations to be considered for parallel scan."), - NULL, + {"min_parallel_table_scan_size", PGC_USERSET, QUERY_TUNING_COST, + gettext_noop("Sets the minimum amount of table data for a parallel scan."), + gettext_noop("If the planner estimates that it will read a number of table pages too small to reach this limit, a parallel scan will not be considered."), GUC_UNIT_BLOCKS, }, - &min_parallel_relation_size, + &min_parallel_table_scan_size, (8 * 1024 * 1024) / BLCKSZ, 0, INT_MAX / 3, NULL, NULL, NULL }, + { + {"min_parallel_index_scan_size", PGC_USERSET, QUERY_TUNING_COST, + gettext_noop("Sets the minimum amount of index data for a parallel scan."), + gettext_noop("If the planner estimates that it will read a number of index pages too small to reach this limit, a parallel scan will not be considered."), + GUC_UNIT_BLOCKS, + }, + &min_parallel_index_scan_size, + (512 * 1024) / BLCKSZ, 0, INT_MAX / 3, + NULL, NULL, NULL + }, + { /* Can't be set in postgresql.conf */ {"server_version_num", PGC_INTERNAL, PRESET_OPTIONS, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 661b0fa9b6..157d775853 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -300,7 +300,8 @@ #cpu_operator_cost = 0.0025 # same scale as above #parallel_tuple_cost = 0.1 # same scale as above #parallel_setup_cost = 1000.0 # same scale as above -#min_parallel_relation_size = 8MB +#min_parallel_table_scan_size = 8MB +#min_parallel_index_scan_size = 512kB #effective_cache_size = 4GB # - Genetic Query Optimizer - diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 81a9be7c67..81e7a4274d 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -22,7 +22,8 @@ */ extern bool enable_geqo; extern int geqo_threshold; -extern int min_parallel_relation_size; +extern int min_parallel_table_scan_size; +extern int min_parallel_index_scan_size; /* Hook for plugins to get control in set_rel_pathlist() */ typedef void (*set_rel_pathlist_hook_type) (PlannerInfo *root, diff --git a/src/test/regress/expected/select_parallel.out b/src/test/regress/expected/select_parallel.out index 8786678f0c..3692d4f1b8 100644 --- a/src/test/regress/expected/select_parallel.out +++ b/src/test/regress/expected/select_parallel.out @@ -9,7 +9,7 @@ begin isolation level repeatable read; -- encourage use of parallel plans set parallel_setup_cost=0; set parallel_tuple_cost=0; -set min_parallel_relation_size=0; +set min_parallel_table_scan_size=0; set max_parallel_workers_per_gather=4; explain (costs off) select count(*) from a_star; diff --git a/src/test/regress/sql/select_parallel.sql b/src/test/regress/sql/select_parallel.sql index def9939d2e..f4f9dd5ab6 100644 --- a/src/test/regress/sql/select_parallel.sql +++ b/src/test/regress/sql/select_parallel.sql @@ -12,7 +12,7 @@ begin isolation level repeatable read; -- encourage use of parallel plans set parallel_setup_cost=0; set parallel_tuple_cost=0; -set min_parallel_relation_size=0; +set min_parallel_table_scan_size=0; set max_parallel_workers_per_gather=4; explain (costs off) -- cgit v1.2.3 From 5262f7a4fc44f651241d2ff1fa688dd664a34874 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Wed, 15 Feb 2017 13:53:24 -0500 Subject: Add optimizer and executor support for parallel index scans. In combination with 569174f1be92be93f5366212cc46960d28a5c5cd, which taught the btree AM how to perform parallel index scans, this allows parallel index scan plans on btree indexes. This infrastructure should be general enough to support parallel index scans for other index AMs as well, if someone updates them to support parallel scans. Amit Kapila, reviewed and tested by Anastasia Lubennikova, Tushar Ahuja, and Haribabu Kommi, and me. --- contrib/bloom/blcost.c | 4 +- contrib/bloom/bloom.h | 2 +- contrib/bloom/blutils.c | 1 + doc/src/sgml/indexam.sgml | 2 + src/backend/access/brin/brin.c | 1 + src/backend/access/gin/ginutil.c | 1 + src/backend/access/gist/gist.c | 1 + src/backend/access/hash/hash.c | 1 + src/backend/access/nbtree/nbtree.c | 1 + src/backend/access/spgist/spgutils.c | 1 + src/backend/executor/execParallel.c | 12 ++ src/backend/executor/nodeIndexscan.c | 153 +++++++++++++++++++++++--- src/backend/optimizer/path/allpaths.c | 4 +- src/backend/optimizer/path/costsize.c | 53 ++++++++- src/backend/optimizer/path/indxpath.c | 67 ++++++++++- src/backend/optimizer/plan/planner.c | 2 +- src/backend/optimizer/util/pathnode.c | 19 ++-- src/backend/optimizer/util/plancat.c | 1 + src/backend/utils/adt/selfuncs.c | 24 +++- src/include/access/amapi.h | 5 +- src/include/executor/nodeIndexscan.h | 4 + src/include/nodes/execnodes.h | 2 + src/include/nodes/relation.h | 1 + src/include/optimizer/cost.h | 2 +- src/include/optimizer/pathnode.h | 3 +- src/include/optimizer/paths.h | 2 + src/include/utils/index_selfuncs.h | 18 ++- src/test/regress/expected/select_parallel.out | 23 ++++ src/test/regress/sql/select_parallel.sql | 11 ++ 29 files changed, 366 insertions(+), 55 deletions(-) (limited to 'doc/src') diff --git a/contrib/bloom/blcost.c b/contrib/bloom/blcost.c index 98a2228edf..ba39f627fd 100644 --- a/contrib/bloom/blcost.c +++ b/contrib/bloom/blcost.c @@ -24,7 +24,8 @@ void blcostestimate(PlannerInfo *root, IndexPath *path, double loop_count, Cost *indexStartupCost, Cost *indexTotalCost, - Selectivity *indexSelectivity, double *indexCorrelation) + Selectivity *indexSelectivity, double *indexCorrelation, + double *indexPages) { IndexOptInfo *index = path->indexinfo; List *qinfos; @@ -45,4 +46,5 @@ blcostestimate(PlannerInfo *root, IndexPath *path, double loop_count, *indexTotalCost = costs.indexTotalCost; *indexSelectivity = costs.indexSelectivity; *indexCorrelation = costs.indexCorrelation; + *indexPages = costs.numIndexPages; } diff --git a/contrib/bloom/bloom.h b/contrib/bloom/bloom.h index 39d8d05c5d..0cfe49aad8 100644 --- a/contrib/bloom/bloom.h +++ b/contrib/bloom/bloom.h @@ -208,6 +208,6 @@ extern bytea *bloptions(Datum reloptions, bool validate); extern void blcostestimate(PlannerInfo *root, IndexPath *path, double loop_count, Cost *indexStartupCost, Cost *indexTotalCost, Selectivity *indexSelectivity, - double *indexCorrelation); + double *indexCorrelation, double *indexPages); #endif diff --git a/contrib/bloom/blutils.c b/contrib/bloom/blutils.c index 858798db85..f2eda67e0a 100644 --- a/contrib/bloom/blutils.c +++ b/contrib/bloom/blutils.c @@ -119,6 +119,7 @@ blhandler(PG_FUNCTION_ARGS) amroutine->amstorage = false; amroutine->amclusterable = false; amroutine->ampredlocks = false; + amroutine->amcanparallel = false; amroutine->amkeytype = InvalidOid; amroutine->ambuild = blbuild; diff --git a/doc/src/sgml/indexam.sgml b/doc/src/sgml/indexam.sgml index 9afd7f6417..401b11598e 100644 --- a/doc/src/sgml/indexam.sgml +++ b/doc/src/sgml/indexam.sgml @@ -110,6 +110,8 @@ typedef struct IndexAmRoutine bool amclusterable; /* does AM handle predicate locks? */ bool ampredlocks; + /* does AM support parallel scan? */ + bool amcanparallel; /* type of data stored in index, or InvalidOid if variable */ Oid amkeytype; diff --git a/src/backend/access/brin/brin.c b/src/backend/access/brin/brin.c index 4ff046b4b0..b22563bf7c 100644 --- a/src/backend/access/brin/brin.c +++ b/src/backend/access/brin/brin.c @@ -93,6 +93,7 @@ brinhandler(PG_FUNCTION_ARGS) amroutine->amstorage = true; amroutine->amclusterable = false; amroutine->ampredlocks = false; + amroutine->amcanparallel = false; amroutine->amkeytype = InvalidOid; amroutine->ambuild = brinbuild; diff --git a/src/backend/access/gin/ginutil.c b/src/backend/access/gin/ginutil.c index a98d4fc397..d03d59da6a 100644 --- a/src/backend/access/gin/ginutil.c +++ b/src/backend/access/gin/ginutil.c @@ -50,6 +50,7 @@ ginhandler(PG_FUNCTION_ARGS) amroutine->amstorage = true; amroutine->amclusterable = false; amroutine->ampredlocks = false; + amroutine->amcanparallel = false; amroutine->amkeytype = InvalidOid; amroutine->ambuild = ginbuild; diff --git a/src/backend/access/gist/gist.c b/src/backend/access/gist/gist.c index 96ead531ea..6593771361 100644 --- a/src/backend/access/gist/gist.c +++ b/src/backend/access/gist/gist.c @@ -71,6 +71,7 @@ gisthandler(PG_FUNCTION_ARGS) amroutine->amstorage = true; amroutine->amclusterable = true; amroutine->ampredlocks = false; + amroutine->amcanparallel = false; amroutine->amkeytype = InvalidOid; amroutine->ambuild = gistbuild; diff --git a/src/backend/access/hash/hash.c b/src/backend/access/hash/hash.c index bca77a80c3..24510e78f5 100644 --- a/src/backend/access/hash/hash.c +++ b/src/backend/access/hash/hash.c @@ -67,6 +67,7 @@ hashhandler(PG_FUNCTION_ARGS) amroutine->amstorage = false; amroutine->amclusterable = false; amroutine->ampredlocks = false; + amroutine->amcanparallel = false; amroutine->amkeytype = INT4OID; amroutine->ambuild = hashbuild; diff --git a/src/backend/access/nbtree/nbtree.c b/src/backend/access/nbtree/nbtree.c index cbc575d5cf..775f2ff1f8 100644 --- a/src/backend/access/nbtree/nbtree.c +++ b/src/backend/access/nbtree/nbtree.c @@ -140,6 +140,7 @@ bthandler(PG_FUNCTION_ARGS) amroutine->amstorage = false; amroutine->amclusterable = true; amroutine->ampredlocks = true; + amroutine->amcanparallel = true; amroutine->amkeytype = InvalidOid; amroutine->ambuild = btbuild; diff --git a/src/backend/access/spgist/spgutils.c b/src/backend/access/spgist/spgutils.c index 78846bec66..e57ac49c6b 100644 --- a/src/backend/access/spgist/spgutils.c +++ b/src/backend/access/spgist/spgutils.c @@ -49,6 +49,7 @@ spghandler(PG_FUNCTION_ARGS) amroutine->amstorage = false; amroutine->amclusterable = false; amroutine->ampredlocks = false; + amroutine->amcanparallel = false; amroutine->amkeytype = InvalidOid; amroutine->ambuild = spgbuild; diff --git a/src/backend/executor/execParallel.c b/src/backend/executor/execParallel.c index 784dbaf590..98d4f1eca7 100644 --- a/src/backend/executor/execParallel.c +++ b/src/backend/executor/execParallel.c @@ -28,6 +28,7 @@ #include "executor/nodeCustom.h" #include "executor/nodeForeignscan.h" #include "executor/nodeSeqscan.h" +#include "executor/nodeIndexscan.h" #include "executor/tqueue.h" #include "nodes/nodeFuncs.h" #include "optimizer/planmain.h" @@ -197,6 +198,10 @@ ExecParallelEstimate(PlanState *planstate, ExecParallelEstimateContext *e) ExecSeqScanEstimate((SeqScanState *) planstate, e->pcxt); break; + case T_IndexScanState: + ExecIndexScanEstimate((IndexScanState *) planstate, + e->pcxt); + break; case T_ForeignScanState: ExecForeignScanEstimate((ForeignScanState *) planstate, e->pcxt); @@ -249,6 +254,10 @@ ExecParallelInitializeDSM(PlanState *planstate, ExecSeqScanInitializeDSM((SeqScanState *) planstate, d->pcxt); break; + case T_IndexScanState: + ExecIndexScanInitializeDSM((IndexScanState *) planstate, + d->pcxt); + break; case T_ForeignScanState: ExecForeignScanInitializeDSM((ForeignScanState *) planstate, d->pcxt); @@ -725,6 +734,9 @@ ExecParallelInitializeWorker(PlanState *planstate, shm_toc *toc) case T_SeqScanState: ExecSeqScanInitializeWorker((SeqScanState *) planstate, toc); break; + case T_IndexScanState: + ExecIndexScanInitializeWorker((IndexScanState *) planstate, toc); + break; case T_ForeignScanState: ExecForeignScanInitializeWorker((ForeignScanState *) planstate, toc); diff --git a/src/backend/executor/nodeIndexscan.c b/src/backend/executor/nodeIndexscan.c index 5734550d2c..0a9dfdbaf3 100644 --- a/src/backend/executor/nodeIndexscan.c +++ b/src/backend/executor/nodeIndexscan.c @@ -22,6 +22,9 @@ * ExecEndIndexScan releases all storage. * ExecIndexMarkPos marks scan position. * ExecIndexRestrPos restores scan position. + * ExecIndexScanEstimate estimates DSM space needed for parallel index scan + * ExecIndexScanInitializeDSM initialize DSM for parallel indexscan + * ExecIndexScanInitializeWorker attach to DSM info in parallel worker */ #include "postgres.h" @@ -514,6 +517,18 @@ ExecIndexScan(IndexScanState *node) void ExecReScanIndexScan(IndexScanState *node) { + bool reset_parallel_scan = true; + + /* + * If we are here to just update the scan keys, then don't reset parallel + * scan. We don't want each of the participating process in the parallel + * scan to update the shared parallel scan state at the start of the scan. + * It is quite possible that one of the participants has already begun + * scanning the index when another has yet to start it. + */ + if (node->iss_NumRuntimeKeys != 0 && !node->iss_RuntimeKeysReady) + reset_parallel_scan = false; + /* * If we are doing runtime key calculations (ie, any of the index key * values weren't simple Consts), compute the new key values. But first, @@ -539,10 +554,21 @@ ExecReScanIndexScan(IndexScanState *node) reorderqueue_pop(node); } - /* reset index scan */ - index_rescan(node->iss_ScanDesc, - node->iss_ScanKeys, node->iss_NumScanKeys, - node->iss_OrderByKeys, node->iss_NumOrderByKeys); + /* + * Reset (parallel) index scan. For parallel-aware nodes, the scan + * descriptor is initialized during actual execution of node and we can + * reach here before that (ex. during execution of nest loop join). So, + * avoid updating the scan descriptor at that time. + */ + if (node->iss_ScanDesc) + { + index_rescan(node->iss_ScanDesc, + node->iss_ScanKeys, node->iss_NumScanKeys, + node->iss_OrderByKeys, node->iss_NumOrderByKeys); + + if (reset_parallel_scan && node->iss_ScanDesc->parallel_scan) + index_parallelrescan(node->iss_ScanDesc); + } node->iss_ReachedEnd = false; ExecScanReScan(&node->ss); @@ -1013,22 +1039,29 @@ ExecInitIndexScan(IndexScan *node, EState *estate, int eflags) } /* - * Initialize scan descriptor. + * for parallel-aware node, we initialize the scan descriptor after + * initializing the shared memory for parallel execution. */ - indexstate->iss_ScanDesc = index_beginscan(currentRelation, - indexstate->iss_RelationDesc, - estate->es_snapshot, - indexstate->iss_NumScanKeys, + if (!node->scan.plan.parallel_aware) + { + /* + * Initialize scan descriptor. + */ + indexstate->iss_ScanDesc = index_beginscan(currentRelation, + indexstate->iss_RelationDesc, + estate->es_snapshot, + indexstate->iss_NumScanKeys, indexstate->iss_NumOrderByKeys); - /* - * If no run-time keys to calculate, go ahead and pass the scankeys to the - * index AM. - */ - if (indexstate->iss_NumRuntimeKeys == 0) - index_rescan(indexstate->iss_ScanDesc, - indexstate->iss_ScanKeys, indexstate->iss_NumScanKeys, + /* + * If no run-time keys to calculate, go ahead and pass the scankeys to + * the index AM. + */ + if (indexstate->iss_NumRuntimeKeys == 0) + index_rescan(indexstate->iss_ScanDesc, + indexstate->iss_ScanKeys, indexstate->iss_NumScanKeys, indexstate->iss_OrderByKeys, indexstate->iss_NumOrderByKeys); + } /* * all done. @@ -1590,3 +1623,91 @@ ExecIndexBuildScanKeys(PlanState *planstate, Relation index, else if (n_array_keys != 0) elog(ERROR, "ScalarArrayOpExpr index qual found where not allowed"); } + +/* ---------------------------------------------------------------- + * Parallel Scan Support + * ---------------------------------------------------------------- + */ + +/* ---------------------------------------------------------------- + * ExecIndexScanEstimate + * + * estimates the space required to serialize indexscan node. + * ---------------------------------------------------------------- + */ +void +ExecIndexScanEstimate(IndexScanState *node, + ParallelContext *pcxt) +{ + EState *estate = node->ss.ps.state; + + node->iss_PscanLen = index_parallelscan_estimate(node->iss_RelationDesc, + estate->es_snapshot); + shm_toc_estimate_chunk(&pcxt->estimator, node->iss_PscanLen); + shm_toc_estimate_keys(&pcxt->estimator, 1); +} + +/* ---------------------------------------------------------------- + * ExecIndexScanInitializeDSM + * + * Set up a parallel index scan descriptor. + * ---------------------------------------------------------------- + */ +void +ExecIndexScanInitializeDSM(IndexScanState *node, + ParallelContext *pcxt) +{ + EState *estate = node->ss.ps.state; + ParallelIndexScanDesc piscan; + + piscan = shm_toc_allocate(pcxt->toc, node->iss_PscanLen); + index_parallelscan_initialize(node->ss.ss_currentRelation, + node->iss_RelationDesc, + estate->es_snapshot, + piscan); + shm_toc_insert(pcxt->toc, node->ss.ps.plan->plan_node_id, piscan); + node->iss_ScanDesc = + index_beginscan_parallel(node->ss.ss_currentRelation, + node->iss_RelationDesc, + node->iss_NumScanKeys, + node->iss_NumOrderByKeys, + piscan); + + /* + * If no run-time keys to calculate, go ahead and pass the scankeys to the + * index AM. + */ + if (node->iss_NumRuntimeKeys == 0) + index_rescan(node->iss_ScanDesc, + node->iss_ScanKeys, node->iss_NumScanKeys, + node->iss_OrderByKeys, node->iss_NumOrderByKeys); +} + +/* ---------------------------------------------------------------- + * ExecIndexScanInitializeWorker + * + * Copy relevant information from TOC into planstate. + * ---------------------------------------------------------------- + */ +void +ExecIndexScanInitializeWorker(IndexScanState *node, shm_toc *toc) +{ + ParallelIndexScanDesc piscan; + + piscan = shm_toc_lookup(toc, node->ss.ps.plan->plan_node_id); + node->iss_ScanDesc = + index_beginscan_parallel(node->ss.ss_currentRelation, + node->iss_RelationDesc, + node->iss_NumScanKeys, + node->iss_NumOrderByKeys, + piscan); + + /* + * If no run-time keys to calculate, go ahead and pass the scankeys to the + * index AM. + */ + if (node->iss_NumRuntimeKeys == 0) + index_rescan(node->iss_ScanDesc, + node->iss_ScanKeys, node->iss_NumScanKeys, + node->iss_OrderByKeys, node->iss_NumOrderByKeys); +} diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index 85505c57d3..eeacf815e3 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -127,8 +127,6 @@ static void subquery_push_qual(Query *subquery, static void recurse_push_qual(Node *setOp, Query *topquery, RangeTblEntry *rte, Index rti, Node *qual); static void remove_unused_subquery_outputs(Query *subquery, RelOptInfo *rel); -static int compute_parallel_worker(RelOptInfo *rel, BlockNumber heap_pages, - BlockNumber index_pages); /* @@ -2885,7 +2883,7 @@ remove_unused_subquery_outputs(Query *subquery, RelOptInfo *rel) * "heap_pages" is the number of pages from the table that we expect to scan. * "index_pages" is the number of pages from the index that we expect to scan. */ -static int +int compute_parallel_worker(RelOptInfo *rel, BlockNumber heap_pages, BlockNumber index_pages) { diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index a43daa744c..d01630f8db 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -391,7 +391,8 @@ cost_gather(GatherPath *path, PlannerInfo *root, * we have to fetch from the table, so they don't reduce the scan cost. */ void -cost_index(IndexPath *path, PlannerInfo *root, double loop_count) +cost_index(IndexPath *path, PlannerInfo *root, double loop_count, + bool partial_path) { IndexOptInfo *index = path->indexinfo; RelOptInfo *baserel = index->rel; @@ -400,6 +401,7 @@ cost_index(IndexPath *path, PlannerInfo *root, double loop_count) List *qpquals; Cost startup_cost = 0; Cost run_cost = 0; + Cost cpu_run_cost = 0; Cost indexStartupCost; Cost indexTotalCost; Selectivity indexSelectivity; @@ -413,6 +415,8 @@ cost_index(IndexPath *path, PlannerInfo *root, double loop_count) Cost cpu_per_tuple; double tuples_fetched; double pages_fetched; + double rand_heap_pages; + double index_pages; /* Should only be applied to base relations */ Assert(IsA(baserel, RelOptInfo) && @@ -459,7 +463,8 @@ cost_index(IndexPath *path, PlannerInfo *root, double loop_count) amcostestimate = (amcostestimate_function) index->amcostestimate; amcostestimate(root, path, loop_count, &indexStartupCost, &indexTotalCost, - &indexSelectivity, &indexCorrelation); + &indexSelectivity, &indexCorrelation, + &index_pages); /* * Save amcostestimate's results for possible use in bitmap scan planning. @@ -526,6 +531,8 @@ cost_index(IndexPath *path, PlannerInfo *root, double loop_count) if (indexonly) pages_fetched = ceil(pages_fetched * (1.0 - baserel->allvisfrac)); + rand_heap_pages = pages_fetched; + max_IO_cost = (pages_fetched * spc_random_page_cost) / loop_count; /* @@ -564,6 +571,8 @@ cost_index(IndexPath *path, PlannerInfo *root, double loop_count) if (indexonly) pages_fetched = ceil(pages_fetched * (1.0 - baserel->allvisfrac)); + rand_heap_pages = pages_fetched; + /* max_IO_cost is for the perfectly uncorrelated case (csquared=0) */ max_IO_cost = pages_fetched * spc_random_page_cost; @@ -583,6 +592,29 @@ cost_index(IndexPath *path, PlannerInfo *root, double loop_count) min_IO_cost = 0; } + if (partial_path) + { + /* + * Estimate the number of parallel workers required to scan index. Use + * the number of heap pages computed considering heap fetches won't be + * sequential as for parallel scans the pages are accessed in random + * order. + */ + path->path.parallel_workers = compute_parallel_worker(baserel, + (BlockNumber) rand_heap_pages, + (BlockNumber) index_pages); + + /* + * Fall out if workers can't be assigned for parallel scan, because in + * such a case this path will be rejected. So there is no benefit in + * doing extra computation. + */ + if (path->path.parallel_workers <= 0) + return; + + path->path.parallel_aware = true; + } + /* * Now interpolate based on estimated index order correlation to get total * disk I/O cost for main table accesses. @@ -602,11 +634,24 @@ cost_index(IndexPath *path, PlannerInfo *root, double loop_count) startup_cost += qpqual_cost.startup; cpu_per_tuple = cpu_tuple_cost + qpqual_cost.per_tuple; - run_cost += cpu_per_tuple * tuples_fetched; + cpu_run_cost += cpu_per_tuple * tuples_fetched; /* tlist eval costs are paid per output row, not per tuple scanned */ startup_cost += path->path.pathtarget->cost.startup; - run_cost += path->path.pathtarget->cost.per_tuple * path->path.rows; + cpu_run_cost += path->path.pathtarget->cost.per_tuple * path->path.rows; + + /* Adjust costing for parallelism, if used. */ + if (path->path.parallel_workers > 0) + { + double parallel_divisor = get_parallel_divisor(&path->path); + + path->path.rows = clamp_row_est(path->path.rows / parallel_divisor); + + /* The CPU cost is divided among all the workers. */ + cpu_run_cost /= parallel_divisor; + } + + run_cost += cpu_run_cost; path->path.startup_cost = startup_cost; path->path.total_cost = startup_cost + run_cost; diff --git a/src/backend/optimizer/path/indxpath.c b/src/backend/optimizer/path/indxpath.c index 5283468988..56eccafd7b 100644 --- a/src/backend/optimizer/path/indxpath.c +++ b/src/backend/optimizer/path/indxpath.c @@ -813,7 +813,7 @@ get_index_paths(PlannerInfo *root, RelOptInfo *rel, /* * build_index_paths * Given an index and a set of index clauses for it, construct zero - * or more IndexPaths. + * or more IndexPaths. It also constructs zero or more partial IndexPaths. * * We return a list of paths because (1) this routine checks some cases * that should cause us to not generate any IndexPath, and (2) in some @@ -1042,8 +1042,41 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, NoMovementScanDirection, index_only_scan, outer_relids, - loop_count); + loop_count, + false); result = lappend(result, ipath); + + /* + * If appropriate, consider parallel index scan. We don't allow + * parallel index scan for bitmap or index only scans. + */ + if (index->amcanparallel && !index_only_scan && + rel->consider_parallel && outer_relids == NULL && + scantype != ST_BITMAPSCAN) + { + ipath = create_index_path(root, index, + index_clauses, + clause_columns, + orderbyclauses, + orderbyclausecols, + useful_pathkeys, + index_is_ordered ? + ForwardScanDirection : + NoMovementScanDirection, + index_only_scan, + outer_relids, + loop_count, + true); + + /* + * if, after costing the path, we find that it's not worth + * using parallel workers, just free it. + */ + if (ipath->path.parallel_workers > 0) + add_partial_path(rel, (Path *) ipath); + else + pfree(ipath); + } } /* @@ -1066,8 +1099,36 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, BackwardScanDirection, index_only_scan, outer_relids, - loop_count); + loop_count, + false); result = lappend(result, ipath); + + /* If appropriate, consider parallel index scan */ + if (index->amcanparallel && !index_only_scan && + rel->consider_parallel && outer_relids == NULL && + scantype != ST_BITMAPSCAN) + { + ipath = create_index_path(root, index, + index_clauses, + clause_columns, + NIL, + NIL, + useful_pathkeys, + BackwardScanDirection, + index_only_scan, + outer_relids, + loop_count, + true); + + /* + * if, after costing the path, we find that it's not worth + * using parallel workers, just free it. + */ + if (ipath->path.parallel_workers > 0) + add_partial_path(rel, (Path *) ipath); + else + pfree(ipath); + } } } diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index abb4f12cea..3d33d46971 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -5333,7 +5333,7 @@ plan_cluster_use_sort(Oid tableOid, Oid indexOid) indexScanPath = create_index_path(root, indexInfo, NIL, NIL, NIL, NIL, NIL, ForwardScanDirection, false, - NULL, 1.0); + NULL, 1.0, false); return (seqScanAndSortPath.total_cost < indexScanPath->path.total_cost); } diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index f440875ceb..324829690d 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -744,10 +744,9 @@ add_path_precheck(RelOptInfo *parent_rel, * As with add_path, we pfree paths that are found to be dominated by * another partial path; this requires that there be no other references to * such paths yet. Hence, GatherPaths must not be created for a rel until - * we're done creating all partial paths for it. We do not currently build - * partial indexscan paths, so there is no need for an exception for - * IndexPaths here; for safety, we instead Assert that a path to be freed - * isn't an IndexPath. + * we're done creating all partial paths for it. Unlike add_path, we don't + * take an exception for IndexPaths as partial index paths won't be + * referenced by partial BitmapHeapPaths. */ void add_partial_path(RelOptInfo *parent_rel, Path *new_path) @@ -826,8 +825,6 @@ add_partial_path(RelOptInfo *parent_rel, Path *new_path) { parent_rel->partial_pathlist = list_delete_cell(parent_rel->partial_pathlist, p1, p1_prev); - /* we should not see IndexPaths here, so always safe to delete */ - Assert(!IsA(old_path, IndexPath)); pfree(old_path); /* p1_prev does not advance */ } @@ -860,8 +857,6 @@ add_partial_path(RelOptInfo *parent_rel, Path *new_path) } else { - /* we should not see IndexPaths here, so always safe to delete */ - Assert(!IsA(new_path, IndexPath)); /* Reject and recycle the new path */ pfree(new_path); } @@ -1005,6 +1000,7 @@ create_samplescan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer * 'required_outer' is the set of outer relids for a parameterized path. * 'loop_count' is the number of repetitions of the indexscan to factor into * estimates of caching behavior. + * 'partial_path' is true if constructing a parallel index scan path. * * Returns the new path node. */ @@ -1019,7 +1015,8 @@ create_index_path(PlannerInfo *root, ScanDirection indexscandir, bool indexonly, Relids required_outer, - double loop_count) + double loop_count, + bool partial_path) { IndexPath *pathnode = makeNode(IndexPath); RelOptInfo *rel = index->rel; @@ -1049,7 +1046,7 @@ create_index_path(PlannerInfo *root, pathnode->indexorderbycols = indexorderbycols; pathnode->indexscandir = indexscandir; - cost_index(pathnode, root, loop_count); + cost_index(pathnode, root, loop_count, partial_path); return pathnode; } @@ -3247,7 +3244,7 @@ reparameterize_path(PlannerInfo *root, Path *path, memcpy(newpath, ipath, sizeof(IndexPath)); newpath->path.param_info = get_baserel_parampathinfo(root, rel, required_outer); - cost_index(newpath, root, loop_count); + cost_index(newpath, root, loop_count, false); return (Path *) newpath; } case T_BitmapHeapScan: diff --git a/src/backend/optimizer/util/plancat.c b/src/backend/optimizer/util/plancat.c index 7836e6b3f8..4ed27054a1 100644 --- a/src/backend/optimizer/util/plancat.c +++ b/src/backend/optimizer/util/plancat.c @@ -241,6 +241,7 @@ get_relation_info(PlannerInfo *root, Oid relationObjectId, bool inhparent, info->amoptionalkey = amroutine->amoptionalkey; info->amsearcharray = amroutine->amsearcharray; info->amsearchnulls = amroutine->amsearchnulls; + info->amcanparallel = amroutine->amcanparallel; info->amhasgettuple = (amroutine->amgettuple != NULL); info->amhasgetbitmap = (amroutine->amgetbitmap != NULL); info->amcostestimate = amroutine->amcostestimate; diff --git a/src/backend/utils/adt/selfuncs.c b/src/backend/utils/adt/selfuncs.c index fa32e9eabe..d14f0f97a8 100644 --- a/src/backend/utils/adt/selfuncs.c +++ b/src/backend/utils/adt/selfuncs.c @@ -6471,7 +6471,8 @@ add_predicate_to_quals(IndexOptInfo *index, List *indexQuals) void btcostestimate(PlannerInfo *root, IndexPath *path, double loop_count, Cost *indexStartupCost, Cost *indexTotalCost, - Selectivity *indexSelectivity, double *indexCorrelation) + Selectivity *indexSelectivity, double *indexCorrelation, + double *indexPages) { IndexOptInfo *index = path->indexinfo; List *qinfos; @@ -6761,12 +6762,14 @@ btcostestimate(PlannerInfo *root, IndexPath *path, double loop_count, *indexTotalCost = costs.indexTotalCost; *indexSelectivity = costs.indexSelectivity; *indexCorrelation = costs.indexCorrelation; + *indexPages = costs.numIndexPages; } void hashcostestimate(PlannerInfo *root, IndexPath *path, double loop_count, Cost *indexStartupCost, Cost *indexTotalCost, - Selectivity *indexSelectivity, double *indexCorrelation) + Selectivity *indexSelectivity, double *indexCorrelation, + double *indexPages) { List *qinfos; GenericCosts costs; @@ -6807,12 +6810,14 @@ hashcostestimate(PlannerInfo *root, IndexPath *path, double loop_count, *indexTotalCost = costs.indexTotalCost; *indexSelectivity = costs.indexSelectivity; *indexCorrelation = costs.indexCorrelation; + *indexPages = costs.numIndexPages; } void gistcostestimate(PlannerInfo *root, IndexPath *path, double loop_count, Cost *indexStartupCost, Cost *indexTotalCost, - Selectivity *indexSelectivity, double *indexCorrelation) + Selectivity *indexSelectivity, double *indexCorrelation, + double *indexPages) { IndexOptInfo *index = path->indexinfo; List *qinfos; @@ -6866,12 +6871,14 @@ gistcostestimate(PlannerInfo *root, IndexPath *path, double loop_count, *indexTotalCost = costs.indexTotalCost; *indexSelectivity = costs.indexSelectivity; *indexCorrelation = costs.indexCorrelation; + *indexPages = costs.numIndexPages; } void spgcostestimate(PlannerInfo *root, IndexPath *path, double loop_count, Cost *indexStartupCost, Cost *indexTotalCost, - Selectivity *indexSelectivity, double *indexCorrelation) + Selectivity *indexSelectivity, double *indexCorrelation, + double *indexPages) { IndexOptInfo *index = path->indexinfo; List *qinfos; @@ -6925,6 +6932,7 @@ spgcostestimate(PlannerInfo *root, IndexPath *path, double loop_count, *indexTotalCost = costs.indexTotalCost; *indexSelectivity = costs.indexSelectivity; *indexCorrelation = costs.indexCorrelation; + *indexPages = costs.numIndexPages; } @@ -7222,7 +7230,8 @@ gincost_scalararrayopexpr(PlannerInfo *root, void gincostestimate(PlannerInfo *root, IndexPath *path, double loop_count, Cost *indexStartupCost, Cost *indexTotalCost, - Selectivity *indexSelectivity, double *indexCorrelation) + Selectivity *indexSelectivity, double *indexCorrelation, + double *indexPages) { IndexOptInfo *index = path->indexinfo; List *indexQuals = path->indexquals; @@ -7537,6 +7546,7 @@ gincostestimate(PlannerInfo *root, IndexPath *path, double loop_count, *indexStartupCost += qual_arg_cost; *indexTotalCost += qual_arg_cost; *indexTotalCost += (numTuples * *indexSelectivity) * (cpu_index_tuple_cost + qual_op_cost); + *indexPages = dataPagesFetched; } /* @@ -7545,7 +7555,8 @@ gincostestimate(PlannerInfo *root, IndexPath *path, double loop_count, void brincostestimate(PlannerInfo *root, IndexPath *path, double loop_count, Cost *indexStartupCost, Cost *indexTotalCost, - Selectivity *indexSelectivity, double *indexCorrelation) + Selectivity *indexSelectivity, double *indexCorrelation, + double *indexPages) { IndexOptInfo *index = path->indexinfo; List *indexQuals = path->indexquals; @@ -7597,6 +7608,7 @@ brincostestimate(PlannerInfo *root, IndexPath *path, double loop_count, *indexStartupCost += qual_arg_cost; *indexTotalCost += qual_arg_cost; *indexTotalCost += (numTuples * *indexSelectivity) * (cpu_index_tuple_cost + qual_op_cost); + *indexPages = index->pages; /* XXX what about pages_per_range? */ } diff --git a/src/include/access/amapi.h b/src/include/access/amapi.h index b0730bfefa..f919cf8b87 100644 --- a/src/include/access/amapi.h +++ b/src/include/access/amapi.h @@ -95,7 +95,8 @@ typedef void (*amcostestimate_function) (struct PlannerInfo *root, Cost *indexStartupCost, Cost *indexTotalCost, Selectivity *indexSelectivity, - double *indexCorrelation); + double *indexCorrelation, + double *indexPages); /* parse index reloptions */ typedef bytea *(*amoptions_function) (Datum reloptions, @@ -188,6 +189,8 @@ typedef struct IndexAmRoutine bool amclusterable; /* does AM handle predicate locks? */ bool ampredlocks; + /* does AM support parallel scan? */ + bool amcanparallel; /* type of data stored in index, or InvalidOid if variable */ Oid amkeytype; diff --git a/src/include/executor/nodeIndexscan.h b/src/include/executor/nodeIndexscan.h index 46d6f45e83..ea3f3a5cc4 100644 --- a/src/include/executor/nodeIndexscan.h +++ b/src/include/executor/nodeIndexscan.h @@ -14,6 +14,7 @@ #ifndef NODEINDEXSCAN_H #define NODEINDEXSCAN_H +#include "access/parallel.h" #include "nodes/execnodes.h" extern IndexScanState *ExecInitIndexScan(IndexScan *node, EState *estate, int eflags); @@ -22,6 +23,9 @@ extern void ExecEndIndexScan(IndexScanState *node); extern void ExecIndexMarkPos(IndexScanState *node); extern void ExecIndexRestrPos(IndexScanState *node); extern void ExecReScanIndexScan(IndexScanState *node); +extern void ExecIndexScanEstimate(IndexScanState *node, ParallelContext *pcxt); +extern void ExecIndexScanInitializeDSM(IndexScanState *node, ParallelContext *pcxt); +extern void ExecIndexScanInitializeWorker(IndexScanState *node, shm_toc *toc); /* * These routines are exported to share code with nodeIndexonlyscan.c and diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 42c6c58ff9..9f41babf35 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1363,6 +1363,7 @@ typedef struct * SortSupport for reordering ORDER BY exprs * OrderByTypByVals is the datatype of order by expression pass-by-value? * OrderByTypLens typlens of the datatypes of order by expressions + * pscan_len size of parallel index scan descriptor * ---------------- */ typedef struct IndexScanState @@ -1389,6 +1390,7 @@ typedef struct IndexScanState SortSupport iss_SortSupport; bool *iss_OrderByTypByVals; int16 *iss_OrderByTypLens; + Size iss_PscanLen; } IndexScanState; /* ---------------- diff --git a/src/include/nodes/relation.h b/src/include/nodes/relation.h index 643be54d40..f7ac6f600f 100644 --- a/src/include/nodes/relation.h +++ b/src/include/nodes/relation.h @@ -629,6 +629,7 @@ typedef struct IndexOptInfo bool amsearchnulls; /* can AM search for NULL/NOT NULL entries? */ bool amhasgettuple; /* does AM have amgettuple interface? */ bool amhasgetbitmap; /* does AM have amgetbitmap interface? */ + bool amcanparallel; /* does AM support parallel scan? */ /* Rather than include amapi.h here, we declare amcostestimate like this */ void (*amcostestimate) (); /* AM's cost estimator */ } IndexOptInfo; diff --git a/src/include/optimizer/cost.h b/src/include/optimizer/cost.h index 0e68264a41..72200fa531 100644 --- a/src/include/optimizer/cost.h +++ b/src/include/optimizer/cost.h @@ -76,7 +76,7 @@ extern void cost_seqscan(Path *path, PlannerInfo *root, RelOptInfo *baserel, extern void cost_samplescan(Path *path, PlannerInfo *root, RelOptInfo *baserel, ParamPathInfo *param_info); extern void cost_index(IndexPath *path, PlannerInfo *root, - double loop_count); + double loop_count, bool partial_path); extern void cost_bitmap_heap_scan(Path *path, PlannerInfo *root, RelOptInfo *baserel, ParamPathInfo *param_info, Path *bitmapqual, double loop_count); diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index 7b41317621..53cad247dc 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -47,7 +47,8 @@ extern IndexPath *create_index_path(PlannerInfo *root, ScanDirection indexscandir, bool indexonly, Relids required_outer, - double loop_count); + double loop_count, + bool partial_path); extern BitmapHeapPath *create_bitmap_heap_path(PlannerInfo *root, RelOptInfo *rel, Path *bitmapqual, diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 81e7a4274d..ebda308c41 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -54,6 +54,8 @@ extern RelOptInfo *standard_join_search(PlannerInfo *root, int levels_needed, List *initial_rels); extern void generate_gather_paths(PlannerInfo *root, RelOptInfo *rel); +extern int compute_parallel_worker(RelOptInfo *rel, BlockNumber heap_pages, + BlockNumber index_pages); #ifdef OPTIMIZER_DEBUG extern void debug_print_rel(PlannerInfo *root, RelOptInfo *rel); diff --git a/src/include/utils/index_selfuncs.h b/src/include/utils/index_selfuncs.h index d3172420f9..17d165ca65 100644 --- a/src/include/utils/index_selfuncs.h +++ b/src/include/utils/index_selfuncs.h @@ -28,41 +28,47 @@ extern void brincostestimate(struct PlannerInfo *root, Cost *indexStartupCost, Cost *indexTotalCost, Selectivity *indexSelectivity, - double *indexCorrelation); + double *indexCorrelation, + double *indexPages); extern void btcostestimate(struct PlannerInfo *root, struct IndexPath *path, double loop_count, Cost *indexStartupCost, Cost *indexTotalCost, Selectivity *indexSelectivity, - double *indexCorrelation); + double *indexCorrelation, + double *indexPages); extern void hashcostestimate(struct PlannerInfo *root, struct IndexPath *path, double loop_count, Cost *indexStartupCost, Cost *indexTotalCost, Selectivity *indexSelectivity, - double *indexCorrelation); + double *indexCorrelation, + double *indexPages); extern void gistcostestimate(struct PlannerInfo *root, struct IndexPath *path, double loop_count, Cost *indexStartupCost, Cost *indexTotalCost, Selectivity *indexSelectivity, - double *indexCorrelation); + double *indexCorrelation, + double *indexPages); extern void spgcostestimate(struct PlannerInfo *root, struct IndexPath *path, double loop_count, Cost *indexStartupCost, Cost *indexTotalCost, Selectivity *indexSelectivity, - double *indexCorrelation); + double *indexCorrelation, + double *indexPages); extern void gincostestimate(struct PlannerInfo *root, struct IndexPath *path, double loop_count, Cost *indexStartupCost, Cost *indexTotalCost, Selectivity *indexSelectivity, - double *indexCorrelation); + double *indexCorrelation, + double *indexPages); #endif /* INDEX_SELFUNCS_H */ diff --git a/src/test/regress/expected/select_parallel.out b/src/test/regress/expected/select_parallel.out index 3692d4f1b8..48fb80e90c 100644 --- a/src/test/regress/expected/select_parallel.out +++ b/src/test/regress/expected/select_parallel.out @@ -125,6 +125,29 @@ select count(*) from tenk1 where (two, four) not in (1 row) alter table tenk2 reset (parallel_workers); +-- test parallel index scans. +set enable_seqscan to off; +set enable_bitmapscan to off; +explain (costs off) + select count((unique1)) from tenk1 where hundred > 1; + QUERY PLAN +-------------------------------------------------------------------- + Finalize Aggregate + -> Gather + Workers Planned: 4 + -> Partial Aggregate + -> Parallel Index Scan using tenk1_hundred on tenk1 + Index Cond: (hundred > 1) +(6 rows) + +select count((unique1)) from tenk1 where hundred > 1; + count +------- + 9800 +(1 row) + +reset enable_seqscan; +reset enable_bitmapscan; set force_parallel_mode=1; explain (costs off) select stringu1::int2 from tenk1 where unique1 = 1; diff --git a/src/test/regress/sql/select_parallel.sql b/src/test/regress/sql/select_parallel.sql index f4f9dd5ab6..f5bc4d1873 100644 --- a/src/test/regress/sql/select_parallel.sql +++ b/src/test/regress/sql/select_parallel.sql @@ -48,6 +48,17 @@ select count(*) from tenk1 where (two, four) not in (select hundred, thousand from tenk2 where thousand > 100); alter table tenk2 reset (parallel_workers); +-- test parallel index scans. +set enable_seqscan to off; +set enable_bitmapscan to off; + +explain (costs off) + select count((unique1)) from tenk1 where hundred > 1; +select count((unique1)) from tenk1 where hundred > 1; + +reset enable_seqscan; +reset enable_bitmapscan; + set force_parallel_mode=1; explain (costs off) -- cgit v1.2.3 From 2b18743614bb526e5a49a542c9c80c96668639cd Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 15 Feb 2017 15:41:09 -0500 Subject: Doc: fix syntax synopsis for INSERT ... ON CONFLICT DO UPDATE. Commit 906bfcad7 adjusted the syntax synopsis for UPDATE, but missed the fact that the INSERT synopsis now contains a duplicate of that. In passing, improve wording and markup about using a table alias to dodge the conflict with use of "excluded" as a special table name. --- doc/src/sgml/ref/insert.sgml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/insert.sgml b/doc/src/sgml/ref/insert.sgml index 9339826818..521216b5d5 100644 --- a/doc/src/sgml/ref/insert.sgml +++ b/doc/src/sgml/ref/insert.sgml @@ -36,7 +36,7 @@ INSERT INTO table_name [ AS column_name = { expression | DEFAULT } | - ( column_name [, ...] ) = ( { expression | DEFAULT } [, ...] ) | + ( column_name [, ...] ) = [ ROW ] ( { expression | DEFAULT } [, ...] ) | ( column_name [, ...] ) = ( sub-SELECT ) } [, ...] [ WHERE condition ] @@ -174,9 +174,9 @@ INSERT INTO table_name [ AS table_name. When an alias is provided, it completely hides the actual name of the table. - This is particularly useful when ON CONFLICT DO - UPDATE targets a table named excluded, since that's - also the name of the special table representing rows proposed + This is particularly useful when ON CONFLICT DO UPDATE + targets a table named excluded, since that will otherwise + be taken as the name of the special table representing rows proposed for insertion. -- cgit v1.2.3 From adb67d67f0f29acb175620ab05be26a146512fa2 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 15 Feb 2017 17:31:02 -0500 Subject: Doc: fix typo in logicaldecoding.sgml. There's no such field as OutputPluginOptions.output_mode; it's actually output_type. Noted by T. Katsumata. Discussion: https://postgr.es/m/20170215072115.6101.29870@wrigleys.postgresql.org --- doc/src/sgml/logicaldecoding.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/logicaldecoding.sgml b/doc/src/sgml/logicaldecoding.sgml index c4dff3b9ff..6e7517d04d 100644 --- a/doc/src/sgml/logicaldecoding.sgml +++ b/doc/src/sgml/logicaldecoding.sgml @@ -419,7 +419,7 @@ CREATE TABLE another_catalog_table(data text) WITH (user_catalog_table = true); data in a data type that can contain arbitrary data (e.g., bytea) is cumbersome. If the output plugin only outputs textual data in the server's encoding, it can declare that by - setting OutputPluginOptions.output_mode + setting OutputPluginOptions.output_type to OUTPUT_PLUGIN_TEXTUAL_OUTPUT instead of OUTPUT_PLUGIN_BINARY_OUTPUT in the startup -- cgit v1.2.3 From 93e6e40574bccf9c6f33c520a4189d3e98e2fd1f Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 15 Feb 2017 18:15:47 -0500 Subject: Formatting and docs corrections for logical decoding output plugins. Make the typedefs for output plugins consistent with project style; they were previously not even consistent with each other as to layout or inclusion of parameter names. Make the documentation look the same, and fix errors therein (missing and misdescribed parameters). Back-patch because of the documentation bugs. --- doc/src/sgml/logicaldecoding.sgml | 59 +++++++++++++-------------------- src/include/replication/output_plugin.h | 33 +++++++----------- 2 files changed, 35 insertions(+), 57 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/logicaldecoding.sgml b/doc/src/sgml/logicaldecoding.sgml index 6e7517d04d..03c2c691d1 100644 --- a/doc/src/sgml/logicaldecoding.sgml +++ b/doc/src/sgml/logicaldecoding.sgml @@ -380,7 +380,7 @@ typedef struct OutputPluginCallbacks LogicalDecodeShutdownCB shutdown_cb; } OutputPluginCallbacks; -typedef void (*LogicalOutputPluginInit)(struct OutputPluginCallbacks *cb); +typedef void (*LogicalOutputPluginInit) (struct OutputPluginCallbacks *cb); The begin_cb, change_cb and commit_cb callbacks are required, @@ -465,11 +465,9 @@ CREATE TABLE another_catalog_table(data text) WITH (user_catalog_table = true); a replication slot is created or asked to stream changes, independent of the number of changes that are ready to be put out. -typedef void (*LogicalDecodeStartupCB) ( - struct LogicalDecodingContext *ctx, - OutputPluginOptions *options, - bool is_init -); +typedef void (*LogicalDecodeStartupCB) (struct LogicalDecodingContext *ctx, + OutputPluginOptions *options, + bool is_init); The is_init parameter will be true when the replication slot is being created and false @@ -504,9 +502,7 @@ typedef struct OutputPluginOptions be used to deallocate resources private to the output plugin. The slot isn't necessarily being dropped, streaming is just being stopped. -typedef void (*LogicalDecodeShutdownCB) ( - struct LogicalDecodingContext *ctx -); +typedef void (*LogicalDecodeShutdownCB) (struct LogicalDecodingContext *ctx); @@ -519,10 +515,8 @@ typedef void (*LogicalDecodeShutdownCB) ( start of a committed transaction has been decoded. Aborted transactions and their contents never get decoded. -typedef void (*LogicalDecodeBeginCB) ( - struct LogicalDecodingContext *, - ReorderBufferTXN *txn -); +typedef void (*LogicalDecodeBeginCB) (struct LogicalDecodingContext *ctx, + ReorderBufferTXN *txn); The txn parameter contains meta information about the transaction, like the time stamp at which it has been committed and @@ -540,10 +534,9 @@ typedef void (*LogicalDecodeBeginCB) ( rows will have been called before this, if there have been any modified rows. -typedef void (*LogicalDecodeCommitCB) ( - struct LogicalDecodingContext *, - ReorderBufferTXN *txn -); +typedef void (*LogicalDecodeCommitCB) (struct LogicalDecodingContext *ctx, + ReorderBufferTXN *txn, + XLogRecPtr commit_lsn); @@ -559,12 +552,10 @@ typedef void (*LogicalDecodeCommitCB) ( several rows at once the callback will be called individually for each row. -typedef void (*LogicalDecodeChangeCB) ( - struct LogicalDecodingContext *ctx, - ReorderBufferTXN *txn, - Relation relation, - ReorderBufferChange *change -); +typedef void (*LogicalDecodeChangeCB) (struct LogicalDecodingContext *ctx, + ReorderBufferTXN *txn, + Relation relation, + ReorderBufferChange *change); The ctx and txn parameters have the same contents as for the begin_cb @@ -594,10 +585,8 @@ typedef void (*LogicalDecodeChangeCB) ( from origin_id is of interest to the output plugin. -typedef bool (*LogicalDecodeFilterByOriginCB) ( - struct LogicalDecodingContext *ctx, - RepNodeId origin_id -); +typedef bool (*LogicalDecodeFilterByOriginCB) (struct LogicalDecodingContext *ctx, + RepOriginId origin_id); The ctx parameter has the same contents as for the other callbacks. No information but the origin is @@ -623,15 +612,13 @@ typedef bool (*LogicalDecodeFilterByOriginCB) ( The optional message_cb callback is called whenever a logical decoding message has been decoded. -typedef void (*LogicalDecodeMessageCB) ( - struct LogicalDecodingContext *, - ReorderBufferTXN *txn, - XLogRecPtr message_lsn, - bool transactional, - const char *prefix, - Size message_size, - const char *message -); +typedef void (*LogicalDecodeMessageCB) (struct LogicalDecodingContext *ctx, + ReorderBufferTXN *txn, + XLogRecPtr message_lsn, + bool transactional, + const char *prefix, + Size message_size, + const char *message); The txn parameter contains meta information about the transaction, like the time stamp at which it has been committed and diff --git a/src/include/replication/output_plugin.h b/src/include/replication/output_plugin.h index 7b5870a744..08e962d0c0 100644 --- a/src/include/replication/output_plugin.h +++ b/src/include/replication/output_plugin.h @@ -41,43 +41,36 @@ typedef void (*LogicalOutputPluginInit) (struct OutputPluginCallbacks *cb); * "is_init" will be set to "true" if the decoding slot just got defined. When * the same slot is used from there one, it will be "false". */ -typedef void (*LogicalDecodeStartupCB) ( - struct LogicalDecodingContext *ctx, +typedef void (*LogicalDecodeStartupCB) (struct LogicalDecodingContext *ctx, OutputPluginOptions *options, - bool is_init -); + bool is_init); /* * Callback called for every (explicit or implicit) BEGIN of a successful * transaction. */ -typedef void (*LogicalDecodeBeginCB) ( - struct LogicalDecodingContext *, +typedef void (*LogicalDecodeBeginCB) (struct LogicalDecodingContext *ctx, ReorderBufferTXN *txn); /* * Callback for every individual change in a successful transaction. */ -typedef void (*LogicalDecodeChangeCB) ( - struct LogicalDecodingContext *, +typedef void (*LogicalDecodeChangeCB) (struct LogicalDecodingContext *ctx, ReorderBufferTXN *txn, Relation relation, - ReorderBufferChange *change -); + ReorderBufferChange *change); /* * Called for every (explicit or implicit) COMMIT of a successful transaction. */ -typedef void (*LogicalDecodeCommitCB) ( - struct LogicalDecodingContext *, +typedef void (*LogicalDecodeCommitCB) (struct LogicalDecodingContext *ctx, ReorderBufferTXN *txn, XLogRecPtr commit_lsn); /* * Called for the generic logical decoding messages. */ -typedef void (*LogicalDecodeMessageCB) ( - struct LogicalDecodingContext *, +typedef void (*LogicalDecodeMessageCB) (struct LogicalDecodingContext *ctx, ReorderBufferTXN *txn, XLogRecPtr message_lsn, bool transactional, @@ -88,16 +81,13 @@ typedef void (*LogicalDecodeMessageCB) ( /* * Filter changes by origin. */ -typedef bool (*LogicalDecodeFilterByOriginCB) ( - struct LogicalDecodingContext *, +typedef bool (*LogicalDecodeFilterByOriginCB) (struct LogicalDecodingContext *ctx, RepOriginId origin_id); /* * Called to shutdown an output plugin. */ -typedef void (*LogicalDecodeShutdownCB) ( - struct LogicalDecodingContext * -); +typedef void (*LogicalDecodeShutdownCB) (struct LogicalDecodingContext *ctx); /* * Output plugin callbacks @@ -113,7 +103,8 @@ typedef struct OutputPluginCallbacks LogicalDecodeShutdownCB shutdown_cb; } OutputPluginCallbacks; -void OutputPluginPrepareWrite(struct LogicalDecodingContext *ctx, bool last_write); -void OutputPluginWrite(struct LogicalDecodingContext *ctx, bool last_write); +/* Functions in replication/logical/logical.c */ +extern void OutputPluginPrepareWrite(struct LogicalDecodingContext *ctx, bool last_write); +extern void OutputPluginWrite(struct LogicalDecodingContext *ctx, bool last_write); #endif /* OUTPUT_PLUGIN_H */ -- cgit v1.2.3 From 3b7673388da3598933ae6c4f9fdc7c79dee05558 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 16 Feb 2017 11:30:07 -0500 Subject: Doc: remove duplicate index entry. This causes a warning with the old html-docs toolchain, though not with the new. I had originally supposed that we needed both entries to get both a primary index entry and a see-also link; but evidently not, as pointed out by Fabien Coelho. Discussion: https://postgr.es/m/alpine.DEB.2.20.1702161616060.5445@lancre --- doc/src/sgml/syntax.sgml | 3 --- 1 file changed, 3 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/syntax.sgml b/doc/src/sgml/syntax.sgml index 4ea667bd52..40f722c18d 100644 --- a/doc/src/sgml/syntax.sgml +++ b/doc/src/sgml/syntax.sgml @@ -1697,9 +1697,6 @@ SELECT string_agg(a ORDER BY a, ',') FROM table; -- incorrect - - median - median percentile -- cgit v1.2.3 From a029d2cf4203f8f240bae4651e62c2358673b9f4 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 17 Feb 2017 16:11:02 -0500 Subject: Document usage of COPT environment variable for adjusting configure flags. Also add to the existing rather half-baked description of PROFILE, which does exactly the same thing, but I think people use it differently. Discussion: https://postgr.es/m/16461.1487361849@sss.pgh.pa.us --- doc/src/sgml/installation.sgml | 28 ++++++++++++++++++++++++++++ src/Makefile.global.in | 5 +++++ 2 files changed, 33 insertions(+) (limited to 'doc/src') diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml index 4431ed75a9..182c801bd5 100644 --- a/doc/src/sgml/installation.sgml +++ b/doc/src/sgml/installation.sgml @@ -1494,6 +1494,26 @@ su - postgres + + Sometimes it is useful to add compiler flags after-the-fact to the set + that were chosen by configure. An important example is + that gcc's + When developing code inside the server, it is recommended to @@ -1514,6 +1534,14 @@ su - postgres + + + The COPT and PROFILE environment variables are + actually handled identically by the PostgreSQL + makefiles. Which to use is a matter of preference, but a common habit + among developers is to use PROFILE for one-time flag + adjustments, while COPT might be kept set all the time. + diff --git a/src/Makefile.global.in b/src/Makefile.global.in index 59bd7996d1..44bfe28f57 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -582,6 +582,11 @@ ifneq ($(CUSTOM_COPT),) COPT= $(CUSTOM_COPT) endif +# +# These variables are meant to be set in the environment of "make" +# to add flags to whatever configure picked. Unlike the ones above, +# they are documented. +# ifdef COPT CFLAGS += $(COPT) LDFLAGS += $(COPT) -- cgit v1.2.3 From 68f3dbc5525a7e78290f7dee8a74f66d5fa738d6 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 17 Feb 2017 18:59:29 -0500 Subject: doc: Fix typos From: Thom Brown --- doc/src/sgml/ref/create_subscription.sgml | 2 +- doc/src/sgml/ref/create_table.sgml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml index 59e5ad00c8..250806f981 100644 --- a/doc/src/sgml/ref/create_subscription.sgml +++ b/doc/src/sgml/ref/create_subscription.sgml @@ -142,7 +142,7 @@ CREATE SUBSCRIPTION subscription_name Create a subscription to a remote server that replicates tables in - the publications mypubclication and + the publications mypublication and insert_only and starts replicating immediately on commit: diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml index e0f7cd9b93..41c08bba74 100644 --- a/doc/src/sgml/ref/create_table.sgml +++ b/doc/src/sgml/ref/create_table.sgml @@ -1539,7 +1539,7 @@ CREATE TABLE measurement_year_month ( CREATE TABLE cities ( city_id bigserial not null, name text not null, - population bigint, + population bigint ) PARTITION BY LIST (left(lower(name), 1)); -- cgit v1.2.3 From a3dc8e495b4967fe07086a700d115c89f4f0add0 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Sun, 19 Feb 2017 21:29:27 +0530 Subject: Make partitions automatically inherit OIDs. Previously, if the parent was specified as WITH OIDS, each child also had to be explicitly specified as WITH OIDS. Amit Langote, per a report from Simon Riggs. Some additional work on the documentation changes by me. Discussion: http://postgr.es/m/CANP8+jJBpWocfKrbJcaf3iBt9E3U=WPE_NC8YE6rye+YJ1sYnQ@mail.gmail.com --- doc/src/sgml/ddl.sgml | 8 ------- doc/src/sgml/ref/create_table.sgml | 34 +++++++++++++++--------------- src/backend/commands/tablecmds.c | 21 +++++++----------- src/test/regress/expected/create_table.out | 18 +++++++++++----- src/test/regress/sql/create_table.sql | 10 +++++---- 5 files changed, 44 insertions(+), 47 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ddl.sgml b/doc/src/sgml/ddl.sgml index f909242e4c..5779eac43d 100644 --- a/doc/src/sgml/ddl.sgml +++ b/doc/src/sgml/ddl.sgml @@ -2861,14 +2861,6 @@ VALUES ('Albany', NULL, NULL, 'NY'); - - - If the partitioned table specified WITH OIDS then - each partition must also specify WITH OIDS. Oids - are not automatically inherited by partitions. - - - One cannot drop a NOT NULL constraint on a diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml index 41c08bba74..bb081ff86f 100644 --- a/doc/src/sgml/ref/create_table.sgml +++ b/doc/src/sgml/ref/create_table.sgml @@ -300,14 +300,18 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI - A partition cannot have columns other than those inherited from the - parent. If the parent is specified WITH OIDS then - the partitions must also explicitly specify WITH OIDS. - Defaults and constraints can optionally be specified for each of the - inherited columns. One can also specify table constraints in addition - to those inherited from the parent. If a check constraint with the name - matching one of the parent's constraint is specified, it is merged with - the latter, provided the specified condition is same. + A partition must have the same column names and types as the partitioned + table to which it belongs. If the parent is specified WITH + OIDS then all partitions must have OIDs; the parent's OID + column will be inherited by all partitions just like any other column. + Modifications to the column names or types of a partitioned table, or + the addition or removal of an OID column, will automatically propagate + to all partitions. CHECK constraints will be inherited + automatically by every partition, but an individual partition may specify + additional CHECK constraints; additional constraints with + the same name and condition as in the parent will be merged with the + parent constraint. Defaults may be specified separately for each + partition. @@ -318,15 +322,11 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI - A partition must have the same column names and types as the table of - which it is a partition. Therefore, modifications to the column names - or types of the partitioned table will automatically propagate to all - children, as will operations such as TRUNCATE which normally affect a - table and all of its inheritance children. It is also possible to - TRUNCATE a partition individually, just as for an inheritance child. - Note that dropping a partition with DROP TABLE - requires taking an ACCESS EXCLUSIVE lock on the - parent table. + Operations such as TRUNCATE which normally affect a table and all of its + inheritance children will cascade to all partitions, but may also be + performed on an individual partition. Note that dropping a partition + with DROP TABLE requires taking an ACCESS + EXCLUSIVE lock on the parent table. diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index f33aa70da6..3cea220421 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -634,19 +634,14 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, relkind == RELKIND_PARTITIONED_TABLE)); descriptor->tdhasoid = (localHasOids || parentOidCount > 0); - if (stmt->partbound) - { - /* If the parent has OIDs, partitions must have them too. */ - if (parentOidCount > 0 && !localHasOids) - ereport(ERROR, - (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("cannot create table without OIDs as partition of table with OIDs"))); - /* If the parent doesn't, partitions must not have them. */ - if (parentOidCount == 0 && localHasOids) - ereport(ERROR, - (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("cannot create table with OIDs as partition of table without OIDs"))); - } + /* + * If a partitioned table doesn't have the system OID column, then none + * of its partitions should have it. + */ + if (stmt->partbound && parentOidCount == 0 && localHasOids) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot create table with OIDs as partition of table without OIDs"))); /* * Find columns with default values and prepare for insertion of the diff --git a/src/test/regress/expected/create_table.out b/src/test/regress/expected/create_table.out index fc92cd92dd..20eb3d35f9 100644 --- a/src/test/regress/expected/create_table.out +++ b/src/test/regress/expected/create_table.out @@ -524,16 +524,24 @@ DROP TABLE temp_parted; CREATE TABLE no_oids_parted ( a int ) PARTITION BY RANGE (a) WITHOUT OIDS; -CREATE TABLE fail_part PARTITION OF no_oids_parted FOR VALUES FROM (1) TO (10 )WITH OIDS; +CREATE TABLE fail_part PARTITION OF no_oids_parted FOR VALUES FROM (1) TO (10) WITH OIDS; ERROR: cannot create table with OIDs as partition of table without OIDs DROP TABLE no_oids_parted; --- likewise, the reverse if also true +-- If the partitioned table has oids, then the partition must have them. +-- If the WITHOUT OIDS option is specified for partition, it is overridden. CREATE TABLE oids_parted ( a int ) PARTITION BY RANGE (a) WITH OIDS; -CREATE TABLE fail_part PARTITION OF oids_parted FOR VALUES FROM (1) TO (10 ) WITHOUT OIDS; -ERROR: cannot create table without OIDs as partition of table with OIDs -DROP TABLE oids_parted; +CREATE TABLE part_forced_oids PARTITION OF oids_parted FOR VALUES FROM (1) TO (10) WITHOUT OIDS; +\d+ part_forced_oids + Table "public.part_forced_oids" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------+---------+-----------+----------+---------+---------+--------------+------------- + a | integer | | not null | | plain | | +Partition of: oids_parted FOR VALUES FROM (1) TO (10) +Has OIDs: yes + +DROP TABLE oids_parted, part_forced_oids; -- check for partition bound overlap and other invalid specifications CREATE TABLE list_parted2 ( a varchar diff --git a/src/test/regress/sql/create_table.sql b/src/test/regress/sql/create_table.sql index 5f25c436ee..f41dd71475 100644 --- a/src/test/regress/sql/create_table.sql +++ b/src/test/regress/sql/create_table.sql @@ -493,15 +493,17 @@ DROP TABLE temp_parted; CREATE TABLE no_oids_parted ( a int ) PARTITION BY RANGE (a) WITHOUT OIDS; -CREATE TABLE fail_part PARTITION OF no_oids_parted FOR VALUES FROM (1) TO (10 )WITH OIDS; +CREATE TABLE fail_part PARTITION OF no_oids_parted FOR VALUES FROM (1) TO (10) WITH OIDS; DROP TABLE no_oids_parted; --- likewise, the reverse if also true +-- If the partitioned table has oids, then the partition must have them. +-- If the WITHOUT OIDS option is specified for partition, it is overridden. CREATE TABLE oids_parted ( a int ) PARTITION BY RANGE (a) WITH OIDS; -CREATE TABLE fail_part PARTITION OF oids_parted FOR VALUES FROM (1) TO (10 ) WITHOUT OIDS; -DROP TABLE oids_parted; +CREATE TABLE part_forced_oids PARTITION OF oids_parted FOR VALUES FROM (1) TO (10) WITHOUT OIDS; +\d+ part_forced_oids +DROP TABLE oids_parted, part_forced_oids; -- check for partition bound overlap and other invalid specifications -- cgit v1.2.3 From 10257fc5ff74487a46594bd8c8c041878f409c17 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 20 Feb 2017 10:05:00 -0500 Subject: Fix documentation of to_char/to_timestamp TZ, tz, OF formatting patterns. These are only supported in to_char, not in the other direction, but the documentation failed to mention that. Also, describe TZ/tz as printing the time zone "abbreviation", not "name", because what they print is elsewhere referred to that way. Per bug #14558. --- doc/src/sgml/func.sgml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index d7738b18b7..1c8202c257 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -6062,15 +6062,18 @@ SELECT regexp_match('abc01234xyz', '(?:(.*?)(\d+)(.*)){1,1}'); TZ - upper case time-zone name + upper case time-zone abbreviation + (only supported in to_char) tz - lower case time-zone name + lower case time-zone abbreviation + (only supported in to_char) OF - time-zone offset + time-zone offset from UTC + (only supported in to_char) -- cgit v1.2.3 From 0bf41dd1908a0c05833168b9972e1c52cb7547b7 Mon Sep 17 00:00:00 2001 From: Simon Riggs Date: Tue, 21 Feb 2017 09:07:15 +0000 Subject: Small correction to BRIN docs Replace incorrect word "index" with "heap" Takayuki Tsunakawa --- doc/src/sgml/brin.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/brin.sgml b/doc/src/sgml/brin.sgml index f51928513c..6448b18e46 100644 --- a/doc/src/sgml/brin.sgml +++ b/doc/src/sgml/brin.sgml @@ -63,7 +63,7 @@ Index Maintenance - At the time of creation, all existing index pages are scanned and a + At the time of creation, all existing heap pages are scanned and a summary index tuple is created for each range, including the possibly-incomplete range at the end. As new pages are filled with data, page ranges that are already -- cgit v1.2.3 From 04aad401867ad3e1519615d8486e32b50dbcb5f5 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 21 Feb 2017 09:27:02 -0500 Subject: Drop support for Python 2.3 There is no specific reason for this right now, but keeping support for old Python versions around indefinitely increases the maintenance burden. The oldest supported Python version is now Python 2.4, which is still shipped in RHEL/CentOS 5 by default. In configure, add a check for the required Python version and give a friendly error message for an old version, instead of relying on an obscure build error later on. --- config/python.m4 | 1 + configure | 4 ++++ configure.in | 3 +++ .../hstore_plpython/expected/hstore_plpython.out | 8 ++------ contrib/hstore_plpython/sql/hstore_plpython.sql | 8 ++------ doc/src/sgml/installation.sgml | 6 +----- src/pl/plpython/expected/plpython_ereport.out | 24 +++++++++------------- src/pl/plpython/plpy_typeio.c | 10 ++------- src/pl/plpython/sql/plpython_ereport.sql | 8 ++------ 9 files changed, 27 insertions(+), 45 deletions(-) (limited to 'doc/src') diff --git a/config/python.m4 b/config/python.m4 index b605212bea..7f775e77d2 100644 --- a/config/python.m4 +++ b/config/python.m4 @@ -31,6 +31,7 @@ else fi AC_MSG_CHECKING([Python configuration directory]) python_majorversion=`${PYTHON} -c "import sys; print(sys.version[[0]])"` +python_minorversion=`${PYTHON} -c "import sys; print(sys.version[[2]])"` python_version=`${PYTHON} -c "import sys; print(sys.version[[:3]])"` python_configdir=`${PYTHON} -c "import distutils.sysconfig; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('LIBPL'))))"` AC_MSG_RESULT([$python_configdir]) diff --git a/configure b/configure index 8468417f69..908109849e 100755 --- a/configure +++ b/configure @@ -7588,6 +7588,7 @@ fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking Python configuration directory" >&5 $as_echo_n "checking Python configuration directory... " >&6; } python_majorversion=`${PYTHON} -c "import sys; print(sys.version[0])"` +python_minorversion=`${PYTHON} -c "import sys; print(sys.version[2])"` python_version=`${PYTHON} -c "import sys; print(sys.version[:3])"` python_configdir=`${PYTHON} -c "import distutils.sysconfig; print(' '.join(filter(None,distutils.sysconfig.get_config_vars('LIBPL'))))"` { $as_echo "$as_me:${as_lineno-$LINENO}: result: $python_configdir" >&5 @@ -7698,6 +7699,9 @@ $as_echo "${python_libspec} ${python_additional_libs}" >&6; } + if test "$python_majorversion" -lt 3 -a "$python_minorversion" -lt 4; then + as_fn_error $? "Python version $python_version is too old (version 2.4 or later is required)" "$LINENO" 5 + fi fi if test "$cross_compiling" = yes && test -z "$with_system_tzdata"; then diff --git a/configure.in b/configure.in index 01b618c931..9e4fb0fa80 100644 --- a/configure.in +++ b/configure.in @@ -927,6 +927,9 @@ fi if test "$with_python" = yes; then PGAC_PATH_PYTHON PGAC_CHECK_PYTHON_EMBED_SETUP + if test "$python_majorversion" -lt 3 -a "$python_minorversion" -lt 4; then + AC_MSG_ERROR([Python version $python_version is too old (version 2.4 or later is required)]) + fi fi if test "$cross_compiling" = yes && test -z "$with_system_tzdata"; then diff --git a/contrib/hstore_plpython/expected/hstore_plpython.out b/contrib/hstore_plpython/expected/hstore_plpython.out index b0025c04a8..df49cd5f37 100644 --- a/contrib/hstore_plpython/expected/hstore_plpython.out +++ b/contrib/hstore_plpython/expected/hstore_plpython.out @@ -6,9 +6,7 @@ LANGUAGE plpythonu TRANSFORM FOR TYPE hstore AS $$ assert isinstance(val, dict) -i = list(val.items()) -i.sort() -plpy.info(i) +plpy.info(sorted(val.items())) return len(val) $$; SELECT test1('aa=>bb, cc=>NULL'::hstore); @@ -24,9 +22,7 @@ LANGUAGE plpython2u TRANSFORM FOR TYPE hstore AS $$ assert isinstance(val, dict) -i = list(val.items()) -i.sort() -plpy.info(i) +plpy.info(sorted(val.items())) return len(val) $$; SELECT test1n('aa=>bb, cc=>NULL'::hstore); diff --git a/contrib/hstore_plpython/sql/hstore_plpython.sql b/contrib/hstore_plpython/sql/hstore_plpython.sql index d55bedaf50..911bbd67fe 100644 --- a/contrib/hstore_plpython/sql/hstore_plpython.sql +++ b/contrib/hstore_plpython/sql/hstore_plpython.sql @@ -7,9 +7,7 @@ LANGUAGE plpythonu TRANSFORM FOR TYPE hstore AS $$ assert isinstance(val, dict) -i = list(val.items()) -i.sort() -plpy.info(i) +plpy.info(sorted(val.items())) return len(val) $$; @@ -22,9 +20,7 @@ LANGUAGE plpython2u TRANSFORM FOR TYPE hstore AS $$ assert isinstance(val, dict) -i = list(val.items()) -i.sort() -plpy.info(i) +plpy.info(sorted(val.items())) return len(val) $$; diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml index 182c801bd5..be0931326b 100644 --- a/doc/src/sgml/installation.sgml +++ b/doc/src/sgml/installation.sgml @@ -193,11 +193,7 @@ su - postgres language, you need a Python installation with the header files and the distutils module. The minimum - required version is Python 2.3. - (To work with function arguments of type numeric, a 2.3.x - installation must include the separately-available cdecimal - module; note the PL/Python regression tests - will not pass if that is missing.) + required version is Python 2.4. Python 3 is supported if it's version 3.1 or later; but see PL/Python documentation]]> diff --git a/src/pl/plpython/expected/plpython_ereport.out b/src/pl/plpython/expected/plpython_ereport.out index 13bd0ab335..1dafd94c72 100644 --- a/src/pl/plpython/expected/plpython_ereport.out +++ b/src/pl/plpython/expected/plpython_ereport.out @@ -94,26 +94,22 @@ kwargs = { "column_name": _column_name, "datatype_name": _datatype_name, "constraint_name": _constraint_name } -# ignore None values - should work on Python2.3 -dict = {} -for k in kwargs: - if kwargs[k] is not None: - dict[k] = kwargs[k] -plpy.error(**dict) +# ignore None values +plpy.error(**dict((k, v) for k, v in iter(kwargs.items()) if v)) $$ LANGUAGE plpythonu; SELECT raise_exception('hello', 'world'); ERROR: plpy.Error: hello DETAIL: world CONTEXT: Traceback (most recent call last): - PL/Python function "raise_exception", line 13, in - plpy.error(**dict) + PL/Python function "raise_exception", line 9, in + plpy.error(**dict((k, v) for k, v in iter(kwargs.items()) if v)) PL/Python function "raise_exception" SELECT raise_exception('message text', 'detail text', _sqlstate => 'YY333'); ERROR: plpy.Error: message text DETAIL: detail text CONTEXT: Traceback (most recent call last): - PL/Python function "raise_exception", line 13, in - plpy.error(**dict) + PL/Python function "raise_exception", line 9, in + plpy.error(**dict((k, v) for k, v in iter(kwargs.items()) if v)) PL/Python function "raise_exception" SELECT raise_exception(_message => 'message text', _detail => 'detail text', @@ -128,8 +124,8 @@ ERROR: plpy.Error: message text DETAIL: detail text HINT: hint text CONTEXT: Traceback (most recent call last): - PL/Python function "raise_exception", line 13, in - plpy.error(**dict) + PL/Python function "raise_exception", line 9, in + plpy.error(**dict((k, v) for k, v in iter(kwargs.items()) if v)) PL/Python function "raise_exception" SELECT raise_exception(_message => 'message text', _hint => 'hint text', @@ -139,8 +135,8 @@ SELECT raise_exception(_message => 'message text', ERROR: plpy.Error: message text HINT: hint text CONTEXT: Traceback (most recent call last): - PL/Python function "raise_exception", line 13, in - plpy.error(**dict) + PL/Python function "raise_exception", line 9, in + plpy.error(**dict((k, v) for k, v in iter(kwargs.items()) if v)) PL/Python function "raise_exception" DO $$ DECLARE diff --git a/src/pl/plpython/plpy_typeio.c b/src/pl/plpython/plpy_typeio.c index b9c6d64baa..06743e46ed 100644 --- a/src/pl/plpython/plpy_typeio.c +++ b/src/pl/plpython/plpy_typeio.c @@ -521,15 +521,9 @@ PLy_input_datum_func2(PLyDatumToOb *arg, MemoryContext arg_mcxt, Oid typeOid, He static PyObject * PLyBool_FromBool(PLyDatumToOb *arg, Datum d) { - /* - * We would like to use Py_RETURN_TRUE and Py_RETURN_FALSE here for - * generating SQL from trigger functions, but those are only supported in - * Python >= 2.4, and we support older versions. - * http://docs.python.org/api/boolObjects.html - */ if (DatumGetBool(d)) - return PyBool_FromLong(1); - return PyBool_FromLong(0); + Py_RETURN_TRUE; + Py_RETURN_FALSE; } static PyObject * diff --git a/src/pl/plpython/sql/plpython_ereport.sql b/src/pl/plpython/sql/plpython_ereport.sql index 2612e93387..889293d33c 100644 --- a/src/pl/plpython/sql/plpython_ereport.sql +++ b/src/pl/plpython/sql/plpython_ereport.sql @@ -55,12 +55,8 @@ kwargs = { "column_name": _column_name, "datatype_name": _datatype_name, "constraint_name": _constraint_name } -# ignore None values - should work on Python2.3 -dict = {} -for k in kwargs: - if kwargs[k] is not None: - dict[k] = kwargs[k] -plpy.error(**dict) +# ignore None values +plpy.error(**dict((k, v) for k, v in iter(kwargs.items()) if v)) $$ LANGUAGE plpythonu; SELECT raise_exception('hello', 'world'); -- cgit v1.2.3 From 7248099c169b40b8f70cdaf8e12d0deaab9b16e2 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 21 Feb 2017 12:34:25 -0500 Subject: doc: Update URL for plr --- doc/src/sgml/external-projects.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/external-projects.sgml b/doc/src/sgml/external-projects.sgml index 88094c19ea..3b51e480f7 100644 --- a/doc/src/sgml/external-projects.sgml +++ b/doc/src/sgml/external-projects.sgml @@ -190,7 +190,7 @@ PL/R R - + -- cgit v1.2.3 From d912dd062b64287adcabab4180abafefd07cea14 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Wed, 22 Feb 2017 06:49:39 +0530 Subject: doc: Add missing comma. Yugo Nagata --- doc/src/sgml/ddl.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ddl.sgml b/doc/src/sgml/ddl.sgml index 5779eac43d..ef0f7cf727 100644 --- a/doc/src/sgml/ddl.sgml +++ b/doc/src/sgml/ddl.sgml @@ -2899,7 +2899,7 @@ VALUES ('Albany', NULL, NULL, 'NY'); - Since primary keys are not supported on partitioned tables + Since primary keys are not supported on partitioned tables, foreign keys referencing partitioned tables are not supported, nor are foreign key references from a partitioned table to some other table. -- cgit v1.2.3 From b6aa17e0ae367afdcea07118e016111af4fa6bc3 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 23 Feb 2017 11:40:12 -0500 Subject: De-support floating-point timestamps. Per discussion, the time has come to do this. The handwriting has been on the wall at least since 9.0 that this would happen someday, whenever it got to be too much of a burden to support the float-timestamp option. The triggering factor now is the discovery that there are multiple bugs in the code that attempts to implement use of integer timestamps in the replication protocol even when the server is built for float timestamps. The internal float timestamps leak into the protocol fields in places. While we could fix the identified bugs, there's a very high risk of introducing more. Trying to build a wall that would positively prevent mixing integer and float timestamps is more complexity than we want to undertake to maintain a long-deprecated option. The fact that these bugs weren't found through testing also indicates a lack of interest in float timestamps. This commit disables configure's --disable-integer-datetimes switch (it'll still accept --enable-integer-datetimes, though), removes direct references to USE_INTEGER_DATETIMES, and removes discussion of float timestamps from the user documentation. A considerable amount of code is rendered dead by this, but removing that will occur as separate mop-up. Discussion: https://postgr.es/m/26788.1487455319@sss.pgh.pa.us --- configure | 18 +++---- configure.in | 12 ++--- doc/src/sgml/config.sgml | 8 ++-- doc/src/sgml/datatype.sgml | 55 +++++----------------- doc/src/sgml/installation.sgml | 22 --------- src/include/c.h | 7 +-- src/include/pg_config.h.in | 4 -- src/include/pg_config.h.win32 | 4 -- src/interfaces/ecpg/include/ecpg_config.h.in | 4 -- src/interfaces/ecpg/include/pgtypes_interval.h | 2 - .../ecpg/test/expected/pgtypeslib-dt_test2.c | 6 +-- .../ecpg/test/expected/pgtypeslib-dt_test2.stdout | 2 + src/interfaces/ecpg/test/pgtypeslib/dt_test2.pgc | 6 +-- src/tools/msvc/Solution.pm | 9 ---- src/tools/msvc/config_default.pl | 1 - 15 files changed, 36 insertions(+), 124 deletions(-) (limited to 'doc/src') diff --git a/configure b/configure index 6a8db7ce2c..bbbe81170a 100755 --- a/configure +++ b/configure @@ -1473,7 +1473,7 @@ Optional Features: --disable-FEATURE do not include FEATURE (same as --enable-FEATURE=no) --enable-FEATURE[=ARG] include FEATURE [ARG=yes] --disable-integer-datetimes - disable 64-bit integer date/time support + obsolete option, no longer supported --enable-nls[=LANGUAGES] enable Native Language Support --disable-rpath do not embed shared library search path in @@ -2984,10 +2984,10 @@ fi # -# 64-bit integer date/time storage: enabled by default. +# 64-bit integer date/time storage is now the only option, but to avoid +# unnecessary breakage of build scripts, continue to accept an explicit +# "--enable-integer-datetimes" switch. # -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with 64-bit integer date/time support" >&5 -$as_echo_n "checking whether to build with 64-bit integer date/time support... " >&6; } # Check whether --enable-integer-datetimes was given. @@ -2995,12 +2995,10 @@ if test "${enable_integer_datetimes+set}" = set; then : enableval=$enable_integer_datetimes; case $enableval in yes) - -$as_echo "#define USE_INTEGER_DATETIMES 1" >>confdefs.h - + : ;; no) - : + as_fn_error $? "--disable-integer-datetimes is no longer supported" "$LINENO" 5 ;; *) as_fn_error $? "no argument expected for --enable-integer-datetimes option" "$LINENO" 5 @@ -3010,13 +3008,9 @@ $as_echo "#define USE_INTEGER_DATETIMES 1" >>confdefs.h else enable_integer_datetimes=yes -$as_echo "#define USE_INTEGER_DATETIMES 1" >>confdefs.h - fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $enable_integer_datetimes" >&5 -$as_echo "$enable_integer_datetimes" >&6; } # diff --git a/configure.in b/configure.in index 01b618c931..dab8ab3b32 100644 --- a/configure.in +++ b/configure.in @@ -122,13 +122,13 @@ PGAC_ARG_REQ(with, libs, [DIRS], [alternative spelling of --with-libraries] # -# 64-bit integer date/time storage: enabled by default. +# 64-bit integer date/time storage is now the only option, but to avoid +# unnecessary breakage of build scripts, continue to accept an explicit +# "--enable-integer-datetimes" switch. # -AC_MSG_CHECKING([whether to build with 64-bit integer date/time support]) -PGAC_ARG_BOOL(enable, integer-datetimes, yes, [disable 64-bit integer date/time support], - [AC_DEFINE([USE_INTEGER_DATETIMES], 1, - [Define to 1 if you want 64-bit integer timestamp and interval support. (--enable-integer-datetimes)])]) -AC_MSG_RESULT([$enable_integer_datetimes]) +PGAC_ARG_BOOL(enable, integer-datetimes, yes, [obsolete option, no longer supported], + [], + [AC_MSG_ERROR([--disable-integer-datetimes is no longer supported])]) # diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 95afc2c483..1b390a257a 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -7695,11 +7695,9 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir' - Reports whether PostgreSQL was built with - support for 64-bit-integer dates and times. This can be - disabled by configuring with --disable-integer-datetimes - when building PostgreSQL. The default value is - on. + Reports whether PostgreSQL was built with support for + 64-bit-integer dates and times. As of PostgreSQL 10, + this is always on. diff --git a/doc/src/sgml/datatype.sgml b/doc/src/sgml/datatype.sgml index 387ba53ef0..35610307d9 100644 --- a/doc/src/sgml/datatype.sgml +++ b/doc/src/sgml/datatype.sgml @@ -1580,7 +1580,7 @@ SELECT E'\\xDEADBEEF'; both date and time (no time zone) 4713 BC 294276 AD - 1 microsecond / 14 digits + 1 microsecond timestamp [ (p) ] with time zone @@ -1588,7 +1588,7 @@ SELECT E'\\xDEADBEEF'; both date and time, with time zone 4713 BC 294276 AD - 1 microsecond / 14 digits + 1 microsecond date @@ -1604,15 +1604,15 @@ SELECT E'\\xDEADBEEF'; time of day (no date) 00:00:00 24:00:00 - 1 microsecond / 14 digits + 1 microsecond time [ (p) ] with time zone 12 bytes - times of day only, with time zone + time of day (no date), with time zone 00:00:00+1459 24:00:00-1459 - 1 microsecond / 14 digits + 1 microsecond interval [ fields ] [ (p) ] @@ -1620,7 +1620,7 @@ SELECT E'\\xDEADBEEF'; time interval -178000000 years 178000000 years - 1 microsecond / 14 digits + 1 microsecond @@ -1643,41 +1643,7 @@ SELECT E'\\xDEADBEEF'; p which specifies the number of fractional digits retained in the seconds field. By default, there is no explicit bound on precision. The allowed range of - p is from 0 to 6 for the - timestamp and interval types. - - - - - When timestamp values are stored as eight-byte integers - (currently the default), microsecond precision is available over - the full range of values. In this case, the internal representation - is the number of microseconds before or after midnight 2000-01-01. - When timestamp values are stored as double precision - floating-point numbers (a deprecated compile-time option), the - internal representation is the number of seconds before or after - midnight 2000-01-01. With this representation, the effective limit - of precision might be less than 6; in practice, - microsecond precision is achieved for dates within a few - years of 2000-01-01, but the precision degrades for dates further - away. Note that using floating-point datetimes allows a larger - range of timestamp values to be represented than - shown above: from 4713 BC up to 5874897 AD. - - - - The same compile-time option also determines whether - time and interval values are stored as - floating-point numbers or eight-byte integers. In the - floating-point case, large interval values degrade in - precision as the size of the interval increases. - - - - - For the time types, the allowed range of - p is from 0 to 6 when eight-byte integer - storage is used, or from 0 to 10 when floating-point storage is used. + p is from 0 to 6. @@ -1760,9 +1726,10 @@ MINUTE TO SECOND specification giving the number of fractional digits in the seconds field. Precision can be specified for time, timestamp, and - interval types. The allowed values are mentioned - above. If no precision is specified in a constant specification, - it defaults to the precision of the literal value. + interval types, and can range from 0 to 6. + If no precision is specified in a constant specification, + it defaults to the precision of the literal value (but not + more than 6 digits). diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml index be0931326b..568995c9f2 100644 --- a/doc/src/sgml/installation.sgml +++ b/doc/src/sgml/installation.sgml @@ -955,28 +955,6 @@ su - postgres - - - - - Disable support for 64-bit integer storage for timestamps and - intervals, and store datetime values as floating-point - numbers instead. Floating-point datetime storage was the - default in PostgreSQL releases - prior to 8.4, but it is now deprecated, because it does not - support microsecond precision for the full range of - timestamp values. However, integer-based - datetime storage requires a 64-bit integer type. Therefore, - this option can be used when no such type is available, or - for compatibility with applications written for prior - versions of PostgreSQL. See - - ]]> - for more information. - - - - diff --git a/src/include/c.h b/src/include/c.h index 91e5baa969..947bd98067 100644 --- a/src/include/c.h +++ b/src/include/c.h @@ -340,10 +340,11 @@ typedef unsigned PG_INT128_TYPE uint128; #define PG_INT64_MAX INT64CONST(0x7FFFFFFFFFFFFFFF) #define PG_UINT64_MAX UINT64CONST(0xFFFFFFFFFFFFFFFF) -/* Select timestamp representation (float8 or int64) */ -#ifdef USE_INTEGER_DATETIMES +/* + * We now always use int64 timestamps, but keep this symbol defined for the + * benefit of external code that might test it. + */ #define HAVE_INT64_TIMESTAMP -#endif /* * Size diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index b9dfdd41c1..8dd73f1d91 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -831,10 +831,6 @@ (--enable-float8-byval) */ #undef USE_FLOAT8_BYVAL -/* Define to 1 if you want 64-bit integer timestamp and interval support. - (--enable-integer-datetimes) */ -#undef USE_INTEGER_DATETIMES - /* Define to 1 to build with LDAP support. (--with-ldap) */ #undef USE_LDAP diff --git a/src/include/pg_config.h.win32 b/src/include/pg_config.h.win32 index 199668c187..fd1af59839 100644 --- a/src/include/pg_config.h.win32 +++ b/src/include/pg_config.h.win32 @@ -625,10 +625,6 @@ /* Define to use /dev/urandom for random number generation */ /* #undef USE_DEV_URANDOM */ -/* Define to 1 if you want 64-bit integer timestamp and interval support. - (--enable-integer-datetimes) */ -/* #undef USE_INTEGER_DATETIMES */ - /* Define to 1 to build with LDAP support. (--with-ldap) */ /* #undef USE_LDAP */ diff --git a/src/interfaces/ecpg/include/ecpg_config.h.in b/src/interfaces/ecpg/include/ecpg_config.h.in index bf99a5e793..736fb08dba 100644 --- a/src/interfaces/ecpg/include/ecpg_config.h.in +++ b/src/interfaces/ecpg/include/ecpg_config.h.in @@ -10,10 +10,6 @@ /* Define to 1 if `long long int' works and is 64 bits. */ #undef HAVE_LONG_LONG_INT_64 -/* Define to 1 if you want 64-bit integer timestamp and interval support. - (--enable-integer-datetimes) */ -#undef USE_INTEGER_DATETIMES - /* Define to 1 to build client libraries as thread-safe code. * (--enable-thread-safety) */ #undef ENABLE_THREAD_SAFETY diff --git a/src/interfaces/ecpg/include/pgtypes_interval.h b/src/interfaces/ecpg/include/pgtypes_interval.h index deac6a2e01..5118ec784d 100644 --- a/src/interfaces/ecpg/include/pgtypes_interval.h +++ b/src/interfaces/ecpg/include/pgtypes_interval.h @@ -20,9 +20,7 @@ typedef long long int int64; #error must have a working 64-bit integer datatype #endif -#ifdef USE_INTEGER_DATETIMES #define HAVE_INT64_TIMESTAMP -#endif #endif /* C_H */ typedef struct diff --git a/src/interfaces/ecpg/test/expected/pgtypeslib-dt_test2.c b/src/interfaces/ecpg/test/expected/pgtypeslib-dt_test2.c index 4277c2615d..b6e77562b2 100644 --- a/src/interfaces/ecpg/test/expected/pgtypeslib-dt_test2.c +++ b/src/interfaces/ecpg/test/expected/pgtypeslib-dt_test2.c @@ -145,10 +145,8 @@ main(void) sprintf(t, "%s %s", dates[i], times[j]); ts1 = PGTYPEStimestamp_from_asc(t, NULL); text = PGTYPEStimestamp_to_asc(ts1); - /* skip outputs sensitive to USE_INTEGER_DATETIMES */ - if (i != 19 || (j != 3 && j != 4)) - printf("TS[%d,%d]: %s\n", - i, j, errno ? "-" : text); + printf("TS[%d,%d]: %s\n", + i, j, errno ? "-" : text); free(text); free(t); } diff --git a/src/interfaces/ecpg/test/expected/pgtypeslib-dt_test2.stdout b/src/interfaces/ecpg/test/expected/pgtypeslib-dt_test2.stdout index 941bffbd81..0fbcce67b6 100644 --- a/src/interfaces/ecpg/test/expected/pgtypeslib-dt_test2.stdout +++ b/src/interfaces/ecpg/test/expected/pgtypeslib-dt_test2.stdout @@ -103,6 +103,8 @@ Date[19]: 0099-01-08 BC (N - F) TS[19,0]: 0099-01-08 00:04:00 BC TS[19,1]: 0099-01-08 01:59:00 BC TS[19,2]: 0099-01-08 13:24:40 BC +TS[19,3]: 0099-01-08 13:24:40.495 BC +TS[19,4]: 0099-01-08 13:24:40.123456 BC Date[20]: - (N - T) Date[21]: - (N - T) interval[0]: @ 1 min diff --git a/src/interfaces/ecpg/test/pgtypeslib/dt_test2.pgc b/src/interfaces/ecpg/test/pgtypeslib/dt_test2.pgc index 0bd1fec109..d519305e18 100644 --- a/src/interfaces/ecpg/test/pgtypeslib/dt_test2.pgc +++ b/src/interfaces/ecpg/test/pgtypeslib/dt_test2.pgc @@ -110,10 +110,8 @@ main(void) sprintf(t, "%s %s", dates[i], times[j]); ts1 = PGTYPEStimestamp_from_asc(t, NULL); text = PGTYPEStimestamp_to_asc(ts1); - /* skip outputs sensitive to USE_INTEGER_DATETIMES */ - if (i != 19 || (j != 3 && j != 4)) - printf("TS[%d,%d]: %s\n", - i, j, errno ? "-" : text); + printf("TS[%d,%d]: %s\n", + i, j, errno ? "-" : text); free(text); free(t); } diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index fbf4da3d68..ff9064f923 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -28,9 +28,6 @@ sub _new $self->DeterminePlatform(); my $bits = $self->{platform} eq 'Win32' ? 32 : 64; - # integer_datetimes is now the default - $options->{integer_datetimes} = 1 - unless exists $options->{integer_datetimes}; $options->{float4byval} = 1 unless exists $options->{float4byval}; $options->{float8byval} = ($bits == 64) @@ -169,8 +166,6 @@ s{PG_VERSION_STR "[^"]+"}{__STRINGIFY(x) #x\n#define __STRINGIFY2(z) __STRINGIFY print O "#ifndef IGNORE_CONFIGURED_SETTINGS\n"; print O "#define USE_ASSERT_CHECKING 1\n" if ($self->{options}->{asserts}); - print O "#define USE_INTEGER_DATETIMES 1\n" - if ($self->{options}->{integer_datetimes}); print O "#define USE_LDAP 1\n" if ($self->{options}->{ldap}); print O "#define HAVE_LIBZ 1\n" if ($self->{options}->{zlib}); print O "#define USE_OPENSSL 1\n" if ($self->{options}->{openssl}); @@ -427,8 +422,6 @@ s{PG_VERSION_STR "[^"]+"}{__STRINGIFY(x) #x\n#define __STRINGIFY2(z) __STRINGIFY #define HAVE_LONG_LONG_INT_64 #define ENABLE_THREAD_SAFETY 1 EOF - print O "#define USE_INTEGER_DATETIMES 1\n" - if ($self->{options}->{integer_datetimes}); print O "#endif\n"; close(O); } @@ -661,8 +654,6 @@ sub GetFakeConfigure my $cfg = '--enable-thread-safety'; $cfg .= ' --enable-cassert' if ($self->{options}->{asserts}); - $cfg .= ' --enable-integer-datetimes' - if ($self->{options}->{integer_datetimes}); $cfg .= ' --enable-nls' if ($self->{options}->{nls}); $cfg .= ' --enable-tap-tests' if ($self->{options}->{tap_tests}); $cfg .= ' --with-ldap' if ($self->{options}->{ldap}); diff --git a/src/tools/msvc/config_default.pl b/src/tools/msvc/config_default.pl index f046687bd0..97f1af8b49 100644 --- a/src/tools/msvc/config_default.pl +++ b/src/tools/msvc/config_default.pl @@ -4,7 +4,6 @@ use warnings; our $config = { asserts => 0, # --enable-cassert - # integer_datetimes=>1, # --enable-integer-datetimes - on is now default # float4byval=>1, # --disable-float4-byval, on by default # float8byval=> $platformbits == 64, # --disable-float8-byval, -- cgit v1.2.3 From d28aafb6dda326688e2f042c95c93ea57963c03c Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 23 Feb 2017 12:23:12 -0500 Subject: Remove pg_control's enableIntTimes field. We don't need it any more. pg_controldata continues to report that date/time type storage is "64-bit integers", but that's now a hard-wired behavior not something it sees in the data. This avoids breaking pg_upgrade, and perhaps other utilities that inspect pg_control this way. Ditto for pg_resetwal. I chose to remove the "bigint_timestamps" output column of pg_control_init(), though, as that function hasn't been around long and probably doesn't have ossified users. Discussion: https://postgr.es/m/26788.1487455319@sss.pgh.pa.us --- doc/src/sgml/func.sgml | 5 ----- src/backend/access/transam/xlog.c | 21 --------------------- src/backend/utils/misc/pg_controldata.c | 23 +++++++++-------------- src/bin/pg_controldata/pg_controldata.c | 3 ++- src/bin/pg_resetwal/pg_resetwal.c | 8 ++------ src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_control.h | 5 +---- src/include/catalog/pg_proc.h | 2 +- 8 files changed, 16 insertions(+), 53 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 1c8202c257..9c53e4288c 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -17655,11 +17655,6 @@ SELECT collation for ('foo' COLLATE "de_DE"); integer - - bigint_timestamps - boolean - - float4_pass_by_value boolean diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f23e108628..ebae9da0f8 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -4379,11 +4379,6 @@ WriteControlFile(void) ControlFile->toast_max_chunk_size = TOAST_MAX_CHUNK_SIZE; ControlFile->loblksize = LOBLKSIZE; -#ifdef HAVE_INT64_TIMESTAMP - ControlFile->enableIntTimes = true; -#else - ControlFile->enableIntTimes = false; -#endif ControlFile->float4ByVal = FLOAT4PASSBYVAL; ControlFile->float8ByVal = FLOAT8PASSBYVAL; @@ -4579,22 +4574,6 @@ ReadControlFile(void) ControlFile->loblksize, (int) LOBLKSIZE), errhint("It looks like you need to recompile or initdb."))); -#ifdef HAVE_INT64_TIMESTAMP - if (ControlFile->enableIntTimes != true) - ereport(FATAL, - (errmsg("database files are incompatible with server"), - errdetail("The database cluster was initialized without HAVE_INT64_TIMESTAMP" - " but the server was compiled with HAVE_INT64_TIMESTAMP."), - errhint("It looks like you need to recompile or initdb."))); -#else - if (ControlFile->enableIntTimes != false) - ereport(FATAL, - (errmsg("database files are incompatible with server"), - errdetail("The database cluster was initialized with HAVE_INT64_TIMESTAMP" - " but the server was compiled without HAVE_INT64_TIMESTAMP."), - errhint("It looks like you need to recompile or initdb."))); -#endif - #ifdef USE_FLOAT4_BYVAL if (ControlFile->float4ByVal != true) ereport(FATAL, diff --git a/src/backend/utils/misc/pg_controldata.c b/src/backend/utils/misc/pg_controldata.c index 93a3d38b1d..d8454111a8 100644 --- a/src/backend/utils/misc/pg_controldata.c +++ b/src/backend/utils/misc/pg_controldata.c @@ -266,8 +266,8 @@ pg_control_recovery(PG_FUNCTION_ARGS) Datum pg_control_init(PG_FUNCTION_ARGS) { - Datum values[13]; - bool nulls[13]; + Datum values[12]; + bool nulls[12]; TupleDesc tupdesc; HeapTuple htup; ControlFileData *ControlFile; @@ -277,7 +277,7 @@ pg_control_init(PG_FUNCTION_ARGS) * Construct a tuple descriptor for the result row. This must match this * function's pg_proc entry! */ - tupdesc = CreateTemplateTupleDesc(13, false); + tupdesc = CreateTemplateTupleDesc(12, false); TupleDescInitEntry(tupdesc, (AttrNumber) 1, "max_data_alignment", INT4OID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 2, "database_block_size", @@ -296,13 +296,11 @@ pg_control_init(PG_FUNCTION_ARGS) INT4OID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 9, "large_object_chunk_size", INT4OID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 10, "bigint_timestamps", + TupleDescInitEntry(tupdesc, (AttrNumber) 10, "float4_pass_by_value", BOOLOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 11, "float4_pass_by_value", + TupleDescInitEntry(tupdesc, (AttrNumber) 11, "float8_pass_by_value", BOOLOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 12, "float8_pass_by_value", - BOOLOID, -1, 0); - TupleDescInitEntry(tupdesc, (AttrNumber) 13, "data_page_checksum_version", + TupleDescInitEntry(tupdesc, (AttrNumber) 12, "data_page_checksum_version", INT4OID, -1, 0); tupdesc = BlessTupleDesc(tupdesc); @@ -339,18 +337,15 @@ pg_control_init(PG_FUNCTION_ARGS) values[8] = Int32GetDatum(ControlFile->loblksize); nulls[8] = false; - values[9] = BoolGetDatum(ControlFile->enableIntTimes); + values[9] = BoolGetDatum(ControlFile->float4ByVal); nulls[9] = false; - values[10] = BoolGetDatum(ControlFile->float4ByVal); + values[10] = BoolGetDatum(ControlFile->float8ByVal); nulls[10] = false; - values[11] = BoolGetDatum(ControlFile->float8ByVal); + values[11] = Int32GetDatum(ControlFile->data_checksum_version); nulls[11] = false; - values[12] = Int32GetDatum(ControlFile->data_checksum_version); - nulls[12] = false; - htup = heap_form_tuple(tupdesc, values, nulls); PG_RETURN_DATUM(HeapTupleGetDatum(htup)); diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c index 20077a6639..f47171d29d 100644 --- a/src/bin/pg_controldata/pg_controldata.c +++ b/src/bin/pg_controldata/pg_controldata.c @@ -293,8 +293,9 @@ main(int argc, char *argv[]) ControlFile->toast_max_chunk_size); printf(_("Size of a large-object chunk: %u\n"), ControlFile->loblksize); + /* This is no longer configurable, but users may still expect to see it: */ printf(_("Date/time type storage: %s\n"), - (ControlFile->enableIntTimes ? _("64-bit integers") : _("floating-point numbers"))); + _("64-bit integers")); printf(_("Float4 argument passing: %s\n"), (ControlFile->float4ByVal ? _("by value") : _("by reference"))); printf(_("Float8 argument passing: %s\n"), diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c index 96b7097f8b..502ea5b290 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c @@ -598,11 +598,6 @@ GuessControlValues(void) ControlFile.indexMaxKeys = INDEX_MAX_KEYS; ControlFile.toast_max_chunk_size = TOAST_MAX_CHUNK_SIZE; ControlFile.loblksize = LOBLKSIZE; -#ifdef HAVE_INT64_TIMESTAMP - ControlFile.enableIntTimes = true; -#else - ControlFile.enableIntTimes = false; -#endif ControlFile.float4ByVal = FLOAT4PASSBYVAL; ControlFile.float8ByVal = FLOAT8PASSBYVAL; @@ -688,8 +683,9 @@ PrintControlValues(bool guessed) ControlFile.toast_max_chunk_size); printf(_("Size of a large-object chunk: %u\n"), ControlFile.loblksize); + /* This is no longer configurable, but users may still expect to see it: */ printf(_("Date/time type storage: %s\n"), - (ControlFile.enableIntTimes ? _("64-bit integers") : _("floating-point numbers"))); + _("64-bit integers")); printf(_("Float4 argument passing: %s\n"), (ControlFile.float4ByVal ? _("by value") : _("by reference"))); printf(_("Float8 argument passing: %s\n"), diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 5f42bde136..9e8c1c2a02 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201702101 +#define CATALOG_VERSION_NO 201702231 #endif diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h index 23731e98a4..e4194b9de1 100644 --- a/src/include/catalog/pg_control.h +++ b/src/include/catalog/pg_control.h @@ -21,7 +21,7 @@ /* Version identifier for this pg_control format */ -#define PG_CONTROL_VERSION 960 +#define PG_CONTROL_VERSION 1001 /* * Body of CheckPoint XLOG records. This is declared here because we keep @@ -215,9 +215,6 @@ typedef struct ControlFileData uint32 toast_max_chunk_size; /* chunk size in TOAST tables */ uint32 loblksize; /* chunk size in pg_largeobject */ - /* flag indicating internal format of timestamp, interval, time */ - bool enableIntTimes; /* int64 storage enabled? */ - /* flags indicating pass-by-value status of various types */ bool float4ByVal; /* float4 pass-by-value? */ bool float8ByVal; /* float8, int8, etc pass-by-value? */ diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index bb7053a942..a4cc86d322 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -5355,7 +5355,7 @@ DESCR("pg_controldata checkpoint state information as a function"); DATA(insert OID = 3443 ( pg_control_recovery PGNSP PGUID 12 1 0 0 0 f f f f t f v s 0 0 2249 "" "{3220,23,3220,3220,16}" "{o,o,o,o,o}" "{min_recovery_end_location,min_recovery_end_timeline,backup_start_location,backup_end_location,end_of_backup_record_required}" _null_ _null_ pg_control_recovery _null_ _null_ _null_ )); DESCR("pg_controldata recovery state information as a function"); -DATA(insert OID = 3444 ( pg_control_init PGNSP PGUID 12 1 0 0 0 f f f f t f v s 0 0 2249 "" "{23,23,23,23,23,23,23,23,23,16,16,16,23}" "{o,o,o,o,o,o,o,o,o,o,o,o,o}" "{max_data_alignment,database_block_size,blocks_per_segment,wal_block_size,bytes_per_wal_segment,max_identifier_length,max_index_columns,max_toast_chunk_size,large_object_chunk_size,bigint_timestamps,float4_pass_by_value,float8_pass_by_value,data_page_checksum_version}" _null_ _null_ pg_control_init _null_ _null_ _null_ )); +DATA(insert OID = 3444 ( pg_control_init PGNSP PGUID 12 1 0 0 0 f f f f t f v s 0 0 2249 "" "{23,23,23,23,23,23,23,23,23,16,16,23}" "{o,o,o,o,o,o,o,o,o,o,o,o}" "{max_data_alignment,database_block_size,blocks_per_segment,wal_block_size,bytes_per_wal_segment,max_identifier_length,max_index_columns,max_toast_chunk_size,large_object_chunk_size,float4_pass_by_value,float8_pass_by_value,data_page_checksum_version}" _null_ _null_ pg_control_init _null_ _null_ _null_ )); DESCR("pg_controldata init state information as a function"); DATA(insert OID = 3445 ( pg_import_system_collations PGNSP PGUID 12 100 0 0 0 f f f f t f v r 2 0 2278 "16 4089" _null_ _null_ "{if_not_exists,schema}" _null_ _null_ pg_import_system_collations _null_ _null_ _null_ )); -- cgit v1.2.3 From 5639ceddcb7f3efa8751b2ba6e50cc1d27cc2a45 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Sat, 25 Feb 2017 12:59:23 -0500 Subject: pg_upgrade docs: clarify instructions on standby extensions Previously the pg_upgrade standby upgrade instructions said not to execute pgcrypto.sql, but it should have referenced the extension command "CREATE EXTENSION pgcrypto". This patch makes that doc change. Reported-by: a private bug report Backpatch-through: 9.4, where standby instructions were added --- doc/src/sgml/ref/pgupgrade.sgml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/pgupgrade.sgml b/doc/src/sgml/ref/pgupgrade.sgml index ad28526296..49aaf513f5 100644 --- a/doc/src/sgml/ref/pgupgrade.sgml +++ b/doc/src/sgml/ref/pgupgrade.sgml @@ -273,7 +273,8 @@ make prefix=/usr/local/pgsql.new install into the new cluster, e.g. pgcrypto.so, whether they are from contrib or some other source. Do not install the schema definitions, e.g. - pgcrypto.sql, because these will be upgraded from the old cluster. + CREATE EXTENSION pgcrypto, because these will be upgraded + from the old cluster. Also, any custom full text search files (dictionary, synonym, thesaurus, stop words) must also be copied to the new cluster. -- cgit v1.2.3 From a315b967cc1bd43ecf3c10ea48b44a4fb0ff8d45 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Sun, 26 Feb 2017 13:36:49 +0530 Subject: Allow custom and foreign scans to have shutdown callbacks. This is expected to be useful mostly when performing such scans in parallel, because in that case it allows (in combination with commit acf555bc53acb589b5a2827e65d655fa8c9adee0) nodes below a Gather to get control just before the DSM segment goes away. KaiGai Kohei, except that I rewrote the documentation. Reviewed by Claudio Freire. Discussion: http://postgr.es/m/CADyhKSXJK0jUJ8rWv4AmKDhsUh124_rEn39eqgfC5D8fu6xVuw@mail.gmail.com --- doc/src/sgml/custom-scan.sgml | 13 +++++++++++++ doc/src/sgml/fdwhandler.sgml | 14 ++++++++++++++ src/backend/executor/execProcnode.c | 6 ++++++ src/backend/executor/nodeCustom.c | 9 +++++++++ src/backend/executor/nodeForeignscan.c | 16 ++++++++++++++++ src/include/executor/nodeCustom.h | 1 + src/include/executor/nodeForeignscan.h | 1 + src/include/foreign/fdwapi.h | 2 ++ src/include/nodes/extensible.h | 1 + 9 files changed, 63 insertions(+) (limited to 'doc/src') diff --git a/doc/src/sgml/custom-scan.sgml b/doc/src/sgml/custom-scan.sgml index 1ca9247124..6159c3a24e 100644 --- a/doc/src/sgml/custom-scan.sgml +++ b/doc/src/sgml/custom-scan.sgml @@ -340,6 +340,19 @@ void (*InitializeWorkerCustomScan) (CustomScanState *node, +void (*ShutdownCustomScan) (CustomScanState *node); + + Release resources when it is anticipated the node will not be executed + to completion. This is not called in all cases; sometimes, + EndCustomScan may be called without this function having + been called first. Since the DSM segment used by parallel query is + destroyed just after this callback is invoked, custom scan providers that + wish to take some action before the DSM segment goes away should implement + this method. + + + + void (*ExplainCustomScan) (CustomScanState *node, List *ancestors, ExplainState *es); diff --git a/doc/src/sgml/fdwhandler.sgml b/doc/src/sgml/fdwhandler.sgml index 0c1db070ed..dbeaab555d 100644 --- a/doc/src/sgml/fdwhandler.sgml +++ b/doc/src/sgml/fdwhandler.sgml @@ -1254,6 +1254,20 @@ InitializeWorkerForeignScan(ForeignScanState *node, shm_toc *toc, This callback is optional, and needs only be supplied if this custom path supports parallel execution. + + + +void +ShutdownForeignScan(ForeignScanState *node); + + Release resources when it is anticipated the node will not be executed + to completion. This is not called in all cases; sometimes, + EndForeignScan may be called without this function having + been called first. Since the DSM segment used by parallel query is + destroyed just after this callback is invoked, foreign data wrappers that + wish to take some action before the DSM segment goes away should implement + this method. + diff --git a/src/backend/executor/execProcnode.c b/src/backend/executor/execProcnode.c index 5ccc2e846d..ef6f35a5a0 100644 --- a/src/backend/executor/execProcnode.c +++ b/src/backend/executor/execProcnode.c @@ -822,6 +822,12 @@ ExecShutdownNode(PlanState *node) case T_GatherState: ExecShutdownGather((GatherState *) node); break; + case T_ForeignScanState: + ExecShutdownForeignScan((ForeignScanState *) node); + break; + case T_CustomScanState: + ExecShutdownCustomScan((CustomScanState *) node); + break; default: break; } diff --git a/src/backend/executor/nodeCustom.c b/src/backend/executor/nodeCustom.c index 16343a56df..d464748290 100644 --- a/src/backend/executor/nodeCustom.c +++ b/src/backend/executor/nodeCustom.c @@ -202,3 +202,12 @@ ExecCustomScanInitializeWorker(CustomScanState *node, shm_toc *toc) methods->InitializeWorkerCustomScan(node, toc, coordinate); } } + +void +ExecShutdownCustomScan(CustomScanState *node) +{ + const CustomExecMethods *methods = node->methods; + + if (methods->ShutdownCustomScan) + methods->ShutdownCustomScan(node); +} diff --git a/src/backend/executor/nodeForeignscan.c b/src/backend/executor/nodeForeignscan.c index 86a77e356c..3b6d1390eb 100644 --- a/src/backend/executor/nodeForeignscan.c +++ b/src/backend/executor/nodeForeignscan.c @@ -353,3 +353,19 @@ ExecForeignScanInitializeWorker(ForeignScanState *node, shm_toc *toc) fdwroutine->InitializeWorkerForeignScan(node, toc, coordinate); } } + +/* ---------------------------------------------------------------- + * ExecShutdownForeignScan + * + * Gives FDW chance to stop asynchronous resource consumption + * and release any resources still held. + * ---------------------------------------------------------------- + */ +void +ExecShutdownForeignScan(ForeignScanState *node) +{ + FdwRoutine *fdwroutine = node->fdwroutine; + + if (fdwroutine->ShutdownForeignScan) + fdwroutine->ShutdownForeignScan(node); +} diff --git a/src/include/executor/nodeCustom.h b/src/include/executor/nodeCustom.h index 19d5d047b5..c2f2ca1eed 100644 --- a/src/include/executor/nodeCustom.h +++ b/src/include/executor/nodeCustom.h @@ -37,5 +37,6 @@ extern void ExecCustomScanInitializeDSM(CustomScanState *node, ParallelContext *pcxt); extern void ExecCustomScanInitializeWorker(CustomScanState *node, shm_toc *toc); +extern void ExecShutdownCustomScan(CustomScanState *node); #endif /* NODECUSTOM_H */ diff --git a/src/include/executor/nodeForeignscan.h b/src/include/executor/nodeForeignscan.h index f0e942a8bc..1b167b8143 100644 --- a/src/include/executor/nodeForeignscan.h +++ b/src/include/executor/nodeForeignscan.h @@ -28,5 +28,6 @@ extern void ExecForeignScanInitializeDSM(ForeignScanState *node, ParallelContext *pcxt); extern void ExecForeignScanInitializeWorker(ForeignScanState *node, shm_toc *toc); +extern void ExecShutdownForeignScan(ForeignScanState *node); #endif /* NODEFOREIGNSCAN_H */ diff --git a/src/include/foreign/fdwapi.h b/src/include/foreign/fdwapi.h index 523d415575..6ca44f734f 100644 --- a/src/include/foreign/fdwapi.h +++ b/src/include/foreign/fdwapi.h @@ -151,6 +151,7 @@ typedef void (*InitializeDSMForeignScan_function) (ForeignScanState *node, typedef void (*InitializeWorkerForeignScan_function) (ForeignScanState *node, shm_toc *toc, void *coordinate); +typedef void (*ShutdownForeignScan_function) (ForeignScanState *node); typedef bool (*IsForeignScanParallelSafe_function) (PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte); @@ -224,6 +225,7 @@ typedef struct FdwRoutine EstimateDSMForeignScan_function EstimateDSMForeignScan; InitializeDSMForeignScan_function InitializeDSMForeignScan; InitializeWorkerForeignScan_function InitializeWorkerForeignScan; + ShutdownForeignScan_function ShutdownForeignScan; } FdwRoutine; diff --git a/src/include/nodes/extensible.h b/src/include/nodes/extensible.h index 7e860b0dca..0b02cc18e9 100644 --- a/src/include/nodes/extensible.h +++ b/src/include/nodes/extensible.h @@ -139,6 +139,7 @@ typedef struct CustomExecMethods void (*InitializeWorkerCustomScan) (CustomScanState *node, shm_toc *toc, void *coordinate); + void (*ShutdownCustomScan) (CustomScanState *node); /* Optional: print additional information in EXPLAIN */ void (*ExplainCustomScan) (CustomScanState *node, -- cgit v1.2.3 From 51e26c9c3d2904b65041fc4a19c72c62508f63d4 Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Sun, 26 Feb 2017 21:27:51 +0100 Subject: Clarify the role of checkpoint at the begininng of base backups Output a message about checkpoint starting in verbose mode of pg_basebackup, and make the documentation state more clearly that this happens. Author: Michael Banck --- doc/src/sgml/backup.sgml | 3 ++- doc/src/sgml/ref/pg_basebackup.sgml | 10 +++++++++- src/bin/pg_basebackup/pg_basebackup.c | 11 +++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml index 5f009ee3d0..12f2a14a5c 100644 --- a/doc/src/sgml/backup.sgml +++ b/doc/src/sgml/backup.sgml @@ -862,7 +862,8 @@ SELECT pg_start_backup('label', false, false); ). This is usually what you want, because it minimizes the impact on query processing. If you want to start the backup as soon as - possible, change the second parameter to true. + possible, change the second parameter to true, which will + issue an immediate checkpoint using as much I/O as available. diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml index c9dd62c141..e1cec9d60f 100644 --- a/doc/src/sgml/ref/pg_basebackup.sgml +++ b/doc/src/sgml/ref/pg_basebackup.sgml @@ -419,7 +419,7 @@ PostgreSQL documentation - Sets checkpoint mode to fast or spread (default) (see ). + Sets checkpoint mode to fast (immediate) or spread (default) (see ). @@ -659,6 +659,14 @@ PostgreSQL documentation Notes + + At the beginning of the backup, a checkpoint needs to be written on the + server the backup is taken from. Especially if the option + --checkpoint=fast is not used, this can take some time + during which pg_basebackup will be appear + to be idle. + + The backup will include all files in the data directory and tablespaces, including the configuration files and any additional files placed in the diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index bc997dc997..761679a4d0 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -1753,6 +1753,14 @@ BaseBackup(void) if (maxrate > 0) maxrate_clause = psprintf("MAX_RATE %u", maxrate); + if (verbose) + fprintf(stderr, + _("%s: initiating base backup, waiting for checkpoint to complete\n"), + progname); + + if (showprogress && !verbose) + fprintf(stderr, "waiting for checkpoint\n"); + basebkp = psprintf("BASE_BACKUP LABEL '%s' %s %s %s %s %s %s", escaped_label, @@ -1790,6 +1798,9 @@ BaseBackup(void) strlcpy(xlogstart, PQgetvalue(res, 0, 0), sizeof(xlogstart)); + if (verbose) + fprintf(stderr, _("%s: checkpoint completed\n"), progname); + /* * 9.3 and later sends the TLI of the starting point. With older servers, * assume it's the same as the latest timeline reported by -- cgit v1.2.3 From 817f2a586342767d3289a320bb1dac5dcbb76979 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 27 Feb 2017 11:20:22 -0500 Subject: Remove PL/Tcl's "module" facility. PL/Tcl has long had a facility whereby Tcl code could be autoloaded from a database table named "pltcl_modules". However, nobody is using it, as evidenced by the recent discovery that it's never been fixed to work with standard_conforming_strings turned on. Moreover, it's rather shaky from a security standpoint, and the table design is very old and crufty (partly because it dates from before we had TOAST). A final problem is that because the table-population scripts depend on the Tcl client library Pgtcl, which we removed from the core distribution in 2004, it's impossible to create a self-contained regression test for the feature. Rather than try to surmount these problems, let's just remove it. A follow-on patch will provide a way to execute user-defined initialization code, similar to features that exist in plperl and plv8. With that, it will be possible to implement this feature or similar ones entirely in userspace, which is where it belongs. Discussion: https://postgr.es/m/22067.1488046447@sss.pgh.pa.us --- doc/src/sgml/pltcl.sgml | 45 ---- src/pl/tcl/Makefile | 5 - src/pl/tcl/modules/.gitignore | 3 - src/pl/tcl/modules/Makefile | 28 -- src/pl/tcl/modules/README | 18 -- src/pl/tcl/modules/pltcl_delmod.in | 117 --------- src/pl/tcl/modules/pltcl_listmod.in | 123 --------- src/pl/tcl/modules/pltcl_loadmod.in | 501 ------------------------------------ src/pl/tcl/modules/unknown.pltcl | 63 ----- src/pl/tcl/pltcl.c | 126 --------- 10 files changed, 1029 deletions(-) delete mode 100644 src/pl/tcl/modules/.gitignore delete mode 100644 src/pl/tcl/modules/Makefile delete mode 100644 src/pl/tcl/modules/README delete mode 100644 src/pl/tcl/modules/pltcl_delmod.in delete mode 100644 src/pl/tcl/modules/pltcl_listmod.in delete mode 100644 src/pl/tcl/modules/pltcl_loadmod.in delete mode 100644 src/pl/tcl/modules/unknown.pltcl (limited to 'doc/src') diff --git a/doc/src/sgml/pltcl.sgml b/doc/src/sgml/pltcl.sgml index 8afaf4ad36..0a693803dd 100644 --- a/doc/src/sgml/pltcl.sgml +++ b/doc/src/sgml/pltcl.sgml @@ -902,51 +902,6 @@ if {[catch { spi_exec $sql_command }]} { - - Modules and the <function>unknown</> Command - - PL/Tcl has support for autoloading Tcl code when used. - It recognizes a special table, pltcl_modules, which - is presumed to contain modules of Tcl code. If this table - exists, the module unknown is fetched from the table - and loaded into the Tcl interpreter immediately before the first - execution of a PL/Tcl function in a database session. (This - happens separately for each Tcl interpreter, if more than one is - used in a session; see .) - - - While the unknown module could actually contain any - initialization script you need, it normally defines a Tcl - unknown procedure that is invoked whenever Tcl does - not recognize an invoked procedure name. PL/Tcl's standard version - of this procedure tries to find a module in pltcl_modules - that will define the required procedure. If one is found, it is - loaded into the interpreter, and then execution is allowed to - proceed with the originally attempted procedure call. A - secondary table pltcl_modfuncs provides an index of - which functions are defined by which modules, so that the lookup - is reasonably quick. - - - The PostgreSQL distribution includes - support scripts to maintain these tables: - pltcl_loadmod, pltcl_listmod, - pltcl_delmod, as well as source for the standard - unknown module in share/unknown.pltcl. This module - must be loaded - into each database initially to support the autoloading mechanism. - - - The tables pltcl_modules and pltcl_modfuncs - must be readable by all, but it is wise to make them owned and - writable only by the database administrator. As a security - precaution, PL/Tcl will ignore pltcl_modules (and thus, - not attempt to load the unknown module) unless it is - owned by a superuser. But update privileges on this table can be - granted to other users, if you trust them sufficiently. - - - Tcl Procedure Names diff --git a/src/pl/tcl/Makefile b/src/pl/tcl/Makefile index 25082ec504..453e7ad2ec 100644 --- a/src/pl/tcl/Makefile +++ b/src/pl/tcl/Makefile @@ -53,7 +53,6 @@ include $(top_srcdir)/src/Makefile.shlib all: all-lib - $(MAKE) -C modules $@ # Force this dependency to be known even without dependency info built: pltcl.o: pltclerrcodes.h @@ -65,14 +64,11 @@ pltclerrcodes.h: $(top_srcdir)/src/backend/utils/errcodes.txt generate-pltclerrc distprep: pltclerrcodes.h install: all install-lib install-data - $(MAKE) -C modules $@ installdirs: installdirs-lib $(MKDIR_P) '$(DESTDIR)$(datadir)/extension' - $(MAKE) -C modules $@ uninstall: uninstall-lib uninstall-data - $(MAKE) -C modules $@ install-data: installdirs $(INSTALL_DATA) $(addprefix $(srcdir)/, $(DATA)) '$(DESTDIR)$(datadir)/extension/' @@ -100,7 +96,6 @@ clean distclean: clean-lib ifeq ($(PORTNAME), win32) rm -f $(tclwithver).def endif - $(MAKE) -C modules $@ maintainer-clean: distclean rm -f pltclerrcodes.h diff --git a/src/pl/tcl/modules/.gitignore b/src/pl/tcl/modules/.gitignore deleted file mode 100644 index 89581887c4..0000000000 --- a/src/pl/tcl/modules/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/pltcl_delmod -/pltcl_listmod -/pltcl_loadmod diff --git a/src/pl/tcl/modules/Makefile b/src/pl/tcl/modules/Makefile deleted file mode 100644 index 8055c61460..0000000000 --- a/src/pl/tcl/modules/Makefile +++ /dev/null @@ -1,28 +0,0 @@ -# src/pl/tcl/modules/Makefile - -subdir = src/pl/tcl/modules -top_builddir = ../../../.. -include $(top_builddir)/src/Makefile.global - -MODULES = pltcl_loadmod pltcl_delmod pltcl_listmod - -all: $(MODULES) - -$(MODULES): %: %.in $(top_builddir)/src/Makefile.global - sed 's,@TCLSH@,$(TCLSH),g' $< >$@ - chmod a+x $@ - -install: all installdirs - $(INSTALL_SCRIPT) pltcl_loadmod '$(DESTDIR)$(bindir)/pltcl_loadmod' - $(INSTALL_SCRIPT) pltcl_delmod '$(DESTDIR)$(bindir)/pltcl_delmod' - $(INSTALL_SCRIPT) pltcl_listmod '$(DESTDIR)$(bindir)/pltcl_listmod' - $(INSTALL_DATA) $(srcdir)/unknown.pltcl '$(DESTDIR)$(datadir)/unknown.pltcl' - -installdirs: - $(MKDIR_P) '$(DESTDIR)$(bindir)' '$(DESTDIR)$(datadir)' - -uninstall: - rm -f '$(DESTDIR)$(bindir)/pltcl_loadmod' '$(DESTDIR)$(bindir)/pltcl_delmod' '$(DESTDIR)$(bindir)/pltcl_listmod' '$(DESTDIR)$(datadir)/unknown.pltcl' - -clean distclean maintainer-clean: - rm -f $(MODULES) diff --git a/src/pl/tcl/modules/README b/src/pl/tcl/modules/README deleted file mode 100644 index 342742c04b..0000000000 --- a/src/pl/tcl/modules/README +++ /dev/null @@ -1,18 +0,0 @@ -src/pl/tcl/modules/README - -Regular Tcl scripts of any size (over 8K :-) can be loaded into -the table pltcl_modules using the pltcl_loadmod script. The script -checks the modules that the procedure names don't overwrite -existing ones before doing anything. They also check for global -variables created at load time. - -All procedures defined in the module files are automatically -added to the table pltcl_modfuncs. This table is used by the -unknown procedure to determine if an unknown command can be -loaded by sourcing a module. In that case the unknown procedure -will silently source in the module and reexecute the original -command that invoked unknown. - -I know, this readme should be more explanatory - but time. - -Jan diff --git a/src/pl/tcl/modules/pltcl_delmod.in b/src/pl/tcl/modules/pltcl_delmod.in deleted file mode 100644 index daa4fac460..0000000000 --- a/src/pl/tcl/modules/pltcl_delmod.in +++ /dev/null @@ -1,117 +0,0 @@ -#! /bin/sh -# src/pl/tcl/modules/pltcl_delmod.in -# -# Start tclsh \ -exec @TCLSH@ "$0" "$@" - -# -# Code still has to be documented -# - -#load /usr/local/pgsql/lib/libpgtcl.so -package require Pgtcl - - -# -# Check for minimum arguments -# -if {$argc < 1} { - puts stderr "" - puts stderr "usage: pltcl_delmod dbname \[options\] modulename \[...\]" - puts stderr "" - puts stderr "options:" - puts stderr " -host hostname" - puts stderr " -port portnumber" - puts stderr "" - exit 1 -} - -# -# Remember database name and initialize options -# -set dbname [lindex $argv 0] -set options "" -set errors 0 -set opt "" -set val "" - -set i 1 -while {$i < $argc} { - if {[string compare [string index [lindex $argv $i] 0] "-"] != 0} { - break; - } - - set opt [lindex $argv $i] - incr i - if {$i >= $argc} { - puts stderr "no value given for option $opt" - incr errors - continue - } - set val [lindex $argv $i] - incr i - - switch -- $opt { - -host { - append options "-host \"$val\" " - } - -port { - append options "-port $val " - } - default { - puts stderr "unknown option '$opt'" - incr errors - } - } -} - -# -# Final syntax check -# -if {$i >= $argc || $errors > 0} { - puts stderr "" - puts stderr "usage: pltcl_delmod dbname \[options\] modulename \[...\]" - puts stderr "" - puts stderr "options:" - puts stderr " -host hostname" - puts stderr " -port portnumber" - puts stderr "" - exit 1 -} - -proc delmodule {conn modname} { - set xname $modname - regsub -all {\\} $xname {\\} xname - regsub -all {'} $xname {''} xname - - set found 0 - pg_select $conn "select * from pltcl_modules where modname = '$xname'" \ - MOD { - set found 1 - break; - } - - if {!$found} { - puts "Module $modname not found in pltcl_modules" - puts "" - return - } - - pg_result \ - [pg_exec $conn "delete from pltcl_modules where modname = '$xname'"] \ - -clear - pg_result \ - [pg_exec $conn "delete from pltcl_modfuncs where modname = '$xname'"] \ - -clear - - puts "Module $modname removed" -} - -set conn [eval pg_connect $dbname $options] - -while {$i < $argc} { - delmodule $conn [lindex $argv $i] - incr i -} - -pg_disconnect $conn diff --git a/src/pl/tcl/modules/pltcl_listmod.in b/src/pl/tcl/modules/pltcl_listmod.in deleted file mode 100644 index 7d930ff0ea..0000000000 --- a/src/pl/tcl/modules/pltcl_listmod.in +++ /dev/null @@ -1,123 +0,0 @@ -#! /bin/sh -# src/pl/tcl/modules/pltcl_listmod.in -# -# Start tclsh \ -exec @TCLSH@ "$0" "$@" - -# -# Code still has to be documented -# - -#load /usr/local/pgsql/lib/libpgtcl.so -package require Pgtcl - - -# -# Check for minimum arguments -# -if {$argc < 1} { - puts stderr "" - puts stderr "usage: pltcl_listmod dbname \[options\] \[modulename \[...\]\]" - puts stderr "" - puts stderr "options:" - puts stderr " -host hostname" - puts stderr " -port portnumber" - puts stderr "" - exit 1 -} - -# -# Remember database name and initialize options -# -set dbname [lindex $argv 0] -set options "" -set errors 0 -set opt "" -set val "" - -set i 1 -while {$i < $argc} { - if {[string compare [string index [lindex $argv $i] 0] "-"] != 0} { - break; - } - - set opt [lindex $argv $i] - incr i - if {$i >= $argc} { - puts stderr "no value given for option $opt" - incr errors - continue - } - set val [lindex $argv $i] - incr i - - switch -- $opt { - -host { - append options "-host \"$val\" " - } - -port { - append options "-port $val " - } - default { - puts stderr "unknown option '$opt'" - incr errors - } - } -} - -# -# Final syntax check -# -if {$errors > 0} { - puts stderr "" - puts stderr "usage: pltcl_listmod dbname \[options\] \[modulename \[...\]\]" - puts stderr "" - puts stderr "options:" - puts stderr " -host hostname" - puts stderr " -port portnumber" - puts stderr "" - exit 1 -} - -proc listmodule {conn modname} { - set xname $modname - regsub -all {\\} $xname {\\} xname - regsub -all {'} $xname {''} xname - - set found 0 - pg_select $conn "select * from pltcl_modules where modname = '$xname'" \ - MOD { - set found 1 - break; - } - - if {!$found} { - puts "Module $modname not found in pltcl_modules" - puts "" - return - } - - puts "Module $modname defines procedures:" - pg_select $conn "select funcname from pltcl_modfuncs \ - where modname = '$xname' order by funcname" FUNC { - puts " $FUNC(funcname)" - } - puts "" -} - -set conn [eval pg_connect $dbname $options] - -if {$i == $argc} { - pg_select $conn "select distinct modname from pltcl_modules \ - order by modname" \ - MOD { - listmodule $conn $MOD(modname) - } -} else { - while {$i < $argc} { - listmodule $conn [lindex $argv $i] - incr i - } -} - -pg_disconnect $conn diff --git a/src/pl/tcl/modules/pltcl_loadmod.in b/src/pl/tcl/modules/pltcl_loadmod.in deleted file mode 100644 index 645c6bbd9c..0000000000 --- a/src/pl/tcl/modules/pltcl_loadmod.in +++ /dev/null @@ -1,501 +0,0 @@ -#! /bin/sh -# Start tclsh \ -exec @TCLSH@ "$0" "$@" - -# -# Code still has to be documented -# - -#load /usr/local/pgsql/lib/libpgtcl.so -package require Pgtcl - - -# -# Check for minimum arguments -# -if {$argc < 2} { - puts stderr "" - puts stderr "usage: pltcl_loadmod dbname \[options\] file \[...\]" - puts stderr "" - puts stderr "options:" - puts stderr " -host hostname" - puts stderr " -port portnumber" - puts stderr "" - exit 1 -} - -# -# Remember database name and initialize options -# -set dbname [lindex $argv 0] -set options "" -set errors 0 -set opt "" -set val "" - -set i 1 -while {$i < $argc} { - if {[string compare [string index [lindex $argv $i] 0] "-"] != 0} { - break; - } - - set opt [lindex $argv $i] - incr i - if {$i >= $argc} { - puts stderr "no value given for option $opt" - incr errors - continue - } - set val [lindex $argv $i] - incr i - - switch -- $opt { - -host { - append options "-host \"$val\" " - } - -port { - append options "-port $val " - } - default { - puts stderr "unknown option '$opt'" - incr errors - } - } -} - -# -# Final syntax check -# -if {$i >= $argc || $errors > 0} { - puts stderr "" - puts stderr "usage: pltcl_loadmod dbname \[options\] file \[...\]" - puts stderr "" - puts stderr "options:" - puts stderr " -host hostname" - puts stderr " -port portnumber" - puts stderr "" - exit 1 -} - - -proc __PLTcl_loadmod_check_table {conn tabname expnames exptypes} { - set attrs [expr [llength $expnames] - 1] - set error 0 - set found 0 - - pg_select $conn "select C.relname, A.attname, A.attnum, T.typname \ - from pg_catalog.pg_class C, pg_catalog.pg_attribute A, pg_catalog.pg_type T \ - where C.relname = '$tabname' \ - and A.attrelid = C.oid \ - and A.attnum > 0 \ - and T.oid = A.atttypid \ - order by attnum" tup { - - incr found - set i $tup(attnum) - - if {$i > $attrs} { - puts stderr "Table $tabname has extra field '$tup(attname)'" - incr error - continue - } - - set xname [lindex $expnames $i] - set xtype [lindex $exptypes $i] - - if {[string compare $tup(attname) $xname] != 0} { - puts stderr "Attribute $i of $tabname has wrong name" - puts stderr " got '$tup(attname)' expected '$xname'" - incr error - } - if {[string compare $tup(typname) $xtype] != 0} { - puts stderr "Attribute $i of $tabname has wrong type" - puts stderr " got '$tup(typname)' expected '$xtype'" - incr error - } - } - - if {$found == 0} { - return 0 - } - - if {$found < $attrs} { - incr found - set miss [lrange $expnames $found end] - puts "Table $tabname doesn't have field(s) $miss" - incr error - } - - if {$error > 0} { - return 2 - } - - return 1 -} - - -proc __PLTcl_loadmod_check_tables {conn} { - upvar #0 __PLTcl_loadmod_status status - - set error 0 - - set names {{} modname modseq modsrc} - set types {{} name int2 text} - - switch [__PLTcl_loadmod_check_table $conn pltcl_modules $names $types] { - 0 { - set status(create_table_modules) 1 - } - 1 { - set status(create_table_modules) 0 - } - 2 { - puts "Error(s) in table pltcl_modules" - incr error - } - } - - set names {{} funcname modname} - set types {{} name name} - - switch [__PLTcl_loadmod_check_table $conn pltcl_modfuncs $names $types] { - 0 { - set status(create_table_modfuncs) 1 - } - 1 { - set status(create_table_modfuncs) 0 - } - 2 { - puts "Error(s) in table pltcl_modfuncs" - incr error - } - } - - if {$status(create_table_modfuncs) && !$status(create_table_modules)} { - puts stderr "Table pltcl_modfuncs doesn't exist but pltcl_modules does" - puts stderr "Either both tables must be present or none." - incr error - } - - if {$status(create_table_modules) && !$status(create_table_modfuncs)} { - puts stderr "Table pltcl_modules doesn't exist but pltcl_modfuncs does" - puts stderr "Either both tables must be present or none." - incr error - } - - if {$error} { - puts stderr "" - puts stderr "Abort" - exit 1 - } - - if {!$status(create_table_modules)} { - __PLTcl_loadmod_read_current $conn - } -} - - -proc __PLTcl_loadmod_read_current {conn} { - upvar #0 __PLTcl_loadmod_status status - upvar #0 __PLTcl_loadmod_modsrc modsrc - upvar #0 __PLTcl_loadmod_funclist funcs - upvar #0 __PLTcl_loadmod_globlist globs - - set errors 0 - - set curmodlist "" - pg_select $conn "select distinct modname from pltcl_modules" mtup { - set mname $mtup(modname); - lappend curmodlist $mname - } - - foreach mname $curmodlist { - set srctext "" - pg_select $conn "select * from pltcl_modules \ - where modname = '$mname' \ - order by modseq" tup { - append srctext $tup(modsrc) - } - - if {[catch { - __PLTcl_loadmod_analyze \ - "Current $mname" \ - $mname \ - $srctext new_globals new_functions - }]} { - incr errors - } - set modsrc($mname) $srctext - set funcs($mname) $new_functions - set globs($mname) $new_globals - } - - if {$errors} { - puts stderr "" - puts stderr "Abort" - exit 1 - } -} - - -proc __PLTcl_loadmod_analyze {modinfo modname srctext v_globals v_functions} { - upvar 1 $v_globals new_g - upvar 1 $v_functions new_f - upvar #0 __PLTcl_loadmod_allfuncs allfuncs - upvar #0 __PLTcl_loadmod_allglobs allglobs - - set errors 0 - - set old_g [info globals] - set old_f [info procs] - set new_g "" - set new_f "" - - if {[catch { - uplevel #0 "$srctext" - } msg]} { - puts "$modinfo: $msg" - incr errors - } - - set cur_g [info globals] - set cur_f [info procs] - - foreach glob $cur_g { - if {[lsearch -exact $old_g $glob] >= 0} { - continue - } - if {[info exists allglobs($glob)]} { - puts stderr "$modinfo: Global $glob previously used in module $allglobs($glob)" - incr errors - } else { - set allglobs($glob) $modname - } - lappend new_g $glob - uplevel #0 unset $glob - } - foreach func $cur_f { - if {[lsearch -exact $old_f $func] >= 0} { - continue - } - if {[info exists allfuncs($func)]} { - puts stderr "$modinfo: Function $func previously defined in module $allfuncs($func)" - incr errors - } else { - set allfuncs($func) $modname - } - lappend new_f $func - rename $func {} - } - - if {$errors} { - return -code error - } - #puts "globs in $modname: $new_g" - #puts "funcs in $modname: $new_f" -} - - -proc __PLTcl_loadmod_create_tables {conn} { - upvar #0 __PLTcl_loadmod_status status - - if {$status(create_table_modules)} { - if {[catch { - set res [pg_exec $conn \ - "create table pltcl_modules ( \ - modname name, \ - modseq int2, \ - modsrc text);"] - } msg]} { - puts stderr "Error creating table pltcl_modules" - puts stderr " $msg" - exit 1 - } - if {[catch { - set res [pg_exec $conn \ - "create index pltcl_modules_i \ - on pltcl_modules using btree \ - (modname name_ops);"] - } msg]} { - puts stderr "Error creating index pltcl_modules_i" - puts stderr " $msg" - exit 1 - } - puts "Table pltcl_modules created" - pg_result $res -clear - } - - if {$status(create_table_modfuncs)} { - if {[catch { - set res [pg_exec $conn \ - "create table pltcl_modfuncs ( \ - funcname name, \ - modname name);"] - } msg]} { - puts stderr "Error creating table pltcl_modfuncs" - puts stderr " $msg" - exit 1 - } - if {[catch { - set res [pg_exec $conn \ - "create index pltcl_modfuncs_i \ - on pltcl_modfuncs using hash \ - (funcname name_ops);"] - } msg]} { - puts stderr "Error creating index pltcl_modfuncs_i" - puts stderr " $msg" - exit 1 - } - puts "Table pltcl_modfuncs created" - pg_result $res -clear - } -} - - -proc __PLTcl_loadmod_read_new {conn} { - upvar #0 __PLTcl_loadmod_status status - upvar #0 __PLTcl_loadmod_modsrc modsrc - upvar #0 __PLTcl_loadmod_funclist funcs - upvar #0 __PLTcl_loadmod_globlist globs - upvar #0 __PLTcl_loadmod_allfuncs allfuncs - upvar #0 __PLTcl_loadmod_allglobs allglobs - upvar #0 __PLTcl_loadmod_modlist modlist - - set errors 0 - - set new_modlist "" - foreach modfile $modlist { - set modname [file rootname [file tail $modfile]] - if {[catch { - set fid [open $modfile "r"] - } msg]} { - puts stderr $msg - incr errors - continue - } - set srctext [read $fid] - close $fid - - if {[info exists modsrc($modname)]} { - if {[string compare $modsrc($modname) $srctext] == 0} { - puts "Module $modname unchanged - ignored" - continue - } - foreach func $funcs($modname) { - unset allfuncs($func) - } - foreach glob $globs($modname) { - unset allglobs($glob) - } - unset funcs($modname) - unset globs($modname) - set modsrc($modname) $srctext - lappend new_modlist $modname - } else { - set modsrc($modname) $srctext - lappend new_modlist $modname - } - - if {[catch { - __PLTcl_loadmod_analyze "New/updated $modname" \ - $modname $srctext new_globals new_funcs - }]} { - incr errors - } - - set funcs($modname) $new_funcs - set globs($modname) $new_globals - } - - if {$errors} { - puts stderr "" - puts stderr "Abort" - exit 1 - } - - set modlist $new_modlist -} - - -proc __PLTcl_loadmod_load_modules {conn} { - upvar #0 __PLTcl_loadmod_modsrc modsrc - upvar #0 __PLTcl_loadmod_funclist funcs - upvar #0 __PLTcl_loadmod_modlist modlist - - set errors 0 - - foreach modname $modlist { - set xname [__PLTcl_loadmod_quote $modname] - - pg_result [pg_exec $conn "begin;"] -clear - - pg_result [pg_exec $conn \ - "delete from pltcl_modules where modname = '$xname'"] -clear - pg_result [pg_exec $conn \ - "delete from pltcl_modfuncs where modname = '$xname'"] -clear - - foreach func $funcs($modname) { - set xfunc [__PLTcl_loadmod_quote $func] - pg_result [ \ - pg_exec $conn "insert into pltcl_modfuncs values ( \ - '$xfunc', '$xname')" \ - ] -clear - } - set i 0 - set srctext $modsrc($modname) - while {[string compare $srctext ""] != 0} { - set xpart [string range $srctext 0 3999] - set xpart [__PLTcl_loadmod_quote $xpart] - set srctext [string range $srctext 4000 end] - - pg_result [ \ - pg_exec $conn "insert into pltcl_modules values ( \ - '$xname', $i, '$xpart')" \ - ] -clear - incr i - } - - pg_result [pg_exec $conn "commit;"] -clear - - puts "Successfully loaded/updated module $modname" - } -} - - -proc __PLTcl_loadmod_quote {s} { - regsub -all {\\} $s {\\\\} s - regsub -all {'} $s {''} s - return $s -} - - -set __PLTcl_loadmod_modlist [lrange $argv $i end] -set __PLTcl_loadmod_modsrc(dummy) "" -set __PLTcl_loadmod_funclist(dummy) "" -set __PLTcl_loadmod_globlist(dummy) "" -set __PLTcl_loadmod_allfuncs(dummy) "" -set __PLTcl_loadmod_allglobs(dummy) "" - -unset __PLTcl_loadmod_modsrc(dummy) -unset __PLTcl_loadmod_funclist(dummy) -unset __PLTcl_loadmod_globlist(dummy) -unset __PLTcl_loadmod_allfuncs(dummy) -unset __PLTcl_loadmod_allglobs(dummy) - - -puts "" - -set __PLTcl_loadmod_conn [eval pg_connect $dbname $options] - -unset i dbname options errors opt val - -__PLTcl_loadmod_check_tables $__PLTcl_loadmod_conn - -__PLTcl_loadmod_read_new $__PLTcl_loadmod_conn - -__PLTcl_loadmod_create_tables $__PLTcl_loadmod_conn -__PLTcl_loadmod_load_modules $__PLTcl_loadmod_conn - -pg_disconnect $__PLTcl_loadmod_conn - -puts "" diff --git a/src/pl/tcl/modules/unknown.pltcl b/src/pl/tcl/modules/unknown.pltcl deleted file mode 100644 index 0729ac1b70..0000000000 --- a/src/pl/tcl/modules/unknown.pltcl +++ /dev/null @@ -1,63 +0,0 @@ -#--------------------------------------------------------------------- -# Support for unknown command -#--------------------------------------------------------------------- - -proc unknown {proname args} { - upvar #0 __PLTcl_unknown_support_plan_modname p_mod - upvar #0 __PLTcl_unknown_support_plan_modsrc p_src - - #----------------------------------------------------------- - # On first call prepare the plans - #----------------------------------------------------------- - if {![info exists p_mod]} { - set p_mod [spi_prepare \ - "select modname from pltcl_modfuncs \ - where funcname = \$1" name] - set p_src [spi_prepare \ - "select modseq, modsrc from pltcl_modules \ - where modname = \$1 \ - order by modseq" name] - } - - #----------------------------------------------------------- - # Lookup the requested function in pltcl_modfuncs - #----------------------------------------------------------- - set n [spi_execp -count 1 $p_mod [list [quote $proname]]] - if {$n != 1} { - #----------------------------------------------------------- - # Not found there either - now it's really unknown - #----------------------------------------------------------- - return -code error "unknown command '$proname'" - } - - #----------------------------------------------------------- - # Collect the source pieces from pltcl_modules - #----------------------------------------------------------- - set src "" - spi_execp $p_src [list [quote $modname]] { - append src $modsrc - } - - #----------------------------------------------------------- - # Load the source into the interpreter - #----------------------------------------------------------- - if {[catch { - uplevel #0 "$src" - } msg]} { - elog NOTICE "pltcl unknown: error while loading module $modname" - elog WARN $msg - } - - #----------------------------------------------------------- - # This should never happen - #----------------------------------------------------------- - if {[catch {info args $proname}]} { - return -code error \ - "unknown command '$proname' (still after loading module $modname)" - } - - #----------------------------------------------------------- - # Finally simulate the initial procedure call - #----------------------------------------------------------- - return [uplevel 1 $proname $args] -} diff --git a/src/pl/tcl/pltcl.c b/src/pl/tcl/pltcl.c index ec5b54ab32..11faa6defe 100644 --- a/src/pl/tcl/pltcl.c +++ b/src/pl/tcl/pltcl.c @@ -255,7 +255,6 @@ void _PG_init(void); static void pltcl_init_interp(pltcl_interp_desc *interp_desc, bool pltrusted); static pltcl_interp_desc *pltcl_fetch_interp(bool pltrusted); -static void pltcl_init_load_unknown(Tcl_Interp *interp); static Datum pltcl_handler(PG_FUNCTION_ARGS, bool pltrusted); @@ -491,11 +490,6 @@ pltcl_init_interp(pltcl_interp_desc *interp_desc, bool pltrusted) pltcl_SPI_execute_plan, NULL, NULL); Tcl_CreateObjCommand(interp, "spi_lastoid", pltcl_SPI_lastoid, NULL, NULL); - - /************************************************************ - * Try to load the unknown procedure from pltcl_modules - ************************************************************/ - pltcl_init_load_unknown(interp); } /********************************************************************** @@ -526,126 +520,6 @@ pltcl_fetch_interp(bool pltrusted) return interp_desc; } -/********************************************************************** - * pltcl_init_load_unknown() - Load the unknown procedure from - * table pltcl_modules (if it exists) - **********************************************************************/ -static void -pltcl_init_load_unknown(Tcl_Interp *interp) -{ - Relation pmrel; - char *pmrelname, - *nspname; - char *buf; - int buflen; - int spi_rc; - int tcl_rc; - Tcl_DString unknown_src; - char *part; - uint64 i; - int fno; - - /************************************************************ - * Check if table pltcl_modules exists - * - * We allow the table to be found anywhere in the search_path. - * This is for backwards compatibility. To ensure that the table - * is trustworthy, we require it to be owned by a superuser. - ************************************************************/ - pmrel = relation_openrv_extended(makeRangeVar(NULL, "pltcl_modules", -1), - AccessShareLock, true); - if (pmrel == NULL) - return; - /* sanity-check the relation kind */ - if (!(pmrel->rd_rel->relkind == RELKIND_RELATION || - pmrel->rd_rel->relkind == RELKIND_MATVIEW || - pmrel->rd_rel->relkind == RELKIND_VIEW)) - { - relation_close(pmrel, AccessShareLock); - return; - } - /* must be owned by superuser, else ignore */ - if (!superuser_arg(pmrel->rd_rel->relowner)) - { - relation_close(pmrel, AccessShareLock); - return; - } - /* get fully qualified table name for use in select command */ - nspname = get_namespace_name(RelationGetNamespace(pmrel)); - if (!nspname) - elog(ERROR, "cache lookup failed for namespace %u", - RelationGetNamespace(pmrel)); - pmrelname = quote_qualified_identifier(nspname, - RelationGetRelationName(pmrel)); - - /************************************************************ - * Read all the rows from it where modname = 'unknown', - * in the order of modseq - ************************************************************/ - buflen = strlen(pmrelname) + 100; - buf = (char *) palloc(buflen); - snprintf(buf, buflen, - "select modsrc from %s where modname = 'unknown' order by modseq", - pmrelname); - - spi_rc = SPI_execute(buf, false, 0); - if (spi_rc != SPI_OK_SELECT) - elog(ERROR, "select from pltcl_modules failed"); - - pfree(buf); - - /************************************************************ - * If there's nothing, module unknown doesn't exist - ************************************************************/ - if (SPI_processed == 0) - { - SPI_freetuptable(SPI_tuptable); - ereport(WARNING, - (errmsg("module \"unknown\" not found in pltcl_modules"))); - relation_close(pmrel, AccessShareLock); - return; - } - - /************************************************************ - * There is a module named unknown. Reassemble the - * source from the modsrc attributes and evaluate - * it in the Tcl interpreter - * - * leave this code as DString - it's only executed once per session - ************************************************************/ - fno = SPI_fnumber(SPI_tuptable->tupdesc, "modsrc"); - Assert(fno > 0); - - Tcl_DStringInit(&unknown_src); - - for (i = 0; i < SPI_processed; i++) - { - part = SPI_getvalue(SPI_tuptable->vals[i], - SPI_tuptable->tupdesc, fno); - if (part != NULL) - { - UTF_BEGIN; - Tcl_DStringAppend(&unknown_src, UTF_E2U(part), -1); - UTF_END; - pfree(part); - } - } - tcl_rc = Tcl_EvalEx(interp, Tcl_DStringValue(&unknown_src), - Tcl_DStringLength(&unknown_src), - TCL_EVAL_GLOBAL); - - Tcl_DStringFree(&unknown_src); - SPI_freetuptable(SPI_tuptable); - - if (tcl_rc != TCL_OK) - ereport(ERROR, - (errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION), - errmsg("could not load module \"unknown\": %s", - utf_u2e(Tcl_GetStringResult(interp))))); - - relation_close(pmrel, AccessShareLock); -} - /********************************************************************** * pltcl_call_handler - This is the only visible function -- cgit v1.2.3 From 9b88f27cb42fe8ff59ddc75e29c005624b8850a2 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 27 Feb 2017 17:20:34 -0500 Subject: Allow index AMs to return either HeapTuple or IndexTuple format during IOS. Previously, only IndexTuple format was supported for the output data of an index-only scan. This is fine for btree, which is just returning a verbatim index tuple anyway. It's not so fine for SP-GiST, which can return reconstructed data that's much larger than a page. To fix, extend the index AM API so that index-only scan data can be returned in either HeapTuple or IndexTuple format. There's other ways we could have done it, but this way avoids an API break for index AMs that aren't concerned with the issue, and it costs little except a couple more fields in IndexScanDescs. I changed both GiST and SP-GiST to use the HeapTuple method. I'm not very clear on whether GiST can reconstruct data that's too large for an IndexTuple, but that seems possible, and it's not much of a code change to fix. Per a complaint from Vik Fearing. Reviewed by Jason Li. Discussion: https://postgr.es/m/49527f79-530d-0bfe-3dad-d183596afa92@2ndquadrant.fr --- doc/src/sgml/indexam.sgml | 16 ++++++++++------ src/backend/access/gist/gistget.c | 17 +++++++++-------- src/backend/access/gist/gistscan.c | 5 +++-- src/backend/access/gist/gistutil.c | 6 +++--- src/backend/access/index/genam.c | 2 ++ src/backend/access/index/indexam.c | 4 ++-- src/backend/access/spgist/spgscan.c | 24 ++++++++++++------------ src/backend/executor/nodeIndexonlyscan.c | 21 +++++++++++++++++++-- src/include/access/gist_private.h | 4 ++-- src/include/access/relscan.h | 9 ++++++++- src/include/access/spgist_private.h | 2 +- 11 files changed, 71 insertions(+), 39 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/indexam.sgml b/doc/src/sgml/indexam.sgml index 401b11598e..ac512588e2 100644 --- a/doc/src/sgml/indexam.sgml +++ b/doc/src/sgml/indexam.sgml @@ -551,15 +551,19 @@ amgettuple (IndexScanDesc scan, If the index supports index-only scans (i.e., amcanreturn returns TRUE for it), - then on success the AM must also check - scan->xs_want_itup, and if that is true it must return - the original indexed data for the index entry, in the form of an + then on success the AM must also check scan->xs_want_itup, + and if that is true it must return the originally indexed data for the + index entry. The data can be returned in the form of an IndexTuple pointer stored at scan->xs_itup, - with tuple descriptor scan->xs_itupdesc. - (Management of the data referenced by the pointer is the access method's + with tuple descriptor scan->xs_itupdesc; or in the form of + a HeapTuple pointer stored at scan->xs_hitup, + with tuple descriptor scan->xs_hitupdesc. (The latter + format should be used when reconstructing data that might possibly not fit + into an IndexTuple.) In either case, + management of the data referenced by the pointer is the access method's responsibility. The data must remain good at least until the next amgettuple, amrescan, or amendscan - call for the scan.) + call for the scan. diff --git a/src/backend/access/gist/gistget.c b/src/backend/access/gist/gistget.c index eea366b1ad..122dc38db5 100644 --- a/src/backend/access/gist/gistget.c +++ b/src/backend/access/gist/gistget.c @@ -441,12 +441,13 @@ gistScanPage(IndexScanDesc scan, GISTSearchItem *pageItem, double *myDistances, so->pageData[so->nPageData].offnum = i; /* - * In an index-only scan, also fetch the data from the tuple. + * In an index-only scan, also fetch the data from the tuple. The + * reconstructed tuples are stored in pageDataCxt. */ if (scan->xs_want_itup) { oldcxt = MemoryContextSwitchTo(so->pageDataCxt); - so->pageData[so->nPageData].ftup = + so->pageData[so->nPageData].recontup = gistFetchTuple(giststate, r, it); MemoryContextSwitchTo(oldcxt); } @@ -478,7 +479,7 @@ gistScanPage(IndexScanDesc scan, GISTSearchItem *pageItem, double *myDistances, * In an index-only scan, also fetch the data from the tuple. */ if (scan->xs_want_itup) - item->data.heap.ftup = gistFetchTuple(giststate, r, it); + item->data.heap.recontup = gistFetchTuple(giststate, r, it); } else { @@ -540,11 +541,11 @@ getNextNearest(IndexScanDesc scan) bool res = false; int i; - if (scan->xs_itup) + if (scan->xs_hitup) { /* free previously returned tuple */ - pfree(scan->xs_itup); - scan->xs_itup = NULL; + pfree(scan->xs_hitup); + scan->xs_hitup = NULL; } do @@ -601,7 +602,7 @@ getNextNearest(IndexScanDesc scan) /* in an index-only scan, also return the reconstructed tuple. */ if (scan->xs_want_itup) - scan->xs_itup = item->data.heap.ftup; + scan->xs_hitup = item->data.heap.recontup; res = true; } else @@ -685,7 +686,7 @@ gistgettuple(IndexScanDesc scan, ScanDirection dir) /* in an index-only scan, also return the reconstructed tuple */ if (scan->xs_want_itup) - scan->xs_itup = so->pageData[so->curPageData].ftup; + scan->xs_hitup = so->pageData[so->curPageData].recontup; so->curPageData++; diff --git a/src/backend/access/gist/gistscan.c b/src/backend/access/gist/gistscan.c index 33b388906a..81ff8fc8b6 100644 --- a/src/backend/access/gist/gistscan.c +++ b/src/backend/access/gist/gistscan.c @@ -155,7 +155,7 @@ gistrescan(IndexScanDesc scan, ScanKey key, int nkeys, * tuple descriptor to represent the returned index tuples and create a * memory context to hold them during the scan. */ - if (scan->xs_want_itup && !scan->xs_itupdesc) + if (scan->xs_want_itup && !scan->xs_hitupdesc) { int natts; int attno; @@ -174,8 +174,9 @@ gistrescan(IndexScanDesc scan, ScanKey key, int nkeys, scan->indexRelation->rd_opcintype[attno - 1], -1, 0); } - scan->xs_itupdesc = so->giststate->fetchTupdesc; + scan->xs_hitupdesc = so->giststate->fetchTupdesc; + /* Also create a memory context that will hold the returned tuples */ so->pageDataCxt = AllocSetContextCreate(so->giststate->scanCxt, "GiST page data context", ALLOCSET_DEFAULT_SIZES); diff --git a/src/backend/access/gist/gistutil.c b/src/backend/access/gist/gistutil.c index f92baedffd..75845ba0e7 100644 --- a/src/backend/access/gist/gistutil.c +++ b/src/backend/access/gist/gistutil.c @@ -624,9 +624,9 @@ gistFetchAtt(GISTSTATE *giststate, int nkey, Datum k, Relation r) /* * Fetch all keys in tuple. - * returns new IndexTuple that contains GISTENTRY with fetched data + * Returns a new HeapTuple containing the originally-indexed data. */ -IndexTuple +HeapTuple gistFetchTuple(GISTSTATE *giststate, Relation r, IndexTuple tuple) { MemoryContext oldcxt = MemoryContextSwitchTo(giststate->tempCxt); @@ -660,7 +660,7 @@ gistFetchTuple(GISTSTATE *giststate, Relation r, IndexTuple tuple) } MemoryContextSwitchTo(oldcxt); - return index_form_tuple(giststate->fetchTupdesc, fetchatt, isnull); + return heap_form_tuple(giststate->fetchTupdesc, fetchatt, isnull); } float diff --git a/src/backend/access/index/genam.c b/src/backend/access/index/genam.c index c4a393f34e..3599476930 100644 --- a/src/backend/access/index/genam.c +++ b/src/backend/access/index/genam.c @@ -119,6 +119,8 @@ RelationGetIndexScan(Relation indexRelation, int nkeys, int norderbys) scan->xs_itup = NULL; scan->xs_itupdesc = NULL; + scan->xs_hitup = NULL; + scan->xs_hitupdesc = NULL; ItemPointerSetInvalid(&scan->xs_ctup.t_self); scan->xs_ctup.t_data = NULL; diff --git a/src/backend/access/index/indexam.c b/src/backend/access/index/indexam.c index 4e7eca73cc..cc5ac8b857 100644 --- a/src/backend/access/index/indexam.c +++ b/src/backend/access/index/indexam.c @@ -535,8 +535,8 @@ index_getnext_tid(IndexScanDesc scan, ScanDirection direction) /* * The AM's amgettuple proc finds the next index entry matching the scan * keys, and puts the TID into scan->xs_ctup.t_self. It should also set - * scan->xs_recheck and possibly scan->xs_itup, though we pay no attention - * to those fields here. + * scan->xs_recheck and possibly scan->xs_itup/scan->xs_hitup, though we + * pay no attention to those fields here. */ found = scan->indexRelation->rd_amroutine->amgettuple(scan, direction); diff --git a/src/backend/access/spgist/spgscan.c b/src/backend/access/spgist/spgscan.c index 139d998600..2d96c0094e 100644 --- a/src/backend/access/spgist/spgscan.c +++ b/src/backend/access/spgist/spgscan.c @@ -92,11 +92,11 @@ resetSpGistScanOpaque(SpGistScanOpaque so) if (so->want_itup) { - /* Must pfree IndexTuples to avoid memory leak */ + /* Must pfree reconstructed tuples to avoid memory leak */ int i; for (i = 0; i < so->nPtrs; i++) - pfree(so->indexTups[i]); + pfree(so->reconTups[i]); } so->iPtr = so->nPtrs = 0; } @@ -195,8 +195,8 @@ spgbeginscan(Relation rel, int keysz, int orderbysz) "SP-GiST search temporary context", ALLOCSET_DEFAULT_SIZES); - /* Set up indexTupDesc and xs_itupdesc in case it's an index-only scan */ - so->indexTupDesc = scan->xs_itupdesc = RelationGetDescr(rel); + /* Set up indexTupDesc and xs_hitupdesc in case it's an index-only scan */ + so->indexTupDesc = scan->xs_hitupdesc = RelationGetDescr(rel); scan->opaque = so; @@ -591,12 +591,12 @@ storeGettuple(SpGistScanOpaque so, ItemPointer heapPtr, if (so->want_itup) { /* - * Reconstruct desired IndexTuple. We have to copy the datum out of - * the temp context anyway, so we may as well create the tuple here. + * Reconstruct index data. We have to copy the datum out of the temp + * context anyway, so we may as well create the tuple here. */ - so->indexTups[so->nPtrs] = index_form_tuple(so->indexTupDesc, - &leafValue, - &isnull); + so->reconTups[so->nPtrs] = heap_form_tuple(so->indexTupDesc, + &leafValue, + &isnull); } so->nPtrs++; } @@ -619,18 +619,18 @@ spggettuple(IndexScanDesc scan, ScanDirection dir) /* continuing to return tuples from a leaf page */ scan->xs_ctup.t_self = so->heapPtrs[so->iPtr]; scan->xs_recheck = so->recheck[so->iPtr]; - scan->xs_itup = so->indexTups[so->iPtr]; + scan->xs_hitup = so->reconTups[so->iPtr]; so->iPtr++; return true; } if (so->want_itup) { - /* Must pfree IndexTuples to avoid memory leak */ + /* Must pfree reconstructed tuples to avoid memory leak */ int i; for (i = 0; i < so->nPtrs; i++) - pfree(so->indexTups[i]); + pfree(so->reconTups[i]); } so->iPtr = so->nPtrs = 0; diff --git a/src/backend/executor/nodeIndexonlyscan.c b/src/backend/executor/nodeIndexonlyscan.c index 66c2ad66d7..4a7f39a7c7 100644 --- a/src/backend/executor/nodeIndexonlyscan.c +++ b/src/backend/executor/nodeIndexonlyscan.c @@ -149,9 +149,26 @@ IndexOnlyNext(IndexOnlyScanState *node) } /* - * Fill the scan tuple slot with data from the index. + * Fill the scan tuple slot with data from the index. This might be + * provided in either HeapTuple or IndexTuple format. Conceivably an + * index AM might fill both fields, in which case we prefer the heap + * format, since it's probably a bit cheaper to fill a slot from. */ - StoreIndexTuple(slot, scandesc->xs_itup, scandesc->xs_itupdesc); + if (scandesc->xs_hitup) + { + /* + * We don't take the trouble to verify that the provided tuple has + * exactly the slot's format, but it seems worth doing a quick + * check on the number of fields. + */ + Assert(slot->tts_tupleDescriptor->natts == + scandesc->xs_hitupdesc->natts); + ExecStoreTuple(scandesc->xs_hitup, slot, InvalidBuffer, false); + } + else if (scandesc->xs_itup) + StoreIndexTuple(slot, scandesc->xs_itup, scandesc->xs_itupdesc); + else + elog(ERROR, "no data returned for index-only scan"); /* * If the index was lossy, we have to recheck the index quals. diff --git a/src/include/access/gist_private.h b/src/include/access/gist_private.h index 5b3303056b..1ad4ed6da7 100644 --- a/src/include/access/gist_private.h +++ b/src/include/access/gist_private.h @@ -119,7 +119,7 @@ typedef struct GISTSearchHeapItem ItemPointerData heapPtr; bool recheck; /* T if quals must be rechecked */ bool recheckDistances; /* T if distances must be rechecked */ - IndexTuple ftup; /* data fetched back from the index, used in + HeapTuple recontup; /* data reconstructed from the index, used in * index-only scans */ OffsetNumber offnum; /* track offset in page to mark tuple as * LP_DEAD */ @@ -477,7 +477,7 @@ extern void gistMakeUnionItVec(GISTSTATE *giststate, IndexTuple *itvec, int len, extern bool gistKeyIsEQ(GISTSTATE *giststate, int attno, Datum a, Datum b); extern void gistDeCompressAtt(GISTSTATE *giststate, Relation r, IndexTuple tuple, Page p, OffsetNumber o, GISTENTRY *attdata, bool *isnull); -extern IndexTuple gistFetchTuple(GISTSTATE *giststate, Relation r, +extern HeapTuple gistFetchTuple(GISTSTATE *giststate, Relation r, IndexTuple tuple); extern void gistMakeUnionKey(GISTSTATE *giststate, int attno, GISTENTRY *entry1, bool isnull1, diff --git a/src/include/access/relscan.h b/src/include/access/relscan.h index ce3ca8d4ac..3fc726d712 100644 --- a/src/include/access/relscan.h +++ b/src/include/access/relscan.h @@ -104,9 +104,16 @@ typedef struct IndexScanDescData /* index access method's private state */ void *opaque; /* access-method-specific info */ - /* in an index-only scan, this is valid after a successful amgettuple */ + /* + * In an index-only scan, a successful amgettuple call must fill either + * xs_itup (and xs_itupdesc) or xs_hitup (and xs_hitupdesc) to provide the + * data returned by the scan. It can fill both, in which case the heap + * format will be used. + */ IndexTuple xs_itup; /* index tuple returned by AM */ TupleDesc xs_itupdesc; /* rowtype descriptor of xs_itup */ + HeapTuple xs_hitup; /* index data returned by AM, as HeapTuple */ + TupleDesc xs_hitupdesc; /* rowtype descriptor of xs_hitup */ /* xs_ctup/xs_cbuf/xs_recheck are valid after a successful index_getnext */ HeapTupleData xs_ctup; /* current heap tuple, if any */ diff --git a/src/include/access/spgist_private.h b/src/include/access/spgist_private.h index e42079b09f..4072c050de 100644 --- a/src/include/access/spgist_private.h +++ b/src/include/access/spgist_private.h @@ -159,7 +159,7 @@ typedef struct SpGistScanOpaqueData int iPtr; /* index for scanning through same */ ItemPointerData heapPtrs[MaxIndexTuplesPerPage]; /* TIDs from cur page */ bool recheck[MaxIndexTuplesPerPage]; /* their recheck flags */ - IndexTuple indexTups[MaxIndexTuplesPerPage]; /* reconstructed tuples */ + HeapTuple reconTups[MaxIndexTuplesPerPage]; /* reconstructed tuples */ /* * Note: using MaxIndexTuplesPerPage above is a bit hokey since -- cgit v1.2.3 From 4461a9bfd1ac4aa1f922e8309e79d097bde9fcb4 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Wed, 1 Mar 2017 19:27:24 -0300 Subject: Create in the functions-xml section This is a small change so that a new XMLTABLE sect3 can be added easily later. Author: Craig Ringer Discussion: https://postgr.es/m/CAFj8pRAgfzMD-LoSmnMGybD0WsEznLHWap8DO79+-GTRAPR4qA@mail.gmail.com --- doc/src/sgml/func.sgml | 104 ++++++++++++++++++++++++++----------------------- 1 file changed, 56 insertions(+), 48 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 9c53e4288c..40af0530a2 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -10328,10 +10328,6 @@ SELECT xml_is_well_formed_document(' Processing XML - - XPath - - To process values of data type xml, PostgreSQL offers the functions xpath and @@ -10339,39 +10335,46 @@ SELECT xml_is_well_formed_document(' + <literal>xpath</literal> + + + XPath + + xpath(xpath, xml , nsarray) - - The function xpath evaluates the XPath - expression xpath (a text value) - against the XML value - xml. It returns an array of XML values - corresponding to the node set produced by the XPath expression. - If the XPath expression returns a scalar value rather than a node set, - a single-element array is returned. - + + The function xpath evaluates the XPath + expression xpath (a text value) + against the XML value + xml. It returns an array of XML values + corresponding to the node set produced by the XPath expression. + If the XPath expression returns a scalar value rather than a node set, + a single-element array is returned. + - - The second argument must be a well formed XML document. In particular, - it must have a single root node element. - + + The second argument must be a well formed XML document. In particular, + it must have a single root node element. + - - The optional third argument of the function is an array of namespace - mappings. This array should be a two-dimensional text array with - the length of the second axis being equal to 2 (i.e., it should be an - array of arrays, each of which consists of exactly 2 elements). - The first element of each array entry is the namespace name (alias), the - second the namespace URI. It is not required that aliases provided in - this array be the same as those being used in the XML document itself (in - other words, both in the XML document and in the xpath - function context, aliases are local). - + + The optional third argument of the function is an array of namespace + mappings. This array should be a two-dimensional text array with + the length of the second axis being equal to 2 (i.e., it should be an + array of arrays, each of which consists of exactly 2 elements). + The first element of each array entry is the namespace name (alias), the + second the namespace URI. It is not required that aliases provided in + this array be the same as those being used in the XML document itself (in + other words, both in the XML document and in the xpath + function context, aliases are local). + - - Example: + + Example: test', ARRAY[ARRAY['my', 'http://example.com']]); @@ -10381,10 +10384,10 @@ SELECT xpath('/my:a/text()', 'test', {test} (1 row) ]]> - + - - To deal with default (anonymous) namespaces, do something like this: + + To deal with default (anonymous) namespaces, do something like this: test', ARRAY[ARRAY['mydefns', 'http://example.com']]); @@ -10394,27 +10397,31 @@ SELECT xpath('//mydefns:b/text()', 'test - + + - - xpath_exists - + + <literal>xpath_exists</literal> + + + xpath_exists + xpath_exists(xpath, xml , nsarray) - - The function xpath_exists is a specialized form - of the xpath function. Instead of returning the - individual XML values that satisfy the XPath, this function returns a - Boolean indicating whether the query was satisfied or not. This - function is equivalent to the standard XMLEXISTS predicate, - except that it also offers support for a namespace mapping argument. - + + The function xpath_exists is a specialized form + of the xpath function. Instead of returning the + individual XML values that satisfy the XPath, this function returns a + Boolean indicating whether the query was satisfied or not. This + function is equivalent to the standard XMLEXISTS predicate, + except that it also offers support for a namespace mapping argument. + - - Example: + + Example: test', ARRAY[ARRAY['my', 'http://example.com']]); @@ -10424,7 +10431,8 @@ SELECT xpath_exists('/my:a/text()', 'test - + + -- cgit v1.2.3 From 3c3bb99330aa9b4c2f6258bfa0265d806bf365c3 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Thu, 2 Mar 2017 17:18:19 +0530 Subject: Don't uselessly rewrite, truncate, VACUUM, or ANALYZE partitioned tables. Also, recursively perform VACUUM and ANALYZE on partitions when the command is applied to a partitioned table. In passing, some related documentation updates. Amit Langote, reviewed by Michael Paquier, Ashutosh Bapat, and by me. Discussion: http://postgr.es/m/47288cf1-f72c-dfc2-5ff0-4af962ae5c1b@lab.ntt.co.jp --- doc/src/sgml/ddl.sgml | 47 ++++++++++++++++++------------------ doc/src/sgml/ref/analyze.sgml | 7 ++++-- doc/src/sgml/ref/vacuum.sgml | 4 +++- src/backend/commands/analyze.c | 37 +++++++++++++++++++---------- src/backend/commands/tablecmds.c | 15 +++++++++--- src/backend/commands/vacuum.c | 51 +++++++++++++++++++++++++++++++++++++--- 6 files changed, 116 insertions(+), 45 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ddl.sgml b/doc/src/sgml/ddl.sgml index ef0f7cf727..09b5b3ff70 100644 --- a/doc/src/sgml/ddl.sgml +++ b/doc/src/sgml/ddl.sgml @@ -3792,8 +3792,7 @@ UNION ALL SELECT * FROM measurement_y2008m01; Caveats - The following caveats apply to partitioned tables implemented using either - method (unless noted otherwise): + The following caveats apply to using inheritance to implement partitioning: @@ -3803,13 +3802,6 @@ UNION ALL SELECT * FROM measurement_y2008m01; partitions and creates and/or modifies associated objects than to write each by hand. - - - This is not a problem with partitioned tables though, as trying to - create a partition that overlaps with one of the existing partitions - results in an error, so it is impossible to end up with partitions - that overlap one another. - @@ -3822,14 +3814,6 @@ UNION ALL SELECT * FROM measurement_y2008m01; on the partition tables, but it makes management of the structure much more complicated. - - - This problem exists even for partitioned tables. An UPDATE - that causes a row to move from one partition to another fails, because - the new value of the row fails to satisfy the implicit partition - constraint of the original partition. This might change in future - releases. - @@ -3840,8 +3824,7 @@ UNION ALL SELECT * FROM measurement_y2008m01; ANALYZE measurement; - will only process the master table. This is true even for partitioned - tables. + will only process the master table. @@ -3852,11 +3835,27 @@ ANALYZE measurement; action is only taken in case of unique violations on the specified target relation, not its child relations. + + + + + The following caveats apply to partitioned tables created with the + explicit syntax: + + + + An UPDATE that causes a row to move from one partition to + another fails, because the new value of the row fails to satisfy the + implicit partition constraint of the original partition. This might + change in future releases. + + + + INSERT statements with ON CONFLICT - clause are currently not allowed on partitioned tables, that is, - cause error when specified. + clause are currently not allowed on partitioned tables. @@ -3864,7 +3863,8 @@ ANALYZE measurement; - The following caveats apply to constraint exclusion: + The following caveats apply to constraint exclusion, which is currently + used by both inheritance and partitioned tables: @@ -3898,8 +3898,7 @@ ANALYZE measurement; during constraint exclusion, so large numbers of partitions are likely to increase query planning time considerably. Partitioning using these techniques will work well with up to perhaps a hundred partitions; - don't try to use many thousands of partitions. This restriction applies - both to inheritance and explicit partitioning syntax. + don't try to use many thousands of partitions. diff --git a/doc/src/sgml/ref/analyze.sgml b/doc/src/sgml/ref/analyze.sgml index 27ab4fca42..49727e22df 100644 --- a/doc/src/sgml/ref/analyze.sgml +++ b/doc/src/sgml/ref/analyze.sgml @@ -63,8 +63,11 @@ ANALYZE [ VERBOSE ] [ table_name [ The name (possibly schema-qualified) of a specific table to - analyze. If omitted, all regular tables (but not foreign tables) - in the current database are analyzed. + analyze. If omitted, all regular tables, partitioned tables, and + and materialized views in the current database are analyzed (but not + foreign tables). If the specified table is a partitioned table, both the + inheritance statistics of the partitioned table as a whole and + statistics of the individual partitions are updated. diff --git a/doc/src/sgml/ref/vacuum.sgml b/doc/src/sgml/ref/vacuum.sgml index f18180a2fa..543ebcf649 100644 --- a/doc/src/sgml/ref/vacuum.sgml +++ b/doc/src/sgml/ref/vacuum.sgml @@ -153,7 +153,9 @@ VACUUM [ FULL ] [ FREEZE ] [ VERBOSE ] ANALYZE [ The name (optionally schema-qualified) of a specific table to - vacuum. Defaults to all tables in the current database. + vacuum. If omitted, all regular tables and materialized views in the + current database are vacuumed. If the specified table is a partitioned + table, all of its leaf partitions are vacuumed. diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c index ed3acb1673..a70c760341 100644 --- a/src/backend/commands/analyze.c +++ b/src/backend/commands/analyze.c @@ -201,8 +201,7 @@ analyze_rel(Oid relid, RangeVar *relation, int options, * locked the relation. */ if (onerel->rd_rel->relkind == RELKIND_RELATION || - onerel->rd_rel->relkind == RELKIND_MATVIEW || - onerel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + onerel->rd_rel->relkind == RELKIND_MATVIEW) { /* Regular table, so we'll use the regular row acquisition function */ acquirefunc = acquire_sample_rows; @@ -234,6 +233,12 @@ analyze_rel(Oid relid, RangeVar *relation, int options, return; } } + else if (onerel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + { + /* + * For partitioned tables, we want to do the recursive ANALYZE below. + */ + } else { /* No need for a WARNING if we already complained during VACUUM */ @@ -253,10 +258,12 @@ analyze_rel(Oid relid, RangeVar *relation, int options, LWLockRelease(ProcArrayLock); /* - * Do the normal non-recursive ANALYZE. + * Do the normal non-recursive ANALYZE. We can skip this for partitioned + * tables, which don't contain any rows. */ - do_analyze_rel(onerel, options, params, va_cols, acquirefunc, relpages, - false, in_outer_xact, elevel); + if (onerel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE) + do_analyze_rel(onerel, options, params, va_cols, acquirefunc, + relpages, false, in_outer_xact, elevel); /* * If there are child tables, do recursive ANALYZE. @@ -1260,6 +1267,7 @@ acquire_inherited_sample_rows(Relation onerel, int elevel, nrels, i; ListCell *lc; + bool has_child; /* * Find all members of inheritance set. We only need AccessShareLock on @@ -1297,6 +1305,7 @@ acquire_inherited_sample_rows(Relation onerel, int elevel, relblocks = (double *) palloc(list_length(tableOIDs) * sizeof(double)); totalblocks = 0; nrels = 0; + has_child = false; foreach(lc, tableOIDs) { Oid childOID = lfirst_oid(lc); @@ -1318,8 +1327,7 @@ acquire_inherited_sample_rows(Relation onerel, int elevel, /* Check table type (MATVIEW can't happen, but might as well allow) */ if (childrel->rd_rel->relkind == RELKIND_RELATION || - childrel->rd_rel->relkind == RELKIND_MATVIEW || - childrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + childrel->rd_rel->relkind == RELKIND_MATVIEW) { /* Regular table, so use the regular row acquisition function */ acquirefunc = acquire_sample_rows; @@ -1351,13 +1359,17 @@ acquire_inherited_sample_rows(Relation onerel, int elevel, } else { - /* ignore, but release the lock on it */ - Assert(childrel != onerel); - heap_close(childrel, AccessShareLock); + /* + * ignore, but release the lock on it. could be a partitioned + * table. + */ + if (childrel != onerel) + heap_close(childrel, AccessShareLock); continue; } /* OK, we'll process this child */ + has_child = true; rels[nrels] = childrel; acquirefuncs[nrels] = acquirefunc; relblocks[nrels] = (double) relpages; @@ -1366,9 +1378,10 @@ acquire_inherited_sample_rows(Relation onerel, int elevel, } /* - * If we don't have at least two tables to consider, fail. + * If we don't have at least one child table to consider, fail. If the + * relation is a partitioned table, it's not counted as a child table. */ - if (nrels < 2) + if (!has_child) { ereport(elevel, (errmsg("skipping analyze of \"%s.%s\" inheritance tree --- this inheritance tree contains no analyzable child tables", diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 3cea220421..317012068b 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -1349,6 +1349,10 @@ ExecuteTruncate(TruncateStmt *stmt) { Relation rel = (Relation) lfirst(cell); + /* Skip partitioned tables as there is nothing to do */ + if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + continue; + /* * Normally, we need a transaction-safe truncation here. However, if * the table was either created in the current (sub)transaction or has @@ -1459,7 +1463,11 @@ truncate_check_rel(Relation rel) { AclResult aclresult; - /* Only allow truncate on regular tables */ + /* + * Only allow truncate on regular tables and partitioned tables (although, + * the latter are only being included here for the following checks; no + * physical truncation will occur in their case.) + */ if (rel->rd_rel->relkind != RELKIND_RELATION && rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE) ereport(ERROR, @@ -4006,8 +4014,9 @@ ATRewriteTables(AlterTableStmt *parsetree, List **wqueue, LOCKMODE lockmode) { AlteredTableInfo *tab = (AlteredTableInfo *) lfirst(ltab); - /* Foreign tables have no storage. */ - if (tab->relkind == RELKIND_FOREIGN_TABLE) + /* Foreign tables have no storage, nor do partitioned tables. */ + if (tab->relkind == RELKIND_FOREIGN_TABLE || + tab->relkind == RELKIND_PARTITIONED_TABLE) continue; /* diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index 812fb4a48f..3a9b965266 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -32,6 +32,7 @@ #include "access/xact.h" #include "catalog/namespace.h" #include "catalog/pg_database.h" +#include "catalog/pg_inherits_fn.h" #include "catalog/pg_namespace.h" #include "commands/cluster.h" #include "commands/vacuum.h" @@ -394,6 +395,9 @@ get_rel_oids(Oid relid, const RangeVar *vacrel) { /* Process a specific relation */ Oid relid; + HeapTuple tuple; + Form_pg_class classForm; + bool include_parts; /* * Since we don't take a lock here, the relation might be gone, or the @@ -406,9 +410,29 @@ get_rel_oids(Oid relid, const RangeVar *vacrel) */ relid = RangeVarGetRelid(vacrel, NoLock, false); - /* Make a relation list entry for this guy */ + /* + * To check whether the relation is a partitioned table, fetch its + * syscache entry. + */ + tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); + if (!HeapTupleIsValid(tuple)) + elog(ERROR, "cache lookup failed for relation %u", relid); + classForm = (Form_pg_class) GETSTRUCT(tuple); + include_parts = (classForm->relkind == RELKIND_PARTITIONED_TABLE); + ReleaseSysCache(tuple); + + /* + * Make relation list entries for this guy and its partitions, if any. + * Note that the list returned by find_all_inheritors() include the + * passed-in OID at its head. Also note that we did not request a + * lock to be taken to match what would be done otherwise. + */ oldcontext = MemoryContextSwitchTo(vac_context); - oid_list = lappend_oid(oid_list, relid); + if (include_parts) + oid_list = list_concat(oid_list, + find_all_inheritors(relid, NoLock, NULL)); + else + oid_list = lappend_oid(oid_list, relid); MemoryContextSwitchTo(oldcontext); } else @@ -429,8 +453,14 @@ get_rel_oids(Oid relid, const RangeVar *vacrel) { Form_pg_class classForm = (Form_pg_class) GETSTRUCT(tuple); + /* + * We include partitioned tables here; depending on which + * operation is to be performed, caller will decide whether to + * process or ignore them. + */ if (classForm->relkind != RELKIND_RELATION && - classForm->relkind != RELKIND_MATVIEW) + classForm->relkind != RELKIND_MATVIEW && + classForm->relkind != RELKIND_PARTITIONED_TABLE) continue; /* Make a relation list entry for this guy */ @@ -1349,6 +1379,21 @@ vacuum_rel(Oid relid, RangeVar *relation, int options, VacuumParams *params) return false; } + /* + * Ignore partitioned tables as there is no work to be done. Since we + * release the lock here, it's possible that any partitions added from + * this point on will not get processed, but that seems harmless. + */ + if (onerel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + { + relation_close(onerel, lmode); + PopActiveSnapshot(); + CommitTransactionCommand(); + + /* It's OK for other commands to look at this table */ + return true; + } + /* * Get a session-level lock too. This will protect our access to the * relation across multiple transactions, so that we can vacuum the -- cgit v1.2.3 From d99706ed5178d7f37ac322e02e8c56e4e5e0e99a Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 2 Mar 2017 09:34:35 -0500 Subject: Update documentation of tsquery_phrase(). Missed in commit 028350f61. Noted by Eiji Seki. --- doc/src/sgml/func.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 40af0530a2..71ad729ab0 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -9618,7 +9618,7 @@ CREATE TYPE rainbow AS ENUM ('red', 'orange', 'yellow', 'green', 'blue', 'purple tsquery make query that searches for query1 followed by - query2 at maximum distance distance + query2 at distance distance tsquery_phrase(to_tsquery('fat'), to_tsquery('cat'), 10) 'fat' <10> 'cat' -- cgit v1.2.3 From be6ed6451c693d9121d357996cbc21b06058b9c1 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 2 Mar 2017 11:09:44 -0500 Subject: Correct old release note item --- doc/src/sgml/release-old.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/release-old.sgml b/doc/src/sgml/release-old.sgml index a480a1d484..266d8719b6 100644 --- a/doc/src/sgml/release-old.sgml +++ b/doc/src/sgml/release-old.sgml @@ -1650,7 +1650,7 @@ operations on bytea columns (Joe) Add variable db_user_namespace for database-local user names (Bruce) SSL improvements (Bear Giles) Make encryption of stored passwords the default (Bruce) -Allow pg_statistic to be reset by calling pg_stat_reset() (Christopher) +Allow statistics collector to be reset by calling pg_stat_reset() (Christopher) Add log_duration parameter (Bruce) Rename debug_print_query to log_statement (Bruce) Rename show_query_stats to show_statement_stats (Bruce) -- cgit v1.2.3 From 19dc233c32f2900e57b8da4f41c0f662ab42e080 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Fri, 3 Mar 2017 11:32:45 +0530 Subject: Add pg_current_logfile() function. The syslogger will write out the current stderr and csvlog names, if it's running and there are any, to a new file in the data directory called "current_logfiles". We take care to remove this file when it might no longer be valid (but not at shutdown). The function pg_current_logfile() can be used to read the entries in the file. Gilles Darold, reviewed and modified by Karl O. Pinc, Michael Paquier, and me. Further review by Álvaro Herrera and Christoph Berg. --- doc/src/sgml/config.sgml | 26 +++++++ doc/src/sgml/func.sgml | 46 ++++++++++++ doc/src/sgml/storage.sgml | 6 ++ src/backend/catalog/system_views.sql | 2 + src/backend/postmaster/postmaster.c | 7 ++ src/backend/postmaster/syslogger.c | 79 ++++++++++++++++++++ src/backend/replication/basebackup.c | 4 ++ src/backend/utils/adt/misc.c | 103 +++++++++++++++++++++++++++ src/bin/pg_basebackup/t/010_pg_basebackup.pl | 6 +- src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_proc.h | 4 ++ src/include/postmaster/syslogger.h | 7 ++ 12 files changed, 288 insertions(+), 4 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 1b390a257a..cd82c04b05 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -4280,6 +4280,11 @@ SELECT * FROM parent WHERE key = 2400; where to log + + current_logfiles + and the log_destination configuration parameter + + @@ -4310,6 +4315,27 @@ SELECT * FROM parent WHERE key = 2400; must be enabled to generate CSV-format log output. + + When either stderr or + csvlog are included, the file + current_logfiles is created to record the location + of the log file(s) currently in use by the logging collector and the + associated logging destination. This provides a convenient way to + find the logs currently in use by the instance. Here is an example of + this file's content: + +stderr pg_log/postgresql.log +csvlog pg_log/postgresql.csv + + + current_logfiles is recreated when a new log file + is created as an effect of rotation, and + when log_destination is reloaded. It is removed when + neither stderr + nor csvlog are included + in log_destination, and when the logging collector is + disabled. + diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 71ad729ab0..9e084adc1a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -15478,6 +15478,13 @@ SELECT * FROM pg_ls_dir('.') WITH ORDINALITY AS t(ls,n); configuration load time + + pg_current_logfile(text) + text + Primary log file name, or log in the requested format, + currently in use by the logging collector + + pg_my_temp_schema() oid @@ -15696,6 +15703,45 @@ SET search_path TO schema , schema, .. the time when the postmaster process re-read the configuration files.) + + pg_current_logfile + + + + Logging + pg_current_logfile function + + + + current_logfiles + and the pg_current_logfile function + + + + Logging + current_logfiles file and the pg_current_logfile + function + + + + pg_current_logfile returns, as text, + the path of the log file(s) currently in use by the logging collector. + The path includes the directory + and the log file name. Log collection must be enabled or the return value + is NULL. When multiple log files exist, each in a + different format, pg_current_logfile called + without arguments returns the path of the file having the first format + found in the ordered list: stderr, csvlog. + NULL is returned when no log file has any of these + formats. To request a specific file format supply, as text, + either csvlog or stderr as the value of the + optional parameter. The return value is NULL when the + log format requested is not a configured + . The + pg_current_logfiles reflects the contents of the + current_logfiles file. + + pg_my_temp_schema diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml index 127b759c14..e0a89861f8 100644 --- a/doc/src/sgml/storage.sgml +++ b/doc/src/sgml/storage.sgml @@ -60,6 +60,12 @@ Item Subdirectory containing per-database subdirectories + + current_logfiles + File recording the log file(s) currently written to by the logging + collector + + global Subdirectory containing cluster-wide tables, such as diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 38be9cf1a0..ada542c530 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1091,6 +1091,8 @@ REVOKE EXECUTE ON FUNCTION pg_wal_replay_pause() FROM public; REVOKE EXECUTE ON FUNCTION pg_wal_replay_resume() FROM public; REVOKE EXECUTE ON FUNCTION pg_rotate_logfile() FROM public; REVOKE EXECUTE ON FUNCTION pg_reload_conf() FROM public; +REVOKE EXECUTE ON FUNCTION pg_current_logfile() FROM public; +REVOKE EXECUTE ON FUNCTION pg_current_logfile(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_reset() FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_reset_shared(text) FROM public; diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 2cf17ac42e..68313424ee 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -1232,6 +1232,13 @@ PostmasterMain(int argc, char *argv[]) */ RemovePromoteSignalFiles(); + /* Remove any outdated file holding the current log filenames. */ + if (unlink(LOG_METAINFO_DATAFILE) < 0 && errno != ENOENT) + ereport(LOG, + (errcode_for_file_access(), + errmsg("could not remove file \"%s\": %m", + LOG_METAINFO_DATAFILE))); + /* * If enabled, start up syslogger collection subprocess */ diff --git a/src/backend/postmaster/syslogger.c b/src/backend/postmaster/syslogger.c index 13a03014eb..aaefdaebad 100644 --- a/src/backend/postmaster/syslogger.c +++ b/src/backend/postmaster/syslogger.c @@ -146,6 +146,7 @@ static char *logfile_getname(pg_time_t timestamp, const char *suffix); static void set_next_rotation_time(void); static void sigHupHandler(SIGNAL_ARGS); static void sigUsr1Handler(SIGNAL_ARGS); +static void update_metainfo_datafile(void); /* @@ -282,6 +283,7 @@ SysLoggerMain(int argc, char *argv[]) currentLogRotationAge = Log_RotationAge; /* set next planned rotation time */ set_next_rotation_time(); + update_metainfo_datafile(); /* main worker loop */ for (;;) @@ -348,6 +350,13 @@ SysLoggerMain(int argc, char *argv[]) rotation_disabled = false; rotation_requested = true; } + + /* + * Force rewriting last log filename when reloading configuration. + * Even if rotation_requested is false, log_destination may have + * been changed and we don't want to wait the next file rotation. + */ + update_metainfo_datafile(); } if (Log_RotationAge > 0 && !rotation_disabled) @@ -1098,6 +1107,8 @@ open_csvlogfile(void) pfree(last_csv_file_name); last_csv_file_name = filename; + + update_metainfo_datafile(); } /* @@ -1268,6 +1279,8 @@ logfile_rotate(bool time_based_rotation, int size_rotation_for) if (csvfilename) pfree(csvfilename); + update_metainfo_datafile(); + set_next_rotation_time(); } @@ -1337,6 +1350,72 @@ set_next_rotation_time(void) next_rotation_time = now; } +/* + * Store the name of the file(s) where the log collector, when enabled, writes + * log messages. Useful for finding the name(s) of the current log file(s) + * when there is time-based logfile rotation. Filenames are stored in a + * temporary file and which is renamed into the final destination for + * atomicity. + */ +static void +update_metainfo_datafile(void) +{ + FILE *fh; + + if (!(Log_destination & LOG_DESTINATION_STDERR) && + !(Log_destination & LOG_DESTINATION_CSVLOG)) + { + if (unlink(LOG_METAINFO_DATAFILE) < 0 && errno != ENOENT) + ereport(LOG, + (errcode_for_file_access(), + errmsg("could not remove file \"%s\": %m", + LOG_METAINFO_DATAFILE))); + return; + } + + if ((fh = logfile_open(LOG_METAINFO_DATAFILE_TMP, "w", true)) == NULL) + { + ereport(LOG, + (errcode_for_file_access(), + errmsg("could not open file \"%s\": %m", + LOG_METAINFO_DATAFILE_TMP))); + return; + } + + if (last_file_name && (Log_destination & LOG_DESTINATION_STDERR)) + { + if (fprintf(fh, "stderr %s\n", last_file_name) < 0) + { + ereport(LOG, + (errcode_for_file_access(), + errmsg("could not write file \"%s\": %m", + LOG_METAINFO_DATAFILE_TMP))); + fclose(fh); + return; + } + } + + if (last_csv_file_name && (Log_destination & LOG_DESTINATION_CSVLOG)) + { + if (fprintf(fh, "csvlog %s\n", last_csv_file_name) < 0) + { + ereport(LOG, + (errcode_for_file_access(), + errmsg("could not write file \"%s\": %m", + LOG_METAINFO_DATAFILE_TMP))); + fclose(fh); + return; + } + } + fclose(fh); + + if (rename(LOG_METAINFO_DATAFILE_TMP, LOG_METAINFO_DATAFILE) != 0) + ereport(LOG, + (errcode_for_file_access(), + errmsg("could not rename file \"%s\" to \"%s\": %m", + LOG_METAINFO_DATAFILE_TMP, LOG_METAINFO_DATAFILE))); +} + /* -------------------------------- * signal handler routines * -------------------------------- diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c index 7414048f4e..e3a7ad5e9a 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c @@ -26,6 +26,7 @@ #include "nodes/pg_list.h" #include "pgtar.h" #include "pgstat.h" +#include "postmaster/syslogger.h" #include "replication/basebackup.h" #include "replication/walsender.h" #include "replication/walsender_private.h" @@ -147,6 +148,9 @@ static const char *excludeFiles[] = /* Skip auto conf temporary file. */ PG_AUTOCONF_FILENAME ".tmp", + /* Skip current log file temporary file */ + LOG_METAINFO_DATAFILE_TMP, + /* * If there's a backup_label or tablespace_map file, it belongs to a * backup started by the user with pg_start_backup(). It is *not* correct diff --git a/src/backend/utils/adt/misc.c b/src/backend/utils/adt/misc.c index 8f7c1f81fd..ff6a25d2b6 100644 --- a/src/backend/utils/adt/misc.c +++ b/src/backend/utils/adt/misc.c @@ -885,3 +885,106 @@ parse_ident(PG_FUNCTION_ARGS) PG_RETURN_DATUM(makeArrayResult(astate, CurrentMemoryContext)); } + +/* + * pg_current_logfile + * + * Report current log file used by log collector by scanning current_logfiles. + */ +Datum +pg_current_logfile(PG_FUNCTION_ARGS) +{ + FILE *fd; + char lbuffer[MAXPGPATH]; + char *logfmt; + char *log_filepath; + char *log_format = lbuffer; + char *nlpos; + + /* The log format parameter is optional */ + if (PG_NARGS() == 0 || PG_ARGISNULL(0)) + logfmt = NULL; + else + { + logfmt = text_to_cstring(PG_GETARG_TEXT_PP(0)); + + if (strcmp(logfmt, "stderr") != 0 && strcmp(logfmt, "csvlog") != 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("log format \"%s\" is not supported", logfmt), + errhint("The supported log formats are \"stderr\" and \"csvlog\"."))); + } + + fd = AllocateFile(LOG_METAINFO_DATAFILE, "r"); + if (fd == NULL) + { + if (errno != ENOENT) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read file \"%s\": %m", + LOG_METAINFO_DATAFILE))); + PG_RETURN_NULL(); + } + + /* + * Read the file to gather current log filename(s) registered by the + * syslogger. + */ + while (fgets(lbuffer, sizeof(lbuffer), fd) != NULL) + { + /* + * Extract log format and log file path from the line; lbuffer == + * log_format, they share storage. + */ + log_filepath = strchr(lbuffer, ' '); + if (log_filepath == NULL) + { + /* + * No space found, file content is corrupted. Return NULL to the + * caller and inform him on the situation. + */ + elog(ERROR, + "missing space character in \"%s\"", LOG_METAINFO_DATAFILE); + break; + } + + *log_filepath = '\0'; + log_filepath++; + nlpos = strchr(log_filepath, '\n'); + if (nlpos == NULL) + { + /* + * No newlinei found, file content is corrupted. Return NULL to + * the caller and inform him on the situation. + */ + elog(ERROR, + "missing newline character in \"%s\"", LOG_METAINFO_DATAFILE); + break; + } + *nlpos = '\0'; + + if (logfmt == NULL || strcmp(logfmt, log_format) == 0) + { + FreeFile(fd); + PG_RETURN_TEXT_P(cstring_to_text(log_filepath)); + } + } + + /* Close the current log filename file. */ + FreeFile(fd); + + PG_RETURN_NULL(); +} + +/* + * Report current log file used by log collector (1 argument version) + * + * note: this wrapper is necessary to pass the sanity check in opr_sanity, + * which checks that all built-in functions that share the implementing C + * function take the same number of arguments + */ +Datum +pg_current_logfile_1arg(PG_FUNCTION_ARGS) +{ + return pg_current_logfile(fcinfo); +} diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/010_pg_basebackup.pl index 29f519d8c9..aafb138fd5 100644 --- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl +++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl @@ -4,7 +4,7 @@ use Cwd; use Config; use PostgresNode; use TestLib; -use Test::More tests => 72; +use Test::More tests => 73; program_help_ok('pg_basebackup'); program_version_ok('pg_basebackup'); @@ -56,7 +56,7 @@ close CONF; $node->restart; # Write some files to test that they are not copied. -foreach my $filename (qw(backup_label tablespace_map postgresql.auto.conf.tmp)) +foreach my $filename (qw(backup_label tablespace_map postgresql.auto.conf.tmp current_logfiles.tmp)) { open FILE, ">>$pgdata/$filename"; print FILE "DONOTCOPY"; @@ -83,7 +83,7 @@ foreach my $dirname (qw(pg_dynshmem pg_notify pg_replslot pg_serial pg_snapshots } # These files should not be copied. -foreach my $filename (qw(postgresql.auto.conf.tmp postmaster.opts postmaster.pid tablespace_map)) +foreach my $filename (qw(postgresql.auto.conf.tmp postmaster.opts postmaster.pid tablespace_map current_logfiles.tmp)) { ok(! -f "$tempdir/backup/$filename", "$filename not copied"); } diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 90456fa668..57fbc9509e 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201703011 +#define CATALOG_VERSION_NO 201703031 #endif diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index 4b9c6e75b0..0c8b5c630d 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -3191,6 +3191,10 @@ DATA(insert OID = 2621 ( pg_reload_conf PGNSP PGUID 12 1 0 0 0 f f f f t f v s DESCR("reload configuration files"); DATA(insert OID = 2622 ( pg_rotate_logfile PGNSP PGUID 12 1 0 0 0 f f f f t f v s 0 0 16 "" _null_ _null_ _null_ _null_ _null_ pg_rotate_logfile _null_ _null_ _null_ )); DESCR("rotate log file"); +DATA(insert OID = 3800 ( pg_current_logfile PGNSP PGUID 12 1 0 0 0 f f f f f f v s 0 0 25 "" _null_ _null_ _null_ _null_ _null_ pg_current_logfile _null_ _null_ _null_ )); +DESCR("current logging collector file location"); +DATA(insert OID = 3801 ( pg_current_logfile PGNSP PGUID 12 1 0 0 0 f f f f f f v s 1 0 25 "25" _null_ _null_ _null_ _null_ _null_ pg_current_logfile_1arg _null_ _null_ _null_ )); +DESCR("current logging collector file location"); DATA(insert OID = 2623 ( pg_stat_file PGNSP PGUID 12 1 0 0 0 f f f f t f v s 1 0 2249 "25" "{25,20,1184,1184,1184,1184,16}" "{i,o,o,o,o,o,o}" "{filename,size,access,modification,change,creation,isdir}" _null_ _null_ pg_stat_file_1arg _null_ _null_ _null_ )); DESCR("get information about file"); diff --git a/src/include/postmaster/syslogger.h b/src/include/postmaster/syslogger.h index c187a5f23e..94d7eac347 100644 --- a/src/include/postmaster/syslogger.h +++ b/src/include/postmaster/syslogger.h @@ -87,4 +87,11 @@ extern void write_syslogger_file(const char *buffer, int count, int dest); extern void SysLoggerMain(int argc, char *argv[]) pg_attribute_noreturn(); #endif +/* + * Name of files saving meta-data information about the log + * files currently in use by the syslogger + */ +#define LOG_METAINFO_DATAFILE "current_logfiles" +#define LOG_METAINFO_DATAFILE_TMP LOG_METAINFO_DATAFILE ".tmp" + #endif /* _SYSLOGGER_H */ -- cgit v1.2.3 From 6f236e1eb8c7601bded96fd96244d676e95b8c26 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 3 Mar 2017 14:13:48 -0500 Subject: psql: Add tab completion for logical replication Add tab completion for publications and subscriptions. Also, to be able to get a list of subscriptions, make pg_subscription world-readable but revoke access to subconninfo using column privileges. From: Michael Paquier --- doc/src/sgml/catalogs.sgml | 5 ++--- src/backend/catalog/system_views.sql | 4 ++++ src/bin/psql/tab-complete.c | 23 ++++++++++++++++++++--- src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_subscription.h | 2 ++ 5 files changed, 29 insertions(+), 7 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 96cb9185c2..41e3e1b547 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -6324,9 +6324,8 @@ - Access to this catalog is restricted from normal users. Normal users can - use the view to get some information - about subscriptions. + Access to the column subconninfo is revoked from + normal users, because it could contain plain-text passwords. diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index ada542c530..ba980de86b 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -900,7 +900,11 @@ CREATE VIEW pg_replication_origin_status AS REVOKE ALL ON pg_replication_origin_status FROM public; +-- All columns of pg_subscription except subconninfo are readable. REVOKE ALL ON pg_subscription FROM public; +GRANT SELECT (subdbid, subname, subowner, subenabled, subslotname, subpublications) + ON pg_subscription TO public; + -- -- We have a few function definitions in here, too. diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index 115cb5ce71..4a65ff5b62 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -845,6 +845,18 @@ static const SchemaQuery Query_for_list_of_matviews = { " FROM pg_catalog.pg_am "\ " WHERE substring(pg_catalog.quote_ident(amname),1,%d)='%s'" +#define Query_for_list_of_publications \ +" SELECT pg_catalog.quote_ident(pubname) "\ +" FROM pg_catalog.pg_publication "\ +" WHERE substring(pg_catalog.quote_ident(pubname),1,%d)='%s'" + +#define Query_for_list_of_subscriptions \ +" SELECT pg_catalog.quote_ident(s.subname) "\ +" FROM pg_catalog.pg_subscription s, pg_catalog.pg_database d "\ +" WHERE substring(pg_catalog.quote_ident(s.subname),1,%d)='%s' "\ +" AND d.datname = pg_catalog.current_database() "\ +" AND s.subdbid = d.oid" + /* the silly-looking length condition is just to eat up the current word */ #define Query_for_list_of_arguments \ "SELECT pg_catalog.oidvectortypes(proargtypes)||')' "\ @@ -985,13 +997,13 @@ static const pgsql_thing_t words_after_create[] = { {"OWNED", NULL, NULL, THING_NO_CREATE}, /* for DROP OWNED BY ... */ {"PARSER", Query_for_list_of_ts_parsers, NULL, THING_NO_SHOW}, {"POLICY", NULL, NULL}, - {"PUBLICATION", NULL, NULL}, + {"PUBLICATION", Query_for_list_of_publications}, {"ROLE", Query_for_list_of_roles}, {"RULE", "SELECT pg_catalog.quote_ident(rulename) FROM pg_catalog.pg_rules WHERE substring(pg_catalog.quote_ident(rulename),1,%d)='%s'"}, {"SCHEMA", Query_for_list_of_schemas}, {"SEQUENCE", NULL, &Query_for_list_of_sequences}, {"SERVER", Query_for_list_of_servers}, - {"SUBSCRIPTION", NULL, NULL}, + {"SUBSCRIPTION", Query_for_list_of_subscriptions}, {"TABLE", NULL, &Query_for_list_of_tables}, {"TABLESPACE", Query_for_list_of_tablespaces}, {"TEMP", NULL, NULL, THING_NO_DROP}, /* for CREATE TEMP TABLE ... */ @@ -2374,8 +2386,13 @@ psql_completion(const char *text, int start, int end) /* CREATE SUBSCRIPTION */ else if (Matches3("CREATE", "SUBSCRIPTION", MatchAny)) COMPLETE_WITH_CONST("CONNECTION"); - else if (Matches5("CREATE", "SUBSCRIPTION", MatchAny, "CONNECTION",MatchAny)) + else if (Matches5("CREATE", "SUBSCRIPTION", MatchAny, "CONNECTION", MatchAny)) COMPLETE_WITH_CONST("PUBLICATION"); + else if (Matches6("CREATE", "SUBSCRIPTION", MatchAny, "CONNECTION", + MatchAny, "PUBLICATION")) + { + /* complete with nothing here as this refers to remote publications */ + } /* Complete "CREATE SUBSCRIPTION ... WITH ( " */ else if (HeadMatches2("CREATE", "SUBSCRIPTION") && TailMatches2("WITH", "(")) COMPLETE_WITH_LIST5("ENABLED", "DISABLED", "CREATE SLOT", diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 57fbc9509e..438378d8fa 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201703031 +#define CATALOG_VERSION_NO 201703032 #endif diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h index 75b618accd..0811880a8f 100644 --- a/src/include/catalog/pg_subscription.h +++ b/src/include/catalog/pg_subscription.h @@ -27,6 +27,8 @@ * seems weird, but the replication launcher process needs to access all of * them to be able to start the workers, so we have to put them in a shared, * nailed catalog. + * + * NOTE: When adding a column, also update system_views.sql. */ CATALOG(pg_subscription,6100) BKI_SHARED_RELATION BKI_ROWTYPE_OID(6101) BKI_SCHEMA_MACRO { -- cgit v1.2.3 From 47b55d4174fd5662cd2f8c40852a20eec5f07c8f Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 3 Mar 2017 15:03:03 -0500 Subject: doc: Put callouts in SQL comments This makes copy-and-pasting the SQL code easier. From: Thomas Munro --- doc/src/sgml/plpgsql.sgml | 10 +++++----- doc/src/sgml/query.sgml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/plpgsql.sgml b/doc/src/sgml/plpgsql.sgml index d3272e1209..d356deb9f5 100644 --- a/doc/src/sgml/plpgsql.sgml +++ b/doc/src/sgml/plpgsql.sgml @@ -5328,14 +5328,14 @@ SELECT * FROM cs_parse_url('http://foobar.com/query.cgi?baz'); CREATE OR REPLACE PROCEDURE cs_create_job(v_job_id IN INTEGER) IS a_running_job_count INTEGER; - PRAGMA AUTONOMOUS_TRANSACTION; + PRAGMA AUTONOMOUS_TRANSACTION; -- BEGIN - LOCK TABLE cs_jobs IN EXCLUSIVE MODE; + LOCK TABLE cs_jobs IN EXCLUSIVE MODE; -- SELECT count(*) INTO a_running_job_count FROM cs_jobs WHERE end_stamp IS NULL; IF a_running_job_count > 0 THEN - COMMIT; -- free lock + COMMIT; -- free lock raise_application_error(-20000, 'Unable to create a new job: a job is currently running.'); END IF; @@ -5402,7 +5402,7 @@ BEGIN SELECT count(*) INTO a_running_job_count FROM cs_jobs WHERE end_stamp IS NULL; IF a_running_job_count > 0 THEN - RAISE EXCEPTION 'Unable to create a new job: a job is currently running'; + RAISE EXCEPTION 'Unable to create a new job: a job is currently running'; -- END IF; DELETE FROM cs_active_job; @@ -5411,7 +5411,7 @@ BEGIN BEGIN INSERT INTO cs_jobs (job_id, start_stamp) VALUES (v_job_id, now()); EXCEPTION - WHEN unique_violation THEN + WHEN unique_violation THEN -- -- don't worry if it already exists END; END; diff --git a/doc/src/sgml/query.sgml b/doc/src/sgml/query.sgml index f4fbf11a8d..98434925df 100644 --- a/doc/src/sgml/query.sgml +++ b/doc/src/sgml/query.sgml @@ -754,7 +754,7 @@ SELECT city, max(temp_lo) SELECT city, max(temp_lo) FROM weather - WHERE city LIKE 'S%' + WHERE city LIKE 'S%' -- GROUP BY city HAVING max(temp_lo) < 40; -- cgit v1.2.3 From 272adf4f9cd67df323ae57ff3dee238b649d3b73 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 3 Mar 2017 23:25:34 -0500 Subject: Disallow CREATE/DROP SUBSCRIPTION in transaction block Disallow CREATE SUBSCRIPTION and DROP SUBSCRIPTION in a transaction block when the replication slot is to be created or dropped, since that cannot be rolled back. based on patch by Masahiko Sawada --- doc/src/sgml/ref/create_subscription.sgml | 5 ++++ doc/src/sgml/ref/drop_subscription.sgml | 4 ++-- src/backend/commands/subscriptioncmds.c | 38 +++++++++++++++++++++++------- src/backend/tcop/utility.c | 5 ++-- src/include/commands/subscriptioncmds.h | 5 ++-- src/test/regress/expected/subscription.out | 12 ++++++++++ src/test/regress/sql/subscription.sql | 12 ++++++++++ 7 files changed, 66 insertions(+), 15 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml index 250806f981..9bed26219c 100644 --- a/doc/src/sgml/ref/create_subscription.sgml +++ b/doc/src/sgml/ref/create_subscription.sgml @@ -51,6 +51,11 @@ CREATE SUBSCRIPTION subscription_name + + CREATE SUBSCRIPTION cannot be executed inside a + transaction block when CREATE SLOT is specified. + + Additional info about subscriptions and logical replication as a whole can is available at and diff --git a/doc/src/sgml/ref/drop_subscription.sgml b/doc/src/sgml/ref/drop_subscription.sgml index 9f2fb93275..4228f1a253 100644 --- a/doc/src/sgml/ref/drop_subscription.sgml +++ b/doc/src/sgml/ref/drop_subscription.sgml @@ -38,8 +38,8 @@ DROP SUBSCRIPTION [ IF EXISTS ] name - The replication worker associated with the subscription will not stop until - after the transaction that issued this command has committed. + DROP SUBSCRIPTION cannot be executed inside a + transaction block when DROP SLOT is specified. diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c index 0e081388c4..0036d99c2e 100644 --- a/src/backend/commands/subscriptioncmds.c +++ b/src/backend/commands/subscriptioncmds.c @@ -18,6 +18,7 @@ #include "access/heapam.h" #include "access/htup_details.h" +#include "access/xact.h" #include "catalog/indexing.h" #include "catalog/objectaccess.h" @@ -204,7 +205,7 @@ publicationListToArray(List *publist) * Create new subscription. */ ObjectAddress -CreateSubscription(CreateSubscriptionStmt *stmt) +CreateSubscription(CreateSubscriptionStmt *stmt, bool isTopLevel) { Relation rel; ObjectAddress myself; @@ -221,6 +222,23 @@ CreateSubscription(CreateSubscriptionStmt *stmt) bool create_slot; List *publications; + /* + * Parse and check options. + * Connection and publication should not be specified here. + */ + parse_subscription_options(stmt->options, NULL, NULL, + &enabled_given, &enabled, + &create_slot, &slotname); + + /* + * Since creating a replication slot is not transactional, rolling back + * the transaction leaves the created replication slot. So we cannot run + * CREATE SUBSCRIPTION inside a transaction block if creating a + * replication slot. + */ + if (create_slot) + PreventTransactionChain(isTopLevel, "CREATE SUBSCRIPTION ... CREATE SLOT"); + if (!superuser()) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), @@ -239,13 +257,6 @@ CreateSubscription(CreateSubscriptionStmt *stmt) stmt->subname))); } - /* - * Parse and check options. - * Connection and publication should not be specified here. - */ - parse_subscription_options(stmt->options, NULL, NULL, - &enabled_given, &enabled, - &create_slot, &slotname); if (slotname == NULL) slotname = stmt->subname; @@ -424,7 +435,7 @@ AlterSubscription(AlterSubscriptionStmt *stmt) * Drop a subscription */ void -DropSubscription(DropSubscriptionStmt *stmt) +DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel) { Relation rel; ObjectAddress myself; @@ -441,6 +452,15 @@ DropSubscription(DropSubscriptionStmt *stmt) WalReceiverConn *wrconn = NULL; StringInfoData cmd; + /* + * Since dropping a replication slot is not transactional, the replication + * slot stays dropped even if the transaction rolls back. So we cannot + * run DROP SUBSCRIPTION inside a transaction block if dropping the + * replication slot. + */ + if (stmt->drop_slot) + PreventTransactionChain(isTopLevel, "DROP SUBSCRIPTION ... DROP SLOT"); + rel = heap_open(SubscriptionRelationId, RowExclusiveLock); tup = SearchSysCache2(SUBSCRIPTIONNAME, MyDatabaseId, diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index 3bc0ae5e7e..20b5273405 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -1609,7 +1609,8 @@ ProcessUtilitySlow(ParseState *pstate, break; case T_CreateSubscriptionStmt: - address = CreateSubscription((CreateSubscriptionStmt *) parsetree); + address = CreateSubscription((CreateSubscriptionStmt *) parsetree, + isTopLevel); break; case T_AlterSubscriptionStmt: @@ -1617,7 +1618,7 @@ ProcessUtilitySlow(ParseState *pstate, break; case T_DropSubscriptionStmt: - DropSubscription((DropSubscriptionStmt *) parsetree); + DropSubscription((DropSubscriptionStmt *) parsetree, isTopLevel); /* no commands stashed for DROP */ commandCollected = true; break; diff --git a/src/include/commands/subscriptioncmds.h b/src/include/commands/subscriptioncmds.h index 127696c60d..1765879333 100644 --- a/src/include/commands/subscriptioncmds.h +++ b/src/include/commands/subscriptioncmds.h @@ -18,9 +18,10 @@ #include "catalog/objectaddress.h" #include "nodes/parsenodes.h" -extern ObjectAddress CreateSubscription(CreateSubscriptionStmt *stmt); +extern ObjectAddress CreateSubscription(CreateSubscriptionStmt *stmt, + bool isTopLevel); extern ObjectAddress AlterSubscription(AlterSubscriptionStmt *stmt); -extern void DropSubscription(DropSubscriptionStmt *stmt); +extern void DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel); extern ObjectAddress AlterSubscriptionOwner(const char *name, Oid newOwnerId); extern void AlterSubscriptionOwner_oid(Oid subid, Oid newOwnerId); diff --git a/src/test/regress/expected/subscription.out b/src/test/regress/expected/subscription.out index cb1ab4e791..a8a61ee8af 100644 --- a/src/test/regress/expected/subscription.out +++ b/src/test/regress/expected/subscription.out @@ -14,6 +14,11 @@ ERROR: syntax error at or near "PUBLICATION" LINE 1: CREATE SUBSCRIPTION testsub PUBLICATION foo; ^ set client_min_messages to error; +-- fail - cannot do CREATE SUBSCRIPTION CREATE SLOT inside transaction block +BEGIN; +CREATE SUBSCRIPTION testsub CONNECTION 'testconn' PUBLICATION testpub WITH (CREATE SLOT); +ERROR: CREATE SUBSCRIPTION ... CREATE SLOT cannot run inside a transaction block +COMMIT; CREATE SUBSCRIPTION testsub CONNECTION 'testconn' PUBLICATION testpub; ERROR: invalid connection string syntax: missing "=" after "testconn" in connection info string @@ -69,6 +74,13 @@ ALTER SUBSCRIPTION testsub RENAME TO testsub_foo; testsub_foo | regress_subscription_user | f | {testpub,testpub1} (1 row) +-- fail - cannot do DROP SUBSCRIPTION DROP SLOT inside transaction block +BEGIN; +DROP SUBSCRIPTION testsub DROP SLOT; +ERROR: DROP SUBSCRIPTION ... DROP SLOT cannot run inside a transaction block +COMMIT; +BEGIN; DROP SUBSCRIPTION testsub_foo NODROP SLOT; +COMMIT; RESET SESSION AUTHORIZATION; DROP ROLE regress_subscription_user; diff --git a/src/test/regress/sql/subscription.sql b/src/test/regress/sql/subscription.sql index fce6069a9c..0b6c8a3f5c 100644 --- a/src/test/regress/sql/subscription.sql +++ b/src/test/regress/sql/subscription.sql @@ -12,6 +12,11 @@ CREATE SUBSCRIPTION testsub CONNECTION 'foo'; CREATE SUBSCRIPTION testsub PUBLICATION foo; set client_min_messages to error; +-- fail - cannot do CREATE SUBSCRIPTION CREATE SLOT inside transaction block +BEGIN; +CREATE SUBSCRIPTION testsub CONNECTION 'testconn' PUBLICATION testpub WITH (CREATE SLOT); +COMMIT; + CREATE SUBSCRIPTION testsub CONNECTION 'testconn' PUBLICATION testpub; CREATE SUBSCRIPTION testsub CONNECTION 'dbname=doesnotexist' PUBLICATION testpub WITH (DISABLED, NOCREATE SLOT); reset client_min_messages; @@ -42,7 +47,14 @@ ALTER SUBSCRIPTION testsub RENAME TO testsub_foo; \dRs +-- fail - cannot do DROP SUBSCRIPTION DROP SLOT inside transaction block +BEGIN; +DROP SUBSCRIPTION testsub DROP SLOT; +COMMIT; + +BEGIN; DROP SUBSCRIPTION testsub_foo NODROP SLOT; +COMMIT; RESET SESSION AUTHORIZATION; DROP ROLE regress_subscription_user; -- cgit v1.2.3 From 6f3a13ff058f15d565a30c16c0c2cb14cc994e42 Mon Sep 17 00:00:00 2001 From: Simon Riggs Date: Mon, 6 Mar 2017 16:48:12 +0530 Subject: Enhance docs for ALTER TABLE lock levels of storage parms As requested by Robert Haas --- doc/src/sgml/ref/alter_table.sgml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/alter_table.sgml b/doc/src/sgml/ref/alter_table.sgml index da431f8369..8b251f9e5d 100644 --- a/doc/src/sgml/ref/alter_table.sgml +++ b/doc/src/sgml/ref/alter_table.sgml @@ -568,10 +568,17 @@ ALTER TABLE [ IF EXISTS ] name That can be done with VACUUM FULL, or one of the forms of ALTER TABLE that forces a table rewrite. + For planner related parameters, changes will take effect from the next + time the table is locked so currently executing queries will not be + affected. - Changing fillfactor and autovacuum storage parameters acquires a SHARE UPDATE EXCLUSIVE lock. + SHARE UPDATE EXCLUSIVE lock will be taken for + fillfactor and autovacuum storage parameters, as well as the + following planner related parameters: + effective_io_concurrency, parallel_workers, seq_page_cost + random_page_cost, n_distinct and n_distinct_inherited. -- cgit v1.2.3 From 583f6c414895e72c710f723fbb3649df664530d7 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 28 Dec 2016 12:00:00 -0500 Subject: Allow dropping multiple functions at once The generic drop support already supported dropping multiple objects of the same kind at once. But the previous representation of function signatures across two grammar symbols and structure members made this cumbersome to do for functions, so it was not supported. Now that function signatures are represented by a single structure, it's trivial to add this support. Same for aggregates and operators. Reviewed-by: Jim Nasby Reviewed-by: Michael Paquier --- doc/src/sgml/ref/drop_aggregate.sgml | 11 +++++-- doc/src/sgml/ref/drop_function.sgml | 8 +++++- doc/src/sgml/ref/drop_operator.sgml | 8 +++++- src/backend/parser/gram.y | 38 ++++++++++++++++--------- src/test/regress/expected/create_function_3.out | 6 ++-- src/test/regress/sql/create_function_3.sql | 2 ++ 6 files changed, 52 insertions(+), 21 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/drop_aggregate.sgml b/doc/src/sgml/ref/drop_aggregate.sgml index c27c5eadf9..631b578df7 100644 --- a/doc/src/sgml/ref/drop_aggregate.sgml +++ b/doc/src/sgml/ref/drop_aggregate.sgml @@ -21,7 +21,7 @@ PostgreSQL documentation -DROP AGGREGATE [ IF EXISTS ] name ( aggregate_signature ) [ CASCADE | RESTRICT ] +DROP AGGREGATE [ IF EXISTS ] name ( aggregate_signature ) [, ...] [ CASCADE | RESTRICT ] where aggregate_signature is: @@ -155,7 +155,14 @@ DROP AGGREGATE myavg(integer); DROP AGGREGATE myrank(VARIADIC "any" ORDER BY VARIADIC "any"); - + + + To remove multiple aggregate functions in one command: + +DROP AGGREGATE myavg(integer), myavg(bigint); + + + Compatibility diff --git a/doc/src/sgml/ref/drop_function.sgml b/doc/src/sgml/ref/drop_function.sgml index 5883d13811..5969b084b4 100644 --- a/doc/src/sgml/ref/drop_function.sgml +++ b/doc/src/sgml/ref/drop_function.sgml @@ -21,7 +21,7 @@ PostgreSQL documentation -DROP FUNCTION [ IF EXISTS ] name ( [ [ argmode ] [ argname ] argtype [, ...] ] ) +DROP FUNCTION [ IF EXISTS ] name ( [ [ argmode ] [ argname ] argtype [, ...] ] ) [, ...] [ CASCADE | RESTRICT ] @@ -134,6 +134,12 @@ DROP FUNCTION [ IF EXISTS ] name ( DROP FUNCTION sqrt(integer); + + + + Drop multiple functions in one command: + +DROP FUNCTION sqrt(integer), sqrt(bigint); diff --git a/doc/src/sgml/ref/drop_operator.sgml b/doc/src/sgml/ref/drop_operator.sgml index 13dd974f38..fc82c3e0e3 100644 --- a/doc/src/sgml/ref/drop_operator.sgml +++ b/doc/src/sgml/ref/drop_operator.sgml @@ -21,7 +21,7 @@ PostgreSQL documentation -DROP OPERATOR [ IF EXISTS ] name ( { left_type | NONE } , { right_type | NONE } ) [ CASCADE | RESTRICT ] +DROP OPERATOR [ IF EXISTS ] name ( { left_type | NONE } , { right_type | NONE } ) [, ...] [ CASCADE | RESTRICT ] @@ -125,6 +125,12 @@ DROP OPERATOR ~ (none, bit); for type bigint: DROP OPERATOR ! (bigint, none); + + + + Remove multiple operators in one command: + +DROP OPERATOR ~ (none, bit), ! (bigint, none); diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 1b06d358b5..542b09b4b0 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -358,7 +358,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); %type privileges privilege_list %type privilege_target %type function_with_argtypes aggregate_with_argtypes operator_with_argtypes -%type function_with_argtypes_list +%type function_with_argtypes_list aggregate_with_argtypes_list operator_with_argtypes_list %type defacl_privilege_target %type DefACLOption %type DefACLOptionList @@ -7495,6 +7495,12 @@ aggregate_with_argtypes: } ; +aggregate_with_argtypes_list: + aggregate_with_argtypes { $$ = list_make1($1); } + | aggregate_with_argtypes_list ',' aggregate_with_argtypes + { $$ = lappend($1, $3); } + ; + createfunc_opt_list: /* Must be at least one to prevent conflict */ createfunc_opt_item { $$ = list_make1($1); } @@ -7676,21 +7682,21 @@ opt_restrict: *****************************************************************************/ RemoveFuncStmt: - DROP FUNCTION function_with_argtypes opt_drop_behavior + DROP FUNCTION function_with_argtypes_list opt_drop_behavior { DropStmt *n = makeNode(DropStmt); n->removeType = OBJECT_FUNCTION; - n->objects = list_make1($3); + n->objects = $3; n->behavior = $4; n->missing_ok = false; n->concurrent = false; $$ = (Node *)n; } - | DROP FUNCTION IF_P EXISTS function_with_argtypes opt_drop_behavior + | DROP FUNCTION IF_P EXISTS function_with_argtypes_list opt_drop_behavior { DropStmt *n = makeNode(DropStmt); n->removeType = OBJECT_FUNCTION; - n->objects = list_make1($5); + n->objects = $5; n->behavior = $6; n->missing_ok = true; n->concurrent = false; @@ -7699,21 +7705,21 @@ RemoveFuncStmt: ; RemoveAggrStmt: - DROP AGGREGATE aggregate_with_argtypes opt_drop_behavior + DROP AGGREGATE aggregate_with_argtypes_list opt_drop_behavior { DropStmt *n = makeNode(DropStmt); n->removeType = OBJECT_AGGREGATE; - n->objects = list_make1($3); + n->objects = $3; n->behavior = $4; n->missing_ok = false; n->concurrent = false; $$ = (Node *)n; } - | DROP AGGREGATE IF_P EXISTS aggregate_with_argtypes opt_drop_behavior + | DROP AGGREGATE IF_P EXISTS aggregate_with_argtypes_list opt_drop_behavior { DropStmt *n = makeNode(DropStmt); n->removeType = OBJECT_AGGREGATE; - n->objects = list_make1($5); + n->objects = $5; n->behavior = $6; n->missing_ok = true; n->concurrent = false; @@ -7722,21 +7728,21 @@ RemoveAggrStmt: ; RemoveOperStmt: - DROP OPERATOR operator_with_argtypes opt_drop_behavior + DROP OPERATOR operator_with_argtypes_list opt_drop_behavior { DropStmt *n = makeNode(DropStmt); n->removeType = OBJECT_OPERATOR; - n->objects = list_make1($3); + n->objects = $3; n->behavior = $4; n->missing_ok = false; n->concurrent = false; $$ = (Node *)n; } - | DROP OPERATOR IF_P EXISTS operator_with_argtypes opt_drop_behavior + | DROP OPERATOR IF_P EXISTS operator_with_argtypes_list opt_drop_behavior { DropStmt *n = makeNode(DropStmt); n->removeType = OBJECT_OPERATOR; - n->objects = list_make1($5); + n->objects = $5; n->behavior = $6; n->missing_ok = true; n->concurrent = false; @@ -7768,6 +7774,12 @@ any_operator: { $$ = lcons(makeString($1), $3); } ; +operator_with_argtypes_list: + operator_with_argtypes { $$ = list_make1($1); } + | operator_with_argtypes_list ',' operator_with_argtypes + { $$ = lappend($1, $3); } + ; + operator_with_argtypes: any_operator oper_argtypes { diff --git a/src/test/regress/expected/create_function_3.out b/src/test/regress/expected/create_function_3.out index 7bb957b51b..cc4e98a1d4 100644 --- a/src/test/regress/expected/create_function_3.out +++ b/src/test/regress/expected/create_function_3.out @@ -217,9 +217,10 @@ SELECT routine_name, ordinal_position, parameter_name, parameter_default functest_is_3 | 2 | b | (7 rows) +DROP FUNCTION functest_IS_1(int, int, text), functest_IS_2(int), functest_IS_3(int); -- Cleanups DROP SCHEMA temp_func_test CASCADE; -NOTICE: drop cascades to 19 other objects +NOTICE: drop cascades to 16 other objects DETAIL: drop cascades to function functest_a_1(text,date) drop cascades to function functest_a_2(text[]) drop cascades to function functest_a_3() @@ -236,8 +237,5 @@ drop cascades to function functext_f_1(integer) drop cascades to function functext_f_2(integer) drop cascades to function functext_f_3(integer) drop cascades to function functext_f_4(integer) -drop cascades to function functest_is_1(integer,integer,text) -drop cascades to function functest_is_2(integer) -drop cascades to function functest_is_3(integer) DROP USER regress_unpriv_user; RESET search_path; diff --git a/src/test/regress/sql/create_function_3.sql b/src/test/regress/sql/create_function_3.sql index 43454ef2f7..66a463b089 100644 --- a/src/test/regress/sql/create_function_3.sql +++ b/src/test/regress/sql/create_function_3.sql @@ -156,6 +156,8 @@ SELECT routine_name, ordinal_position, parameter_name, parameter_default WHERE routine_schema = 'temp_func_test' AND routine_name ~ '^functest_is_' ORDER BY 1, 2; +DROP FUNCTION functest_IS_1(int, int, text), functest_IS_2(int), functest_IS_3(int); + -- Cleanups DROP SCHEMA temp_func_test CASCADE; -- cgit v1.2.3 From e6477a8134ace06ef3a45a7ce15813cd398e72d8 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 21 Feb 2017 23:10:07 -0500 Subject: Combine several DROP variants into generic DropStmt Combine DROP of FOREIGN DATA WRAPPER, SERVER, POLICY, RULE, and TRIGGER into generic DropStmt grammar. Reviewed-by: Jim Nasby Reviewed-by: Michael Paquier --- doc/src/sgml/ref/drop_foreign_data_wrapper.sgml | 2 +- doc/src/sgml/ref/drop_server.sgml | 2 +- src/backend/parser/gram.y | 173 +++++------------------- 3 files changed, 35 insertions(+), 142 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/drop_foreign_data_wrapper.sgml b/doc/src/sgml/ref/drop_foreign_data_wrapper.sgml index 824d72c176..702cc021db 100644 --- a/doc/src/sgml/ref/drop_foreign_data_wrapper.sgml +++ b/doc/src/sgml/ref/drop_foreign_data_wrapper.sgml @@ -21,7 +21,7 @@ PostgreSQL documentation -DROP FOREIGN DATA WRAPPER [ IF EXISTS ] name [ CASCADE | RESTRICT ] +DROP FOREIGN DATA WRAPPER [ IF EXISTS ] name [, ...] [ CASCADE | RESTRICT ] diff --git a/doc/src/sgml/ref/drop_server.sgml b/doc/src/sgml/ref/drop_server.sgml index f08dd7767d..42acdd41dc 100644 --- a/doc/src/sgml/ref/drop_server.sgml +++ b/doc/src/sgml/ref/drop_server.sgml @@ -21,7 +21,7 @@ PostgreSQL documentation -DROP SERVER [ IF EXISTS ] name [ CASCADE | RESTRICT ] +DROP SERVER [ IF EXISTS ] name [, ...] [ CASCADE | RESTRICT ] diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 542b09b4b0..083124ed3b 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -263,10 +263,10 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); CreateUserStmt CreateUserMappingStmt CreateRoleStmt CreatePolicyStmt CreatedbStmt DeclareCursorStmt DefineStmt DeleteStmt DiscardStmt DoStmt DropGroupStmt DropOpClassStmt DropOpFamilyStmt DropPLangStmt DropStmt - DropAssertStmt DropTrigStmt DropRuleStmt DropCastStmt DropRoleStmt - DropPolicyStmt DropUserStmt DropdbStmt DropTableSpaceStmt DropFdwStmt + DropAssertStmt DropCastStmt DropRoleStmt + DropUserStmt DropdbStmt DropTableSpaceStmt DropTransformStmt - DropForeignServerStmt DropUserMappingStmt ExplainStmt FetchStmt + DropUserMappingStmt ExplainStmt FetchStmt GrantStmt GrantRoleStmt ImportForeignSchemaStmt IndexStmt InsertStmt ListenStmt LoadStmt LockStmt NotifyStmt ExplainableStmt PreparableStmt CreateFunctionStmt AlterFunctionStmt ReindexStmt RemoveAggrStmt @@ -440,7 +440,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); %type copy_from opt_program %type opt_column event cursor_options opt_hold opt_set_data -%type drop_type_any_name drop_type_name +%type drop_type_any_name drop_type_name drop_type_name_on_any_name comment_type_any_name comment_type_name security_label_type_any_name security_label_type_name @@ -885,20 +885,15 @@ stmt : | DoStmt | DropAssertStmt | DropCastStmt - | DropFdwStmt - | DropForeignServerStmt | DropGroupStmt | DropOpClassStmt | DropOpFamilyStmt | DropOwnedStmt - | DropPolicyStmt | DropPLangStmt - | DropRuleStmt | DropStmt | DropSubscriptionStmt | DropTableSpaceStmt | DropTransformStmt - | DropTrigStmt | DropRoleStmt | DropUserStmt | DropUserMappingStmt @@ -4511,35 +4506,6 @@ opt_fdw_options: | /*EMPTY*/ { $$ = NIL; } ; -/***************************************************************************** - * - * QUERY : - * DROP FOREIGN DATA WRAPPER name - * - ****************************************************************************/ - -DropFdwStmt: DROP FOREIGN DATA_P WRAPPER name opt_drop_behavior - { - DropStmt *n = makeNode(DropStmt); - n->removeType = OBJECT_FDW; - n->objects = list_make1(makeString($5)); - n->missing_ok = false; - n->behavior = $6; - n->concurrent = false; - $$ = (Node *) n; - } - | DROP FOREIGN DATA_P WRAPPER IF_P EXISTS name opt_drop_behavior - { - DropStmt *n = makeNode(DropStmt); - n->removeType = OBJECT_FDW; - n->objects = list_make1(makeString($7)); - n->missing_ok = true; - n->behavior = $8; - n->concurrent = false; - $$ = (Node *) n; - } - ; - /***************************************************************************** * * QUERY : @@ -4671,35 +4637,6 @@ opt_foreign_server_version: | /*EMPTY*/ { $$ = NULL; } ; -/***************************************************************************** - * - * QUERY : - * DROP SERVER name - * - ****************************************************************************/ - -DropForeignServerStmt: DROP SERVER name opt_drop_behavior - { - DropStmt *n = makeNode(DropStmt); - n->removeType = OBJECT_FOREIGN_SERVER; - n->objects = list_make1(makeString($3)); - n->missing_ok = false; - n->behavior = $4; - n->concurrent = false; - $$ = (Node *) n; - } - | DROP SERVER IF_P EXISTS name opt_drop_behavior - { - DropStmt *n = makeNode(DropStmt); - n->removeType = OBJECT_FOREIGN_SERVER; - n->objects = list_make1(makeString($5)); - n->missing_ok = true; - n->behavior = $6; - n->concurrent = false; - $$ = (Node *) n; - } - ; - /***************************************************************************** * * QUERY : @@ -4975,7 +4912,6 @@ AlterUserMappingStmt: ALTER USER MAPPING FOR auth_ident SERVER name alter_generi * [USING (qual)] [WITH CHECK (with check qual)] * ALTER POLICY name ON table [TO role, ...] * [USING (qual)] [WITH CHECK (with check qual)] - * DROP POLICY name ON table * *****************************************************************************/ @@ -5010,29 +4946,6 @@ AlterPolicyStmt: } ; -DropPolicyStmt: - DROP POLICY name ON any_name opt_drop_behavior - { - DropStmt *n = makeNode(DropStmt); - n->removeType = OBJECT_POLICY; - n->objects = list_make1(lappend($5, makeString($3))); - n->behavior = $6; - n->missing_ok = false; - n->concurrent = false; - $$ = (Node *) n; - } - | DROP POLICY IF_P EXISTS name ON any_name opt_drop_behavior - { - DropStmt *n = makeNode(DropStmt); - n->removeType = OBJECT_POLICY; - n->objects = list_make1(lappend($7, makeString($5))); - n->behavior = $8; - n->missing_ok = true; - n->concurrent = false; - $$ = (Node *) n; - } - ; - RowSecurityOptionalExpr: USING '(' a_expr ')' { $$ = $3; } | /* EMPTY */ { $$ = NULL; } @@ -5105,7 +5018,6 @@ CreateAmStmt: CREATE ACCESS METHOD name TYPE_P INDEX HANDLER handler_name * * QUERIES : * CREATE TRIGGER ... - * DROP TRIGGER ... * *****************************************************************************/ @@ -5332,30 +5244,6 @@ ConstraintAttributeElem: ; -DropTrigStmt: - DROP TRIGGER name ON any_name opt_drop_behavior - { - DropStmt *n = makeNode(DropStmt); - n->removeType = OBJECT_TRIGGER; - n->objects = list_make1(lappend($5, makeString($3))); - n->behavior = $6; - n->missing_ok = false; - n->concurrent = false; - $$ = (Node *) n; - } - | DROP TRIGGER IF_P EXISTS name ON any_name opt_drop_behavior - { - DropStmt *n = makeNode(DropStmt); - n->removeType = OBJECT_TRIGGER; - n->objects = list_make1(lappend($7, makeString($5))); - n->behavior = $8; - n->missing_ok = true; - n->concurrent = false; - $$ = (Node *) n; - } - ; - - /***************************************************************************** * * QUERIES : @@ -6034,6 +5922,26 @@ DropStmt: DROP drop_type_any_name IF_P EXISTS any_name_list opt_drop_behavior n->concurrent = false; $$ = (Node *)n; } + | DROP drop_type_name_on_any_name name ON any_name opt_drop_behavior + { + DropStmt *n = makeNode(DropStmt); + n->removeType = $2; + n->objects = list_make1(lappend($5, makeString($3))); + n->behavior = $6; + n->missing_ok = false; + n->concurrent = false; + $$ = (Node *) n; + } + | DROP drop_type_name_on_any_name IF_P EXISTS name ON any_name opt_drop_behavior + { + DropStmt *n = makeNode(DropStmt); + n->removeType = $2; + n->objects = list_make1(lappend($7, makeString($5))); + n->behavior = $8; + n->missing_ok = true; + n->concurrent = false; + $$ = (Node *) n; + } | DROP TYPE_P type_name_list opt_drop_behavior { DropStmt *n = makeNode(DropStmt); @@ -6117,8 +6025,17 @@ drop_type_name: ACCESS METHOD { $$ = OBJECT_ACCESS_METHOD; } | EVENT TRIGGER { $$ = OBJECT_EVENT_TRIGGER; } | EXTENSION { $$ = OBJECT_EXTENSION; } + | FOREIGN DATA_P WRAPPER { $$ = OBJECT_FDW; } | PUBLICATION { $$ = OBJECT_PUBLICATION; } | SCHEMA { $$ = OBJECT_SCHEMA; } + | SERVER { $$ = OBJECT_FOREIGN_SERVER; } + ; + +/* object types attached to a table */ +drop_type_name_on_any_name: + POLICY { $$ = OBJECT_POLICY; } + | RULE { $$ = OBJECT_RULE; } + | TRIGGER { $$ = OBJECT_TRIGGER; } ; any_name_list: @@ -9277,30 +9194,6 @@ opt_instead: ; -DropRuleStmt: - DROP RULE name ON any_name opt_drop_behavior - { - DropStmt *n = makeNode(DropStmt); - n->removeType = OBJECT_RULE; - n->objects = list_make1(lappend($5, makeString($3))); - n->behavior = $6; - n->missing_ok = false; - n->concurrent = false; - $$ = (Node *) n; - } - | DROP RULE IF_P EXISTS name ON any_name opt_drop_behavior - { - DropStmt *n = makeNode(DropStmt); - n->removeType = OBJECT_RULE; - n->objects = list_make1(lappend($7, makeString($5))); - n->behavior = $8; - n->missing_ok = true; - n->concurrent = false; - $$ = (Node *) n; - } - ; - - /***************************************************************************** * * QUERY: -- cgit v1.2.3 From 818fd4a67d610991757b610755e3065fb99d80a5 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Tue, 7 Mar 2017 14:25:40 +0200 Subject: Support SCRAM-SHA-256 authentication (RFC 5802 and 7677). This introduces a new generic SASL authentication method, similar to the GSS and SSPI methods. The server first tells the client which SASL authentication mechanism to use, and then the mechanism-specific SASL messages are exchanged in AuthenticationSASLcontinue and PasswordMessage messages. Only SCRAM-SHA-256 is supported at the moment, but this allows adding more SASL mechanisms in the future, without changing the overall protocol. Support for channel binding, aka SCRAM-SHA-256-PLUS is left for later. The SASLPrep algorithm, for pre-processing the password, is not yet implemented. That could cause trouble, if you use a password with non-ASCII characters, and a client library that does implement SASLprep. That will hopefully be added later. Authorization identities, as specified in the SCRAM-SHA-256 specification, are ignored. SET SESSION AUTHORIZATION provides more or less the same functionality, anyway. If a user doesn't exist, perform a "mock" authentication, by constructing an authentic-looking challenge on the fly. The challenge is derived from a new system-wide random value, "mock authentication nonce", which is created at initdb, and stored in the control file. We go through these motions, in order to not give away the information on whether the user exists, to unauthenticated users. Bumps PG_CONTROL_VERSION, because of the new field in control file. Patch by Michael Paquier and Heikki Linnakangas, reviewed at different stages by Robert Haas, Stephen Frost, David Steele, Aleksander Alekseev, and many others. Discussion: https://www.postgresql.org/message-id/CAB7nPqRbR3GmFYdedCAhzukfKrgBLTLtMvENOmPrVWREsZkF8g%40mail.gmail.com Discussion: https://www.postgresql.org/message-id/CAB7nPqSMXU35g%3DW9X74HVeQp0uvgJxvYOuA4A-A3M%2B0wfEBv-w%40mail.gmail.com Discussion: https://www.postgresql.org/message-id/55192AFE.6080106@iki.fi --- contrib/pgcrypto/.gitignore | 4 - contrib/pgcrypto/Makefile | 11 +- doc/src/sgml/catalogs.sgml | 25 +- doc/src/sgml/client-auth.sgml | 59 +- doc/src/sgml/config.sgml | 7 +- doc/src/sgml/protocol.sgml | 147 +++- doc/src/sgml/ref/create_role.sgml | 17 +- src/backend/access/transam/xlog.c | 24 + src/backend/commands/user.c | 14 +- src/backend/libpq/Makefile | 2 +- src/backend/libpq/auth-scram.c | 1032 +++++++++++++++++++++++++ src/backend/libpq/auth.c | 136 ++++ src/backend/libpq/crypt.c | 27 +- src/backend/libpq/hba.c | 3 + src/backend/libpq/pg_hba.conf.sample | 8 +- src/backend/utils/misc/guc.c | 1 + src/backend/utils/misc/postgresql.conf.sample | 2 +- src/bin/initdb/initdb.c | 21 +- src/bin/pg_controldata/pg_controldata.c | 12 +- src/common/Makefile | 6 +- src/common/base64.c | 199 +++++ src/common/scram-common.c | 196 +++++ src/include/access/xlog.h | 1 + src/include/catalog/pg_control.h | 11 +- src/include/common/base64.h | 19 + src/include/common/scram-common.h | 62 ++ src/include/libpq/crypt.h | 3 +- src/include/libpq/hba.h | 1 + src/include/libpq/pqcomm.h | 2 + src/include/libpq/scram.h | 35 + src/interfaces/libpq/.gitignore | 5 + src/interfaces/libpq/Makefile | 20 +- src/interfaces/libpq/fe-auth-scram.c | 640 +++++++++++++++ src/interfaces/libpq/fe-auth.c | 112 +++ src/interfaces/libpq/fe-auth.h | 8 + src/interfaces/libpq/fe-connect.c | 52 ++ src/interfaces/libpq/libpq-int.h | 7 +- src/tools/msvc/Mkvcbuild.pm | 12 +- 38 files changed, 2866 insertions(+), 77 deletions(-) create mode 100644 src/backend/libpq/auth-scram.c create mode 100644 src/common/base64.c create mode 100644 src/common/scram-common.c create mode 100644 src/include/common/base64.h create mode 100644 src/include/common/scram-common.h create mode 100644 src/include/libpq/scram.h create mode 100644 src/interfaces/libpq/fe-auth-scram.c (limited to 'doc/src') diff --git a/contrib/pgcrypto/.gitignore b/contrib/pgcrypto/.gitignore index 30619bfbbf..5dcb3ff972 100644 --- a/contrib/pgcrypto/.gitignore +++ b/contrib/pgcrypto/.gitignore @@ -1,7 +1,3 @@ -# Source file copied from src/common -/sha2.c -/sha2_openssl.c - # Generated subdirectories /log/ /results/ diff --git a/contrib/pgcrypto/Makefile b/contrib/pgcrypto/Makefile index 14e74f899c..573bc6df79 100644 --- a/contrib/pgcrypto/Makefile +++ b/contrib/pgcrypto/Makefile @@ -1,10 +1,10 @@ # contrib/pgcrypto/Makefile -INT_SRCS = md5.c sha1.c sha2.c internal.c internal-sha2.c blf.c rijndael.c \ +INT_SRCS = md5.c sha1.c internal.c internal-sha2.c blf.c rijndael.c \ pgp-mpi-internal.c imath.c INT_TESTS = sha2 -OSSL_SRCS = openssl.c pgp-mpi-openssl.c sha2_openssl.c +OSSL_SRCS = openssl.c pgp-mpi-openssl.c OSSL_TESTS = sha2 des 3des cast5 ZLIB_TST = pgp-compression @@ -59,13 +59,6 @@ SHLIB_LINK += $(filter -leay32, $(LIBS)) SHLIB_LINK += -lws2_32 endif -# Compiling pgcrypto with those two raw files is necessary as long -# as none of their routines are used by the backend code. Note doing -# so can either result in library loading failures or linking resolution -# failures at compilation depending on the environment used. -sha2.c sha2_openssl.c: % : $(top_srcdir)/src/common/% - rm -f $@ && $(LN_S) $< . - rijndael.o: rijndael.tbl rijndael.tbl: diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 41e3e1b547..28cdabe6fe 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -1334,14 +1334,8 @@ rolpassword text - Password (possibly encrypted); null if none. If the password - is encrypted, this column will begin with the string md5 - followed by a 32-character hexadecimal MD5 hash. The MD5 hash - will be of the user's password concatenated to their user name. - For example, if user joe has password xyzzy, - PostgreSQL will store the md5 hash of - xyzzyjoe. A password that does not follow that - format is assumed to be unencrypted. + Password (possibly encrypted); null if none. The format depends + on the form of encryption used. @@ -1355,6 +1349,21 @@
+ + For an MD5 encrypted password, rolpassword + column will begin with the string md5 followed by a + 32-character hexadecimal MD5 hash. The MD5 hash will be of the user's + password concatenated to their user name. For example, if user + joe has password xyzzy, PostgreSQL + will store the md5 hash of xyzzyjoe. If the password is + encrypted with SCRAM-SHA-256, it consists of 5 fields separated by colons. + The first field is the constant scram-sha-256, to + identify the password as a SCRAM-SHA-256 verifier. The second field is a + salt, Base64-encoded, and the third field is the number of iterations used + to generate the password. The fourth field and fifth field are the stored + key and server key, respectively, in hexadecimal format. A password that + does not follow either of those formats is assumed to be unencrypted. + diff --git a/doc/src/sgml/client-auth.sgml b/doc/src/sgml/client-auth.sgml index 231fc40fc3..bbd52a5418 100644 --- a/doc/src/sgml/client-auth.sgml +++ b/doc/src/sgml/client-auth.sgml @@ -422,6 +422,17 @@ hostnossl database user + + scram + + + Perform SCRAM-SHA-256 authentication to verify the user's + password. + See for details. + + + + password @@ -673,13 +684,19 @@ host postgres all 192.168.93.0/24 ident # "postgres" if the user's password is correctly supplied. # # TYPE DATABASE USER ADDRESS METHOD -host postgres all 192.168.12.10/32 md5 +host postgres all 192.168.12.10/32 scram # Allow any user from hosts in the example.com domain to connect to # any database if the user's password is correctly supplied. # +# Most users use SCRAM authentication, but some users use older clients +# that don't support SCRAM authentication, and need to be able to log +# in using MD5 authentication. Such users are put in the @md5users +# group, everyone else must use SCRAM. +# # TYPE DATABASE USER ADDRESS METHOD -host all all .example.com md5 +host all @md5users .example.com md5 +host all all .example.com scram # In the absence of preceding "host" lines, these two lines will # reject all connections from 192.168.54.1 (since that entry will be @@ -907,21 +924,37 @@ omicron bryanh guest1 - The password-based authentication methods are md5 - and password. These methods operate + The password-based authentication methods are scram + md5 and password. These methods operate similarly except for the way that the password is sent across the - connection, namely MD5-hashed and clear-text respectively. + connection. + + + + Plain password sends the password in clear-text, and is + therefore vulnerable to password sniffing attacks. It should + always be avoided if possible. If the connection is protected by SSL + encryption then password can be used safely, though. + (Though SSL certificate authentication might be a better choice if one + is depending on using SSL). + + + + + scram performs SCRAM-SHA-256 authentication, as described + in RFC5802. It + is a challenge-response scheme, that prevents password sniffing on + untrusted connections. It is more secure than the md5 + method, but might not be supported by older clients. - If you are at all concerned about password - sniffing attacks then md5 is preferred. - Plain password should always be avoided if possible. - However, md5 cannot be used with the feature. If the connection is - protected by SSL encryption then password can be used - safely (though SSL certificate authentication might be a better - choice if one is depending on using SSL). + In md5, the client sends a hash of a random challenge, + generated by the server, and the password. It prevents password sniffing, + but is less secure than scram, and provides no protection + if an attacker manages to steal the password hash from the server. + md5 cannot be used with the feature. diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index cd82c04b05..1881236726 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -1193,9 +1193,10 @@ include_dir 'conf.d' password is to be encrypted. The default value is md5, which stores the password as an MD5 hash. Setting this to plain stores it in plaintext. on and off are also accepted, as - aliases for md5 and plain, respectively. - - + aliases for md5 and plain, respectively. Setting + this parameter to scram will encrypt the password with + SCRAM-SHA-256. + diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index 589b881ef2..3d6e8eed43 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -228,11 +228,11 @@ The server then sends an appropriate authentication request message, to which the frontend must reply with an appropriate authentication response message (such as a password). - For all authentication methods except GSSAPI and SSPI, there is at most - one request and one response. In some methods, no response + For all authentication methods except GSSAPI, SSPI and SASL, there is at + most one request and one response. In some methods, no response at all is needed from the frontend, and so no authentication request - occurs. For GSSAPI and SSPI, multiple exchanges of packets may be needed - to complete the authentication. + occurs. For GSSAPI, SSPI and SASL, multiple exchanges of packets may be + needed to complete the authentication. @@ -366,6 +366,35 @@ + + AuthenticationSASL + + + The frontend must now initiate a SASL negotiation, using the SASL + mechanism specified in the message. The frontend will send a + PasswordMessage with the first part of the SASL data stream in + response to this. If further messages are needed, the server will + respond with AuthenticationSASLContinue. + + + + + + AuthenticationSASLContinue + + + This message contains the response data from the previous step + of SASL negotiation (AuthenticationSASL, or a previous + AuthenticationSASLContinue). If the SASL data in this message + indicates more data is needed to complete the authentication, + the frontend must send that data as another PasswordMessage. If + SASL authentication is completed by this message, the server + will next send AuthenticationOk to indicate successful authentication + or ErrorResponse to indicate failure. + + + + @@ -2782,6 +2811,114 @@ AuthenticationGSSContinue (B) + + +AuthenticationSASL (B) + + + + + + + + Byte1('R') + + + + Identifies the message as an authentication request. + + + + + + Int32 + + + + Length of message contents in bytes, including self. + + + + + + Int32(10) + + + + Specifies that SASL authentication is started. + + + + + + String + + + + Name of a SASL authentication mechanism. + + + + + + + + + + + +AuthenticationSASLContinue (B) + + + + + + + + Byte1('R') + + + + Identifies the message as an authentication request. + + + + + + Int32 + + + + Length of message contents in bytes, including self. + + + + + + Int32(11) + + + + Specifies that this message contains SASL-mechanism specific + data. + + + + + + Byten + + + + SASL data, specific to the SASL mechanism being used. + + + + + + + + @@ -4544,7 +4681,7 @@ PasswordMessage (F) Identifies the message as a password response. Note that - this is also used for GSSAPI and SSPI response messages + this is also used for GSSAPI, SSPI and SASL response messages (which is really a design error, since the contained data is not a null-terminated string in that case, but can be arbitrary binary data). diff --git a/doc/src/sgml/ref/create_role.sgml b/doc/src/sgml/ref/create_role.sgml index 2ae576ede6..99d1c8336c 100644 --- a/doc/src/sgml/ref/create_role.sgml +++ b/doc/src/sgml/ref/create_role.sgml @@ -231,12 +231,17 @@ CREATE ROLE name [ [ WITH ] .) If the - presented password string is already in MD5-encrypted format, - then it is stored encrypted as-is, regardless of whether - ENCRYPTED or UNENCRYPTED is specified - (since the system cannot decrypt the specified encrypted - password string). This allows reloading of encrypted - passwords during dump/restore. + presented password string is already in MD5-encrypted or + SCRAM-encrypted format, then it is stored encrypted as-is, + regardless of whether ENCRYPTED or UNENCRYPTED + is specified (since the system cannot decrypt the specified encrypted + password string). This allows reloading of encrypted passwords + during dump/restore. + + + + Note that older clients might lack support for the SCRAM + authentication mechanism. diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 897358342d..744360c769 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -65,6 +65,7 @@ #include "storage/reinit.h" #include "storage/smgr.h" #include "storage/spin.h" +#include "utils/backend_random.h" #include "utils/builtins.h" #include "utils/guc.h" #include "utils/memutils.h" @@ -4664,6 +4665,16 @@ GetSystemIdentifier(void) return ControlFile->system_identifier; } +/* + * Returns the random nonce from control file. + */ +char * +GetMockAuthenticationNonce(void) +{ + Assert(ControlFile != NULL); + return ControlFile->mock_authentication_nonce; +} + /* * Are checksums enabled for data pages? */ @@ -4914,6 +4925,7 @@ BootStrapXLOG(void) char *recptr; bool use_existent; uint64 sysidentifier; + char mock_auth_nonce[MOCK_AUTH_NONCE_LEN]; struct timeval tv; pg_crc32c crc; @@ -4934,6 +4946,17 @@ BootStrapXLOG(void) sysidentifier |= ((uint64) tv.tv_usec) << 12; sysidentifier |= getpid() & 0xFFF; + /* + * Generate a random nonce. This is used for authentication requests + * that will fail because the user does not exist. The nonce is used to + * create a genuine-looking password challenge for the non-existent user, + * in lieu of an actual stored password. + */ + if (!pg_backend_random(mock_auth_nonce, MOCK_AUTH_NONCE_LEN)) + ereport(PANIC, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("could not generation secret authorization token"))); + /* First timeline ID is always 1 */ ThisTimeLineID = 1; @@ -5040,6 +5063,7 @@ BootStrapXLOG(void) memset(ControlFile, 0, sizeof(ControlFileData)); /* Initialize pg_control status fields */ ControlFile->system_identifier = sysidentifier; + memcpy(ControlFile->mock_authentication_nonce, mock_auth_nonce, MOCK_AUTH_NONCE_LEN); ControlFile->state = DB_SHUTDOWNED; ControlFile->time = checkPoint.time; ControlFile->checkPoint = checkPoint.redo; diff --git a/src/backend/commands/user.c b/src/backend/commands/user.c index 994c093250..14b9779144 100644 --- a/src/backend/commands/user.c +++ b/src/backend/commands/user.c @@ -139,7 +139,12 @@ CreateRole(ParseState *pstate, CreateRoleStmt *stmt) parser_errposition(pstate, defel->location))); dpassword = defel; if (strcmp(defel->defname, "encryptedPassword") == 0) - password_type = PASSWORD_TYPE_MD5; + { + if (Password_encryption == PASSWORD_TYPE_SCRAM) + password_type = PASSWORD_TYPE_SCRAM; + else + password_type = PASSWORD_TYPE_MD5; + } else if (strcmp(defel->defname, "unencryptedPassword") == 0) password_type = PASSWORD_TYPE_PLAINTEXT; } @@ -542,7 +547,12 @@ AlterRole(AlterRoleStmt *stmt) errmsg("conflicting or redundant options"))); dpassword = defel; if (strcmp(defel->defname, "encryptedPassword") == 0) - password_type = PASSWORD_TYPE_MD5; + { + if (Password_encryption == PASSWORD_TYPE_SCRAM) + password_type = PASSWORD_TYPE_SCRAM; + else + password_type = PASSWORD_TYPE_MD5; + } else if (strcmp(defel->defname, "unencryptedPassword") == 0) password_type = PASSWORD_TYPE_PLAINTEXT; } diff --git a/src/backend/libpq/Makefile b/src/backend/libpq/Makefile index 1bdd8adde2..7fa2b02743 100644 --- a/src/backend/libpq/Makefile +++ b/src/backend/libpq/Makefile @@ -15,7 +15,7 @@ include $(top_builddir)/src/Makefile.global # be-fsstubs is here for historical reasons, probably belongs elsewhere OBJS = be-fsstubs.o be-secure.o auth.o crypt.o hba.o ifaddr.o pqcomm.o \ - pqformat.o pqmq.o pqsignal.o + pqformat.o pqmq.o pqsignal.o auth-scram.o ifeq ($(with_openssl),yes) OBJS += be-secure-openssl.o diff --git a/src/backend/libpq/auth-scram.c b/src/backend/libpq/auth-scram.c new file mode 100644 index 0000000000..cc4e84403f --- /dev/null +++ b/src/backend/libpq/auth-scram.c @@ -0,0 +1,1032 @@ +/*------------------------------------------------------------------------- + * + * auth-scram.c + * Server-side implementation of the SASL SCRAM-SHA-256 mechanism. + * + * See the following RFCs for more details: + * - RFC 5802: https://tools.ietf.org/html/rfc5802 + * - RFC 7677: https://tools.ietf.org/html/rfc7677 + * + * Here are some differences: + * + * - Username from the authentication exchange is not used. The client + * should send an empty string as the username. + * - Password is not processed with the SASLprep algorithm. + * - Channel binding is not supported yet. + * + * The password stored in pg_authid consists of the salt, iteration count, + * StoredKey and ServerKey. + * + * On error handling: + * + * Don't reveal user information to an unauthenticated client. We don't + * want an attacker to be able to probe whether a particular username is + * valid. In SCRAM, the server has to read the salt and iteration count + * from the user's password verifier, and send it to the client. To avoid + * revealing whether a user exists, when the client tries to authenticate + * with a username that doesn't exist, or doesn't have a valid SCRAM + * verifier in pg_authid, we create a fake salt and iteration count + * on-the-fly, and proceed with the authentication with that. In the end, + * we'll reject the attempt, as if an incorrect password was given. When + * we are performing a "mock" authentication, the 'doomed' flag in + * scram_state is set. + * + * In the error messages, avoid printing strings from the client, unless + * you check that they are pure ASCII. We don't want an unauthenticated + * attacker to be able to spam the logs with characters that are not valid + * to the encoding being used, whatever that is. We cannot avoid that in + * general, after logging in, but let's do what we can here. + * + * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/backend/libpq/auth-scram.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include + +#include "access/xlog.h" +#include "catalog/pg_authid.h" +#include "catalog/pg_control.h" +#include "common/base64.h" +#include "common/scram-common.h" +#include "common/sha2.h" +#include "libpq/auth.h" +#include "libpq/crypt.h" +#include "libpq/scram.h" +#include "miscadmin.h" +#include "utils/backend_random.h" +#include "utils/builtins.h" +#include "utils/timestamp.h" + +/* + * Status data for a SCRAM authentication exchange. This should be kept + * internal to this file. + */ +typedef enum +{ + SCRAM_AUTH_INIT, + SCRAM_AUTH_SALT_SENT, + SCRAM_AUTH_FINISHED +} scram_state_enum; + +typedef struct +{ + scram_state_enum state; + + const char *username; /* username from startup packet */ + + char *salt; /* base64-encoded */ + int iterations; + uint8 StoredKey[SCRAM_KEY_LEN]; + uint8 ServerKey[SCRAM_KEY_LEN]; + + /* Fields of the first message from client */ + char *client_first_message_bare; + char *client_username; + char *client_nonce; + + /* Fields from the last message from client */ + char *client_final_message_without_proof; + char *client_final_nonce; + char ClientProof[SCRAM_KEY_LEN]; + + /* Fields generated in the server */ + char *server_first_message; + char *server_nonce; + + /* + * If something goes wrong during the authentication, or we are performing + * a "mock" authentication (see comments at top of file), the 'doomed' + * flag is set. A reason for the failure, for the server log, is put in + * 'logdetail'. + */ + bool doomed; + char *logdetail; +} scram_state; + +static void read_client_first_message(scram_state *state, char *input); +static void read_client_final_message(scram_state *state, char *input); +static char *build_server_first_message(scram_state *state); +static char *build_server_final_message(scram_state *state); +static bool verify_client_proof(scram_state *state); +static bool verify_final_nonce(scram_state *state); +static bool parse_scram_verifier(const char *verifier, char **salt, + int *iterations, uint8 *stored_key, uint8 *server_key); +static void mock_scram_verifier(const char *username, char **salt, int *iterations, + uint8 *stored_key, uint8 *server_key); +static bool is_scram_printable(char *p); +static char *sanitize_char(char c); +static char *scram_MockSalt(const char *username); + +/* + * pg_be_scram_init + * + * Initialize a new SCRAM authentication exchange status tracker. This + * needs to be called before doing any exchange. It will be filled later + * after the beginning of the exchange with verifier data. + * + * 'username' is the provided by the client. 'shadow_pass' is the role's + * password verifier, from pg_authid.rolpassword. If 'doomed' is true, the + * authentication must fail, as if an incorrect password was given. + * 'shadow_pass' may be NULL, when 'doomed' is set. + */ +void * +pg_be_scram_init(const char *username, const char *shadow_pass, bool doomed) +{ + scram_state *state; + int password_type; + + state = (scram_state *) palloc0(sizeof(scram_state)); + state->state = SCRAM_AUTH_INIT; + state->username = username; + + /* + * Perform sanity checks on the provided password after catalog lookup. + * The authentication is bound to fail if the lookup itself failed or if + * the password stored is MD5-encrypted. Authentication is possible for + * users with a valid plain password though. + */ + + if (shadow_pass == NULL || doomed) + password_type = -1; + else + password_type = get_password_type(shadow_pass); + + if (password_type == PASSWORD_TYPE_SCRAM) + { + if (!parse_scram_verifier(shadow_pass, &state->salt, &state->iterations, + state->StoredKey, state->ServerKey)) + { + /* + * The password looked like a SCRAM verifier, but could not be + * parsed. + */ + elog(LOG, "invalid SCRAM verifier for user \"%s\"", username); + doomed = true; + } + } + else if (password_type == PASSWORD_TYPE_PLAINTEXT) + { + char *verifier; + + /* + * The password provided is in plain format, in which case a fresh + * SCRAM verifier can be generated and used for the rest of the + * processing. + */ + verifier = scram_build_verifier(username, shadow_pass, 0); + + (void) parse_scram_verifier(verifier, &state->salt, &state->iterations, + state->StoredKey, state->ServerKey); + pfree(verifier); + } + else + doomed = true; + + if (doomed) + { + /* + * We don't have a valid SCRAM verifier, nor could we generate one, or + * the caller requested us to perform a dummy authentication. + * + * The authentication is bound to fail, but to avoid revealing + * information to the attacker, go through the motions with a fake + * SCRAM verifier, and fail as if the password was incorrect. + */ + state->logdetail = psprintf(_("User \"%s\" does not have a valid SCRAM verifier."), + state->username); + mock_scram_verifier(username, &state->salt, &state->iterations, + state->StoredKey, state->ServerKey); + } + state->doomed = doomed; + + return state; +} + +/* + * Continue a SCRAM authentication exchange. + * + * The next message to send to client is saved in "output", for a length + * of "outputlen". In the case of an error, optionally store a palloc'd + * string at *logdetail that will be sent to the postmaster log (but not + * the client). + */ +int +pg_be_scram_exchange(void *opaq, char *input, int inputlen, + char **output, int *outputlen, char **logdetail) +{ + scram_state *state = (scram_state *) opaq; + int result; + + *output = NULL; + + /* + * Check that the input length agrees with the string length of the input. + * We can ignore inputlen after this. + */ + if (inputlen == 0) + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + (errmsg("malformed SCRAM message (empty message)")))); + if (inputlen != strlen(input)) + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + (errmsg("malformed SCRAM message (length mismatch)")))); + + switch (state->state) + { + case SCRAM_AUTH_INIT: + + /* + * Initialization phase. Receive the first message from client + * and be sure that it parsed correctly. Then send the challenge + * to the client. + */ + read_client_first_message(state, input); + + /* prepare message to send challenge */ + *output = build_server_first_message(state); + + state->state = SCRAM_AUTH_SALT_SENT; + result = SASL_EXCHANGE_CONTINUE; + break; + + case SCRAM_AUTH_SALT_SENT: + + /* + * Final phase for the server. Receive the response to the + * challenge previously sent, verify, and let the client know that + * everything went well (or not). + */ + read_client_final_message(state, input); + + if (!verify_final_nonce(state)) + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + (errmsg("invalid SCRAM response (nonce mismatch)")))); + + /* + * Now check the final nonce and the client proof. + * + * If we performed a "mock" authentication that we knew would fail + * from the get go, this is where we fail. + * + * NB: the order of these checks is intentional. We calculate the + * client proof even in a mock authentication, even though it's + * bound to fail, to thwart timing attacks to determine if a role + * with the given name exists or not. + */ + if (!verify_client_proof(state) || state->doomed) + { + /* + * Signal invalid-proof, although the real reason might also + * be e.g. that the password has expired, or the user doesn't + * exist. "e=other-error" might be more correct, but + * "e=invalid-proof" is more likely to give a nice error + * message to the user. + */ + *output = psprintf("e=invalid-proof"); + result = SASL_EXCHANGE_FAILURE; + break; + } + + /* Build final message for client */ + *output = build_server_final_message(state); + + /* Success! */ + result = SASL_EXCHANGE_SUCCESS; + state->state = SCRAM_AUTH_FINISHED; + break; + + default: + elog(ERROR, "invalid SCRAM exchange state"); + result = SASL_EXCHANGE_FAILURE; + } + + if (result == SASL_EXCHANGE_FAILURE && state->logdetail && logdetail) + *logdetail = state->logdetail; + + if (*output) + *outputlen = strlen(*output); + + return result; +} + +/* + * Construct a verifier string for SCRAM, stored in pg_authid.rolpassword. + * + * If iterations is 0, default number of iterations is used. The result is + * palloc'd, so caller is responsible for freeing it. + */ +char * +scram_build_verifier(const char *username, const char *password, + int iterations) +{ + uint8 keybuf[SCRAM_KEY_LEN + 1]; + char storedkey_hex[SCRAM_KEY_LEN * 2 + 1]; + char serverkey_hex[SCRAM_KEY_LEN * 2 + 1]; + char salt[SCRAM_SALT_LEN]; + char *encoded_salt; + int encoded_len; + + if (iterations <= 0) + iterations = SCRAM_ITERATIONS_DEFAULT; + + if (!pg_backend_random(salt, SCRAM_SALT_LEN)) + { + ereport(LOG, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("could not generate random salt"))); + return NULL; + } + + encoded_salt = palloc(pg_b64_enc_len(SCRAM_SALT_LEN) + 1); + encoded_len = pg_b64_encode(salt, SCRAM_SALT_LEN, encoded_salt); + encoded_salt[encoded_len] = '\0'; + + /* Calculate StoredKey, and encode it in hex */ + scram_ClientOrServerKey(password, salt, SCRAM_SALT_LEN, + iterations, SCRAM_CLIENT_KEY_NAME, keybuf); + scram_H(keybuf, SCRAM_KEY_LEN, keybuf); /* StoredKey */ + (void) hex_encode((const char *) keybuf, SCRAM_KEY_LEN, storedkey_hex); + storedkey_hex[SCRAM_KEY_LEN * 2] = '\0'; + + /* And same for ServerKey */ + scram_ClientOrServerKey(password, salt, SCRAM_SALT_LEN, iterations, + SCRAM_SERVER_KEY_NAME, keybuf); + (void) hex_encode((const char *) keybuf, SCRAM_KEY_LEN, serverkey_hex); + serverkey_hex[SCRAM_KEY_LEN * 2] = '\0'; + + return psprintf("scram-sha-256:%s:%d:%s:%s", encoded_salt, iterations, storedkey_hex, serverkey_hex); +} + + +/* + * Check if given verifier can be used for SCRAM authentication. + * + * Returns true if it is a SCRAM verifier, and false otherwise. + */ +bool +is_scram_verifier(const char *verifier) +{ + char *salt = NULL; + int iterations; + uint8 stored_key[SCRAM_KEY_LEN]; + uint8 server_key[SCRAM_KEY_LEN]; + bool result; + + result = parse_scram_verifier(verifier, &salt, &iterations, stored_key, server_key); + if (salt) + pfree(salt); + + return result; +} + + +/* + * Parse and validate format of given SCRAM verifier. + * + * Returns true if the SCRAM verifier has been parsed, and false otherwise. + */ +static bool +parse_scram_verifier(const char *verifier, char **salt, int *iterations, + uint8 *stored_key, uint8 *server_key) +{ + char *v; + char *p; + + /* + * The verifier is of form: + * + * scram-sha-256:::: + */ + if (strncmp(verifier, "scram-sha-256:", strlen("scram-sha-256:")) != 0) + return false; + + v = pstrdup(verifier + strlen("scram-sha-256:")); + + /* salt */ + if ((p = strtok(v, ":")) == NULL) + goto invalid_verifier; + *salt = pstrdup(p); + + /* iterations */ + if ((p = strtok(NULL, ":")) == NULL) + goto invalid_verifier; + errno = 0; + *iterations = strtol(p, &p, SCRAM_ITERATION_LEN); + if (*p || errno != 0) + goto invalid_verifier; + + /* storedkey */ + if ((p = strtok(NULL, ":")) == NULL) + goto invalid_verifier; + if (strlen(p) != SCRAM_KEY_LEN * 2) + goto invalid_verifier; + + hex_decode(p, SCRAM_KEY_LEN * 2, (char *) stored_key); + + /* serverkey */ + if ((p = strtok(NULL, ":")) == NULL) + goto invalid_verifier; + if (strlen(p) != SCRAM_KEY_LEN * 2) + goto invalid_verifier; + hex_decode(p, SCRAM_KEY_LEN * 2, (char *) server_key); + + pfree(v); + return true; + +invalid_verifier: + pfree(v); + return false; +} + +static void +mock_scram_verifier(const char *username, char **salt, int *iterations, + uint8 *stored_key, uint8 *server_key) +{ + char *raw_salt; + char *encoded_salt; + int encoded_len; + + /* Generate deterministic salt */ + raw_salt = scram_MockSalt(username); + + encoded_salt = (char *) palloc(pg_b64_enc_len(SCRAM_SALT_LEN) + 1); + encoded_len = pg_b64_encode(raw_salt, SCRAM_SALT_LEN, encoded_salt); + encoded_salt[encoded_len] = '\0'; + + *salt = encoded_salt; + *iterations = SCRAM_ITERATIONS_DEFAULT; + + /* StoredKey and ServerKey are not used in a doomed authentication */ + memset(stored_key, 0, SCRAM_KEY_LEN); + memset(server_key, 0, SCRAM_KEY_LEN); +} + +/* + * Read the value in a given SASL exchange message for given attribute. + */ +static char * +read_attr_value(char **input, char attr) +{ + char *begin = *input; + char *end; + + if (*begin != attr) + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + (errmsg("malformed SCRAM message (attribute '%c' expected, %s found)", + attr, sanitize_char(*begin))))); + begin++; + + if (*begin != '=') + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + (errmsg("malformed SCRAM message (expected = in attr %c)", attr)))); + begin++; + + end = begin; + while (*end && *end != ',') + end++; + + if (*end) + { + *end = '\0'; + *input = end + 1; + } + else + *input = end; + + return begin; +} + +static bool +is_scram_printable(char *p) +{ + /*------ + * Printable characters, as defined by SCRAM spec: (RFC 5802) + * + * printable = %x21-2B / %x2D-7E + * ;; Printable ASCII except ",". + * ;; Note that any "printable" is also + * ;; a valid "value". + *------ + */ + for (; *p; p++) + { + if (*p < 0x21 || *p > 0x7E || *p == 0x2C /* comma */ ) + return false; + } + return true; +} + +/* + * Convert an arbitrary byte to printable form. For error messages. + * + * If it's a printable ASCII character, print it as a single character. + * otherwise, print it in hex. + * + * The returned pointer points to a static buffer. + */ +static char * +sanitize_char(char c) +{ + static char buf[5]; + + if (c >= 0x21 && c <= 0x7E) + snprintf(buf, sizeof(buf), "'%c'", c); + else + snprintf(buf, sizeof(buf), "0x%02x", c); + return buf; +} + +/* + * Read the next attribute and value in a SASL exchange message. + * + * Returns NULL if there is attribute. + */ +static char * +read_any_attr(char **input, char *attr_p) +{ + char *begin = *input; + char *end; + char attr = *begin; + + /*------ + * attr-val = ALPHA "=" value + * ;; Generic syntax of any attribute sent + * ;; by server or client + *------ + */ + if (!((attr >= 'A' && attr <= 'Z') || + (attr >= 'a' && attr <= 'z'))) + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + (errmsg("malformed SCRAM message (attribute expected, invalid char %s found)", + sanitize_char(attr))))); + if (attr_p) + *attr_p = attr; + begin++; + + if (*begin != '=') + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + (errmsg("malformed SCRAM message (expected = in attr %c)", attr)))); + begin++; + + end = begin; + while (*end && *end != ',') + end++; + + if (*end) + { + *end = '\0'; + *input = end + 1; + } + else + *input = end; + + return begin; +} + +/* + * Read and parse the first message from client in the context of a SASL + * authentication exchange message. + * + * At this stage, any errors will be reported directly with ereport(ERROR). + */ +static void +read_client_first_message(scram_state *state, char *input) +{ + input = pstrdup(input); + + /*------ + * The syntax for the client-first-message is: (RFC 5802) + * + * saslname = 1*(value-safe-char / "=2C" / "=3D") + * ;; Conforms to . + * + * authzid = "a=" saslname + * ;; Protocol specific. + * + * cb-name = 1*(ALPHA / DIGIT / "." / "-") + * ;; See RFC 5056, Section 7. + * ;; E.g., "tls-server-end-point" or + * ;; "tls-unique". + * + * gs2-cbind-flag = ("p=" cb-name) / "n" / "y" + * ;; "n" -> client doesn't support channel binding. + * ;; "y" -> client does support channel binding + * ;; but thinks the server does not. + * ;; "p" -> client requires channel binding. + * ;; The selected channel binding follows "p=". + * + * gs2-header = gs2-cbind-flag "," [ authzid ] "," + * ;; GS2 header for SCRAM + * ;; (the actual GS2 header includes an optional + * ;; flag to indicate that the GSS mechanism is not + * ;; "standard", but since SCRAM is "standard", we + * ;; don't include that flag). + * + * username = "n=" saslname + * ;; Usernames are prepared using SASLprep. + * + * reserved-mext = "m=" 1*(value-char) + * ;; Reserved for signaling mandatory extensions. + * ;; The exact syntax will be defined in + * ;; the future. + * + * nonce = "r=" c-nonce [s-nonce] + * ;; Second part provided by server. + * + * c-nonce = printable + * + * client-first-message-bare = + * [reserved-mext ","] + * username "," nonce ["," extensions] + * + * client-first-message = + * gs2-header client-first-message-bare + * + * For example: + * n,,n=user,r=fyko+d2lbbFgONRv9qkxdawL + * + * The "n,," in the beginning means that the client doesn't support + * channel binding, and no authzid is given. "n=user" is the username. + * However, in PostgreSQL the username is sent in the startup packet, and + * the username in the SCRAM exchange is ignored. libpq always sends it + * as an empty string. The last part, "r=fyko+d2lbbFgONRv9qkxdawL" is + * the client nonce. + *------ + */ + + /* read gs2-cbind-flag */ + switch (*input) + { + case 'n': + /* Client does not support channel binding */ + input++; + break; + case 'y': + /* Client supports channel binding, but we're not doing it today */ + input++; + break; + case 'p': + + /* + * Client requires channel binding. We don't support it. + * + * RFC 5802 specifies a particular error code, + * e=server-does-support-channel-binding, for this. But it can + * only be sent in the server-final message, and we don't want to + * go through the motions of the authentication, knowing it will + * fail, just to send that error message. + */ + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("client requires SCRAM channel binding, but it is not supported"))); + default: + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + (errmsg("malformed SCRAM message (unexpected channel-binding flag %s)", + sanitize_char(*input))))); + } + if (*input != ',') + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("malformed SCRAM message (comma expected, got %s)", + sanitize_char(*input)))); + input++; + + /* + * Forbid optional authzid (authorization identity). We don't support it. + */ + if (*input == 'a') + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("client uses authorization identity, but it is not supported"))); + if (*input != ',') + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("malformed SCRAM message (unexpected attribute %s in client-first-message)", + sanitize_char(*input)))); + input++; + + state->client_first_message_bare = pstrdup(input); + + /* + * Any mandatory extensions would go here. We don't support any. + * + * RFC 5802 specifies error code "e=extensions-not-supported" for this, + * but it can only be sent in the server-final message. We prefer to fail + * immediately (which the RFC also allows). + */ + if (*input == 'm') + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("client requires mandatory SCRAM extension"))); + + /* + * Read username. Note: this is ignored. We use the username from the + * startup message instead, still it is kept around if provided as it + * proves to be useful for debugging purposes. + */ + state->client_username = read_attr_value(&input, 'n'); + + /* read nonce and check that it is made of only printable characters */ + state->client_nonce = read_attr_value(&input, 'r'); + if (!is_scram_printable(state->client_nonce)) + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("non-printable characters in SCRAM nonce"))); + + /* + * There can be any number of optional extensions after this. We don't + * support any extensions, so ignore them. + */ + while (*input != '\0') + read_any_attr(&input, NULL); + + /* success! */ +} + +/* + * Verify the final nonce contained in the last message received from + * client in an exchange. + */ +static bool +verify_final_nonce(scram_state *state) +{ + int client_nonce_len = strlen(state->client_nonce); + int server_nonce_len = strlen(state->server_nonce); + int final_nonce_len = strlen(state->client_final_nonce); + + if (final_nonce_len != client_nonce_len + server_nonce_len) + return false; + if (memcmp(state->client_final_nonce, state->client_nonce, client_nonce_len) != 0) + return false; + if (memcmp(state->client_final_nonce + client_nonce_len, state->server_nonce, server_nonce_len) != 0) + return false; + + return true; +} + +/* + * Verify the client proof contained in the last message received from + * client in an exchange. + */ +static bool +verify_client_proof(scram_state *state) +{ + uint8 ClientSignature[SCRAM_KEY_LEN]; + uint8 ClientKey[SCRAM_KEY_LEN]; + uint8 client_StoredKey[SCRAM_KEY_LEN]; + scram_HMAC_ctx ctx; + int i; + + /* calculate ClientSignature */ + scram_HMAC_init(&ctx, state->StoredKey, SCRAM_KEY_LEN); + scram_HMAC_update(&ctx, + state->client_first_message_bare, + strlen(state->client_first_message_bare)); + scram_HMAC_update(&ctx, ",", 1); + scram_HMAC_update(&ctx, + state->server_first_message, + strlen(state->server_first_message)); + scram_HMAC_update(&ctx, ",", 1); + scram_HMAC_update(&ctx, + state->client_final_message_without_proof, + strlen(state->client_final_message_without_proof)); + scram_HMAC_final(ClientSignature, &ctx); + + /* Extract the ClientKey that the client calculated from the proof */ + for (i = 0; i < SCRAM_KEY_LEN; i++) + ClientKey[i] = state->ClientProof[i] ^ ClientSignature[i]; + + /* Hash it one more time, and compare with StoredKey */ + scram_H(ClientKey, SCRAM_KEY_LEN, client_StoredKey); + + if (memcmp(client_StoredKey, state->StoredKey, SCRAM_KEY_LEN) != 0) + return false; + + return true; +} + +/* + * Build the first server-side message sent to the client in a SASL + * communication exchange. + */ +static char * +build_server_first_message(scram_state *state) +{ + /*------ + * The syntax for the server-first-message is: (RFC 5802) + * + * server-first-message = + * [reserved-mext ","] nonce "," salt "," + * iteration-count ["," extensions] + * + * nonce = "r=" c-nonce [s-nonce] + * ;; Second part provided by server. + * + * c-nonce = printable + * + * s-nonce = printable + * + * salt = "s=" base64 + * + * iteration-count = "i=" posit-number + * ;; A positive number. + * + * Example: + * + * r=fyko+d2lbbFgONRv9qkxdawL3rfcNHYJY1ZVvWVs7j,s=QSXCR+Q6sek8bf92,i=4096 + *------ + */ + + /* + * Per the spec, the nonce may consist of any printable ASCII characters. + * For convenience, however, we don't use the whole range available, + * rather, we generate some random bytes, and base64 encode them. + */ + char raw_nonce[SCRAM_RAW_NONCE_LEN]; + int encoded_len; + + if (!pg_backend_random(raw_nonce, SCRAM_RAW_NONCE_LEN)) + ereport(COMMERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("could not generate random nonce"))); + + state->server_nonce = palloc(pg_b64_enc_len(SCRAM_RAW_NONCE_LEN) + 1); + encoded_len = pg_b64_encode(raw_nonce, SCRAM_RAW_NONCE_LEN, state->server_nonce); + state->server_nonce[encoded_len] = '\0'; + + state->server_first_message = + psprintf("r=%s%s,s=%s,i=%u", + state->client_nonce, state->server_nonce, + state->salt, state->iterations); + + return state->server_first_message; +} + + +/* + * Read and parse the final message received from client. + */ +static void +read_client_final_message(scram_state *state, char *input) +{ + char attr; + char *channel_binding; + char *value; + char *begin, + *proof; + char *p; + char *client_proof; + + begin = p = pstrdup(input); + + /*------ + * The syntax for the server-first-message is: (RFC 5802) + * + * gs2-header = gs2-cbind-flag "," [ authzid ] "," + * ;; GS2 header for SCRAM + * ;; (the actual GS2 header includes an optional + * ;; flag to indicate that the GSS mechanism is not + * ;; "standard", but since SCRAM is "standard", we + * ;; don't include that flag). + * + * cbind-input = gs2-header [ cbind-data ] + * ;; cbind-data MUST be present for + * ;; gs2-cbind-flag of "p" and MUST be absent + * ;; for "y" or "n". + * + * channel-binding = "c=" base64 + * ;; base64 encoding of cbind-input. + * + * proof = "p=" base64 + * + * client-final-message-without-proof = + * channel-binding "," nonce ["," + * extensions] + * + * client-final-message = + * client-final-message-without-proof "," proof + *------ + */ + + /* + * Read channel-binding. We don't support channel binding, so it's + * expected to always be "biws", which is "n,,", base64-encoded. + */ + channel_binding = read_attr_value(&p, 'c'); + if (strcmp(channel_binding, "biws") != 0) + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + (errmsg("unexpected SCRAM channel-binding attribute in client-final-message")))); + state->client_final_nonce = read_attr_value(&p, 'r'); + + /* ignore optional extensions */ + do + { + proof = p - 1; + value = read_any_attr(&p, &attr); + } while (attr != 'p'); + + client_proof = palloc(pg_b64_dec_len(strlen(value))); + if (pg_b64_decode(value, strlen(value), client_proof) != SCRAM_KEY_LEN) + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + (errmsg("malformed SCRAM message (malformed proof in client-final-message")))); + memcpy(state->ClientProof, client_proof, SCRAM_KEY_LEN); + pfree(client_proof); + + if (*p != '\0') + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + (errmsg("malformed SCRAM message (garbage at end of client-final-message)")))); + + state->client_final_message_without_proof = palloc(proof - begin + 1); + memcpy(state->client_final_message_without_proof, input, proof - begin); + state->client_final_message_without_proof[proof - begin] = '\0'; +} + +/* + * Build the final server-side message of an exchange. + */ +static char * +build_server_final_message(scram_state *state) +{ + uint8 ServerSignature[SCRAM_KEY_LEN]; + char *server_signature_base64; + int siglen; + scram_HMAC_ctx ctx; + + /* calculate ServerSignature */ + scram_HMAC_init(&ctx, state->ServerKey, SCRAM_KEY_LEN); + scram_HMAC_update(&ctx, + state->client_first_message_bare, + strlen(state->client_first_message_bare)); + scram_HMAC_update(&ctx, ",", 1); + scram_HMAC_update(&ctx, + state->server_first_message, + strlen(state->server_first_message)); + scram_HMAC_update(&ctx, ",", 1); + scram_HMAC_update(&ctx, + state->client_final_message_without_proof, + strlen(state->client_final_message_without_proof)); + scram_HMAC_final(ServerSignature, &ctx); + + server_signature_base64 = palloc(pg_b64_enc_len(SCRAM_KEY_LEN) + 1); + siglen = pg_b64_encode((const char *) ServerSignature, + SCRAM_KEY_LEN, server_signature_base64); + server_signature_base64[siglen] = '\0'; + + /*------ + * The syntax for the server-final-message is: (RFC 5802) + * + * verifier = "v=" base64 + * ;; base-64 encoded ServerSignature. + * + * server-final-message = (server-error / verifier) + * ["," extensions] + * + *------ + */ + return psprintf("v=%s", server_signature_base64); +} + + +/* + * Determinisitcally generate salt for mock authentication, using a SHA256 + * hash based on the username and a cluster-level secret key. Returns a + * pointer to a static buffer of size SCRAM_SALT_LEN. + */ +static char * +scram_MockSalt(const char *username) +{ + pg_sha256_ctx ctx; + static uint8 sha_digest[PG_SHA256_DIGEST_LENGTH]; + char *mock_auth_nonce = GetMockAuthenticationNonce(); + + /* + * Generate salt using a SHA256 hash of the username and the cluster's + * mock authentication nonce. (This works as long as the salt length is + * not larger the SHA256 digest length. If the salt is smaller, the caller + * will just ignore the extra data)) + */ + StaticAssertStmt(PG_SHA256_DIGEST_LENGTH >= SCRAM_SALT_LEN, + "salt length greater than SHA256 digest length"); + + pg_sha256_init(&ctx); + pg_sha256_update(&ctx, (uint8 *) username, strlen(username)); + pg_sha256_update(&ctx, (uint8 *) mock_auth_nonce, MOCK_AUTH_NONCE_LEN); + pg_sha256_final(&ctx, sha_digest); + + return (char *) sha_digest; +} diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c index 824e40837b..ebf10bbbae 100644 --- a/src/backend/libpq/auth.c +++ b/src/backend/libpq/auth.c @@ -30,10 +30,12 @@ #include "libpq/crypt.h" #include "libpq/libpq.h" #include "libpq/pqformat.h" +#include "libpq/scram.h" #include "miscadmin.h" #include "replication/walsender.h" #include "storage/ipc.h" #include "utils/backend_random.h" +#include "utils/timestamp.h" /*---------------------------------------------------------------- @@ -197,6 +199,12 @@ static int pg_SSPI_make_upn(char *accountname, static int CheckRADIUSAuth(Port *port); +/*---------------------------------------------------------------- + * SASL authentication + *---------------------------------------------------------------- + */ +static int CheckSASLAuth(Port *port, char **logdetail); + /* * Maximum accepted size of GSS and SSPI authentication tokens. * @@ -212,6 +220,13 @@ static int CheckRADIUSAuth(Port *port); */ #define PG_MAX_AUTH_TOKEN_LENGTH 65535 +/* + * Maximum accepted size of SASL messages. + * + * The messages that the server or libpq generate are much smaller than this, + * but have some headroom. + */ +#define PG_MAX_SASL_MESSAGE_LENGTH 1024 /*---------------------------------------------------------------- * Global authentication functions @@ -275,6 +290,7 @@ auth_failed(Port *port, int status, char *logdetail) break; case uaPassword: case uaMD5: + case uaSASL: errstr = gettext_noop("password authentication failed for user \"%s\""); /* We use it to indicate if a .pgpass password failed. */ errcode_return = ERRCODE_INVALID_PASSWORD; @@ -542,6 +558,10 @@ ClientAuthentication(Port *port) status = CheckPasswordAuth(port, &logdetail); break; + case uaSASL: + status = CheckSASLAuth(port, &logdetail); + break; + case uaPAM: #ifdef USE_PAM status = CheckPAMAuth(port, port->user_name, ""); @@ -762,6 +782,122 @@ CheckPasswordAuth(Port *port, char **logdetail) return result; } +/*---------------------------------------------------------------- + * SASL authentication system + *---------------------------------------------------------------- + */ +static int +CheckSASLAuth(Port *port, char **logdetail) +{ + int mtype; + StringInfoData buf; + void *scram_opaq; + char *output = NULL; + int outputlen = 0; + int result; + char *shadow_pass; + bool doomed = false; + + /* + * SASL auth is not supported for protocol versions before 3, because it + * relies on the overall message length word to determine the SASL payload + * size in AuthenticationSASLContinue and PasswordMessage messages. (We + * used to have a hard rule that protocol messages must be parsable + * without relying on the length word, but we hardly care about older + * protocol version anymore.) + */ + if (PG_PROTOCOL_MAJOR(FrontendProtocol) < 3) + ereport(FATAL, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("SASL authentication is not supported in protocol version 2"))); + + /* + * Send first the authentication request to user. + */ + sendAuthRequest(port, AUTH_REQ_SASL, SCRAM_SHA256_NAME, + strlen(SCRAM_SHA256_NAME) + 1); + + /* + * If the user doesn't exist, or doesn't have a valid password, or it's + * expired, we still go through the motions of SASL authentication, but + * tell the authentication method that the authentication is "doomed". + * That is, it's going to fail, no matter what. + * + * This is because we don't want to reveal to an attacker what usernames + * are valid, nor which users have a valid password. + */ + if (get_role_password(port->user_name, &shadow_pass, logdetail) != STATUS_OK) + doomed = true; + + /* Initialize the status tracker for message exchanges */ + scram_opaq = pg_be_scram_init(port->user_name, shadow_pass, doomed); + + /* + * Loop through SASL message exchange. This exchange can consist of + * multiple messages sent in both directions. First message is always + * from the client. All messages from client to server are password + * packets (type 'p'). + */ + do + { + pq_startmsgread(); + mtype = pq_getbyte(); + if (mtype != 'p') + { + /* Only log error if client didn't disconnect. */ + if (mtype != EOF) + { + ereport(COMMERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("expected SASL response, got message type %d", + mtype))); + return STATUS_ERROR; + } + else + return STATUS_EOF; + } + + /* Get the actual SASL message */ + initStringInfo(&buf); + if (pq_getmessage(&buf, PG_MAX_SASL_MESSAGE_LENGTH)) + { + /* EOF - pq_getmessage already logged error */ + pfree(buf.data); + return STATUS_ERROR; + } + + elog(DEBUG4, "Processing received SASL token of length %d", buf.len); + + /* + * we pass 'logdetail' as NULL when doing a mock authentication, + * because we should already have a better error message in that case + */ + result = pg_be_scram_exchange(scram_opaq, buf.data, buf.len, + &output, &outputlen, + doomed ? NULL : logdetail); + + /* input buffer no longer used */ + pfree(buf.data); + + if (outputlen > 0) + { + /* + * Negotiation generated data to be sent to the client. + */ + elog(DEBUG4, "sending SASL response token of length %u", outputlen); + + sendAuthRequest(port, AUTH_REQ_SASL_CONT, output, outputlen); + } + } while (result == SASL_EXCHANGE_CONTINUE); + + /* Oops, Something bad happened */ + if (result != SASL_EXCHANGE_SUCCESS) + { + return STATUS_ERROR; + } + + return STATUS_OK; +} /*---------------------------------------------------------------- diff --git a/src/backend/libpq/crypt.c b/src/backend/libpq/crypt.c index e7dd212355..bd3e936d38 100644 --- a/src/backend/libpq/crypt.c +++ b/src/backend/libpq/crypt.c @@ -21,6 +21,7 @@ #include "catalog/pg_authid.h" #include "common/md5.h" #include "libpq/crypt.h" +#include "libpq/scram.h" #include "miscadmin.h" #include "utils/builtins.h" #include "utils/syscache.h" @@ -111,6 +112,8 @@ get_password_type(const char *shadow_pass) { if (strncmp(shadow_pass, "md5", 3) == 0 && strlen(shadow_pass) == MD5_PASSWD_LEN) return PASSWORD_TYPE_MD5; + if (strncmp(shadow_pass, "scram-sha-256:", strlen("scram-sha-256:")) == 0) + return PASSWORD_TYPE_SCRAM; return PASSWORD_TYPE_PLAINTEXT; } @@ -150,7 +153,29 @@ encrypt_password(PasswordType target_type, const char *role, elog(ERROR, "password encryption failed"); return encrypted_password; + case PASSWORD_TYPE_SCRAM: + + /* + * cannot convert a SCRAM verifier to an MD5 hash, so fall + * through to save the SCRAM verifier instead. + */ + case PASSWORD_TYPE_MD5: + return pstrdup(password); + } + + case PASSWORD_TYPE_SCRAM: + switch (guessed_type) + { + case PASSWORD_TYPE_PLAINTEXT: + return scram_build_verifier(role, password, 0); + case PASSWORD_TYPE_MD5: + + /* + * cannot convert an MD5 hash to a SCRAM verifier, so fall + * through to save the MD5 hash instead. + */ + case PASSWORD_TYPE_SCRAM: return pstrdup(password); } } @@ -160,7 +185,7 @@ encrypt_password(PasswordType target_type, const char *role, * handle every combination of source and target password types. */ elog(ERROR, "cannot encrypt password to requested type"); - return NULL; /* keep compiler quiet */ + return NULL; /* keep compiler quiet */ } /* diff --git a/src/backend/libpq/hba.c b/src/backend/libpq/hba.c index 323bfa858d..3817d249c4 100644 --- a/src/backend/libpq/hba.c +++ b/src/backend/libpq/hba.c @@ -125,6 +125,7 @@ static const char *const UserAuthName[] = "ident", "password", "md5", + "scram", "gss", "sspi", "pam", @@ -1323,6 +1324,8 @@ parse_hba_line(TokenizedLine *tok_line, int elevel) } parsedline->auth_method = uaMD5; } + else if (strcmp(token->string, "scram") == 0) + parsedline->auth_method = uaSASL; else if (strcmp(token->string, "pam") == 0) #ifdef USE_PAM parsedline->auth_method = uaPAM; diff --git a/src/backend/libpq/pg_hba.conf.sample b/src/backend/libpq/pg_hba.conf.sample index e0fbfcb026..73f7973ea2 100644 --- a/src/backend/libpq/pg_hba.conf.sample +++ b/src/backend/libpq/pg_hba.conf.sample @@ -42,10 +42,10 @@ # or "samenet" to match any address in any subnet that the server is # directly connected to. # -# METHOD can be "trust", "reject", "md5", "password", "gss", "sspi", -# "ident", "peer", "pam", "ldap", "radius" or "cert". Note that -# "password" sends passwords in clear text; "md5" is preferred since -# it sends encrypted passwords. +# METHOD can be "trust", "reject", "md5", "password", "scram", "gss", +# "sspi", "ident", "peer", "pam", "ldap", "radius" or "cert". Note that +# "password" sends passwords in clear text; "md5" or "scram" are preferred +# since they send encrypted passwords. # # OPTIONS are a set of options for the authentication in the format # NAME=VALUE. The available options depend on the different diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 0707f66631..f8b073d8a9 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -409,6 +409,7 @@ static const struct config_enum_entry force_parallel_mode_options[] = { static const struct config_enum_entry password_encryption_options[] = { {"plain", PASSWORD_TYPE_PLAINTEXT, false}, {"md5", PASSWORD_TYPE_MD5, false}, + {"scram", PASSWORD_TYPE_SCRAM, false}, {"off", PASSWORD_TYPE_PLAINTEXT, false}, {"on", PASSWORD_TYPE_MD5, false}, {"true", PASSWORD_TYPE_MD5, true}, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 157d775853..891b16e483 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -84,7 +84,7 @@ #ssl_key_file = 'server.key' #ssl_ca_file = '' #ssl_crl_file = '' -#password_encryption = md5 # md5 or plain +#password_encryption = md5 # md5, scram or plain #db_user_namespace = off #row_security = on diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index 1ed0d20504..4968fc783e 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -75,7 +75,7 @@ extern const char *select_default_timezone(const char *share_path); static const char *const auth_methods_host[] = { - "trust", "reject", "md5", "password", "ident", "radius", + "trust", "reject", "md5", "password", "scram", "ident", "radius", #ifdef ENABLE_GSS "gss", #endif @@ -97,7 +97,7 @@ static const char *const auth_methods_host[] = { NULL }; static const char *const auth_methods_local[] = { - "trust", "reject", "md5", "password", "peer", "radius", + "trust", "reject", "md5", "scram", "password", "peer", "radius", #ifdef USE_PAM "pam", "pam ", #endif @@ -1128,6 +1128,14 @@ setup_config(void) "#update_process_title = off"); #endif + if (strcmp(authmethodlocal, "scram") == 0 || + strcmp(authmethodhost, "scram") == 0) + { + conflines = replace_token(conflines, + "#password_encryption = md5", + "password_encryption = scram"); + } + snprintf(path, sizeof(path), "%s/postgresql.conf", pg_data); writefile(path, conflines); @@ -2307,14 +2315,17 @@ static void check_need_password(const char *authmethodlocal, const char *authmethodhost) { if ((strcmp(authmethodlocal, "md5") == 0 || - strcmp(authmethodlocal, "password") == 0) && + strcmp(authmethodlocal, "password") == 0 || + strcmp(authmethodlocal, "scram") == 0) && (strcmp(authmethodhost, "md5") == 0 || - strcmp(authmethodhost, "password") == 0) && + strcmp(authmethodhost, "password") == 0 || + strcmp(authmethodlocal, "scram") == 0) && !(pwprompt || pwfilename)) { fprintf(stderr, _("%s: must specify a password for the superuser to enable %s authentication\n"), progname, (strcmp(authmethodlocal, "md5") == 0 || - strcmp(authmethodlocal, "password") == 0) + strcmp(authmethodlocal, "password") == 0 || + strcmp(authmethodlocal, "scram") == 0) ? authmethodlocal : authmethodhost); exit(1); diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c index f47171d29d..2ea893179a 100644 --- a/src/bin/pg_controldata/pg_controldata.c +++ b/src/bin/pg_controldata/pg_controldata.c @@ -92,11 +92,13 @@ main(int argc, char *argv[]) char pgctime_str[128]; char ckpttime_str[128]; char sysident_str[32]; + char mock_auth_nonce_str[MOCK_AUTH_NONCE_LEN * 2 + 1]; const char *strftime_fmt = "%c"; const char *progname; XLogSegNo segno; char xlogfilename[MAXFNAMELEN]; int c; + int i; set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_controldata")); @@ -186,11 +188,15 @@ main(int argc, char *argv[]) XLogFileName(xlogfilename, ControlFile->checkPointCopy.ThisTimeLineID, segno); /* - * Format system_identifier separately to keep platform-dependent format - * code out of the translatable message string. + * Format system_identifier and mock_authentication_nonce separately to + * keep platform-dependent format code out of the translatable message + * string. */ snprintf(sysident_str, sizeof(sysident_str), UINT64_FORMAT, ControlFile->system_identifier); + for (i = 0; i < MOCK_AUTH_NONCE_LEN; i++) + snprintf(&mock_auth_nonce_str[i * 2], 3, "%02x", + (unsigned char) ControlFile->mock_authentication_nonce[i]); printf(_("pg_control version number: %u\n"), ControlFile->pg_control_version); @@ -302,5 +308,7 @@ main(int argc, char *argv[]) (ControlFile->float8ByVal ? _("by value") : _("by reference"))); printf(_("Data page checksum version: %u\n"), ControlFile->data_checksum_version); + printf(_("Mock authentication nonce: %s\n"), + mock_auth_nonce_str); return 0; } diff --git a/src/common/Makefile b/src/common/Makefile index 5ddfff8b44..971ddd5ea7 100644 --- a/src/common/Makefile +++ b/src/common/Makefile @@ -40,9 +40,9 @@ override CPPFLAGS += -DVAL_LDFLAGS_EX="\"$(LDFLAGS_EX)\"" override CPPFLAGS += -DVAL_LDFLAGS_SL="\"$(LDFLAGS_SL)\"" override CPPFLAGS += -DVAL_LIBS="\"$(LIBS)\"" -OBJS_COMMON = config_info.o controldata_utils.o exec.o ip.o keywords.o \ - md5.o pg_lzcompress.o pgfnames.o psprintf.o relpath.o rmtree.o \ - string.o username.o wait_error.o +OBJS_COMMON = base64.o config_info.o controldata_utils.o exec.o ip.o \ + keywords.o md5.o pg_lzcompress.o pgfnames.o psprintf.o relpath.o \ + rmtree.o scram-common.o string.o username.o wait_error.o ifeq ($(with_openssl),yes) OBJS_COMMON += sha2_openssl.o diff --git a/src/common/base64.c b/src/common/base64.c new file mode 100644 index 0000000000..96406b191d --- /dev/null +++ b/src/common/base64.c @@ -0,0 +1,199 @@ +/*------------------------------------------------------------------------- + * + * base64.c + * Encoding and decoding routines for base64 without whitespace. + * + * Copyright (c) 2001-2016, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/common/base64.c + * + *------------------------------------------------------------------------- + */ + +#ifndef FRONTEND +#include "postgres.h" +#else +#include "postgres_fe.h" +#endif + +#include "common/base64.h" + +/* + * BASE64 + */ + +static const char _base64[] = +"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +static const int8 b64lookup[128] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1, + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, + -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1, +}; + +/* + * pg_b64_encode + * + * Encode into base64 the given string. Returns the length of the encoded + * string. + */ +int +pg_b64_encode(const char *src, int len, char *dst) +{ + char *p; + const char *s, + *end = src + len; + int pos = 2; + uint32 buf = 0; + + s = src; + p = dst; + + while (s < end) + { + buf |= (unsigned char) *s << (pos << 3); + pos--; + s++; + + /* write it out */ + if (pos < 0) + { + *p++ = _base64[(buf >> 18) & 0x3f]; + *p++ = _base64[(buf >> 12) & 0x3f]; + *p++ = _base64[(buf >> 6) & 0x3f]; + *p++ = _base64[buf & 0x3f]; + + pos = 2; + buf = 0; + } + } + if (pos != 2) + { + *p++ = _base64[(buf >> 18) & 0x3f]; + *p++ = _base64[(buf >> 12) & 0x3f]; + *p++ = (pos == 0) ? _base64[(buf >> 6) & 0x3f] : '='; + *p++ = '='; + } + + return p - dst; +} + +/* + * pg_b64_decode + * + * Decode the given base64 string. Returns the length of the decoded + * string on success, and -1 in the event of an error. + */ +int +pg_b64_decode(const char *src, int len, char *dst) +{ + const char *srcend = src + len, + *s = src; + char *p = dst; + char c; + int b = 0; + uint32 buf = 0; + int pos = 0, + end = 0; + + while (s < srcend) + { + c = *s++; + + /* Leave if a whitespace is found */ + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') + return -1; + + if (c == '=') + { + /* end sequence */ + if (!end) + { + if (pos == 2) + end = 1; + else if (pos == 3) + end = 2; + else + { + /* + * Unexpected "=" character found while decoding base64 + * sequence. + */ + return -1; + } + } + b = 0; + } + else + { + b = -1; + if (c > 0 && c < 127) + b = b64lookup[(unsigned char) c]; + if (b < 0) + { + /* invalid symbol found */ + return -1; + } + } + /* add it to buffer */ + buf = (buf << 6) + b; + pos++; + if (pos == 4) + { + *p++ = (buf >> 16) & 255; + if (end == 0 || end > 1) + *p++ = (buf >> 8) & 255; + if (end == 0 || end > 2) + *p++ = buf & 255; + buf = 0; + pos = 0; + } + } + + if (pos != 0) + { + /* + * base64 end sequence is invalid. Input data is missing padding, is + * truncated or is otherwise corrupted. + */ + return -1; + } + + return p - dst; +} + +/* + * pg_b64_enc_len + * + * Returns to caller the length of the string if it were encoded with + * base64 based on the length provided by caller. This is useful to + * estimate how large a buffer allocation needs to be done before doing + * the actual encoding. + */ +int +pg_b64_enc_len(int srclen) +{ + /* 3 bytes will be converted to 4 */ + return (srclen + 2) * 4 / 3; +} + +/* + * pg_b64_dec_len + * + * Returns to caller the length of the string if it were to be decoded + * with base64, based on the length given by caller. This is useful to + * estimate how large a buffer allocation needs to be done before doing + * the actual decoding. + */ +int +pg_b64_dec_len(int srclen) +{ + return (srclen * 3) >> 2; +} diff --git a/src/common/scram-common.c b/src/common/scram-common.c new file mode 100644 index 0000000000..4a1c3809cf --- /dev/null +++ b/src/common/scram-common.c @@ -0,0 +1,196 @@ +/*------------------------------------------------------------------------- + * scram-common.c + * Shared frontend/backend code for SCRAM authentication + * + * This contains the common low-level functions needed in both frontend and + * backend, for implement the Salted Challenge Response Authentication + * Mechanism (SCRAM), per IETF's RFC 5802. + * + * Portions Copyright (c) 2016, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/common/scram-common.c + * + *------------------------------------------------------------------------- + */ +#ifndef FRONTEND +#include "postgres.h" +#include "utils/memutils.h" +#else +#include "postgres_fe.h" +#endif + +#include "common/scram-common.h" + +#define HMAC_IPAD 0x36 +#define HMAC_OPAD 0x5C + +/* + * Calculate HMAC per RFC2104. + * + * The hash function used is SHA-256. + */ +void +scram_HMAC_init(scram_HMAC_ctx *ctx, const uint8 *key, int keylen) +{ + uint8 k_ipad[SHA256_HMAC_B]; + int i; + uint8 keybuf[SCRAM_KEY_LEN]; + + /* + * If the key is longer than the block size (64 bytes for SHA-256), pass + * it through SHA-256 once to shrink it down. + */ + if (keylen > SHA256_HMAC_B) + { + pg_sha256_ctx sha256_ctx; + + pg_sha256_init(&sha256_ctx); + pg_sha256_update(&sha256_ctx, key, keylen); + pg_sha256_final(&sha256_ctx, keybuf); + key = keybuf; + keylen = SCRAM_KEY_LEN; + } + + memset(k_ipad, HMAC_IPAD, SHA256_HMAC_B); + memset(ctx->k_opad, HMAC_OPAD, SHA256_HMAC_B); + + for (i = 0; i < keylen; i++) + { + k_ipad[i] ^= key[i]; + ctx->k_opad[i] ^= key[i]; + } + + /* tmp = H(K XOR ipad, text) */ + pg_sha256_init(&ctx->sha256ctx); + pg_sha256_update(&ctx->sha256ctx, k_ipad, SHA256_HMAC_B); +} + +/* + * Update HMAC calculation + * The hash function used is SHA-256. + */ +void +scram_HMAC_update(scram_HMAC_ctx *ctx, const char *str, int slen) +{ + pg_sha256_update(&ctx->sha256ctx, (const uint8 *) str, slen); +} + +/* + * Finalize HMAC calculation. + * The hash function used is SHA-256. + */ +void +scram_HMAC_final(uint8 *result, scram_HMAC_ctx *ctx) +{ + uint8 h[SCRAM_KEY_LEN]; + + pg_sha256_final(&ctx->sha256ctx, h); + + /* H(K XOR opad, tmp) */ + pg_sha256_init(&ctx->sha256ctx); + pg_sha256_update(&ctx->sha256ctx, ctx->k_opad, SHA256_HMAC_B); + pg_sha256_update(&ctx->sha256ctx, h, SCRAM_KEY_LEN); + pg_sha256_final(&ctx->sha256ctx, result); +} + +/* + * Iterate hash calculation of HMAC entry using given salt. + * scram_Hi() is essentially PBKDF2 (see RFC2898) with HMAC() as the + * pseudorandom function. + */ +static void +scram_Hi(const char *str, const char *salt, int saltlen, int iterations, uint8 *result) +{ + int str_len = strlen(str); + uint32 one = htonl(1); + int i, + j; + uint8 Ui[SCRAM_KEY_LEN]; + uint8 Ui_prev[SCRAM_KEY_LEN]; + scram_HMAC_ctx hmac_ctx; + + /* First iteration */ + scram_HMAC_init(&hmac_ctx, (uint8 *) str, str_len); + scram_HMAC_update(&hmac_ctx, salt, saltlen); + scram_HMAC_update(&hmac_ctx, (char *) &one, sizeof(uint32)); + scram_HMAC_final(Ui_prev, &hmac_ctx); + memcpy(result, Ui_prev, SCRAM_KEY_LEN); + + /* Subsequent iterations */ + for (i = 2; i <= iterations; i++) + { + scram_HMAC_init(&hmac_ctx, (uint8 *) str, str_len); + scram_HMAC_update(&hmac_ctx, (const char *) Ui_prev, SCRAM_KEY_LEN); + scram_HMAC_final(Ui, &hmac_ctx); + for (j = 0; j < SCRAM_KEY_LEN; j++) + result[j] ^= Ui[j]; + memcpy(Ui_prev, Ui, SCRAM_KEY_LEN); + } +} + + +/* + * Calculate SHA-256 hash for a NULL-terminated string. (The NULL terminator is + * not included in the hash). + */ +void +scram_H(const uint8 *input, int len, uint8 *result) +{ + pg_sha256_ctx ctx; + + pg_sha256_init(&ctx); + pg_sha256_update(&ctx, input, len); + pg_sha256_final(&ctx, result); +} + +/* + * Normalize a password for SCRAM authentication. + */ +static void +scram_Normalize(const char *password, char *result) +{ + /* + * XXX: Here SASLprep should be applied on password. However, per RFC5802, + * it is required that the password is encoded in UTF-8, something that is + * not guaranteed in this protocol. We may want to revisit this + * normalization function once encoding functions are available as well in + * the frontend in order to be able to encode properly this string, and + * then apply SASLprep on it. + */ + memcpy(result, password, strlen(password) + 1); +} + +/* + * Encrypt password for SCRAM authentication. This basically applies the + * normalization of the password and a hash calculation using the salt + * value given by caller. + */ +static void +scram_SaltedPassword(const char *password, const char *salt, int saltlen, int iterations, + uint8 *result) +{ + char *pwbuf; + + pwbuf = (char *) malloc(strlen(password) + 1); + scram_Normalize(password, pwbuf); + scram_Hi(pwbuf, salt, saltlen, iterations, result); + free(pwbuf); +} + +/* + * Calculate ClientKey or ServerKey. + */ +void +scram_ClientOrServerKey(const char *password, + const char *salt, int saltlen, int iterations, + const char *keystr, uint8 *result) +{ + uint8 keybuf[SCRAM_KEY_LEN]; + scram_HMAC_ctx ctx; + + scram_SaltedPassword(password, salt, saltlen, iterations, keybuf); + scram_HMAC_init(&ctx, keybuf, SCRAM_KEY_LEN); + scram_HMAC_update(&ctx, keystr, strlen(keystr)); + scram_HMAC_final(result, &ctx); +} diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index 9f036c72d8..104ee7dd5e 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -256,6 +256,7 @@ extern char *XLogFileNameP(TimeLineID tli, XLogSegNo segno); extern void UpdateControlFile(void); extern uint64 GetSystemIdentifier(void); +extern char *GetMockAuthenticationNonce(void); extern bool DataChecksumsEnabled(void); extern XLogRecPtr GetFakeLSNForUnloggedRel(void); extern Size XLOGShmemSize(void); diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h index e4194b9de1..3a25cc84b2 100644 --- a/src/include/catalog/pg_control.h +++ b/src/include/catalog/pg_control.h @@ -20,8 +20,10 @@ #include "port/pg_crc32c.h" +#define MOCK_AUTH_NONCE_LEN 32 + /* Version identifier for this pg_control format */ -#define PG_CONTROL_VERSION 1001 +#define PG_CONTROL_VERSION 1002 /* * Body of CheckPoint XLOG records. This is declared here because we keep @@ -222,6 +224,13 @@ typedef struct ControlFileData /* Are data pages protected by checksums? Zero if no checksum version */ uint32 data_checksum_version; + /* + * Random nonce, used in authentication requests that need to proceed + * based on values that are cluster-unique, like a SASL exchange that + * failed at an early stage. + */ + char mock_authentication_nonce[MOCK_AUTH_NONCE_LEN]; + /* CRC of all above ... MUST BE LAST! */ pg_crc32c crc; } ControlFileData; diff --git a/src/include/common/base64.h b/src/include/common/base64.h new file mode 100644 index 0000000000..47c28c3b62 --- /dev/null +++ b/src/include/common/base64.h @@ -0,0 +1,19 @@ +/* + * base64.h + * Encoding and decoding routines for base64 without whitespace + * support. + * + * Portions Copyright (c) 2001-2016, PostgreSQL Global Development Group + * + * src/include/common/base64.h + */ +#ifndef BASE64_H +#define BASE64_H + +/* base 64 */ +extern int pg_b64_encode(const char *src, int len, char *dst); +extern int pg_b64_decode(const char *src, int len, char *dst); +extern int pg_b64_enc_len(int srclen); +extern int pg_b64_dec_len(int srclen); + +#endif /* BASE64_H */ diff --git a/src/include/common/scram-common.h b/src/include/common/scram-common.h new file mode 100644 index 0000000000..14bb053832 --- /dev/null +++ b/src/include/common/scram-common.h @@ -0,0 +1,62 @@ +/*------------------------------------------------------------------------- + * + * scram-common.h + * Declarations for helper functions used for SCRAM authentication + * + * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/common/relpath.h + * + *------------------------------------------------------------------------- + */ +#ifndef SCRAM_COMMON_H +#define SCRAM_COMMON_H + +#include "common/sha2.h" + +/* Length of SCRAM keys (client and server) */ +#define SCRAM_KEY_LEN PG_SHA256_DIGEST_LENGTH + +/* length of HMAC */ +#define SHA256_HMAC_B PG_SHA256_BLOCK_LENGTH + +/* + * Size of random nonce generated in the authentication exchange. This + * is in "raw" number of bytes, the actual nonces sent over the wire are + * encoded using only ASCII-printable characters. + */ +#define SCRAM_RAW_NONCE_LEN 10 + +/* length of salt when generating new verifiers */ +#define SCRAM_SALT_LEN 10 + +/* number of bytes used when sending iteration number during exchange */ +#define SCRAM_ITERATION_LEN 10 + +/* default number of iterations when generating verifier */ +#define SCRAM_ITERATIONS_DEFAULT 4096 + +/* Base name of keys used for proof generation */ +#define SCRAM_SERVER_KEY_NAME "Server Key" +#define SCRAM_CLIENT_KEY_NAME "Client Key" + +/* + * Context data for HMAC used in SCRAM authentication. + */ +typedef struct +{ + pg_sha256_ctx sha256ctx; + uint8 k_opad[SHA256_HMAC_B]; +} scram_HMAC_ctx; + +extern void scram_HMAC_init(scram_HMAC_ctx *ctx, const uint8 *key, int keylen); +extern void scram_HMAC_update(scram_HMAC_ctx *ctx, const char *str, int slen); +extern void scram_HMAC_final(uint8 *result, scram_HMAC_ctx *ctx); + +extern void scram_H(const uint8 *str, int len, uint8 *result); +extern void scram_ClientOrServerKey(const char *password, const char *salt, + int saltlen, int iterations, + const char *keystr, uint8 *result); + +#endif /* SCRAM_COMMON_H */ diff --git a/src/include/libpq/crypt.h b/src/include/libpq/crypt.h index f94bc6339b..0502d6a0e5 100644 --- a/src/include/libpq/crypt.h +++ b/src/include/libpq/crypt.h @@ -24,7 +24,8 @@ typedef enum PasswordType { PASSWORD_TYPE_PLAINTEXT = 0, - PASSWORD_TYPE_MD5 + PASSWORD_TYPE_MD5, + PASSWORD_TYPE_SCRAM } PasswordType; extern PasswordType get_password_type(const char *shadow_pass); diff --git a/src/include/libpq/hba.h b/src/include/libpq/hba.h index 748a072854..8f55edb16a 100644 --- a/src/include/libpq/hba.h +++ b/src/include/libpq/hba.h @@ -30,6 +30,7 @@ typedef enum UserAuth uaIdent, uaPassword, uaMD5, + uaSASL, uaGSS, uaSSPI, uaPAM, diff --git a/src/include/libpq/pqcomm.h b/src/include/libpq/pqcomm.h index de9ccc63b1..5441aaa93a 100644 --- a/src/include/libpq/pqcomm.h +++ b/src/include/libpq/pqcomm.h @@ -172,6 +172,8 @@ extern bool Db_user_namespace; #define AUTH_REQ_GSS 7 /* GSSAPI without wrap() */ #define AUTH_REQ_GSS_CONT 8 /* Continue GSS exchanges */ #define AUTH_REQ_SSPI 9 /* SSPI negotiate without wrap() */ +#define AUTH_REQ_SASL 10 /* SASL */ +#define AUTH_REQ_SASL_CONT 11 /* continue SASL exchange */ typedef uint32 AuthRequest; diff --git a/src/include/libpq/scram.h b/src/include/libpq/scram.h new file mode 100644 index 0000000000..563462fb1c --- /dev/null +++ b/src/include/libpq/scram.h @@ -0,0 +1,35 @@ +/*------------------------------------------------------------------------- + * + * scram.h + * Interface to libpq/scram.c + * + * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/libpq/scram.h + * + *------------------------------------------------------------------------- + */ +#ifndef PG_SCRAM_H +#define PG_SCRAM_H + +/* Name of SCRAM-SHA-256 per IANA */ +#define SCRAM_SHA256_NAME "SCRAM-SHA-256" + +/* Status codes for message exchange */ +#define SASL_EXCHANGE_CONTINUE 0 +#define SASL_EXCHANGE_SUCCESS 1 +#define SASL_EXCHANGE_FAILURE 2 + +/* Routines dedicated to authentication */ +extern void *pg_be_scram_init(const char *username, const char *shadow_pass, bool doomed); +extern int pg_be_scram_exchange(void *opaq, char *input, int inputlen, + char **output, int *outputlen, char **logdetail); + +/* Routines to handle and check SCRAM-SHA-256 verifier */ +extern char *scram_build_verifier(const char *username, + const char *password, + int iterations); +extern bool is_scram_verifier(const char *verifier); + +#endif /* PG_SCRAM_H */ diff --git a/src/interfaces/libpq/.gitignore b/src/interfaces/libpq/.gitignore index cb96af7176..2224ada731 100644 --- a/src/interfaces/libpq/.gitignore +++ b/src/interfaces/libpq/.gitignore @@ -1,4 +1,5 @@ /exports.list +/base64.c /chklocale.c /crypt.c /getaddrinfo.c @@ -7,8 +8,12 @@ /inet_net_ntop.c /noblock.c /open.c +/pg_strong_random.c /pgstrcasecmp.c /pqsignal.c +/scram-common.c +/sha2.c +/sha2_openssl.c /snprintf.c /strerror.c /strlcpy.c diff --git a/src/interfaces/libpq/Makefile b/src/interfaces/libpq/Makefile index 4b1e552d16..792232db49 100644 --- a/src/interfaces/libpq/Makefile +++ b/src/interfaces/libpq/Makefile @@ -31,7 +31,7 @@ LIBS := $(LIBS:-lpgport=) # We can't use Makefile variables here because the MSVC build system scrapes # OBJS from this file. -OBJS= fe-auth.o fe-connect.o fe-exec.o fe-misc.o fe-print.o fe-lobj.o \ +OBJS= fe-auth.o fe-auth-scram.o fe-connect.o fe-exec.o fe-misc.o fe-print.o fe-lobj.o \ fe-protocol2.o fe-protocol3.o pqexpbuffer.o fe-secure.o \ libpq-events.o # libpgport C files we always use @@ -39,13 +39,22 @@ OBJS += chklocale.o inet_net_ntop.o noblock.o pgstrcasecmp.o pqsignal.o \ thread.o # libpgport C files that are needed if identified by configure OBJS += $(filter crypt.o getaddrinfo.o getpeereid.o inet_aton.o open.o system.o snprintf.o strerror.o strlcpy.o win32error.o win32setlocale.o, $(LIBOBJS)) + +ifeq ($(enable_strong_random), yes) +OBJS += pg_strong_random.o +else +OBJS += erand48.o +endif + # src/backend/utils/mb OBJS += encnames.o wchar.o # src/common -OBJS += ip.o md5.o +OBJS += base64.o ip.o md5.o scram-common.o ifeq ($(with_openssl),yes) -OBJS += fe-secure-openssl.o +OBJS += fe-secure-openssl.o sha2_openssl.o +else +OBJS += sha2.o endif ifeq ($(PORTNAME), cygwin) @@ -93,7 +102,7 @@ backend_src = $(top_srcdir)/src/backend # For some libpgport modules, this only happens if configure decides # the module is needed (see filter hack in OBJS, above). -chklocale.c crypt.c getaddrinfo.c getpeereid.c inet_aton.c inet_net_ntop.c noblock.c open.c system.c pgsleep.c pgstrcasecmp.c pqsignal.c snprintf.c strerror.c strlcpy.c thread.c win32error.c win32setlocale.c: % : $(top_srcdir)/src/port/% +chklocale.c crypt.c erand48.c getaddrinfo.c getpeereid.c inet_aton.c inet_net_ntop.c noblock.c open.c system.c pgsleep.c pg_strong_random.c pgstrcasecmp.c pqsignal.c snprintf.c strerror.c strlcpy.c thread.c win32error.c win32setlocale.c: % : $(top_srcdir)/src/port/% rm -f $@ && $(LN_S) $< . ip.c md5.c: % : $(top_srcdir)/src/common/% @@ -102,6 +111,9 @@ ip.c md5.c: % : $(top_srcdir)/src/common/% encnames.c wchar.c: % : $(backend_src)/utils/mb/% rm -f $@ && $(LN_S) $< . +base64.c scram-common.c sha2.c sha2_openssl.c: % : $(top_srcdir)/src/common/% + rm -f $@ && $(LN_S) $< . + distprep: libpq-dist.rc diff --git a/src/interfaces/libpq/fe-auth-scram.c b/src/interfaces/libpq/fe-auth-scram.c new file mode 100644 index 0000000000..14331f8d61 --- /dev/null +++ b/src/interfaces/libpq/fe-auth-scram.c @@ -0,0 +1,640 @@ +/*------------------------------------------------------------------------- + * + * fe-auth-scram.c + * The front-end (client) implementation of SCRAM authentication. + * + * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/interfaces/libpq/fe-auth-scram.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres_fe.h" + +#include "common/base64.h" +#include "common/scram-common.h" +#include "fe-auth.h" + +/* These are needed for getpid(), in the fallback implementation */ +#ifndef HAVE_STRONG_RANDOM +#include +#include +#endif + +/* + * Status of exchange messages used for SCRAM authentication via the + * SASL protocol. + */ +typedef enum +{ + FE_SCRAM_INIT, + FE_SCRAM_NONCE_SENT, + FE_SCRAM_PROOF_SENT, + FE_SCRAM_FINISHED +} fe_scram_state_enum; + +typedef struct +{ + fe_scram_state_enum state; + + /* These are supplied by the user */ + const char *username; + const char *password; + + /* We construct these */ + char *client_nonce; + char *client_first_message_bare; + char *client_final_message_without_proof; + + /* These come from the server-first message */ + char *server_first_message; + char *salt; + int saltlen; + int iterations; + char *nonce; + + /* These come from the server-final message */ + char *server_final_message; + char ServerProof[SCRAM_KEY_LEN]; +} fe_scram_state; + +static bool read_server_first_message(fe_scram_state *state, char *input, + PQExpBuffer errormessage); +static bool read_server_final_message(fe_scram_state *state, char *input, + PQExpBuffer errormessage); +static char *build_client_first_message(fe_scram_state *state, + PQExpBuffer errormessage); +static char *build_client_final_message(fe_scram_state *state, + PQExpBuffer errormessage); +static bool verify_server_proof(fe_scram_state *state); +static void calculate_client_proof(fe_scram_state *state, + const char *client_final_message_without_proof, + uint8 *result); +static bool pg_frontend_random(char *dst, int len); + +/* + * Initialize SCRAM exchange status. + */ +void * +pg_fe_scram_init(const char *username, const char *password) +{ + fe_scram_state *state; + + state = (fe_scram_state *) malloc(sizeof(fe_scram_state)); + if (!state) + return NULL; + memset(state, 0, sizeof(fe_scram_state)); + state->state = FE_SCRAM_INIT; + state->username = username; + state->password = password; + + return state; +} + +/* + * Free SCRAM exchange status + */ +void +pg_fe_scram_free(void *opaq) +{ + fe_scram_state *state = (fe_scram_state *) opaq; + + /* client messages */ + if (state->client_nonce) + free(state->client_nonce); + if (state->client_first_message_bare) + free(state->client_first_message_bare); + if (state->client_final_message_without_proof) + free(state->client_final_message_without_proof); + + /* first message from server */ + if (state->server_first_message) + free(state->server_first_message); + if (state->salt) + free(state->salt); + if (state->nonce) + free(state->nonce); + + /* final message from server */ + if (state->server_final_message) + free(state->server_final_message); + + free(state); +} + +/* + * Exchange a SCRAM message with backend. + */ +void +pg_fe_scram_exchange(void *opaq, char *input, int inputlen, + char **output, int *outputlen, + bool *done, bool *success, PQExpBuffer errorMessage) +{ + fe_scram_state *state = (fe_scram_state *) opaq; + + *done = false; + *success = false; + *output = NULL; + *outputlen = 0; + + /* + * Check that the input length agrees with the string length of the input. + * We can ignore inputlen after this. + */ + if (state->state != FE_SCRAM_INIT) + { + if (inputlen == 0) + { + printfPQExpBuffer(errorMessage, + libpq_gettext("malformed SCRAM message (empty message)\n")); + goto error; + } + if (inputlen != strlen(input)) + { + printfPQExpBuffer(errorMessage, + libpq_gettext("malformed SCRAM message (length mismatch)\n")); + goto error; + } + } + + switch (state->state) + { + case FE_SCRAM_INIT: + /* Begin the SCRAM handshake, by sending client nonce */ + *output = build_client_first_message(state, errorMessage); + if (*output == NULL) + goto error; + + *outputlen = strlen(*output); + *done = false; + state->state = FE_SCRAM_NONCE_SENT; + break; + + case FE_SCRAM_NONCE_SENT: + /* Receive salt and server nonce, send response. */ + if (!read_server_first_message(state, input, errorMessage)) + goto error; + + *output = build_client_final_message(state, errorMessage); + if (*output == NULL) + goto error; + + *outputlen = strlen(*output); + *done = false; + state->state = FE_SCRAM_PROOF_SENT; + break; + + case FE_SCRAM_PROOF_SENT: + /* Receive server proof */ + if (!read_server_final_message(state, input, errorMessage)) + goto error; + + /* + * Verify server proof, to make sure we're talking to the genuine + * server. XXX: A fake server could simply not require + * authentication, though. There is currently no option in libpq + * to reject a connection, if SCRAM authentication did not happen. + */ + if (verify_server_proof(state)) + *success = true; + else + { + *success = false; + printfPQExpBuffer(errorMessage, + libpq_gettext("invalid server proof\n")); + } + *done = true; + state->state = FE_SCRAM_FINISHED; + break; + + default: + /* shouldn't happen */ + printfPQExpBuffer(errorMessage, + libpq_gettext("invalid SCRAM exchange state\n")); + goto error; + } + return; + +error: + *done = true; + *success = false; + return; +} + +/* + * Read value for an attribute part of a SASL message. + */ +static char * +read_attr_value(char **input, char attr, PQExpBuffer errorMessage) +{ + char *begin = *input; + char *end; + + if (*begin != attr) + { + printfPQExpBuffer(errorMessage, + libpq_gettext("malformed SCRAM message (%c expected)\n"), + attr); + return NULL; + } + begin++; + + if (*begin != '=') + { + printfPQExpBuffer(errorMessage, + libpq_gettext("malformed SCRAM message (expected = in attr '%c')\n"), + attr); + return NULL; + } + begin++; + + end = begin; + while (*end && *end != ',') + end++; + + if (*end) + { + *end = '\0'; + *input = end + 1; + } + else + *input = end; + + return begin; +} + +/* + * Build the first exchange message sent by the client. + */ +static char * +build_client_first_message(fe_scram_state *state, PQExpBuffer errormessage) +{ + char raw_nonce[SCRAM_RAW_NONCE_LEN + 1]; + char *buf; + char buflen; + int encoded_len; + + /* + * Generate a "raw" nonce. This is converted to ASCII-printable form by + * base64-encoding it. + */ + if (!pg_frontend_random(raw_nonce, SCRAM_RAW_NONCE_LEN)) + { + printfPQExpBuffer(errormessage, + libpq_gettext("failed to generate nonce\n")); + return NULL; + } + + state->client_nonce = malloc(pg_b64_enc_len(SCRAM_RAW_NONCE_LEN) + 1); + if (state->client_nonce == NULL) + { + printfPQExpBuffer(errormessage, + libpq_gettext("out of memory\n")); + return NULL; + } + encoded_len = pg_b64_encode(raw_nonce, SCRAM_RAW_NONCE_LEN, state->client_nonce); + state->client_nonce[encoded_len] = '\0'; + + /* + * Generate message. The username is left empty as the backend uses the + * value provided by the startup packet. Also, as this username is not + * prepared with SASLprep, the message parsing would fail if it includes + * '=' or ',' characters. + */ + buflen = 8 + strlen(state->client_nonce) + 1; + buf = malloc(buflen); + if (buf == NULL) + { + printfPQExpBuffer(errormessage, + libpq_gettext("out of memory\n")); + return NULL; + } + snprintf(buf, buflen, "n,,n=,r=%s", state->client_nonce); + + state->client_first_message_bare = strdup(buf + 3); + if (!state->client_first_message_bare) + { + free(buf); + printfPQExpBuffer(errormessage, + libpq_gettext("out of memory\n")); + return NULL; + } + + return buf; +} + +/* + * Build the final exchange message sent from the client. + */ +static char * +build_client_final_message(fe_scram_state *state, PQExpBuffer errormessage) +{ + PQExpBufferData buf; + uint8 client_proof[SCRAM_KEY_LEN]; + char *result; + + initPQExpBuffer(&buf); + + /* + * Construct client-final-message-without-proof. We need to remember it + * for verifying the server proof in the final step of authentication. + */ + appendPQExpBuffer(&buf, "c=biws,r=%s", state->nonce); + if (PQExpBufferDataBroken(buf)) + goto oom_error; + + state->client_final_message_without_proof = strdup(buf.data); + if (state->client_final_message_without_proof == NULL) + goto oom_error; + + /* Append proof to it, to form client-final-message. */ + calculate_client_proof(state, + state->client_final_message_without_proof, + client_proof); + + appendPQExpBuffer(&buf, ",p="); + if (!enlargePQExpBuffer(&buf, pg_b64_enc_len(SCRAM_KEY_LEN))) + goto oom_error; + buf.len += pg_b64_encode((char *) client_proof, + SCRAM_KEY_LEN, + buf.data + buf.len); + buf.data[buf.len] = '\0'; + + result = strdup(buf.data); + if (result == NULL) + goto oom_error; + + termPQExpBuffer(&buf); + return result; + +oom_error: + termPQExpBuffer(&buf); + printfPQExpBuffer(errormessage, + libpq_gettext("out of memory\n")); + return NULL; +} + +/* + * Read the first exchange message coming from the server. + */ +static bool +read_server_first_message(fe_scram_state *state, char *input, + PQExpBuffer errormessage) +{ + char *iterations_str; + char *endptr; + char *encoded_salt; + char *nonce; + + state->server_first_message = strdup(input); + if (state->server_first_message == NULL) + { + printfPQExpBuffer(errormessage, + libpq_gettext("out of memory\n")); + return false; + } + + /* parse the message */ + nonce = read_attr_value(&input, 'r', errormessage); + if (nonce == NULL) + { + /* read_attr_value() has generated an error string */ + return false; + } + + /* Verify immediately that the server used our part of the nonce */ + if (strncmp(nonce, state->client_nonce, strlen(state->client_nonce)) != 0) + { + printfPQExpBuffer(errormessage, + libpq_gettext("invalid SCRAM response (nonce mismatch)\n")); + return false; + } + + state->nonce = strdup(nonce); + if (state->nonce == NULL) + { + printfPQExpBuffer(errormessage, + libpq_gettext("out of memory\n")); + return false; + } + + encoded_salt = read_attr_value(&input, 's', errormessage); + if (encoded_salt == NULL) + { + /* read_attr_value() has generated an error string */ + return false; + } + state->salt = malloc(pg_b64_dec_len(strlen(encoded_salt))); + if (state->salt == NULL) + { + printfPQExpBuffer(errormessage, + libpq_gettext("out of memory\n")); + return false; + } + state->saltlen = pg_b64_decode(encoded_salt, + strlen(encoded_salt), + state->salt); + + iterations_str = read_attr_value(&input, 'i', errormessage); + if (iterations_str == NULL) + { + /* read_attr_value() has generated an error string */ + return false; + } + state->iterations = strtol(iterations_str, &endptr, SCRAM_ITERATION_LEN); + if (*endptr != '\0' || state->iterations < 1) + { + printfPQExpBuffer(errormessage, + libpq_gettext("malformed SCRAM message (invalid iteration count)\n")); + return false; + } + + if (*input != '\0') + printfPQExpBuffer(errormessage, + libpq_gettext("malformed SCRAM message (garbage at end of server-first-message)\n")); + + return true; +} + +/* + * Read the final exchange message coming from the server. + */ +static bool +read_server_final_message(fe_scram_state *state, + char *input, + PQExpBuffer errormessage) +{ + char *encoded_server_proof; + int server_proof_len; + + state->server_final_message = strdup(input); + if (!state->server_final_message) + { + printfPQExpBuffer(errormessage, + libpq_gettext("out of memory\n")); + return false; + } + + /* Check for error result. */ + if (*input == 'e') + { + char *errmsg = read_attr_value(&input, 'e', errormessage); + + printfPQExpBuffer(errormessage, + libpq_gettext("error received from server in SASL exchange: %s\n"), + errmsg); + return false; + } + + /* Parse the message. */ + encoded_server_proof = read_attr_value(&input, 'v', errormessage); + if (encoded_server_proof == NULL) + { + /* read_attr_value() has generated an error message */ + return false; + } + + if (*input != '\0') + printfPQExpBuffer(errormessage, + libpq_gettext("malformed SCRAM message (garbage at end of server-final-message)\n")); + + server_proof_len = pg_b64_decode(encoded_server_proof, + strlen(encoded_server_proof), + state->ServerProof); + if (server_proof_len != SCRAM_KEY_LEN) + { + printfPQExpBuffer(errormessage, + libpq_gettext("malformed SCRAM message (invalid server proof)\n")); + return false; + } + + return true; +} + +/* + * Calculate the client proof, part of the final exchange message sent + * by the client. + */ +static void +calculate_client_proof(fe_scram_state *state, + const char *client_final_message_without_proof, + uint8 *result) +{ + uint8 StoredKey[SCRAM_KEY_LEN]; + uint8 ClientKey[SCRAM_KEY_LEN]; + uint8 ClientSignature[SCRAM_KEY_LEN]; + int i; + scram_HMAC_ctx ctx; + + scram_ClientOrServerKey(state->password, state->salt, state->saltlen, + state->iterations, SCRAM_CLIENT_KEY_NAME, ClientKey); + scram_H(ClientKey, SCRAM_KEY_LEN, StoredKey); + + scram_HMAC_init(&ctx, StoredKey, SCRAM_KEY_LEN); + scram_HMAC_update(&ctx, + state->client_first_message_bare, + strlen(state->client_first_message_bare)); + scram_HMAC_update(&ctx, ",", 1); + scram_HMAC_update(&ctx, + state->server_first_message, + strlen(state->server_first_message)); + scram_HMAC_update(&ctx, ",", 1); + scram_HMAC_update(&ctx, + client_final_message_without_proof, + strlen(client_final_message_without_proof)); + scram_HMAC_final(ClientSignature, &ctx); + + for (i = 0; i < SCRAM_KEY_LEN; i++) + result[i] = ClientKey[i] ^ ClientSignature[i]; +} + +/* + * Validate the server proof, received as part of the final exchange message + * received from the server. + */ +static bool +verify_server_proof(fe_scram_state *state) +{ + uint8 ServerSignature[SCRAM_KEY_LEN]; + uint8 ServerKey[SCRAM_KEY_LEN]; + scram_HMAC_ctx ctx; + + scram_ClientOrServerKey(state->password, state->salt, state->saltlen, + state->iterations, SCRAM_SERVER_KEY_NAME, + ServerKey); + + /* calculate ServerSignature */ + scram_HMAC_init(&ctx, ServerKey, SCRAM_KEY_LEN); + scram_HMAC_update(&ctx, + state->client_first_message_bare, + strlen(state->client_first_message_bare)); + scram_HMAC_update(&ctx, ",", 1); + scram_HMAC_update(&ctx, + state->server_first_message, + strlen(state->server_first_message)); + scram_HMAC_update(&ctx, ",", 1); + scram_HMAC_update(&ctx, + state->client_final_message_without_proof, + strlen(state->client_final_message_without_proof)); + scram_HMAC_final(ServerSignature, &ctx); + + if (memcmp(ServerSignature, state->ServerProof, SCRAM_KEY_LEN) != 0) + return false; + + return true; +} + +/* + * Random number generator. + */ +static bool +pg_frontend_random(char *dst, int len) +{ +#ifdef HAVE_STRONG_RANDOM + return pg_strong_random(dst, len); +#else + int i; + char *end = dst + len; + + static unsigned short seed[3]; + static int mypid = 0; + + pglock_thread(); + + if (mypid != getpid()) + { + struct timeval now; + + gettimeofday(&now, NULL); + + seed[0] = now.tv_sec ^ getpid(); + seed[1] = (unsigned short) (now.tv_usec); + seed[2] = (unsigned short) (now.tv_usec >> 16); + } + + for (i = 0; dst < end; i++) + { + uint32 r; + int j; + + /* + * pg_jrand48 returns a 32-bit integer. Fill the next 4 bytes from + * it. + */ + r = (uint32) pg_jrand48(seed); + + for (j = 0; j < 4 && dst < end; j++) + { + *(dst++) = (char) (r & 0xFF); + r >>= 8; + } + } + + pgunlock_thread(); + + return true; +#endif +} diff --git a/src/interfaces/libpq/fe-auth.c b/src/interfaces/libpq/fe-auth.c index b47a16e3d0..c69260b522 100644 --- a/src/interfaces/libpq/fe-auth.c +++ b/src/interfaces/libpq/fe-auth.c @@ -40,6 +40,7 @@ #include "common/md5.h" #include "libpq-fe.h" +#include "libpq/scram.h" #include "fe-auth.h" @@ -432,6 +433,87 @@ pg_SSPI_startup(PGconn *conn, int use_negotiate) } #endif /* ENABLE_SSPI */ +/* + * Initialize SASL authentication exchange. + */ +static bool +pg_SASL_init(PGconn *conn, const char *auth_mechanism) +{ + /* + * Check the authentication mechanism (only SCRAM-SHA-256 is supported at + * the moment.) + */ + if (strcmp(auth_mechanism, SCRAM_SHA256_NAME) == 0) + { + char *password = conn->connhost[conn->whichhost].password; + + if (password == NULL) + password = conn->pgpass; + conn->password_needed = true; + if (password == NULL || password == '\0') + { + printfPQExpBuffer(&conn->errorMessage, + PQnoPasswordSupplied); + return STATUS_ERROR; + } + + conn->sasl_state = pg_fe_scram_init(conn->pguser, password); + if (!conn->sasl_state) + { + printfPQExpBuffer(&conn->errorMessage, + libpq_gettext("out of memory\n")); + return STATUS_ERROR; + } + + return STATUS_OK; + } + else + { + printfPQExpBuffer(&conn->errorMessage, + libpq_gettext("SASL authentication mechanism %s not supported\n"), + (char *) conn->auth_req_inbuf); + return STATUS_ERROR; + } +} + +/* + * Exchange a message for SASL communication protocol with the backend. + * This should be used after calling pg_SASL_init to set up the status of + * the protocol. + */ +static int +pg_SASL_exchange(PGconn *conn) +{ + char *output; + int outputlen; + bool done; + bool success; + int res; + + pg_fe_scram_exchange(conn->sasl_state, + conn->auth_req_inbuf, conn->auth_req_inlen, + &output, &outputlen, + &done, &success, &conn->errorMessage); + if (outputlen != 0) + { + /* + * Send the SASL response to the server. We don't care if it's the + * first or subsequent packet, just send the same kind of password + * packet. + */ + res = pqPacketSend(conn, 'p', output, outputlen); + free(output); + + if (res != STATUS_OK) + return STATUS_ERROR; + } + + if (done && !success) + return STATUS_ERROR; + + return STATUS_OK; +} + /* * Respond to AUTH_REQ_SCM_CREDS challenge. * @@ -707,6 +789,36 @@ pg_fe_sendauth(AuthRequest areq, PGconn *conn) break; } + case AUTH_REQ_SASL: + + /* + * The request contains the name (as assigned by IANA) of the + * authentication mechanism. + */ + if (pg_SASL_init(conn, conn->auth_req_inbuf) != STATUS_OK) + { + /* pg_SASL_init already set the error message */ + return STATUS_ERROR; + } + /* fall through */ + + case AUTH_REQ_SASL_CONT: + if (conn->sasl_state == NULL) + { + printfPQExpBuffer(&conn->errorMessage, + "fe_sendauth: invalid authentication request from server: AUTH_REQ_SASL_CONT without AUTH_REQ_SASL\n"); + return STATUS_ERROR; + } + if (pg_SASL_exchange(conn) != STATUS_OK) + { + /* Use error message, if set already */ + if (conn->errorMessage.len == 0) + printfPQExpBuffer(&conn->errorMessage, + "fe_sendauth: error in SASL authentication\n"); + return STATUS_ERROR; + } + break; + case AUTH_REQ_SCM_CREDS: if (pg_local_sendauth(conn) != STATUS_OK) return STATUS_ERROR; diff --git a/src/interfaces/libpq/fe-auth.h b/src/interfaces/libpq/fe-auth.h index 0aa6a28768..204790cea4 100644 --- a/src/interfaces/libpq/fe-auth.h +++ b/src/interfaces/libpq/fe-auth.h @@ -18,7 +18,15 @@ #include "libpq-int.h" +/* Prototypes for functions in fe-auth.c */ extern int pg_fe_sendauth(AuthRequest areq, PGconn *conn); extern char *pg_fe_getauthname(PQExpBuffer errorMessage); +/* Prototypes for functions in fe-auth-scram.c */ +extern void *pg_fe_scram_init(const char *username, const char *password); +extern void pg_fe_scram_free(void *opaq); +extern void pg_fe_scram_exchange(void *opaq, char *input, int inputlen, + char **output, int *outputlen, + bool *done, bool *success, PQExpBuffer errorMessage); + #endif /* FE_AUTH_H */ diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index 685f355ab3..c1814f5fe4 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -2720,6 +2720,49 @@ keep_going: /* We will come back to here until there is } } #endif + /* Get additional payload for SASL, if any */ + if ((areq == AUTH_REQ_SASL || + areq == AUTH_REQ_SASL_CONT) && + msgLength > 4) + { + int llen = msgLength - 4; + + /* + * We can be called repeatedly for the same buffer. Avoid + * re-allocating the buffer in this case - just re-use the + * old buffer. + */ + if (llen != conn->auth_req_inlen) + { + if (conn->auth_req_inbuf) + { + free(conn->auth_req_inbuf); + conn->auth_req_inbuf = NULL; + } + + conn->auth_req_inlen = llen; + conn->auth_req_inbuf = malloc(llen + 1); + if (!conn->auth_req_inbuf) + { + printfPQExpBuffer(&conn->errorMessage, + libpq_gettext("out of memory allocating SASL buffer (%d)"), + llen); + goto error_return; + } + } + + if (pqGetnchar(conn->auth_req_inbuf, llen, conn)) + { + /* We'll come back when there is more data. */ + return PGRES_POLLING_READING; + } + + /* + * For safety and convenience, always ensure the in-buffer + * is NULL-terminated. + */ + conn->auth_req_inbuf[llen] = '\0'; + } /* * OK, we successfully read the message; mark data consumed @@ -3506,6 +3549,15 @@ closePGconn(PGconn *conn) conn->sspictx = NULL; } #endif + if (conn->sasl_state) + { + /* + * XXX: if support for more authentication mechanisms is added, this + * needs to call the right 'free' function. + */ + pg_fe_scram_free(conn->sasl_state); + conn->sasl_state = NULL; + } } /* diff --git a/src/interfaces/libpq/libpq-int.h b/src/interfaces/libpq/libpq-int.h index 24242da221..360956d6eb 100644 --- a/src/interfaces/libpq/libpq-int.h +++ b/src/interfaces/libpq/libpq-int.h @@ -453,7 +453,12 @@ struct pg_conn PGresult *result; /* result being constructed */ PGresult *next_result; /* next result (used in single-row mode) */ - /* Assorted state for SSL, GSS, etc */ + /* Buffer to hold incoming authentication request data */ + char *auth_req_inbuf; + int auth_req_inlen; + + /* Assorted state for SASL, SSL, GSS, etc */ + void *sasl_state; #ifdef USE_SSL bool allow_ssl_try; /* Allowed to try SSL negotiation */ diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index 2824711767..12f73f344c 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -110,9 +110,9 @@ sub mkvcbuild } our @pgcommonallfiles = qw( - config_info.c controldata_utils.c exec.c ip.c keywords.c + base64.c config_info.c controldata_utils.c exec.c ip.c keywords.c md5.c pg_lzcompress.c pgfnames.c psprintf.c relpath.c rmtree.c - string.c username.c wait_error.c); + scram-common.c string.c username.c wait_error.c); if ($solution->{options}->{openssl}) { @@ -233,10 +233,16 @@ sub mkvcbuild $libpq->AddReference($libpgport); # The OBJS scraper doesn't know about ifdefs, so remove fe-secure-openssl.c - # if building without OpenSSL + # and sha2_openssl.c if building without OpenSSL, and remove sha2.c if + # building with OpenSSL. if (!$solution->{options}->{openssl}) { $libpq->RemoveFile('src/interfaces/libpq/fe-secure-openssl.c'); + $libpq->RemoveFile('src/common/sha2_openssl.c'); + } + else + { + $libpq->RemoveFile('src/common/sha2.c'); } my $libpqwalreceiver = -- cgit v1.2.3 From 9a83d56b38c870ce47b7651385ff2add583bf136 Mon Sep 17 00:00:00 2001 From: Simon Riggs Date: Tue, 7 Mar 2017 22:00:54 +0800 Subject: Allow pg_dumpall to dump roles w/o user passwords Add new option --no-role-passwords which dumps roles without passwords. Since we don’t need passwords, we choose to use pg_roles in preference to pg_authid since access may be restricted for security reasons in some configrations. Robins Tharakan and Simon Riggs --- doc/src/sgml/ref/pg_dumpall.sgml | 13 ++++ src/bin/pg_dump/pg_dumpall.c | 132 +++++++++++++++++++++++++-------------- 2 files changed, 97 insertions(+), 48 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml index 97168a0815..afbadce247 100644 --- a/doc/src/sgml/ref/pg_dumpall.sgml +++ b/doc/src/sgml/ref/pg_dumpall.sgml @@ -332,6 +332,19 @@ PostgreSQL documentation + + + + + Do not dump passwords for roles. When restored, roles will have a NULL + password and authentication will always fail until the password is reset. + Since password values aren't needed when this option is specified we + use the catalog view pg_roles in preference to pg_authid, since access + to pg_authid may be restricted by security policy. + + + + diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index f4b4d7bd9b..81ed924b9f 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -74,8 +74,13 @@ static int no_tablespaces = 0; static int use_setsessauth = 0; static int no_security_labels = 0; static int no_unlogged_table_data = 0; +static int no_role_passwords = 0; static int server_version; +static char role_catalog[10]; +#define PG_AUTHID "pg_authid" +#define PG_ROLES "pg_roles " + static FILE *OPF; static char *filename = NULL; @@ -123,6 +128,7 @@ main(int argc, char *argv[]) {"use-set-session-authorization", no_argument, &use_setsessauth, 1}, {"no-security-labels", no_argument, &no_security_labels, 1}, {"no-unlogged-table-data", no_argument, &no_unlogged_table_data, 1}, + {"no-role-passwords", no_argument, &no_role_passwords, 1}, {NULL, 0, NULL, 0} }; @@ -342,6 +348,25 @@ main(int argc, char *argv[]) exit_nicely(1); } + if (no_role_passwords && binary_upgrade) + { + fprintf(stderr, _("%s: options --no-role-passwords and --binary-upgrade cannot be used together\n"), + progname); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), + progname); + exit_nicely(1); + } + + /* + * If password values are not required in the dump, switch to + * using pg_roles which is equally useful, just more likely + * to have unrestricted access than pg_authid. + */ + if (no_role_passwords) + sprintf(role_catalog, "%s", PG_ROLES); + else + sprintf(role_catalog, "%s", PG_AUTHID); + /* Add long options to the pg_dump argument list */ if (binary_upgrade) appendPQExpBufferStr(pgdumpopts, " --binary-upgrade"); @@ -563,6 +588,7 @@ help(void) printf(_(" --no-security-labels do not dump security label assignments\n")); printf(_(" --no-tablespaces do not dump tablespace assignments\n")); printf(_(" --no-unlogged-table-data do not dump unlogged table data\n")); + printf(_(" --no-role-passwords do not dump passwords for roles\n")); printf(_(" --quote-all-identifiers quote all identifiers, even if not key words\n")); printf(_(" --use-set-session-authorization\n" " use SET SESSION AUTHORIZATION commands instead of\n" @@ -590,23 +616,24 @@ help(void) static void dropRoles(PGconn *conn) { + PQExpBuffer buf = createPQExpBuffer(); PGresult *res; int i_rolname; int i; if (server_version >= 90600) - res = executeQuery(conn, + printfPQExpBuffer(buf, "SELECT rolname " - "FROM pg_authid " + "FROM %s " "WHERE rolname !~ '^pg_' " - "ORDER BY 1"); + "ORDER BY 1", role_catalog); else if (server_version >= 80100) - res = executeQuery(conn, + printfPQExpBuffer(buf, "SELECT rolname " - "FROM pg_authid " - "ORDER BY 1"); + "FROM %s " + "ORDER BY 1", role_catalog); else - res = executeQuery(conn, + printfPQExpBuffer(buf, "SELECT usename as rolname " "FROM pg_shadow " "UNION " @@ -614,6 +641,8 @@ dropRoles(PGconn *conn) "FROM pg_group " "ORDER BY 1"); + res = executeQuery(conn, buf->data); + i_rolname = PQfnumber(res, "rolname"); if (PQntuples(res) > 0) @@ -631,6 +660,7 @@ dropRoles(PGconn *conn) } PQclear(res); + destroyPQExpBuffer(buf); fprintf(OPF, "\n\n"); } @@ -666,21 +696,21 @@ dumpRoles(PGconn *conn) "rolcreaterole, rolcreatedb, " "rolcanlogin, rolconnlimit, rolpassword, " "rolvaliduntil, rolreplication, rolbypassrls, " - "pg_catalog.shobj_description(oid, 'pg_authid') as rolcomment, " + "pg_catalog.shobj_description(oid, '%s') as rolcomment, " "rolname = current_user AS is_current_user " - "FROM pg_authid " + "FROM %s " "WHERE rolname !~ '^pg_' " - "ORDER BY 2"); + "ORDER BY 2", role_catalog, role_catalog); else if (server_version >= 90500) printfPQExpBuffer(buf, "SELECT oid, rolname, rolsuper, rolinherit, " "rolcreaterole, rolcreatedb, " "rolcanlogin, rolconnlimit, rolpassword, " "rolvaliduntil, rolreplication, rolbypassrls, " - "pg_catalog.shobj_description(oid, 'pg_authid') as rolcomment, " + "pg_catalog.shobj_description(oid, '%s') as rolcomment, " "rolname = current_user AS is_current_user " - "FROM pg_authid " - "ORDER BY 2"); + "FROM %s " + "ORDER BY 2", role_catalog, role_catalog); else if (server_version >= 90100) printfPQExpBuffer(buf, "SELECT oid, rolname, rolsuper, rolinherit, " @@ -688,10 +718,10 @@ dumpRoles(PGconn *conn) "rolcanlogin, rolconnlimit, rolpassword, " "rolvaliduntil, rolreplication, " "false as rolbypassrls, " - "pg_catalog.shobj_description(oid, 'pg_authid') as rolcomment, " + "pg_catalog.shobj_description(oid, '%s') as rolcomment, " "rolname = current_user AS is_current_user " - "FROM pg_authid " - "ORDER BY 2"); + "FROM %s " + "ORDER BY 2", role_catalog, role_catalog); else if (server_version >= 80200) printfPQExpBuffer(buf, "SELECT oid, rolname, rolsuper, rolinherit, " @@ -699,10 +729,10 @@ dumpRoles(PGconn *conn) "rolcanlogin, rolconnlimit, rolpassword, " "rolvaliduntil, false as rolreplication, " "false as rolbypassrls, " - "pg_catalog.shobj_description(oid, 'pg_authid') as rolcomment, " + "pg_catalog.shobj_description(oid, '%s') as rolcomment, " "rolname = current_user AS is_current_user " - "FROM pg_authid " - "ORDER BY 2"); + "FROM %s " + "ORDER BY 2", role_catalog, role_catalog); else if (server_version >= 80100) printfPQExpBuffer(buf, "SELECT oid, rolname, rolsuper, rolinherit, " @@ -712,8 +742,8 @@ dumpRoles(PGconn *conn) "false as rolbypassrls, " "null as rolcomment, " "rolname = current_user AS is_current_user " - "FROM pg_authid " - "ORDER BY 2"); + "FROM %s " + "ORDER BY 2", role_catalog); else printfPQExpBuffer(buf, "SELECT 0 as oid, usename as rolname, " @@ -846,7 +876,8 @@ dumpRoles(PGconn *conn) appendPQExpBuffer(buf, " CONNECTION LIMIT %s", PQgetvalue(res, i, i_rolconnlimit)); - if (!PQgetisnull(res, i, i_rolpassword)) + + if (!PQgetisnull(res, i, i_rolpassword) && !no_role_passwords) { appendPQExpBufferStr(buf, " PASSWORD "); appendStringLiteralConn(buf, PQgetvalue(res, i, i_rolpassword), conn); @@ -897,19 +928,21 @@ dumpRoles(PGconn *conn) static void dumpRoleMembership(PGconn *conn) { + PQExpBuffer buf = createPQExpBuffer(); PGresult *res; int i; - res = executeQuery(conn, "SELECT ur.rolname AS roleid, " + printfPQExpBuffer(buf, "SELECT ur.rolname AS roleid, " "um.rolname AS member, " "a.admin_option, " "ug.rolname AS grantor " "FROM pg_auth_members a " - "LEFT JOIN pg_authid ur on ur.oid = a.roleid " - "LEFT JOIN pg_authid um on um.oid = a.member " - "LEFT JOIN pg_authid ug on ug.oid = a.grantor " + "LEFT JOIN %s ur on ur.oid = a.roleid " + "LEFT JOIN %s um on um.oid = a.member " + "LEFT JOIN %s ug on ug.oid = a.grantor " "WHERE NOT (ur.rolname ~ '^pg_' AND um.rolname ~ '^pg_')" - "ORDER BY 1,2,3"); + "ORDER BY 1,2,3", role_catalog, role_catalog, role_catalog); + res = executeQuery(conn, buf->data); if (PQntuples(res) > 0) fprintf(OPF, "--\n-- Role memberships\n--\n\n"); @@ -939,6 +972,7 @@ dumpRoleMembership(PGconn *conn) } PQclear(res); + destroyPQExpBuffer(buf); fprintf(OPF, "\n\n"); } @@ -1298,9 +1332,9 @@ dumpCreateDB(PGconn *conn) * databases. */ if (server_version >= 90600) - res = executeQuery(conn, + printfPQExpBuffer(buf, "SELECT datname, " - "coalesce(rolname, (select rolname from pg_authid where oid=(select datdba from pg_database where datname='template0'))), " + "coalesce(rolname, (select rolname from %s where oid=(select datdba from pg_database where datname='template0'))), " "pg_encoding_to_char(d.encoding), " "datcollate, datctype, datfrozenxid, datminmxid, " "datistemplate, " @@ -1314,43 +1348,43 @@ dumpCreateDB(PGconn *conn) "AS rdatacl, " "datconnlimit, " "(SELECT spcname FROM pg_tablespace t WHERE t.oid = d.dattablespace) AS dattablespace " - "FROM pg_database d LEFT JOIN pg_authid u ON (datdba = u.oid) " - "WHERE datallowconn ORDER BY 1"); + "FROM pg_database d LEFT JOIN %s u ON (datdba = u.oid) " + "WHERE datallowconn ORDER BY 1", role_catalog, role_catalog); else if (server_version >= 90300) - res = executeQuery(conn, + printfPQExpBuffer(buf, "SELECT datname, " - "coalesce(rolname, (select rolname from pg_authid where oid=(select datdba from pg_database where datname='template0'))), " + "coalesce(rolname, (select rolname from %s where oid=(select datdba from pg_database where datname='template0'))), " "pg_encoding_to_char(d.encoding), " "datcollate, datctype, datfrozenxid, datminmxid, " "datistemplate, datacl, '' as rdatacl, " "datconnlimit, " "(SELECT spcname FROM pg_tablespace t WHERE t.oid = d.dattablespace) AS dattablespace " - "FROM pg_database d LEFT JOIN pg_authid u ON (datdba = u.oid) " - "WHERE datallowconn ORDER BY 1"); + "FROM pg_database d LEFT JOIN %s u ON (datdba = u.oid) " + "WHERE datallowconn ORDER BY 1", role_catalog, role_catalog); else if (server_version >= 80400) - res = executeQuery(conn, + printfPQExpBuffer(buf, "SELECT datname, " - "coalesce(rolname, (select rolname from pg_authid where oid=(select datdba from pg_database where datname='template0'))), " + "coalesce(rolname, (select rolname from %s where oid=(select datdba from pg_database where datname='template0'))), " "pg_encoding_to_char(d.encoding), " "datcollate, datctype, datfrozenxid, 0 AS datminmxid, " "datistemplate, datacl, '' as rdatacl, " "datconnlimit, " "(SELECT spcname FROM pg_tablespace t WHERE t.oid = d.dattablespace) AS dattablespace " - "FROM pg_database d LEFT JOIN pg_authid u ON (datdba = u.oid) " - "WHERE datallowconn ORDER BY 1"); + "FROM pg_database d LEFT JOIN %s u ON (datdba = u.oid) " + "WHERE datallowconn ORDER BY 1", role_catalog, role_catalog); else if (server_version >= 80100) - res = executeQuery(conn, + printfPQExpBuffer(buf, "SELECT datname, " - "coalesce(rolname, (select rolname from pg_authid where oid=(select datdba from pg_database where datname='template0'))), " + "coalesce(rolname, (select rolname from %s where oid=(select datdba from pg_database where datname='template0'))), " "pg_encoding_to_char(d.encoding), " "null::text AS datcollate, null::text AS datctype, datfrozenxid, 0 AS datminmxid, " "datistemplate, datacl, '' as rdatacl, " "datconnlimit, " "(SELECT spcname FROM pg_tablespace t WHERE t.oid = d.dattablespace) AS dattablespace " - "FROM pg_database d LEFT JOIN pg_authid u ON (datdba = u.oid) " - "WHERE datallowconn ORDER BY 1"); + "FROM pg_database d LEFT JOIN %s u ON (datdba = u.oid) " + "WHERE datallowconn ORDER BY 1", role_catalog, role_catalog); else - res = executeQuery(conn, + printfPQExpBuffer(buf, "SELECT datname, " "coalesce(usename, (select usename from pg_shadow where usesysid=(select datdba from pg_database where datname='template0'))), " "pg_encoding_to_char(d.encoding), " @@ -1361,6 +1395,8 @@ dumpCreateDB(PGconn *conn) "FROM pg_database d LEFT JOIN pg_shadow u ON (datdba = usesysid) " "WHERE datallowconn ORDER BY 1"); + res = executeQuery(conn, buf->data); + for (i = 0; i < PQntuples(res); i++) { char *dbname = PQgetvalue(res, i, 0); @@ -1557,9 +1593,9 @@ dumpUserConfig(PGconn *conn, const char *username) if (server_version >= 90000) printfPQExpBuffer(buf, "SELECT setconfig[%d] FROM pg_db_role_setting WHERE " "setdatabase = 0 AND setrole = " - "(SELECT oid FROM pg_authid WHERE rolname = ", count); + "(SELECT oid FROM %s WHERE rolname = ", count, role_catalog); else if (server_version >= 80100) - printfPQExpBuffer(buf, "SELECT rolconfig[%d] FROM pg_authid WHERE rolname = ", count); + printfPQExpBuffer(buf, "SELECT rolconfig[%d] FROM %s WHERE rolname = ", count, role_catalog); else printfPQExpBuffer(buf, "SELECT useconfig[%d] FROM pg_shadow WHERE usename = ", count); appendStringLiteralConn(buf, username, conn); @@ -1597,8 +1633,8 @@ dumpDbRoleConfig(PGconn *conn) int i; printfPQExpBuffer(buf, "SELECT rolname, datname, unnest(setconfig) " - "FROM pg_db_role_setting, pg_authid, pg_database " - "WHERE setrole = pg_authid.oid AND setdatabase = pg_database.oid"); + "FROM pg_db_role_setting, %s u, pg_database " + "WHERE setrole = u.oid AND setdatabase = pg_database.oid", role_catalog); res = executeQuery(conn, buf->data); if (PQntuples(res) > 0) -- cgit v1.2.3 From b2678efd43f17db7dfa04e0ca076ea01275cd9bc Mon Sep 17 00:00:00 2001 From: Stephen Frost Date: Tue, 7 Mar 2017 09:31:52 -0500 Subject: psql: Add \gx command It can often be useful to use expanded mode output (\x) for just a single query. Introduce a \gx which acts exactly like \g except that it will force expanded output mode for that one \gx call. This is simpler than having to use \x as a toggle and also means that the user doesn't have to worry about the current state of the expanded variable, or resetting it later, to ensure a given query is always returned in expanded mode. Primairly Christoph's patch, though I did tweak the documentation and help text a bit, and re-indented the tab completion section. Author: Christoph Berg Reviewed By: Daniel Verite Discussion: https://postgr.es/m/20170127132737.6skslelaf4txs6iw%40msg.credativ.de --- doc/src/sgml/ref/psql-ref.sgml | 12 ++++++++++++ src/bin/psql/command.c | 9 +++++++-- src/bin/psql/common.c | 7 +++++++ src/bin/psql/help.c | 1 + src/bin/psql/settings.h | 1 + src/bin/psql/tab-complete.c | 11 ++++++----- src/test/regress/expected/psql.out | 23 +++++++++++++++++++++++ src/test/regress/sql/psql.sql | 7 +++++++ 8 files changed, 64 insertions(+), 7 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml index ae58708aae..2a9c412020 100644 --- a/doc/src/sgml/ref/psql-ref.sgml +++ b/doc/src/sgml/ref/psql-ref.sgml @@ -1890,6 +1890,18 @@ Tue Oct 26 21:40:57 CEST 1999 + + \gx [ filename ] + \gx [ |command ] + + + \gx is equivalent to \g, but + forces expanded output mode for this query. See \x. + + + + + \gexec diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c index a52adc8186..07efc27a69 100644 --- a/src/bin/psql/command.c +++ b/src/bin/psql/command.c @@ -906,8 +906,11 @@ exec_command(const char *cmd, free(fname); } - /* \g [filename] -- send query, optionally with output to file/pipe */ - else if (strcmp(cmd, "g") == 0) + /* + * \g [filename] -- send query, optionally with output to file/pipe + * \gx [filename] -- same as \g, with expanded mode forced + */ + else if (strcmp(cmd, "g") == 0 || strcmp(cmd, "gx") == 0) { char *fname = psql_scan_slash_option(scan_state, OT_FILEPIPE, NULL, false); @@ -920,6 +923,8 @@ exec_command(const char *cmd, pset.gfname = pg_strdup(fname); } free(fname); + if (strcmp(cmd, "gx") == 0) + pset.g_expanded = true; status = PSQL_CMD_SEND; } diff --git a/src/bin/psql/common.c b/src/bin/psql/common.c index 5349c39411..1aa56ab3a2 100644 --- a/src/bin/psql/common.c +++ b/src/bin/psql/common.c @@ -770,6 +770,10 @@ PrintQueryTuples(const PGresult *results) { printQueryOpt my_popt = pset.popt; + /* one-shot expanded output requested via \gx */ + if (pset.g_expanded) + my_popt.topt.expanded = 1; + /* write output to \g argument, if any */ if (pset.gfname) { @@ -1410,6 +1414,9 @@ sendquery_cleanup: pset.gfname = NULL; } + /* reset \gx's expanded-mode flag */ + pset.g_expanded = false; + /* reset \gset trigger */ if (pset.gset_prefix) { diff --git a/src/bin/psql/help.c b/src/bin/psql/help.c index 91cf0be46a..ba14df0344 100644 --- a/src/bin/psql/help.c +++ b/src/bin/psql/help.c @@ -173,6 +173,7 @@ slashUsage(unsigned short int pager) fprintf(output, _(" \\copyright show PostgreSQL usage and distribution terms\n")); fprintf(output, _(" \\errverbose show most recent error message at maximum verbosity\n")); fprintf(output, _(" \\g [FILE] or ; execute query (and send results to file or |pipe)\n")); + fprintf(output, _(" \\gx [FILE] as \\g, but forces expanded output mode\n")); fprintf(output, _(" \\gexec execute query, then execute each value in its result\n")); fprintf(output, _(" \\gset [PREFIX] execute query and store results in psql variables\n")); fprintf(output, _(" \\q quit psql\n")); diff --git a/src/bin/psql/settings.h b/src/bin/psql/settings.h index 195f5a1184..70ff1812c8 100644 --- a/src/bin/psql/settings.h +++ b/src/bin/psql/settings.h @@ -91,6 +91,7 @@ typedef struct _psqlSettings printQueryOpt popt; char *gfname; /* one-shot file output argument for \g */ + bool g_expanded; /* one-shot expanded output requested via \gx */ char *gset_prefix; /* one-shot prefix argument for \gset */ bool gexec_flag; /* one-shot flag to execute query's results */ bool crosstab_flag; /* one-shot request to crosstab results */ diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index 4a65ff5b62..121a492e6d 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -1375,11 +1375,12 @@ psql_completion(const char *text, int start, int end) "\\dm", "\\dn", "\\do", "\\dO", "\\dp", "\\drds", "\\ds", "\\dS", "\\dt", "\\dT", "\\dv", "\\du", "\\dx", "\\dy", "\\e", "\\echo", "\\ef", "\\encoding", "\\errverbose", "\\ev", - "\\f", "\\g", "\\gexec", "\\gset", "\\h", "\\help", "\\H", "\\i", "\\ir", "\\l", - "\\lo_import", "\\lo_export", "\\lo_list", "\\lo_unlink", - "\\o", "\\p", "\\password", "\\prompt", "\\pset", "\\q", "\\qecho", "\\r", - "\\s", "\\set", "\\setenv", "\\sf", "\\sv", "\\t", "\\T", - "\\timing", "\\unset", "\\x", "\\w", "\\watch", "\\z", "\\!", NULL + "\\f", "\\g", "\\gexec", "\\gset", "\\gx", "\\h", "\\help", "\\H", + "\\i", "\\ir", "\\l", "\\lo_import", "\\lo_export", "\\lo_list", + "\\lo_unlink", "\\o", "\\p", "\\password", "\\prompt", "\\pset", "\\q", + "\\qecho", "\\r", "\\s", "\\set", "\\setenv", "\\sf", "\\sv", "\\t", + "\\T", "\\timing", "\\unset", "\\x", "\\w", "\\watch", "\\z", "\\!", + NULL }; (void) end; /* "end" is not used */ diff --git a/src/test/regress/expected/psql.out b/src/test/regress/expected/psql.out index 026a4f0c83..eb7f197b12 100644 --- a/src/test/regress/expected/psql.out +++ b/src/test/regress/expected/psql.out @@ -28,6 +28,29 @@ on \unset ON_ERROR_ROLLBACK \echo :ON_ERROR_ROLLBACK off +-- \g and \gx +SELECT 1 as one, 2 as two \g + one | two +-----+----- + 1 | 2 +(1 row) + +\gx +-[ RECORD 1 ] +one | 1 +two | 2 + +SELECT 3 as three, 4 as four \gx +-[ RECORD 1 ] +three | 3 +four | 4 + +\g + three | four +-------+------ + 3 | 4 +(1 row) + -- \gset select 10 as test01, 20 as test02, 'Hello' as test03 \gset pref01_ \echo :pref01_test01 :pref01_test02 :pref01_test03 diff --git a/src/test/regress/sql/psql.sql b/src/test/regress/sql/psql.sql index d823d11b95..8f8e17a87c 100644 --- a/src/test/regress/sql/psql.sql +++ b/src/test/regress/sql/psql.sql @@ -21,6 +21,13 @@ \unset ON_ERROR_ROLLBACK \echo :ON_ERROR_ROLLBACK +-- \g and \gx + +SELECT 1 as one, 2 as two \g +\gx +SELECT 3 as three, 4 as four \gx +\g + -- \gset select 10 as test01, 20 as test02, 'Hello' as test03 \gset pref01_ -- cgit v1.2.3 From 889a3f4892b271c502f74082a352447408f6292d Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Tue, 7 Mar 2017 10:57:46 -0500 Subject: Document what values postgres_fdw sets for each parameter it sets. David Rader, reviewed by me. --- doc/src/sgml/postgres-fdw.sgml | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/postgres-fdw.sgml b/doc/src/sgml/postgres-fdw.sgml index b31f3731e4..7a9b655d36 100644 --- a/doc/src/sgml/postgres-fdw.sgml +++ b/doc/src/sgml/postgres-fdw.sgml @@ -532,11 +532,32 @@ postgres_fdw likewise establishes remote session settings - for the parameters , - , , - and . These are less likely - to be problematic than search_path, but can be handled - with function SET options if the need arises. + for various parameters: + + + + is set to UTC + + + + + is set to ISO + + + + + is set to postgres + + + + + is set to 3 for remote + servers 9.0 and newer and is set to 2 for older versions + + + + These are less likely to be problematic than search_path, but + can be handled with function SET options if the need arises. -- cgit v1.2.3 From 508dabaf39e1f66cbe3fc54d2b66f010aa59c66b Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Tue, 7 Mar 2017 11:18:56 -0500 Subject: Remove duplicated word. Amit Langote --- doc/src/sgml/ref/analyze.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/analyze.sgml b/doc/src/sgml/ref/analyze.sgml index 49727e22df..45dee101df 100644 --- a/doc/src/sgml/ref/analyze.sgml +++ b/doc/src/sgml/ref/analyze.sgml @@ -64,7 +64,7 @@ ANALYZE [ VERBOSE ] [ table_name [ The name (possibly schema-qualified) of a specific table to analyze. If omitted, all regular tables, partitioned tables, and - and materialized views in the current database are analyzed (but not + materialized views in the current database are analyzed (but not foreign tables). If the specified table is a partitioned table, both the inheritance statistics of the partitioned table as a whole and statistics of the individual partitions are updated. -- cgit v1.2.3 From 0d2b1f305dc78d536d80cfb4bb2ac4d7104453db Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 7 Mar 2017 12:40:44 -0500 Subject: Invent start_proc parameters for PL/Tcl. Define GUCs pltcl.start_proc and pltclu.start_proc. When set to a nonempty value at the time a new Tcl interpreter is created, the parameterless pltcl or pltclu function named by the GUC is called to allow user-controlled initialization to occur within the interpreter. This is modeled on plv8's start_proc parameter, and also has much in common with plperl's on_init feature. It allows users to fully replace the "modules" feature that was removed in commit 817f2a586. Since an initializer function could subvert later Tcl code in nearly arbitrary ways, mark both GUCs as SUSET for now. It would be nice to find a way to relax that someday; but the corresponding GUCs in plperl are also SUSET, and there's not been much complaint. Discussion: https://postgr.es/m/22067.1488046447@sss.pgh.pa.us --- doc/src/sgml/pltcl.sgml | 74 ++++++++++++++ src/pl/tcl/Makefile | 2 +- src/pl/tcl/expected/pltcl_start_proc.out | 31 ++++++ src/pl/tcl/pltcl.c | 167 +++++++++++++++++++++++++++++-- src/pl/tcl/sql/pltcl_start_proc.sql | 21 ++++ 5 files changed, 286 insertions(+), 9 deletions(-) create mode 100644 src/pl/tcl/expected/pltcl_start_proc.out create mode 100644 src/pl/tcl/sql/pltcl_start_proc.sql (limited to 'doc/src') diff --git a/doc/src/sgml/pltcl.sgml b/doc/src/sgml/pltcl.sgml index 0a693803dd..ad216dd5b7 100644 --- a/doc/src/sgml/pltcl.sgml +++ b/doc/src/sgml/pltcl.sgml @@ -902,6 +902,80 @@ if {[catch { spi_exec $sql_command }]} { + + PL/Tcl Configuration + + + This section lists configuration parameters that + affect PL/Tcl. + + + + + + + pltcl.start_proc (string) + + pltcl.start_proc configuration parameter + + + + + This parameter, if set to a nonempty string, specifies the name + (possibly schema-qualified) of a parameterless PL/Tcl function that + is to be executed whenever a new Tcl interpreter is created for + PL/Tcl. Such a function can perform per-session initialization, such + as loading additional Tcl code. A new Tcl interpreter is created + when a PL/Tcl function is first executed in a database session, or + when an additional interpreter has to be created because a PL/Tcl + function is called by a new SQL role. + + + + The referenced function must be written in the pltcl + language, and must not be marked SECURITY DEFINER. + (These restrictions ensure that it runs in the interpreter it's + supposed to initialize.) The current user must have permission to + call it, too. + + + + If the function fails with an error it will abort the function call + that caused the new interpreter to be created and propagate out to + the calling query, causing the current transaction or subtransaction + to be aborted. Any actions already done within Tcl won't be undone; + however, that interpreter won't be used again. If the language is + used again the initialization will be attempted again within a fresh + Tcl interpreter. + + + + Only superusers can change this setting. Although this setting + can be changed within a session, such changes will not affect Tcl + interpreters that have already been created. + + + + + + + pltclu.start_proc (string) + + pltclu.start_proc configuration parameter + + + + + This parameter is exactly like pltcl.start_proc, + except that it applies to PL/TclU. The referenced function must + be written in the pltclu language. + + + + + + + Tcl Procedure Names diff --git a/src/pl/tcl/Makefile b/src/pl/tcl/Makefile index 453e7ad2ec..1096c4faf0 100644 --- a/src/pl/tcl/Makefile +++ b/src/pl/tcl/Makefile @@ -28,7 +28,7 @@ DATA = pltcl.control pltcl--1.0.sql pltcl--unpackaged--1.0.sql \ pltclu.control pltclu--1.0.sql pltclu--unpackaged--1.0.sql REGRESS_OPTS = --dbname=$(PL_TESTDB) --load-extension=pltcl -REGRESS = pltcl_setup pltcl_queries pltcl_unicode +REGRESS = pltcl_setup pltcl_queries pltcl_start_proc pltcl_unicode # Tcl on win32 ships with import libraries only for Microsoft Visual C++, # which are not compatible with mingw gcc. Therefore we need to build a diff --git a/src/pl/tcl/expected/pltcl_start_proc.out b/src/pl/tcl/expected/pltcl_start_proc.out new file mode 100644 index 0000000000..9946cd9652 --- /dev/null +++ b/src/pl/tcl/expected/pltcl_start_proc.out @@ -0,0 +1,31 @@ +-- +-- Test start_proc execution +-- +SET pltcl.start_proc = 'no_such_function'; +select tcl_int4add(1, 2); +ERROR: function no_such_function() does not exist +CONTEXT: processing pltcl.start_proc parameter +select tcl_int4add(1, 2); +ERROR: function no_such_function() does not exist +CONTEXT: processing pltcl.start_proc parameter +create function tcl_initialize() returns void as +$$ elog NOTICE "in tcl_initialize" $$ language pltcl SECURITY DEFINER; +SET pltcl.start_proc = 'public.tcl_initialize'; +select tcl_int4add(1, 2); -- fail +ERROR: function "public.tcl_initialize" must not be SECURITY DEFINER +CONTEXT: processing pltcl.start_proc parameter +create or replace function tcl_initialize() returns void as +$$ elog NOTICE "in tcl_initialize" $$ language pltcl; +select tcl_int4add(1, 2); +NOTICE: in tcl_initialize + tcl_int4add +------------- + 3 +(1 row) + +select tcl_int4add(1, 2); + tcl_int4add +------------- + 3 +(1 row) + diff --git a/src/pl/tcl/pltcl.c b/src/pl/tcl/pltcl.c index 11faa6defe..2cf7e6619b 100644 --- a/src/pl/tcl/pltcl.c +++ b/src/pl/tcl/pltcl.c @@ -15,6 +15,7 @@ #include "access/htup_details.h" #include "access/xact.h" +#include "catalog/objectaccess.h" #include "catalog/pg_proc.h" #include "catalog/pg_type.h" #include "commands/event_trigger.h" @@ -25,11 +26,14 @@ #include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/makefuncs.h" +#include "parser/parse_func.h" #include "parser/parse_type.h" +#include "pgstat.h" #include "tcop/tcopprot.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "utils/memutils.h" +#include "utils/regproc.h" #include "utils/rel.h" #include "utils/syscache.h" #include "utils/typcache.h" @@ -226,6 +230,8 @@ typedef struct pltcl_call_state /********************************************************************** * Global data **********************************************************************/ +static char *pltcl_start_proc = NULL; +static char *pltclu_start_proc = NULL; static bool pltcl_pm_init_done = false; static Tcl_Interp *pltcl_hold_interp = NULL; static HTAB *pltcl_interp_htab = NULL; @@ -253,8 +259,11 @@ static const TclExceptionNameMap exception_name_map[] = { **********************************************************************/ void _PG_init(void); -static void pltcl_init_interp(pltcl_interp_desc *interp_desc, bool pltrusted); -static pltcl_interp_desc *pltcl_fetch_interp(bool pltrusted); +static void pltcl_init_interp(pltcl_interp_desc *interp_desc, + Oid prolang, bool pltrusted); +static pltcl_interp_desc *pltcl_fetch_interp(Oid prolang, bool pltrusted); +static void call_pltcl_start_proc(Oid prolang, bool pltrusted); +static void start_proc_error_callback(void *arg); static Datum pltcl_handler(PG_FUNCTION_ARGS, bool pltrusted); @@ -441,6 +450,24 @@ _PG_init(void) &hash_ctl, HASH_ELEM | HASH_BLOBS); + /************************************************************ + * Define PL/Tcl's custom GUCs + ************************************************************/ + DefineCustomStringVariable("pltcl.start_proc", + gettext_noop("PL/Tcl function to call once when pltcl is first used."), + NULL, + &pltcl_start_proc, + NULL, + PGC_SUSET, 0, + NULL, NULL, NULL); + DefineCustomStringVariable("pltclu.start_proc", + gettext_noop("PL/TclU function to call once when pltclu is first used."), + NULL, + &pltclu_start_proc, + NULL, + PGC_SUSET, 0, + NULL, NULL, NULL); + pltcl_pm_init_done = true; } @@ -448,7 +475,7 @@ _PG_init(void) * pltcl_init_interp() - initialize a new Tcl interpreter **********************************************************************/ static void -pltcl_init_interp(pltcl_interp_desc *interp_desc, bool pltrusted) +pltcl_init_interp(pltcl_interp_desc *interp_desc, Oid prolang, bool pltrusted) { Tcl_Interp *interp; char interpname[32]; @@ -462,7 +489,6 @@ pltcl_init_interp(pltcl_interp_desc *interp_desc, bool pltrusted) if ((interp = Tcl_CreateSlave(pltcl_hold_interp, interpname, pltrusted ? 1 : 0)) == NULL) elog(ERROR, "could not create slave Tcl interpreter"); - interp_desc->interp = interp; /************************************************************ * Initialize the query hash table associated with interpreter @@ -490,16 +516,35 @@ pltcl_init_interp(pltcl_interp_desc *interp_desc, bool pltrusted) pltcl_SPI_execute_plan, NULL, NULL); Tcl_CreateObjCommand(interp, "spi_lastoid", pltcl_SPI_lastoid, NULL, NULL); + + /************************************************************ + * Call the appropriate start_proc, if there is one. + * + * We must set interp_desc->interp before the call, else the start_proc + * won't find the interpreter it's supposed to use. But, if the + * start_proc fails, we want to abandon use of the interpreter. + ************************************************************/ + PG_TRY(); + { + interp_desc->interp = interp; + call_pltcl_start_proc(prolang, pltrusted); + } + PG_CATCH(); + { + interp_desc->interp = NULL; + Tcl_DeleteInterp(interp); + PG_RE_THROW(); + } + PG_END_TRY(); } /********************************************************************** * pltcl_fetch_interp() - fetch the Tcl interpreter to use for a function * * This also takes care of any on-first-use initialization required. - * Note: we assume caller has already connected to SPI. **********************************************************************/ static pltcl_interp_desc * -pltcl_fetch_interp(bool pltrusted) +pltcl_fetch_interp(Oid prolang, bool pltrusted) { Oid user_id; pltcl_interp_desc *interp_desc; @@ -515,12 +560,117 @@ pltcl_fetch_interp(bool pltrusted) HASH_ENTER, &found); if (!found) - pltcl_init_interp(interp_desc, pltrusted); + interp_desc->interp = NULL; + + /* If we haven't yet successfully made an interpreter, try to do that */ + if (!interp_desc->interp) + pltcl_init_interp(interp_desc, prolang, pltrusted); return interp_desc; } +/********************************************************************** + * call_pltcl_start_proc() - Call user-defined initialization proc, if any + **********************************************************************/ +static void +call_pltcl_start_proc(Oid prolang, bool pltrusted) +{ + char *start_proc; + const char *gucname; + ErrorContextCallback errcallback; + List *namelist; + Oid fargtypes[1]; /* dummy */ + Oid procOid; + HeapTuple procTup; + Form_pg_proc procStruct; + AclResult aclresult; + FmgrInfo finfo; + FunctionCallInfoData fcinfo; + PgStat_FunctionCallUsage fcusage; + + /* select appropriate GUC */ + start_proc = pltrusted ? pltcl_start_proc : pltclu_start_proc; + gucname = pltrusted ? "pltcl.start_proc" : "pltclu.start_proc"; + + /* Nothing to do if it's empty or unset */ + if (start_proc == NULL || start_proc[0] == '\0') + return; + + /* Set up errcontext callback to make errors more helpful */ + errcallback.callback = start_proc_error_callback; + errcallback.arg = (void *) gucname; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + + /* Parse possibly-qualified identifier and look up the function */ + namelist = stringToQualifiedNameList(start_proc); + procOid = LookupFuncName(namelist, 0, fargtypes, false); + + /* Current user must have permission to call function */ + aclresult = pg_proc_aclcheck(procOid, GetUserId(), ACL_EXECUTE); + if (aclresult != ACLCHECK_OK) + aclcheck_error(aclresult, ACL_KIND_PROC, start_proc); + + /* Get the function's pg_proc entry */ + procTup = SearchSysCache1(PROCOID, ObjectIdGetDatum(procOid)); + if (!HeapTupleIsValid(procTup)) + elog(ERROR, "cache lookup failed for function %u", procOid); + procStruct = (Form_pg_proc) GETSTRUCT(procTup); + + /* It must be same language as the function we're currently calling */ + if (procStruct->prolang != prolang) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("function \"%s\" is in the wrong language", + start_proc))); + + /* + * It must not be SECURITY DEFINER, either. This together with the + * language match check ensures that the function will execute in the same + * Tcl interpreter we just finished initializing. + */ + if (procStruct->prosecdef) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("function \"%s\" must not be SECURITY DEFINER", + start_proc))); + + /* A-OK */ + ReleaseSysCache(procTup); + + /* + * Call the function using the normal SQL function call mechanism. We + * could perhaps cheat and jump directly to pltcl_handler(), but it seems + * better to do it this way so that the call is exposed to, eg, call + * statistics collection. + */ + InvokeFunctionExecuteHook(procOid); + fmgr_info(procOid, &finfo); + InitFunctionCallInfoData(fcinfo, &finfo, + 0, + InvalidOid, NULL, NULL); + pgstat_init_function_usage(&fcinfo, &fcusage); + (void) FunctionCallInvoke(&fcinfo); + pgstat_end_function_usage(&fcusage, true); + + /* Pop the error context stack */ + error_context_stack = errcallback.previous; +} + +/* + * Error context callback for errors occurring during start_proc processing. + */ +static void +start_proc_error_callback(void *arg) +{ + const char *gucname = (const char *) arg; + + /* translator: %s is "pltcl.start_proc" or "pltclu.start_proc" */ + errcontext("processing %s parameter", gucname); +} + + /********************************************************************** * pltcl_call_handler - This is the only visible function * of the PL interpreter. The PostgreSQL @@ -1319,7 +1469,8 @@ compile_pltcl_function(Oid fn_oid, Oid tgreloid, /************************************************************ * Identify the interpreter to use for the function ************************************************************/ - prodesc->interp_desc = pltcl_fetch_interp(prodesc->lanpltrusted); + prodesc->interp_desc = pltcl_fetch_interp(procStruct->prolang, + prodesc->lanpltrusted); interp = prodesc->interp_desc->interp; /************************************************************ diff --git a/src/pl/tcl/sql/pltcl_start_proc.sql b/src/pl/tcl/sql/pltcl_start_proc.sql new file mode 100644 index 0000000000..7a8e68e266 --- /dev/null +++ b/src/pl/tcl/sql/pltcl_start_proc.sql @@ -0,0 +1,21 @@ +-- +-- Test start_proc execution +-- + +SET pltcl.start_proc = 'no_such_function'; + +select tcl_int4add(1, 2); +select tcl_int4add(1, 2); + +create function tcl_initialize() returns void as +$$ elog NOTICE "in tcl_initialize" $$ language pltcl SECURITY DEFINER; + +SET pltcl.start_proc = 'public.tcl_initialize'; + +select tcl_int4add(1, 2); -- fail + +create or replace function tcl_initialize() returns void as +$$ elog NOTICE "in tcl_initialize" $$ language pltcl; + +select tcl_int4add(1, 2); +select tcl_int4add(1, 2); -- cgit v1.2.3 From b7fa016d68b51bc385b75f9d5cffef79e5671981 Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Tue, 7 Mar 2017 22:45:45 -0500 Subject: Fix grammar Reported by Jeremy Finzel --- doc/src/sgml/wal.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/wal.sgml b/doc/src/sgml/wal.sgml index 346aa769a8..1468ba52a4 100644 --- a/doc/src/sgml/wal.sgml +++ b/doc/src/sgml/wal.sgml @@ -569,7 +569,7 @@ removes the rest. The estimate is based on a moving average of the number of WAL files used in previous checkpoint cycles. The moving average is increased immediately if the actual usage exceeds the estimate, so it - accommodates peak usage rather average usage to some extent. + accommodates peak usage rather than average usage to some extent. min_wal_size puts a minimum on the amount of WAL files recycled for future usage; that much WAL is always recycled for future use, even if the system is idle and the WAL usage estimate suggests that little -- cgit v1.2.3 From 98e6e89040a0534ca26914c66cae9dd49ef62ad9 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Wed, 8 Mar 2017 08:02:03 -0500 Subject: tidbitmap: Support shared iteration. When a shared iterator is used, each call to tbm_shared_iterate() returns a result that has not yet been returned to any process attached to the shared iterator. In other words, each cooperating processes gets a disjoint subset of the full result set, but all results are returned exactly once. This is infrastructure for parallel bitmap heap scan. Dilip Kumar. The larger patch set of which this is a part has been reviewed and tested by (at least) Andres Freund, Amit Khandekar, Tushar Ahuja, Rafia Sabih, Haribabu Kommi, and Thomas Munro. Discussion: http://postgr.es/m/CAFiTN-uc4=0WxRGfCzs-xfkMYcSEWUC-Fon6thkJGjkh9i=13A@mail.gmail.com --- doc/src/sgml/monitoring.sgml | 6 +- src/backend/access/gin/ginget.c | 2 +- src/backend/executor/nodeBitmapIndexscan.c | 2 +- src/backend/executor/nodeBitmapOr.c | 2 +- src/backend/nodes/tidbitmap.c | 551 +++++++++++++++++++++++++++-- src/backend/storage/lmgr/lwlock.c | 1 + src/include/nodes/tidbitmap.h | 10 +- src/include/storage/lwlock.h | 1 + 8 files changed, 536 insertions(+), 39 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index fad5cb05b9..27ed35f0a7 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -826,7 +826,7 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser - LWLock + LWLock ShmemIndexLock Waiting to find or allocate space in shared memory. @@ -1081,6 +1081,10 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser parallel_query_dsa Waiting for parallel query dynamic shared memory allocation lock. + + tbm + Waiting for TBM shared iterator lock. + Lock relation diff --git a/src/backend/access/gin/ginget.c b/src/backend/access/gin/ginget.c index 60f005ce2b..87cd9eaa9f 100644 --- a/src/backend/access/gin/ginget.c +++ b/src/backend/access/gin/ginget.c @@ -120,7 +120,7 @@ collectMatchBitmap(GinBtreeData *btree, GinBtreeStack *stack, Form_pg_attribute attr; /* Initialize empty bitmap result */ - scanEntry->matchBitmap = tbm_create(work_mem * 1024L); + scanEntry->matchBitmap = tbm_create(work_mem * 1024L, NULL); /* Null query cannot partial-match anything */ if (scanEntry->isPartialMatch && diff --git a/src/backend/executor/nodeBitmapIndexscan.c b/src/backend/executor/nodeBitmapIndexscan.c index 4274e9abcb..94bb289f1b 100644 --- a/src/backend/executor/nodeBitmapIndexscan.c +++ b/src/backend/executor/nodeBitmapIndexscan.c @@ -78,7 +78,7 @@ MultiExecBitmapIndexScan(BitmapIndexScanState *node) else { /* XXX should we use less than work_mem for this? */ - tbm = tbm_create(work_mem * 1024L); + tbm = tbm_create(work_mem * 1024L, NULL); } /* diff --git a/src/backend/executor/nodeBitmapOr.c b/src/backend/executor/nodeBitmapOr.c index 3c6155b7c3..1d280beddc 100644 --- a/src/backend/executor/nodeBitmapOr.c +++ b/src/backend/executor/nodeBitmapOr.c @@ -129,7 +129,7 @@ MultiExecBitmapOr(BitmapOrState *node) if (result == NULL) /* first subplan */ { /* XXX should we use less than work_mem for this? */ - result = tbm_create(work_mem * 1024L); + result = tbm_create(work_mem * 1024L, NULL); } ((BitmapIndexScanState *) subnode)->biss_result = result; diff --git a/src/backend/nodes/tidbitmap.c b/src/backend/nodes/tidbitmap.c index 0885812420..9dcef32704 100644 --- a/src/backend/nodes/tidbitmap.c +++ b/src/backend/nodes/tidbitmap.c @@ -43,6 +43,8 @@ #include "access/htup_details.h" #include "nodes/bitmapset.h" #include "nodes/tidbitmap.h" +#include "storage/lwlock.h" +#include "utils/dsa.h" /* * The maximum number of tuples per page is not large (typically 256 with @@ -102,6 +104,15 @@ typedef struct PagetableEntry bitmapword words[Max(WORDS_PER_PAGE, WORDS_PER_CHUNK)]; } PagetableEntry; +/* + * Holds array of pagetable entries. + */ +typedef struct PTEntryArray +{ + pg_atomic_uint32 refcount; /* no. of iterator attached */ + PagetableEntry ptentry[FLEXIBLE_ARRAY_MEMBER]; +} PTEntryArray; + /* * We want to avoid the overhead of creating the hashtable, which is * comparatively large, when not necessary. Particularly when we are using a @@ -120,6 +131,16 @@ typedef enum TBM_HASH /* pagetable is valid, entry1 is not */ } TBMStatus; +/* + * Current iterating state of the TBM. + */ +typedef enum +{ + TBM_NOT_ITERATING, /* not yet converted to page and chunk array */ + TBM_ITERATING_PRIVATE, /* converted to local page and chunk array */ + TBM_ITERATING_SHARED /* converted to shared page and chunk array */ +} TBMIteratingState; + /* * Here is the representation for a whole TIDBitMap: */ @@ -133,12 +154,17 @@ struct TIDBitmap int maxentries; /* limit on same to meet maxbytes */ int npages; /* number of exact entries in pagetable */ int nchunks; /* number of lossy entries in pagetable */ - bool iterating; /* tbm_begin_iterate called? */ + TBMIteratingState iterating; /* tbm_begin_iterate called? */ uint32 lossify_start; /* offset to start lossifying hashtable at */ PagetableEntry entry1; /* used when status == TBM_ONE_PAGE */ /* these are valid when iterating is true: */ PagetableEntry **spages; /* sorted exact-page list, or NULL */ PagetableEntry **schunks; /* sorted lossy-chunk list, or NULL */ + dsa_pointer dsapagetable; /* dsa_pointer to the element array */ + dsa_pointer dsapagetableold; /* dsa_pointer to the old element array */ + dsa_pointer ptpages; /* dsa_pointer to the page array */ + dsa_pointer ptchunks; /* dsa_pointer to the chunk array */ + dsa_area *dsa; /* reference to per-query dsa area */ }; /* @@ -156,6 +182,46 @@ struct TBMIterator TBMIterateResult output; /* MUST BE LAST (because variable-size) */ }; +/* + * Holds the shared members of the iterator so that multiple processes + * can jointly iterate. + */ +typedef struct TBMSharedIteratorState +{ + int nentries; /* number of entries in pagetable */ + int maxentries; /* limit on same to meet maxbytes */ + int npages; /* number of exact entries in pagetable */ + int nchunks; /* number of lossy entries in pagetable */ + dsa_pointer pagetable; /* dsa pointers to head of pagetable data */ + dsa_pointer spages; /* dsa pointer to page array */ + dsa_pointer schunks; /* dsa pointer to chunk array */ + LWLock lock; /* lock to protect below members */ + int spageptr; /* next spages index */ + int schunkptr; /* next schunks index */ + int schunkbit; /* next bit to check in current schunk */ +} TBMSharedIteratorState; + +/* + * pagetable iteration array. + */ +typedef struct PTIterationArray +{ + pg_atomic_uint32 refcount; /* no. of iterator attached */ + int index[FLEXIBLE_ARRAY_MEMBER]; /* index array */ +} PTIterationArray; + +/* + * same as TBMIterator, but it is used for joint iteration, therefore this + * also holds a reference to the shared state. + */ +struct TBMSharedIterator +{ + TBMSharedIteratorState *state; /* shared state */ + PTEntryArray *ptbase; /* pagetable element array */ + PTIterationArray *ptpages; /* sorted exact page index list */ + PTIterationArray *ptchunks; /* sorted lossy page index list */ + TBMIterateResult output; /* MUST BE LAST (because variable-size) */ +}; /* Local function prototypes */ static void tbm_union_page(TIDBitmap *a, const PagetableEntry *bpage); @@ -168,6 +234,8 @@ static bool tbm_page_is_lossy(const TIDBitmap *tbm, BlockNumber pageno); static void tbm_mark_page_lossy(TIDBitmap *tbm, BlockNumber pageno); static void tbm_lossify(TIDBitmap *tbm); static int tbm_comparator(const void *left, const void *right); +static int tbm_shared_comparator(const void *left, const void *right, + void *arg); /* * Simple inline murmur hash implementation for the exact width required, for @@ -187,6 +255,7 @@ hash_blockno(BlockNumber b) } /* define hashtable mapping block numbers to PagetableEntry's */ +#define SH_USE_NONDEFAULT_ALLOCATOR #define SH_PREFIX pagetable #define SH_ELEMENT_TYPE PagetableEntry #define SH_KEY_TYPE BlockNumber @@ -204,10 +273,12 @@ hash_blockno(BlockNumber b) * * The bitmap will live in the memory context that is CurrentMemoryContext * at the time of this call. It will be limited to (approximately) maxbytes - * total memory consumption. + * total memory consumption. If the DSA passed to this function is not NULL + * then the memory for storing elements of the underlying page table will + * be allocated from the DSA. */ TIDBitmap * -tbm_create(long maxbytes) +tbm_create(long maxbytes, dsa_area *dsa) { TIDBitmap *tbm; long nbuckets; @@ -230,6 +301,7 @@ tbm_create(long maxbytes) nbuckets = Max(nbuckets, 16); /* sanity limit */ tbm->maxentries = (int) nbuckets; tbm->lossify_start = 0; + tbm->dsa = dsa; return tbm; } @@ -244,7 +316,7 @@ tbm_create_pagetable(TIDBitmap *tbm) Assert(tbm->status != TBM_HASH); Assert(tbm->pagetable == NULL); - tbm->pagetable = pagetable_create(tbm->mcxt, 128, NULL); + tbm->pagetable = pagetable_create(tbm->mcxt, 128, tbm); /* If entry1 is valid, push it into the hashtable */ if (tbm->status == TBM_ONE_PAGE) @@ -280,6 +352,40 @@ tbm_free(TIDBitmap *tbm) pfree(tbm); } +/* + * tbm_free_shared_area - free shared state + * + * Free shared iterator state, Also free shared pagetable and iterator arrays + * memory if they are not referred by any of the shared iterator i.e recount + * is becomes 0. + */ +void +tbm_free_shared_area(dsa_area *dsa, dsa_pointer dp) +{ + TBMSharedIteratorState *istate = dsa_get_address(dsa, dp); + PTEntryArray *ptbase = dsa_get_address(dsa, istate->pagetable); + PTIterationArray *ptpages; + PTIterationArray *ptchunks; + + if (pg_atomic_sub_fetch_u32(&ptbase->refcount, 1) == 0) + dsa_free(dsa, istate->pagetable); + + if (istate->spages) + { + ptpages = dsa_get_address(dsa, istate->spages); + if (pg_atomic_sub_fetch_u32(&ptpages->refcount, 1) == 0) + dsa_free(dsa, istate->spages); + } + if (istate->schunks) + { + ptchunks = dsa_get_address(dsa, istate->schunks); + if (pg_atomic_sub_fetch_u32(&ptchunks->refcount, 1) == 0) + dsa_free(dsa, istate->schunks); + } + + dsa_free(dsa, dp); +} + /* * tbm_add_tuples - add some tuple IDs to a TIDBitmap * @@ -294,7 +400,7 @@ tbm_add_tuples(TIDBitmap *tbm, const ItemPointer tids, int ntids, PagetableEntry *page = NULL; /* only valid when currblk is valid */ int i; - Assert(!tbm->iterating); + Assert(tbm->iterating == TBM_NOT_ITERATING); for (i = 0; i < ntids; i++) { BlockNumber blk = ItemPointerGetBlockNumber(tids + i); @@ -603,6 +709,8 @@ tbm_begin_iterate(TIDBitmap *tbm) { TBMIterator *iterator; + Assert(tbm->iterating != TBM_ITERATING_SHARED); + /* * Create the TBMIterator struct, with enough trailing space to serve the * needs of the TBMIterateResult sub-struct. @@ -624,7 +732,7 @@ tbm_begin_iterate(TIDBitmap *tbm) * attached to the bitmap not the iterator, so they can be used by more * than one iterator. */ - if (tbm->status == TBM_HASH && !tbm->iterating) + if (tbm->status == TBM_HASH && tbm->iterating == TBM_NOT_ITERATING) { pagetable_iterator i; PagetableEntry *page; @@ -659,11 +767,210 @@ tbm_begin_iterate(TIDBitmap *tbm) tbm_comparator); } - tbm->iterating = true; + tbm->iterating = TBM_ITERATING_PRIVATE; return iterator; } +/* + * tbm_prepare_shared_iterate - prepare shared iteration state for a TIDBitmap. + * + * The necessary shared state will be allocated from the DSA passed to + * tbm_create, so that multiple processes can attach to it and iterate jointly. + * + * This will convert the pagetable hash into page and chunk array of the index + * into pagetable array. + */ +dsa_pointer +tbm_prepare_shared_iterate(TIDBitmap *tbm) +{ + dsa_pointer dp; + TBMSharedIteratorState *istate; + PTEntryArray *ptbase; + PTIterationArray *ptpages; + PTIterationArray *ptchunks; + + Assert(tbm->dsa != NULL); + Assert(tbm->iterating != TBM_ITERATING_PRIVATE); + + /* + * Allocate TBMSharedIteratorState from DSA to hold the shared members and + * lock, this will also be used by multiple worker for shared iterate. + */ + dp = dsa_allocate(tbm->dsa, sizeof(TBMSharedIteratorState)); + istate = dsa_get_address(tbm->dsa, dp); + + /* + * If we're not already iterating, create and fill the sorted page lists. + * (If we are, the sorted page lists are already stored in the TIDBitmap, + * and we can just reuse them.) + */ + if (tbm->iterating == TBM_NOT_ITERATING) + { + pagetable_iterator i; + PagetableEntry *page; + int idx; + int npages; + int nchunks; + + /* + * Allocate the page and chunk array memory from the DSA to share + * across multiple processes. + */ + if (tbm->npages) + { + tbm->ptpages = dsa_allocate(tbm->dsa, sizeof(PTIterationArray) + + tbm->npages * sizeof(int)); + ptpages = dsa_get_address(tbm->dsa, tbm->ptpages); + pg_atomic_init_u32(&ptpages->refcount, 0); + } + if (tbm->nchunks) + { + tbm->ptchunks = dsa_allocate(tbm->dsa, sizeof(PTIterationArray) + + tbm->nchunks * sizeof(int)); + ptchunks = dsa_get_address(tbm->dsa, tbm->ptchunks); + pg_atomic_init_u32(&ptchunks->refcount, 0); + } + + /* + * If TBM status is TBM_HASH then iterate over the pagetable and + * convert it to page and chunk arrays. But if it's in the + * TBM_ONE_PAGE mode then directly allocate the space for one entry + * from the DSA. + */ + npages = nchunks = 0; + if (tbm->status == TBM_HASH) + { + ptbase = dsa_get_address(tbm->dsa, tbm->dsapagetable); + + pagetable_start_iterate(tbm->pagetable, &i); + while ((page = pagetable_iterate(tbm->pagetable, &i)) != NULL) + { + idx = page - ptbase->ptentry; + if (page->ischunk) + ptchunks->index[nchunks++] = idx; + else + ptpages->index[npages++] = idx; + } + + Assert(npages == tbm->npages); + Assert(nchunks == tbm->nchunks); + } + else + { + /* + * In one page mode allocate the space for one pagetable entry and + * directly store its index i.e. 0 in page array + */ + tbm->dsapagetable = dsa_allocate(tbm->dsa, sizeof(PTEntryArray) + + sizeof(PagetableEntry)); + ptbase = dsa_get_address(tbm->dsa, tbm->dsapagetable); + ptpages->index[0] = 0; + } + + pg_atomic_init_u32(&ptbase->refcount, 0); + + if (npages > 1) + qsort_arg((void *) (ptpages->index), npages, sizeof(int), + tbm_shared_comparator, (void *) ptbase->ptentry); + if (nchunks > 1) + qsort_arg((void *) (ptchunks->index), nchunks, sizeof(int), + tbm_shared_comparator, (void *) ptbase->ptentry); + } + + /* + * Store the TBM members in the shared state so that we can share them + * across multiple processes. + */ + istate->nentries = tbm->nentries; + istate->maxentries = tbm->maxentries; + istate->npages = tbm->npages; + istate->nchunks = tbm->nchunks; + istate->pagetable = tbm->dsapagetable; + istate->spages = tbm->ptpages; + istate->schunks = tbm->ptchunks; + + ptbase = dsa_get_address(tbm->dsa, tbm->dsapagetable); + ptpages = dsa_get_address(tbm->dsa, tbm->ptpages); + ptchunks = dsa_get_address(tbm->dsa, tbm->ptchunks); + + /* + * For every shared iterator, referring to pagetable and iterator array, + * increase the refcount by 1 so that while freeing the shared iterator + * we don't free pagetable and iterator array until its refcount becomes 0. + */ + pg_atomic_add_fetch_u32(&ptbase->refcount, 1); + if (ptpages) + pg_atomic_add_fetch_u32(&ptpages->refcount, 1); + if (ptchunks) + pg_atomic_add_fetch_u32(&ptchunks->refcount, 1); + + /* Initialize the iterator lock */ + LWLockInitialize(&istate->lock, LWTRANCHE_TBM); + + /* Initialize the shared iterator state */ + istate->schunkbit = 0; + istate->schunkptr = 0; + istate->spageptr = 0; + + tbm->iterating = TBM_ITERATING_SHARED; + + return dp; +} + +/* + * tbm_extract_page_tuple - extract the tuple offsets from a page + * + * The extracted offsets are stored into TBMIterateResult. + */ +static inline int +tbm_extract_page_tuple(PagetableEntry *page, TBMIterateResult *output) +{ + int wordnum; + int ntuples = 0; + + for (wordnum = 0; wordnum < WORDS_PER_PAGE; wordnum++) + { + bitmapword w = page->words[wordnum]; + + if (w != 0) + { + int off = wordnum * BITS_PER_BITMAPWORD + 1; + + while (w != 0) + { + if (w & 1) + output->offsets[ntuples++] = (OffsetNumber) off; + off++; + w >>= 1; + } + } + } + + return ntuples; +} + +/* + * tbm_advance_schunkbit - Advance the chunkbit + */ +static inline void +tbm_advance_schunkbit(PagetableEntry *chunk, int *schunkbitp) +{ + int schunkbit = *schunkbitp; + + while (schunkbit < PAGES_PER_CHUNK) + { + int wordnum = WORDNUM(schunkbit); + int bitnum = BITNUM(schunkbit); + + if ((chunk->words[wordnum] & ((bitmapword) 1 << bitnum)) != 0) + break; + schunkbit++; + } + + *schunkbitp = schunkbit; +} + /* * tbm_iterate - scan through next page of a TIDBitmap * @@ -682,7 +989,7 @@ tbm_iterate(TBMIterator *iterator) TIDBitmap *tbm = iterator->tbm; TBMIterateResult *output = &(iterator->output); - Assert(tbm->iterating); + Assert(tbm->iterating == TBM_ITERATING_PRIVATE); /* * If lossy chunk pages remain, make sure we've advanced schunkptr/ @@ -693,15 +1000,7 @@ tbm_iterate(TBMIterator *iterator) PagetableEntry *chunk = tbm->schunks[iterator->schunkptr]; int schunkbit = iterator->schunkbit; - while (schunkbit < PAGES_PER_CHUNK) - { - int wordnum = WORDNUM(schunkbit); - int bitnum = BITNUM(schunkbit); - - if ((chunk->words[wordnum] & ((bitmapword) 1 << bitnum)) != 0) - break; - schunkbit++; - } + tbm_advance_schunkbit(chunk, &schunkbit); if (schunkbit < PAGES_PER_CHUNK) { iterator->schunkbit = schunkbit; @@ -738,7 +1037,6 @@ tbm_iterate(TBMIterator *iterator) { PagetableEntry *page; int ntuples; - int wordnum; /* In ONE_PAGE state, we don't allocate an spages[] array */ if (tbm->status == TBM_ONE_PAGE) @@ -747,31 +1045,102 @@ tbm_iterate(TBMIterator *iterator) page = tbm->spages[iterator->spageptr]; /* scan bitmap to extract individual offset numbers */ - ntuples = 0; - for (wordnum = 0; wordnum < WORDS_PER_PAGE; wordnum++) + ntuples = tbm_extract_page_tuple(page, output); + output->blockno = page->blockno; + output->ntuples = ntuples; + output->recheck = page->recheck; + iterator->spageptr++; + return output; + } + + /* Nothing more in the bitmap */ + return NULL; +} + +/* + * tbm_shared_iterate - scan through next page of a TIDBitmap + * + * As above, but this will iterate using an iterator which is shared + * across multiple processes. We need to acquire the iterator LWLock, + * before accessing the shared members. + */ +TBMIterateResult * +tbm_shared_iterate(TBMSharedIterator *iterator) +{ + TBMIterateResult *output = &iterator->output; + TBMSharedIteratorState *istate = iterator->state; + PagetableEntry *ptbase = iterator->ptbase->ptentry; + int *idxpages = iterator->ptpages->index; + int *idxchunks = iterator->ptchunks->index; + + /* Acquire the LWLock before accessing the shared members */ + LWLockAcquire(&istate->lock, LW_EXCLUSIVE); + + /* + * If lossy chunk pages remain, make sure we've advanced schunkptr/ + * schunkbit to the next set bit. + */ + while (istate->schunkptr < istate->nchunks) + { + PagetableEntry *chunk = &ptbase[idxchunks[istate->schunkptr]]; + int schunkbit = istate->schunkbit; + + tbm_advance_schunkbit(chunk, &schunkbit); + if (schunkbit < PAGES_PER_CHUNK) { - bitmapword w = page->words[wordnum]; + istate->schunkbit = schunkbit; + break; + } + /* advance to next chunk */ + istate->schunkptr++; + istate->schunkbit = 0; + } - if (w != 0) - { - int off = wordnum * BITS_PER_BITMAPWORD + 1; + /* + * If both chunk and per-page data remain, must output the numerically + * earlier page. + */ + if (istate->schunkptr < istate->nchunks) + { + PagetableEntry *chunk = &ptbase[idxchunks[istate->schunkptr]]; + PagetableEntry *page = &ptbase[idxpages[istate->spageptr]]; + BlockNumber chunk_blockno; - while (w != 0) - { - if (w & 1) - output->offsets[ntuples++] = (OffsetNumber) off; - off++; - w >>= 1; - } - } + chunk_blockno = chunk->blockno + istate->schunkbit; + + if (istate->spageptr >= istate->npages || + chunk_blockno < page->blockno) + { + /* Return a lossy page indicator from the chunk */ + output->blockno = chunk_blockno; + output->ntuples = -1; + output->recheck = true; + istate->schunkbit++; + + LWLockRelease(&istate->lock); + return output; } + } + + if (istate->spageptr < istate->npages) + { + PagetableEntry *page = &ptbase[idxpages[istate->spageptr]]; + int ntuples; + + /* scan bitmap to extract individual offset numbers */ + ntuples = tbm_extract_page_tuple(page, output); output->blockno = page->blockno; output->ntuples = ntuples; output->recheck = page->recheck; - iterator->spageptr++; + istate->spageptr++; + + LWLockRelease(&istate->lock); + return output; } + LWLockRelease(&istate->lock); + /* Nothing more in the bitmap */ return NULL; } @@ -789,6 +1158,18 @@ tbm_end_iterate(TBMIterator *iterator) pfree(iterator); } +/* + * tbm_end_shared_iterate - finish a shared iteration over a TIDBitmap + * + * This doesn't free any of the shared state associated with the iterator, + * just our backend-private state. + */ +void +tbm_end_shared_iterate(TBMSharedIterator *iterator) +{ + pfree(iterator); +} + /* * tbm_find_pageentry - find a PagetableEntry for the pageno * @@ -995,7 +1376,7 @@ tbm_lossify(TIDBitmap *tbm) * push nentries down to significantly less than maxentries, or else we'll * just end up doing this again very soon. We shoot for maxentries/2. */ - Assert(!tbm->iterating); + Assert(tbm->iterating == TBM_NOT_ITERATING); Assert(tbm->status == TBM_HASH); pagetable_start_iterate_at(tbm->pagetable, &i, tbm->lossify_start); @@ -1061,3 +1442,105 @@ tbm_comparator(const void *left, const void *right) return 1; return 0; } + +/* + * As above, but this will get index into PagetableEntry array. Therefore, + * it needs to get actual PagetableEntry using the index before comparing the + * blockno. + */ +static int +tbm_shared_comparator(const void *left, const void *right, void *arg) +{ + PagetableEntry *base = (PagetableEntry *) arg; + PagetableEntry *lpage = &base[*(int *) left]; + PagetableEntry *rpage = &base[*(int *) right]; + + if (lpage->blockno < rpage->blockno) + return -1; + else if (lpage->blockno > rpage->blockno) + return 1; + return 0; +} + +/* + * tbm_attach_shared_iterate + * + * Allocate a backend-private iterator and attach the shared iterator state + * to it so that multiple processed can iterate jointly. + * + * We also converts the DSA pointers to local pointers and store them into + * our private iterator. + */ +TBMSharedIterator * +tbm_attach_shared_iterate(dsa_area *dsa, dsa_pointer dp) +{ + TBMSharedIterator *iterator; + TBMSharedIteratorState *istate; + + /* + * Create the TBMSharedIterator struct, with enough trailing space to + * serve the needs of the TBMIterateResult sub-struct. + */ + iterator = (TBMSharedIterator *) palloc(sizeof(TBMSharedIterator) + + MAX_TUPLES_PER_PAGE * sizeof(OffsetNumber)); + + istate = (TBMSharedIteratorState *) dsa_get_address(dsa, dp); + + iterator->state = istate; + + iterator->ptbase = dsa_get_address(dsa, istate->pagetable); + + if (istate->npages) + iterator->ptpages = dsa_get_address(dsa, istate->spages); + if (istate->nchunks) + iterator->ptchunks = dsa_get_address(dsa, istate->schunks); + + return iterator; +} + +/* + * pagetable_allocate + * + * Callback function for allocating the memory for hashtable elements. + * Allocate memory for hashtable elements, using DSA if available. + */ +static inline void * +pagetable_allocate(pagetable_hash *pagetable, Size size) +{ + TIDBitmap *tbm = (TIDBitmap *) pagetable->private_data; + PTEntryArray *ptbase; + + if (tbm->dsa == NULL) + return MemoryContextAllocExtended(pagetable->ctx, size, + MCXT_ALLOC_HUGE | MCXT_ALLOC_ZERO); + + /* + * Save the dsapagetable reference in dsapagetableold before allocating + * new memory so that pagetable_free can free the old entry. + */ + tbm->dsapagetableold = tbm->dsapagetable; + tbm->dsapagetable = dsa_allocate0(tbm->dsa, sizeof(PTEntryArray) + size); + + ptbase = dsa_get_address(tbm->dsa, tbm->dsapagetable); + return ptbase->ptentry; +} + +/* + * pagetable_free + * + * Callback function for freeing hash table elements. + */ +static inline void +pagetable_free(pagetable_hash *pagetable, void *pointer) +{ + TIDBitmap *tbm = (TIDBitmap *) pagetable->private_data; + + /* pfree the input pointer if DSA is not available */ + if (tbm->dsa == NULL) + pfree(pointer); + else if (DsaPointerIsValid(tbm->dsapagetableold)) + { + dsa_free(tbm->dsa, tbm->dsapagetableold); + tbm->dsapagetableold = InvalidDsaPointer; + } +} diff --git a/src/backend/storage/lmgr/lwlock.c b/src/backend/storage/lmgr/lwlock.c index ab81d94b51..3e133941f4 100644 --- a/src/backend/storage/lmgr/lwlock.c +++ b/src/backend/storage/lmgr/lwlock.c @@ -510,6 +510,7 @@ RegisterLWLockTranches(void) "predicate_lock_manager"); LWLockRegisterTranche(LWTRANCHE_PARALLEL_QUERY_DSA, "parallel_query_dsa"); + LWLockRegisterTranche(LWTRANCHE_TBM, "tbm"); /* Register named tranches. */ for (i = 0; i < NamedLWLockTrancheRequests; i++) diff --git a/src/include/nodes/tidbitmap.h b/src/include/nodes/tidbitmap.h index 14992e09f8..87f4bb7c91 100644 --- a/src/include/nodes/tidbitmap.h +++ b/src/include/nodes/tidbitmap.h @@ -23,6 +23,7 @@ #define TIDBITMAP_H #include "storage/itemptr.h" +#include "utils/dsa.h" /* @@ -33,6 +34,7 @@ typedef struct TIDBitmap TIDBitmap; /* Likewise, TBMIterator is private */ typedef struct TBMIterator TBMIterator; +typedef struct TBMSharedIterator TBMSharedIterator; /* Result structure for tbm_iterate */ typedef struct @@ -46,8 +48,9 @@ typedef struct /* function prototypes in nodes/tidbitmap.c */ -extern TIDBitmap *tbm_create(long maxbytes); +extern TIDBitmap *tbm_create(long maxbytes, dsa_area *dsa); extern void tbm_free(TIDBitmap *tbm); +extern void tbm_free_shared_area(dsa_area *dsa, dsa_pointer dp); extern void tbm_add_tuples(TIDBitmap *tbm, const ItemPointer tids, int ntids, @@ -60,7 +63,12 @@ extern void tbm_intersect(TIDBitmap *a, const TIDBitmap *b); extern bool tbm_is_empty(const TIDBitmap *tbm); extern TBMIterator *tbm_begin_iterate(TIDBitmap *tbm); +extern dsa_pointer tbm_prepare_shared_iterate(TIDBitmap *tbm); extern TBMIterateResult *tbm_iterate(TBMIterator *iterator); +extern TBMIterateResult *tbm_shared_iterate(TBMSharedIterator *iterator); extern void tbm_end_iterate(TBMIterator *iterator); +extern void tbm_end_shared_iterate(TBMSharedIterator *iterator); +extern TBMSharedIterator *tbm_attach_shared_iterate(dsa_area *dsa, + dsa_pointer dp); #endif /* TIDBITMAP_H */ diff --git a/src/include/storage/lwlock.h b/src/include/storage/lwlock.h index 8bd93c3b81..0cd45bb6d8 100644 --- a/src/include/storage/lwlock.h +++ b/src/include/storage/lwlock.h @@ -212,6 +212,7 @@ typedef enum BuiltinTrancheIds LWTRANCHE_LOCK_MANAGER, LWTRANCHE_PREDICATE_LOCK_MANAGER, LWTRANCHE_PARALLEL_QUERY_DSA, + LWTRANCHE_TBM, LWTRANCHE_FIRST_USER_DEFINED } BuiltinTrancheIds; -- cgit v1.2.3 From 044d9efb6c97d65247a1287e7676de0ee75b3cfe Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Mon, 27 Feb 2017 10:44:59 -0500 Subject: Create INSTALL file via XSLT As before, create an INSTALL.html file for processing with lynx, but use xsltproc and a new XSLT stylesheet instead of jade and DSSSL. Replacing jade with xsltproc removes jade from the requirements for distribution building. Reviewed-by: Magnus Hagander --- doc/src/sgml/.gitignore | 1 + doc/src/sgml/Makefile | 35 ++++++++------ doc/src/sgml/stylesheet-text.xsl | 98 ++++++++++++++++++++++++++++++++++++++++ doc/src/sgml/stylesheet.dsl | 43 ------------------ 4 files changed, 120 insertions(+), 57 deletions(-) create mode 100644 doc/src/sgml/stylesheet-text.xsl (limited to 'doc/src') diff --git a/doc/src/sgml/.gitignore b/doc/src/sgml/.gitignore index 2f0329c15f..cdeace2991 100644 --- a/doc/src/sgml/.gitignore +++ b/doc/src/sgml/.gitignore @@ -7,6 +7,7 @@ /man-stamp # Other popular build targets /INSTALL +/INSTALL.xml /postgres-US.pdf /postgres-A4.pdf /postgres.html diff --git a/doc/src/sgml/Makefile b/doc/src/sgml/Makefile index fe7ca65cd4..774d35de20 100644 --- a/doc/src/sgml/Makefile +++ b/doc/src/sgml/Makefile @@ -217,10 +217,9 @@ postgres.pdf: ## -## Semi-automatic generation of some text files. +## Generation of some text files. ## -JADE.text = $(JADE) $(JADEFLAGS) $(SGMLINCLUDE) $(CATALOG) -d stylesheet.dsl -i output-text -t sgml ICONV = iconv LYNX = lynx @@ -233,10 +232,15 @@ LYNX = lynx # locale support and is very picky about locale name spelling. The # below has been finely tuned to run on FreeBSD and Linux/glibc. INSTALL: % : %.html - $(PERL) -p -e 's/ $@ + $(PERL) -p -e 's, $@ -INSTALL.html: standalone-install.sgml installation.sgml version.sgml - $(JADE.text) -V nochunks standalone-install.sgml installation.sgml > $@ +INSTALL.html: %.html : stylesheet-text.xsl %.xml + $(XMLLINT) --noout --valid $*.xml + $(XSLTPROC) $(XSLTPROCFLAGS) $(XSLTPROC_HTML_FLAGS) $^ >$@ + +INSTALL.xml: standalone-install.sgml installation.sgml version.sgml + $(OSX) -D. -x lower $(filter-out version.sgml,$^) >$@.tmp + $(call mangle-xml,chapter) ## @@ -247,12 +251,15 @@ INSTALL.html: standalone-install.sgml installation.sgml version.sgml # if we try to do "make all" in a VPATH build without the explicit # $(srcdir) on the postgres.sgml dependency in this rule. GNU make bug? postgres.xml: $(srcdir)/postgres.sgml $(ALMOSTALLSGML) - $(OSX) -D. -x lower -i include-xslt-index $< >postgres.xmltmp - $(PERL) -p -e 's/\[(aacute|acirc|aelig|agrave|amp|aring|atilde|auml|bull|copy|eacute|egrave|gt|iacute|lt|mdash|nbsp|ntilde|oacute|ocirc|oslash|ouml|pi|quot|scaron|uuml) *\]/\&\1;/gi;' \ - -e '$$_ .= qq{\n} if $$. == 1;' \ - $@ - rm postgres.xmltmp -# ' hello Emacs + $(OSX) -D. -x lower -i include-xslt-index $< >$@.tmp + $(call mangle-xml,book) + +define mangle-xml +$(PERL) -p -e 's/\[(aacute|acirc|aelig|agrave|amp|aring|atilde|auml|bull|copy|eacute|egrave|gt|iacute|lt|mdash|nbsp|ntilde|oacute|ocirc|oslash|ouml|pi|quot|scaron|uuml) *\]/\&\1;/gi;' \ + -e '$$_ .= qq{\n} if $$. == 1;' \ + <$@.tmp > $@ +rm $@.tmp +endef ifeq ($(STYLE),website) XSLTPROC_HTML_FLAGS += --param website.stylesheet 1 @@ -386,13 +393,13 @@ check-tabs: # This allows removing some files from the distribution tarballs while # keeping the dependencies satisfied. .SECONDARY: postgres.xml $(GENERATED_SGML) HTML.index -.SECONDARY: INSTALL.html +.SECONDARY: INSTALL.html INSTALL.xml .SECONDARY: %-A4.tex-ps %-US.tex-ps %-A4.tex-pdf %-US.tex-pdf clean: # text --- these are shipped, but not in this directory rm -f INSTALL - rm -f INSTALL.html + rm -f INSTALL.html INSTALL.xml # single-page output rm -f postgres.html postgres.txt # print @@ -400,7 +407,7 @@ clean: # index rm -f HTML.index $(GENERATED_SGML) # XSLT - rm -f postgres.xml postgres.xmltmp htmlhelp.hhp toc.hhc index.hhk *.fo + rm -f postgres.xml *.tmp htmlhelp.hhp toc.hhc index.hhk *.fo # EPUB rm -f postgres.epub # Texinfo diff --git a/doc/src/sgml/stylesheet-text.xsl b/doc/src/sgml/stylesheet-text.xsl new file mode 100644 index 0000000000..476b871870 --- /dev/null +++ b/doc/src/sgml/stylesheet-text.xsl @@ -0,0 +1,98 @@ + + + + + + + + + + + + + + + + + + +
  • + + + + +
    • +     + + + + + * + + * + + + + + + + + + + + + + + +
    + + + + + + +

    + + + : + +

    +     + + +
    +     + + + + +
    + +     + + +
    + +     + + +
    + +     + +     diff --git a/doc/src/sgml/stylesheet.dsl b/doc/src/sgml/stylesheet.dsl index 61d2963b17..05cab9a5f3 100644 --- a/doc/src/sgml/stylesheet.dsl +++ b/doc/src/sgml/stylesheet.dsl @@ -4,7 +4,6 @@ - @@ -14,10 +13,6 @@ ]]> - -]]> - ]> @@ -796,44 +791,6 @@ ]]> - - - - - - - -- cgit v1.2.3 From fcec6caafa2346b6c9d3ad5065e417733bd63cd9 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Wed, 8 Mar 2017 12:39:37 -0300 Subject: Support XMLTABLE query expression XMLTABLE is defined by the SQL/XML standard as a feature that allows turning XML-formatted data into relational form, so that it can be used as a in the FROM clause of a query. This new construct provides significant simplicity and performance benefit for XML data processing; what in a client-side custom implementation was reported to take 20 minutes can be executed in 400ms using XMLTABLE. (The same functionality was said to take 10 seconds using nested PostgreSQL XPath function calls, and 5 seconds using XMLReader under PL/Python). The implemented syntax deviates slightly from what the standard requires. First, the standard indicates that the PASSING clause is optional and that multiple XML input documents may be given to it; we make it mandatory and accept a single document only. Second, we don't currently support a default namespace to be specified. This implementation relies on a new executor node based on a hardcoded method table. (Because the grammar is fixed, there is no extensibility in the current approach; further constructs can be implemented on top of this such as JSON_TABLE, but they require changes to core code.) Author: Pavel Stehule, Álvaro Herrera Extensively reviewed by: Craig Ringer Discussion: https://postgr.es/m/CAFj8pRAgfzMD-LoSmnMGybD0WsEznLHWap8DO79+-GTRAPR4qA@mail.gmail.com --- contrib/pg_stat_statements/pg_stat_statements.c | 12 + doc/src/sgml/func.sgml | 203 ++++++++- src/backend/commands/explain.c | 24 + src/backend/executor/Makefile | 3 +- src/backend/executor/execAmi.c | 6 + src/backend/executor/execProcnode.c | 14 + src/backend/executor/nodeTableFuncscan.c | 502 +++++++++++++++++++++ src/backend/nodes/copyfuncs.c | 91 ++++ src/backend/nodes/equalfuncs.c | 60 +++ src/backend/nodes/makefuncs.c | 8 +- src/backend/nodes/nodeFuncs.c | 66 +++ src/backend/nodes/outfuncs.c | 73 ++++ src/backend/nodes/print.c | 4 + src/backend/nodes/readfuncs.c | 47 ++ src/backend/optimizer/path/allpaths.c | 37 ++ src/backend/optimizer/path/costsize.c | 81 ++++ src/backend/optimizer/plan/createplan.c | 77 +++- src/backend/optimizer/plan/initsplan.c | 2 + src/backend/optimizer/plan/planner.c | 37 +- src/backend/optimizer/plan/setrefs.c | 14 + src/backend/optimizer/plan/subselect.c | 6 + src/backend/optimizer/prep/prepjointree.c | 6 + src/backend/optimizer/util/pathnode.c | 26 ++ src/backend/optimizer/util/plancat.c | 6 +- src/backend/optimizer/util/relnode.c | 5 +- src/backend/parser/analyze.c | 9 + src/backend/parser/gram.y | 189 +++++++- src/backend/parser/parse_clause.c | 245 ++++++++++- src/backend/parser/parse_coerce.c | 33 +- src/backend/parser/parse_relation.c | 79 +++- src/backend/parser/parse_target.c | 7 + src/backend/rewrite/rewriteHandler.c | 4 + src/backend/utils/adt/ruleutils.c | 133 +++++- src/backend/utils/adt/xml.c | 558 +++++++++++++++++++++++- src/include/catalog/catversion.h | 2 +- src/include/executor/nodeTableFuncscan.h | 24 + src/include/executor/tablefunc.h | 67 +++ src/include/nodes/execnodes.h | 25 ++ src/include/nodes/nodes.h | 5 + src/include/nodes/parsenodes.h | 39 ++ src/include/nodes/plannodes.h | 10 + src/include/nodes/primnodes.h | 22 + src/include/optimizer/cost.h | 5 + src/include/optimizer/pathnode.h | 4 + src/include/parser/kwlist.h | 3 + src/include/parser/parse_coerce.h | 4 + src/include/parser/parse_relation.h | 5 + src/include/utils/xml.h | 3 + src/test/regress/expected/xml.out | 504 +++++++++++++++++++++ src/test/regress/expected/xml_1.out | 475 ++++++++++++++++++++ src/test/regress/expected/xml_2.out | 504 +++++++++++++++++++++ src/test/regress/sql/xml.sql | 288 ++++++++++++ 52 files changed, 4606 insertions(+), 50 deletions(-) create mode 100644 src/backend/executor/nodeTableFuncscan.c create mode 100644 src/include/executor/nodeTableFuncscan.h create mode 100644 src/include/executor/tablefunc.h (limited to 'doc/src') diff --git a/contrib/pg_stat_statements/pg_stat_statements.c b/contrib/pg_stat_statements/pg_stat_statements.c index 62dec8768a..221ac98d4a 100644 --- a/contrib/pg_stat_statements/pg_stat_statements.c +++ b/contrib/pg_stat_statements/pg_stat_statements.c @@ -2400,6 +2400,9 @@ JumbleRangeTable(pgssJumbleState *jstate, List *rtable) case RTE_FUNCTION: JumbleExpr(jstate, (Node *) rte->functions); break; + case RTE_TABLEFUNC: + JumbleExpr(jstate, (Node *) rte->tablefunc); + break; case RTE_VALUES: JumbleExpr(jstate, (Node *) rte->values_lists); break; @@ -2868,6 +2871,15 @@ JumbleExpr(pgssJumbleState *jstate, Node *node) JumbleExpr(jstate, rtfunc->funcexpr); } break; + case T_TableFunc: + { + TableFunc *tablefunc = (TableFunc *) node; + + JumbleExpr(jstate, tablefunc->docexpr); + JumbleExpr(jstate, tablefunc->rowexpr); + JumbleExpr(jstate, (Node *) tablefunc->colexprs); + } + break; case T_TableSampleClause: { TableSampleClause *tsc = (TableSampleClause *) node; diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 9e084adc1a..583b3b241a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -10332,7 +10332,8 @@ SELECT xml_is_well_formed_document(' @@ -10430,6 +10431,206 @@ SELECT xpath_exists('/my:a/text()', 'test + + + + + <literal>xmltable</literal> + + + xmltable + + + + table function + XMLTABLE + + + +xmltable( XMLNAMESPACES(namespace uri AS namespace name, ...) + row_expression PASSING BY REF document_expression BY REF + COLUMNS name { type PATH column_expression DEFAULT default_expression NOT NULL | NULL + | FOR ORDINALITY } + , ... +) + + + + The xmltable function produces a table based + on the given XML value, an XPath filter to extract rows, and an + optional set of column definitions. + + + + The optional XMLNAMESPACES clause is a comma-separated + list of namespaces. It specifies the XML namespaces used in + the document and their aliases. A default namespace specification + is not currently supported. + + + + The required row_expression argument is an XPath + expression that is evaluated against the supplied XML document to + obtain an ordered sequence of XML nodes. This sequence is what + xmltable transforms into output rows. + + + + document_expression provides the XML document to + operate on. + The BY REF clauses have no effect in PostgreSQL, + but are allowed for SQL conformance and compatibility with other + implementations. + The argument must be a well-formed XML document; fragments/forests + are not accepted. + + + + The mandatory COLUMNS clause specifies the list + of columns in the output table. + If the COLUMNS clause is omitted, the rows in the result + set contain a single column of type xml containing the + data matched by row_expression. + If COLUMNS is specified, each entry describes a + single column. + See the syntax summary above for the format. + The column name and type are required; the path, default and + nullability clauses are optional. + + + + A column marked FOR ORDINALITY will be populated + with row numbers matching the order in which the + output rows appeared in the original input XML document. + At most one column may be marked FOR ORDINALITY. + + + + The column_expression for a column is an XPath expression + that is evaluated for each row, relative to the result of the + row_expression, to find the value of the column. + If no column_expression is given, then the column name + is used as an implicit path. + + + + If a column's XPath expression returns multiple elements, an error + is raised. + If the expression matches an empty tag, the result is an + empty string (not NULL). + Any xsi:nil attributes are ignored. + + + + The text body of the XML matched by the column_expression + is used as the column value. Multiple text() nodes + within an element are concatenated in order. Any child elements, + processing instructions, and comments are ignored, but the text contents + of child elements are concatenated to the result. + Note that the whitespace-only text() node between two non-text + elements is preserved, and that leading whitespace on a text() + node is not flattened. + + + + If the path expression does not match for a given row but + default_expression is specified, the value resulting + from evaluating that expression is used. + If no DEFAULT clause is given for the column, + the field will be set to NULL. + It is possible for a default_expression to reference + the value of output columns that appear prior to it in the column list, + so the default of one column may be based on the value of another + column. + + + + Columns may be marked NOT NULL. If the + column_expression for a NOT NULL column + does not match anything and there is no DEFAULT or the + default_expression also evaluates to null, an error + is reported. + + + + Unlike regular PostgreSQL functions, column_expression + and default_expression are not evaluated to a simple + value before calling the function. + column_expression is normally evaluated + exactly once per input row, and default_expression + is evaluated each time a default is needed for a field. + If the expression qualifies as stable or immutable the repeat + evaluation may be skipped. + Effectively xmltable behaves more like a subquery than a + function call. + This means that you can usefully use volatile functions like + nextval in default_expression, and + column_expression may depend on other parts of the + XML document. + + + + Examples: + + + AU + Australia + + + JP + Japan + Shinzo Abe + 145935 + + + SG + Singapore + 697 + + +$$ AS data; + +SELECT xmltable.* + FROM xmldata, + XMLTABLE('//ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + ordinality FOR ORDINALITY, + "COUNTRY_NAME" text, + country_id text PATH 'COUNTRY_ID', + size_sq_km float PATH 'SIZE[@unit = "sq_km"]', + size_other text PATH + 'concat(SIZE[@unit!="sq_km"], " ", SIZE[@unit!="sq_km"]/@unit)', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified') ; + + id | ordinality | COUNTRY_NAME | country_id | size_sq_km | size_other | premier_name +----+------------+--------------+------------+------------+--------------+--------------- + 1 | 1 | Australia | AU | | | not specified + 5 | 2 | Japan | JP | | 145935 sq_mi | Shinzo Abe + 6 | 3 | Singapore | SG | 697 | | not specified +]]> + + The following example shows concatenation of multiple text() nodes, + usage of the column name as XPath filter, and the treatment of whitespace, + XML comments and processing instructions: + + + Hello2a2 bbbxxxCC + +$$ AS data; + +SELECT xmltable.* + FROM xmlelements, XMLTABLE('/root' PASSING data COLUMNS element text); + element +---------------------- + Hello2a2 bbbCC ]]> diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index c9e0a3e42d..7a8d36c8db 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -781,6 +781,7 @@ ExplainPreScanNode(PlanState *planstate, Bitmapset **rels_used) case T_TidScan: case T_SubqueryScan: case T_FunctionScan: + case T_TableFuncScan: case T_ValuesScan: case T_CteScan: case T_WorkTableScan: @@ -926,6 +927,9 @@ ExplainNode(PlanState *planstate, List *ancestors, case T_FunctionScan: pname = sname = "Function Scan"; break; + case T_TableFuncScan: + pname = sname = "Table Function Scan"; + break; case T_ValuesScan: pname = sname = "Values Scan"; break; @@ -1103,6 +1107,7 @@ ExplainNode(PlanState *planstate, List *ancestors, case T_TidScan: case T_SubqueryScan: case T_FunctionScan: + case T_TableFuncScan: case T_ValuesScan: case T_CteScan: case T_WorkTableScan: @@ -1416,6 +1421,20 @@ ExplainNode(PlanState *planstate, List *ancestors, show_instrumentation_count("Rows Removed by Filter", 1, planstate, es); break; + case T_TableFuncScan: + if (es->verbose) + { + TableFunc *tablefunc = ((TableFuncScan *) plan)->tablefunc; + + show_expression((Node *) tablefunc, + "Table Function Call", planstate, ancestors, + es->verbose, es); + } + show_scan_qual(plan->qual, "Filter", planstate, ancestors, es); + if (plan->qual) + show_instrumentation_count("Rows Removed by Filter", 1, + planstate, es); + break; case T_TidScan: { /* @@ -2593,6 +2612,11 @@ ExplainTargetRel(Plan *plan, Index rti, ExplainState *es) objecttag = "Function Name"; } break; + case T_TableFuncScan: + Assert(rte->rtekind == RTE_TABLEFUNC); + objectname = "xmltable"; + objecttag = "Table Function Name"; + break; case T_ValuesScan: Assert(rte->rtekind == RTE_VALUES); break; diff --git a/src/backend/executor/Makefile b/src/backend/executor/Makefile index 2a2b7eb9bd..a9893c2b22 100644 --- a/src/backend/executor/Makefile +++ b/src/backend/executor/Makefile @@ -26,6 +26,7 @@ OBJS = execAmi.o execCurrent.o execGrouping.o execIndexing.o execJunk.o \ nodeSamplescan.o nodeSeqscan.o nodeSetOp.o nodeSort.o nodeUnique.o \ nodeValuesscan.o nodeCtescan.o nodeWorktablescan.o \ nodeGroup.o nodeSubplan.o nodeSubqueryscan.o nodeTidscan.o \ - nodeForeignscan.o nodeWindowAgg.o tstoreReceiver.o tqueue.o spi.o + nodeForeignscan.o nodeWindowAgg.o tstoreReceiver.o tqueue.o spi.o \ + nodeTableFuncscan.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/executor/execAmi.c b/src/backend/executor/execAmi.c index d3802079f5..5d59f95a91 100644 --- a/src/backend/executor/execAmi.c +++ b/src/backend/executor/execAmi.c @@ -48,6 +48,7 @@ #include "executor/nodeSort.h" #include "executor/nodeSubplan.h" #include "executor/nodeSubqueryscan.h" +#include "executor/nodeTableFuncscan.h" #include "executor/nodeTidscan.h" #include "executor/nodeUnique.h" #include "executor/nodeValuesscan.h" @@ -198,6 +199,10 @@ ExecReScan(PlanState *node) ExecReScanFunctionScan((FunctionScanState *) node); break; + case T_TableFuncScanState: + ExecReScanTableFuncScan((TableFuncScanState *) node); + break; + case T_ValuesScanState: ExecReScanValuesScan((ValuesScanState *) node); break; @@ -564,6 +569,7 @@ ExecMaterializesOutput(NodeTag plantype) { case T_Material: case T_FunctionScan: + case T_TableFuncScan: case T_CteScan: case T_WorkTableScan: case T_Sort: diff --git a/src/backend/executor/execProcnode.c b/src/backend/executor/execProcnode.c index ef6f35a5a0..468f50e6a6 100644 --- a/src/backend/executor/execProcnode.c +++ b/src/backend/executor/execProcnode.c @@ -110,6 +110,7 @@ #include "executor/nodeSort.h" #include "executor/nodeSubplan.h" #include "executor/nodeSubqueryscan.h" +#include "executor/nodeTableFuncscan.h" #include "executor/nodeTidscan.h" #include "executor/nodeUnique.h" #include "executor/nodeValuesscan.h" @@ -239,6 +240,11 @@ ExecInitNode(Plan *node, EState *estate, int eflags) estate, eflags); break; + case T_TableFuncScan: + result = (PlanState *) ExecInitTableFuncScan((TableFuncScan *) node, + estate, eflags); + break; + case T_ValuesScan: result = (PlanState *) ExecInitValuesScan((ValuesScan *) node, estate, eflags); @@ -459,6 +465,10 @@ ExecProcNode(PlanState *node) result = ExecFunctionScan((FunctionScanState *) node); break; + case T_TableFuncScanState: + result = ExecTableFuncScan((TableFuncScanState *) node); + break; + case T_ValuesScanState: result = ExecValuesScan((ValuesScanState *) node); break; @@ -715,6 +725,10 @@ ExecEndNode(PlanState *node) ExecEndFunctionScan((FunctionScanState *) node); break; + case T_TableFuncScanState: + ExecEndTableFuncScan((TableFuncScanState *) node); + break; + case T_ValuesScanState: ExecEndValuesScan((ValuesScanState *) node); break; diff --git a/src/backend/executor/nodeTableFuncscan.c b/src/backend/executor/nodeTableFuncscan.c new file mode 100644 index 0000000000..628f1ba074 --- /dev/null +++ b/src/backend/executor/nodeTableFuncscan.c @@ -0,0 +1,502 @@ +/*------------------------------------------------------------------------- + * + * nodeTableFuncscan.c + * Support routines for scanning RangeTableFunc (XMLTABLE like functions). + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/executor/nodeTableFuncscan.c + * + *------------------------------------------------------------------------- + */ +/* + * INTERFACE ROUTINES + * ExecTableFuncscan scans a function. + * ExecFunctionNext retrieve next tuple in sequential order. + * ExecInitTableFuncscan creates and initializes a TableFuncscan node. + * ExecEndTableFuncscan releases any storage allocated. + * ExecReScanTableFuncscan rescans the function + */ +#include "postgres.h" + +#include "nodes/execnodes.h" +#include "executor/executor.h" +#include "executor/nodeTableFuncscan.h" +#include "executor/tablefunc.h" +#include "miscadmin.h" +#include "utils/builtins.h" +#include "utils/lsyscache.h" +#include "utils/memutils.h" +#include "utils/xml.h" + + +static TupleTableSlot *TableFuncNext(TableFuncScanState *node); +static bool TableFuncRecheck(TableFuncScanState *node, TupleTableSlot *slot); + +static void tfuncFetchRows(TableFuncScanState *tstate, ExprContext *econtext); +static void tfuncInitialize(TableFuncScanState *tstate, ExprContext *econtext, Datum doc); +static void tfuncLoadRows(TableFuncScanState *tstate, ExprContext *econtext); + +/* ---------------------------------------------------------------- + * Scan Support + * ---------------------------------------------------------------- + */ +/* ---------------------------------------------------------------- + * TableFuncNext + * + * This is a workhorse for ExecTableFuncscan + * ---------------------------------------------------------------- + */ +static TupleTableSlot * +TableFuncNext(TableFuncScanState *node) +{ + TupleTableSlot *scanslot; + + scanslot = node->ss.ss_ScanTupleSlot; + + /* + * If first time through, read all tuples from function and put them in a + * tuplestore. Subsequent calls just fetch tuples from tuplestore. + */ + if (node->tupstore == NULL) + tfuncFetchRows(node, node->ss.ps.ps_ExprContext); + + /* + * Get the next tuple from tuplestore. + */ + (void) tuplestore_gettupleslot(node->tupstore, + true, + false, + scanslot); + return scanslot; +} + +/* + * TableFuncRecheck -- access method routine to recheck a tuple in EvalPlanQual + */ +static bool +TableFuncRecheck(TableFuncScanState *node, TupleTableSlot *slot) +{ + /* nothing to check */ + return true; +} + +/* ---------------------------------------------------------------- + * ExecTableFuncscan(node) + * + * Scans the function sequentially and returns the next qualifying + * tuple. + * We call the ExecScan() routine and pass it the appropriate + * access method functions. + * ---------------------------------------------------------------- + */ +TupleTableSlot * +ExecTableFuncScan(TableFuncScanState *node) +{ + return ExecScan(&node->ss, + (ExecScanAccessMtd) TableFuncNext, + (ExecScanRecheckMtd) TableFuncRecheck); +} + +/* ---------------------------------------------------------------- + * ExecInitTableFuncscan + * ---------------------------------------------------------------- + */ +TableFuncScanState * +ExecInitTableFuncScan(TableFuncScan *node, EState *estate, int eflags) +{ + TableFuncScanState *scanstate; + TableFunc *tf = node->tablefunc; + TupleDesc tupdesc; + int i; + + /* check for unsupported flags */ + Assert(!(eflags & EXEC_FLAG_MARK)); + + /* + * TableFuncscan should not have any children. + */ + Assert(outerPlan(node) == NULL); + Assert(innerPlan(node) == NULL); + + /* + * create new ScanState for node + */ + scanstate = makeNode(TableFuncScanState); + scanstate->ss.ps.plan = (Plan *) node; + scanstate->ss.ps.state = estate; + + /* + * Miscellaneous initialization + * + * create expression context for node + */ + ExecAssignExprContext(estate, &scanstate->ss.ps); + + /* + * initialize child expressions + */ + scanstate->ss.ps.targetlist = (List *) + ExecInitExpr((Expr *) node->scan.plan.targetlist, + (PlanState *) scanstate); + scanstate->ss.ps.qual = (List *) + ExecInitExpr((Expr *) node->scan.plan.qual, + (PlanState *) scanstate); + + /* + * tuple table initialization + */ + ExecInitResultTupleSlot(estate, &scanstate->ss.ps); + ExecInitScanTupleSlot(estate, &scanstate->ss); + + /* + * initialize source tuple type + */ + tupdesc = BuildDescFromLists(tf->colnames, + tf->coltypes, + tf->coltypmods, + tf->colcollations); + + ExecAssignScanType(&scanstate->ss, tupdesc); + + /* + * Initialize result tuple type and projection info. + */ + ExecAssignResultTypeFromTL(&scanstate->ss.ps); + ExecAssignScanProjectionInfo(&scanstate->ss); + + /* Only XMLTABLE is supported currently */ + scanstate->routine = &XmlTableRoutine; + + scanstate->perValueCxt = + AllocSetContextCreate(CurrentMemoryContext, + "TableFunc per value context", + ALLOCSET_DEFAULT_SIZES); + scanstate->opaque = NULL; /* initialized at runtime */ + + scanstate->ns_names = tf->ns_names; + + scanstate->ns_uris = (List *) + ExecInitExpr((Expr *) tf->ns_uris, (PlanState *) scanstate); + scanstate->docexpr = + ExecInitExpr((Expr *) tf->docexpr, (PlanState *) scanstate); + scanstate->rowexpr = + ExecInitExpr((Expr *) tf->rowexpr, (PlanState *) scanstate); + scanstate->colexprs = (List *) + ExecInitExpr((Expr *) tf->colexprs, (PlanState *) scanstate); + scanstate->coldefexprs = (List *) + ExecInitExpr((Expr *) tf->coldefexprs, (PlanState *) scanstate); + + scanstate->notnulls = tf->notnulls; + + /* these are allocated now and initialized later */ + scanstate->in_functions = palloc(sizeof(FmgrInfo) * tupdesc->natts); + scanstate->typioparams = palloc(sizeof(Oid) * tupdesc->natts); + + /* + * Fill in the necessary fmgr infos. + */ + for (i = 0; i < tupdesc->natts; i++) + { + Oid in_funcid; + + getTypeInputInfo(tupdesc->attrs[i]->atttypid, + &in_funcid, &scanstate->typioparams[i]); + fmgr_info(in_funcid, &scanstate->in_functions[i]); + } + + return scanstate; +} + +/* ---------------------------------------------------------------- + * ExecEndTableFuncscan + * + * frees any storage allocated through C routines. + * ---------------------------------------------------------------- + */ +void +ExecEndTableFuncScan(TableFuncScanState *node) +{ + /* + * Free the exprcontext + */ + ExecFreeExprContext(&node->ss.ps); + + /* + * clean out the tuple table + */ + ExecClearTuple(node->ss.ps.ps_ResultTupleSlot); + ExecClearTuple(node->ss.ss_ScanTupleSlot); + + /* + * Release tuplestore resources + */ + if (node->tupstore != NULL) + tuplestore_end(node->tupstore); + node->tupstore = NULL; +} + +/* ---------------------------------------------------------------- + * ExecReScanTableFuncscan + * + * Rescans the relation. + * ---------------------------------------------------------------- + */ +void +ExecReScanTableFuncScan(TableFuncScanState *node) +{ + Bitmapset *chgparam = node->ss.ps.chgParam; + + ExecClearTuple(node->ss.ps.ps_ResultTupleSlot); + ExecScanReScan(&node->ss); + + /* + * Recompute when parameters are changed. + */ + if (chgparam) + { + if (node->tupstore != NULL) + { + tuplestore_end(node->tupstore); + node->tupstore = NULL; + } + } + + if (node->tupstore != NULL) + tuplestore_rescan(node->tupstore); +} + +/* ---------------------------------------------------------------- + * tfuncFetchRows + * + * Read rows from a TableFunc producer + * ---------------------------------------------------------------- + */ +static void +tfuncFetchRows(TableFuncScanState *tstate, ExprContext *econtext) +{ + const TableFuncRoutine *routine = tstate->routine; + MemoryContext oldcxt; + Datum value; + bool isnull; + + Assert(tstate->opaque == NULL); + + /* build tuplestore for the result */ + oldcxt = MemoryContextSwitchTo(econtext->ecxt_per_query_memory); + tstate->tupstore = tuplestore_begin_heap(false, false, work_mem); + + PG_TRY(); + { + routine->InitOpaque(tstate, + tstate->ss.ss_ScanTupleSlot->tts_tupleDescriptor->natts); + + /* + * If evaluating the document expression returns NULL, the table + * expression is empty and we return immediately. + */ + value = ExecEvalExpr(tstate->docexpr, econtext, &isnull); + + if (!isnull) + { + /* otherwise, pass the document value to the table builder */ + tfuncInitialize(tstate, econtext, value); + + /* initialize ordinality counter */ + tstate->ordinal = 1; + + /* Load all rows into the tuplestore, and we're done */ + tfuncLoadRows(tstate, econtext); + } + } + PG_CATCH(); + { + if (tstate->opaque != NULL) + routine->DestroyOpaque(tstate); + PG_RE_THROW(); + } + PG_END_TRY(); + + /* return to original memory context, and clean up */ + MemoryContextSwitchTo(oldcxt); + + if (tstate->opaque != NULL) + { + routine->DestroyOpaque(tstate); + tstate->opaque = NULL; + } + + return; +} + +/* + * Fill in namespace declarations, the row filter, and column filters in a + * table expression builder context. + */ +static void +tfuncInitialize(TableFuncScanState *tstate, ExprContext *econtext, Datum doc) +{ + const TableFuncRoutine *routine = tstate->routine; + TupleDesc tupdesc; + ListCell *lc1, + *lc2; + bool isnull; + int colno; + Datum value; + int ordinalitycol = + ((TableFuncScan *) (tstate->ss.ps.plan))->tablefunc->ordinalitycol; + + /* + * Install the document as a possibly-toasted Datum into the tablefunc + * context. + */ + routine->SetDocument(tstate, doc); + + /* Evaluate namespace specifications */ + forboth(lc1, tstate->ns_uris, lc2, tstate->ns_names) + { + ExprState *expr = (ExprState *) lfirst(lc1); + char *ns_name = strVal(lfirst(lc2)); + char *ns_uri; + + value = ExecEvalExpr((ExprState *) expr, econtext, &isnull); + if (isnull) + ereport(ERROR, + (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("namespace URI must not be null"))); + ns_uri = TextDatumGetCString(value); + + routine->SetNamespace(tstate, ns_name, ns_uri); + } + + /* Install the row filter expression into the table builder context */ + value = ExecEvalExpr(tstate->rowexpr, econtext, &isnull); + if (isnull) + ereport(ERROR, + (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("row filter expression must not be null"))); + + routine->SetRowFilter(tstate, TextDatumGetCString(value)); + + /* + * Install the column filter expressions into the table builder context. + * If an expression is given, use that; otherwise the column name itself + * is the column filter. + */ + colno = 0; + tupdesc = tstate->ss.ss_ScanTupleSlot->tts_tupleDescriptor; + foreach(lc1, tstate->colexprs) + { + char *colfilter; + + if (colno != ordinalitycol) + { + ExprState *colexpr = lfirst(lc1); + + if (colexpr != NULL) + { + value = ExecEvalExpr(colexpr, econtext, &isnull); + if (isnull) + ereport(ERROR, + (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("column filter expression must not be null"), + errdetail("Filter for column \"%s\" is null.", + NameStr(tupdesc->attrs[colno]->attname)))); + colfilter = TextDatumGetCString(value); + } + else + colfilter = NameStr(tupdesc->attrs[colno]->attname); + + routine->SetColumnFilter(tstate, colfilter, colno); + } + + colno++; + } +} + +/* + * Load all the rows from the TableFunc table builder into a tuplestore. + */ +static void +tfuncLoadRows(TableFuncScanState *tstate, ExprContext *econtext) +{ + const TableFuncRoutine *routine = tstate->routine; + TupleTableSlot *slot = tstate->ss.ss_ScanTupleSlot; + TupleDesc tupdesc = slot->tts_tupleDescriptor; + Datum *values = slot->tts_values; + bool *nulls = slot->tts_isnull; + int natts = tupdesc->natts; + MemoryContext oldcxt; + int ordinalitycol; + + ordinalitycol = + ((TableFuncScan *) (tstate->ss.ps.plan))->tablefunc->ordinalitycol; + oldcxt = MemoryContextSwitchTo(tstate->perValueCxt); + + /* + * Keep requesting rows from the table builder until there aren't any. + */ + while (routine->FetchRow(tstate)) + { + ListCell *cell = list_head(tstate->coldefexprs); + int colno; + + ExecClearTuple(tstate->ss.ss_ScanTupleSlot); + + /* + * Obtain the value of each column for this row, installing them into the + * slot; then add the tuple to the tuplestore. + */ + for (colno = 0; colno < natts; colno++) + { + if (colno == ordinalitycol) + { + /* Fast path for ordinality column */ + values[colno] = Int32GetDatum(tstate->ordinal++); + nulls[colno] = false; + } + else + { + bool isnull; + + values[colno] = routine->GetValue(tstate, + colno, + tupdesc->attrs[colno]->atttypid, + tupdesc->attrs[colno]->atttypmod, + &isnull); + + /* No value? Evaluate and apply the default, if any */ + if (isnull && cell != NULL) + { + ExprState *coldefexpr = (ExprState *) lfirst(cell); + + if (coldefexpr != NULL) + values[colno] = ExecEvalExpr(coldefexpr, econtext, + &isnull); + } + + /* Verify a possible NOT NULL constraint */ + if (isnull && bms_is_member(colno, tstate->notnulls)) + ereport(ERROR, + (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("null is not allowed in column \"%s\"", + NameStr(tupdesc->attrs[colno]->attname)))); + + nulls[colno] = isnull; + } + + /* advance list of default expressions */ + if (cell != NULL) + cell = lnext(cell); + } + + tuplestore_putvalues(tstate->tupstore, tupdesc, values, nulls); + + MemoryContextReset(tstate->perValueCxt); + } + + MemoryContextSwitchTo(oldcxt); +} diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index bb2a8a3586..b3eac06c50 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -587,6 +587,27 @@ _copyFunctionScan(const FunctionScan *from) return newnode; } +/* + * _copyTableFuncScan + */ +static TableFuncScan * +_copyTableFuncScan(const TableFuncScan *from) +{ + TableFuncScan *newnode = makeNode(TableFuncScan); + + /* + * copy node superclass fields + */ + CopyScanFields((const Scan *) from, (Scan *) newnode); + + /* + * copy remainder of node + */ + COPY_NODE_FIELD(tablefunc); + + return newnode; +} + /* * _copyValuesScan */ @@ -1138,6 +1159,31 @@ _copyRangeVar(const RangeVar *from) return newnode; } +/* + * _copyTableFunc + */ +static TableFunc * +_copyTableFunc(const TableFunc *from) +{ + TableFunc *newnode = makeNode(TableFunc); + + COPY_NODE_FIELD(ns_names); + COPY_NODE_FIELD(ns_uris); + COPY_NODE_FIELD(docexpr); + COPY_NODE_FIELD(rowexpr); + COPY_NODE_FIELD(colnames); + COPY_NODE_FIELD(coltypes); + COPY_NODE_FIELD(coltypmods); + COPY_NODE_FIELD(colcollations); + COPY_NODE_FIELD(colexprs); + COPY_NODE_FIELD(coldefexprs); + COPY_BITMAPSET_FIELD(notnulls); + COPY_SCALAR_FIELD(ordinalitycol); + COPY_LOCATION_FIELD(location); + + return newnode; +} + /* * _copyIntoClause */ @@ -2169,6 +2215,7 @@ _copyRangeTblEntry(const RangeTblEntry *from) COPY_NODE_FIELD(joinaliasvars); COPY_NODE_FIELD(functions); COPY_SCALAR_FIELD(funcordinality); + COPY_NODE_FIELD(tablefunc); COPY_NODE_FIELD(values_lists); COPY_STRING_FIELD(ctename); COPY_SCALAR_FIELD(ctelevelsup); @@ -2590,6 +2637,38 @@ _copyRangeTableSample(const RangeTableSample *from) return newnode; } +static RangeTableFunc * +_copyRangeTableFunc(const RangeTableFunc *from) +{ + RangeTableFunc *newnode = makeNode(RangeTableFunc); + + COPY_SCALAR_FIELD(lateral); + COPY_NODE_FIELD(docexpr); + COPY_NODE_FIELD(rowexpr); + COPY_NODE_FIELD(namespaces); + COPY_NODE_FIELD(columns); + COPY_NODE_FIELD(alias); + COPY_LOCATION_FIELD(location); + + return newnode; +} + +static RangeTableFuncCol * +_copyRangeTableFuncCol(const RangeTableFuncCol *from) +{ + RangeTableFuncCol *newnode = makeNode(RangeTableFuncCol); + + COPY_STRING_FIELD(colname); + COPY_NODE_FIELD(typeName); + COPY_SCALAR_FIELD(for_ordinality); + COPY_SCALAR_FIELD(is_not_null); + COPY_NODE_FIELD(colexpr); + COPY_NODE_FIELD(coldefexpr); + COPY_LOCATION_FIELD(location); + + return newnode; +} + static TypeCast * _copyTypeCast(const TypeCast *from) { @@ -4540,6 +4619,9 @@ copyObject(const void *from) case T_FunctionScan: retval = _copyFunctionScan(from); break; + case T_TableFuncScan: + retval = _copyTableFuncScan(from); + break; case T_ValuesScan: retval = _copyValuesScan(from); break; @@ -4616,6 +4698,9 @@ copyObject(const void *from) case T_RangeVar: retval = _copyRangeVar(from); break; + case T_TableFunc: + retval = _copyTableFunc(from); + break; case T_IntoClause: retval = _copyIntoClause(from); break; @@ -5210,6 +5295,12 @@ copyObject(const void *from) case T_RangeTableSample: retval = _copyRangeTableSample(from); break; + case T_RangeTableFunc: + retval = _copyRangeTableFunc(from); + break; + case T_RangeTableFuncCol: + retval = _copyRangeTableFuncCol(from); + break; case T_TypeName: retval = _copyTypeName(from); break; diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index 9fa83b9453..54e9c983a0 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -116,6 +116,26 @@ _equalRangeVar(const RangeVar *a, const RangeVar *b) return true; } +static bool +_equalTableFunc(const TableFunc *a, const TableFunc *b) +{ + COMPARE_NODE_FIELD(ns_names); + COMPARE_NODE_FIELD(ns_uris); + COMPARE_NODE_FIELD(docexpr); + COMPARE_NODE_FIELD(rowexpr); + COMPARE_NODE_FIELD(colnames); + COMPARE_NODE_FIELD(coltypes); + COMPARE_NODE_FIELD(coltypes); + COMPARE_NODE_FIELD(colcollations); + COMPARE_NODE_FIELD(colexprs); + COMPARE_NODE_FIELD(coldefexprs); + COMPARE_BITMAPSET_FIELD(notnulls); + COMPARE_SCALAR_FIELD(ordinalitycol); + COMPARE_LOCATION_FIELD(location); + + return true; +} + static bool _equalIntoClause(const IntoClause *a, const IntoClause *b) { @@ -2419,6 +2439,36 @@ _equalRangeTableSample(const RangeTableSample *a, const RangeTableSample *b) return true; } +static bool +_equalRangeTableFunc(const RangeTableFunc *a, const RangeTableFunc *b) +{ + COMPARE_SCALAR_FIELD(lateral); + COMPARE_NODE_FIELD(docexpr); + COMPARE_NODE_FIELD(rowexpr); + COMPARE_NODE_FIELD(namespaces); + COMPARE_NODE_FIELD(columns); + COMPARE_NODE_FIELD(alias); + COMPARE_LOCATION_FIELD(location); + + return true; +} + +static bool +_equalRangeTableFuncCol(const RangeTableFuncCol *a, const RangeTableFuncCol *b) +{ + COMPARE_STRING_FIELD(colname); + COMPARE_NODE_FIELD(typeName); + COMPARE_SCALAR_FIELD(for_ordinality); + COMPARE_NODE_FIELD(typeName); + COMPARE_SCALAR_FIELD(is_not_null); + COMPARE_NODE_FIELD(colexpr); + COMPARE_NODE_FIELD(coldefexpr); + COMPARE_LOCATION_FIELD(location); + + return true; +} + + static bool _equalIndexElem(const IndexElem *a, const IndexElem *b) { @@ -2521,6 +2571,7 @@ _equalRangeTblEntry(const RangeTblEntry *a, const RangeTblEntry *b) COMPARE_SCALAR_FIELD(jointype); COMPARE_NODE_FIELD(joinaliasvars); COMPARE_NODE_FIELD(functions); + COMPARE_NODE_FIELD(tablefunc); COMPARE_SCALAR_FIELD(funcordinality); COMPARE_NODE_FIELD(values_lists); COMPARE_STRING_FIELD(ctename); @@ -2887,6 +2938,9 @@ equal(const void *a, const void *b) case T_RangeVar: retval = _equalRangeVar(a, b); break; + case T_TableFunc: + retval = _equalTableFunc(a, b); + break; case T_IntoClause: retval = _equalIntoClause(a, b); break; @@ -3468,6 +3522,12 @@ equal(const void *a, const void *b) case T_RangeTableSample: retval = _equalRangeTableSample(a, b); break; + case T_RangeTableFunc: + retval = _equalRangeTableFunc(a, b); + break; + case T_RangeTableFuncCol: + retval = _equalRangeTableFuncCol(a, b); + break; case T_TypeName: retval = _equalTypeName(a, b); break; diff --git a/src/backend/nodes/makefuncs.c b/src/backend/nodes/makefuncs.c index 7586cce56a..e88d82f3b0 100644 --- a/src/backend/nodes/makefuncs.c +++ b/src/backend/nodes/makefuncs.c @@ -210,10 +210,10 @@ makeWholeRowVar(RangeTblEntry *rte, default: /* - * RTE is a join, subselect, or VALUES. We represent this as a - * whole-row Var of RECORD type. (Note that in most cases the Var - * will be expanded to a RowExpr during planning, but that is not - * our concern here.) + * RTE is a join, subselect, tablefunc, or VALUES. We represent + * this as a whole-row Var of RECORD type. (Note that in most + * cases the Var will be expanded to a RowExpr during planning, + * but that is not our concern here.) */ result = makeVar(varno, InvalidAttrNumber, diff --git a/src/backend/nodes/nodeFuncs.c b/src/backend/nodes/nodeFuncs.c index 71b24a079c..6e52eb7231 100644 --- a/src/backend/nodes/nodeFuncs.c +++ b/src/backend/nodes/nodeFuncs.c @@ -1212,6 +1212,9 @@ exprLocation(const Node *expr) case T_RangeVar: loc = ((const RangeVar *) expr)->location; break; + case T_TableFunc: + loc = ((const TableFunc *) expr)->location; + break; case T_Var: loc = ((const Var *) expr)->location; break; @@ -2211,6 +2214,22 @@ expression_tree_walker(Node *node, return true; } break; + case T_TableFunc: + { + TableFunc *tf = (TableFunc *) node; + + if (walker(tf->ns_uris, context)) + return true; + if (walker(tf->docexpr, context)) + return true; + if (walker(tf->rowexpr, context)) + return true; + if (walker(tf->colexprs, context)) + return true; + if (walker(tf->coldefexprs, context)) + return true; + } + break; default: elog(ERROR, "unrecognized node type: %d", (int) nodeTag(node)); @@ -2318,6 +2337,10 @@ range_table_walker(List *rtable, if (walker(rte->functions, context)) return true; break; + case RTE_TABLEFUNC: + if (walker(rte->tablefunc, context)) + return true; + break; case RTE_VALUES: if (walker(rte->values_lists, context)) return true; @@ -3007,6 +3030,20 @@ expression_tree_mutator(Node *node, return (Node *) newnode; } break; + case T_TableFunc: + { + TableFunc *tf = (TableFunc *) node; + TableFunc *newnode; + + FLATCOPY(newnode, tf, TableFunc); + MUTATE(newnode->ns_uris, tf->ns_uris, List *); + MUTATE(newnode->docexpr, tf->docexpr, Node *); + MUTATE(newnode->rowexpr, tf->rowexpr, Node *); + MUTATE(newnode->colexprs, tf->colexprs, List *); + MUTATE(newnode->coldefexprs, tf->coldefexprs, List *); + return (Node *) newnode; + } + break; default: elog(ERROR, "unrecognized node type: %d", (int) nodeTag(node)); @@ -3124,6 +3161,9 @@ range_table_mutator(List *rtable, case RTE_FUNCTION: MUTATE(newrte->functions, rte->functions, List *); break; + case RTE_TABLEFUNC: + MUTATE(newrte->tablefunc, rte->tablefunc, TableFunc *); + break; case RTE_VALUES: MUTATE(newrte->values_lists, rte->values_lists, List *); break; @@ -3548,6 +3588,32 @@ raw_expression_tree_walker(Node *node, return true; } break; + case T_RangeTableFunc: + { + RangeTableFunc *rtf = (RangeTableFunc *) node; + + if (walker(rtf->docexpr, context)) + return true; + if (walker(rtf->rowexpr, context)) + return true; + if (walker(rtf->namespaces, context)) + return true; + if (walker(rtf->columns, context)) + return true; + if (walker(rtf->alias, context)) + return true; + } + break; + case T_RangeTableFuncCol: + { + RangeTableFuncCol *rtfc = (RangeTableFuncCol *) node; + + if (walker(rtfc->colexpr, context)) + return true; + if (walker(rtfc->coldefexpr, context)) + return true; + } + break; case T_TypeName: { TypeName *tn = (TypeName *) node; diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index b3802b4428..d4297d11b1 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -565,6 +565,16 @@ _outFunctionScan(StringInfo str, const FunctionScan *node) WRITE_BOOL_FIELD(funcordinality); } +static void +_outTableFuncScan(StringInfo str, const TableFuncScan *node) +{ + WRITE_NODE_TYPE("TABLEFUNCSCAN"); + + _outScanInfo(str, (const Scan *) node); + + WRITE_NODE_FIELD(tablefunc); +} + static void _outValuesScan(StringInfo str, const ValuesScan *node) { @@ -955,6 +965,26 @@ _outRangeVar(StringInfo str, const RangeVar *node) WRITE_LOCATION_FIELD(location); } +static void +_outTableFunc(StringInfo str, const TableFunc *node) +{ + WRITE_NODE_TYPE("TABLEFUNC"); + + WRITE_NODE_FIELD(ns_names); + WRITE_NODE_FIELD(ns_uris); + WRITE_NODE_FIELD(docexpr); + WRITE_NODE_FIELD(rowexpr); + WRITE_NODE_FIELD(colnames); + WRITE_NODE_FIELD(coltypes); + WRITE_NODE_FIELD(coltypmods); + WRITE_NODE_FIELD(colcollations); + WRITE_NODE_FIELD(colexprs); + WRITE_NODE_FIELD(coldefexprs); + WRITE_BITMAPSET_FIELD(notnulls); + WRITE_INT_FIELD(ordinalitycol); + WRITE_LOCATION_FIELD(location); +} + static void _outIntoClause(StringInfo str, const IntoClause *node) { @@ -2869,6 +2899,9 @@ _outRangeTblEntry(StringInfo str, const RangeTblEntry *node) WRITE_NODE_FIELD(functions); WRITE_BOOL_FIELD(funcordinality); break; + case RTE_TABLEFUNC: + WRITE_NODE_FIELD(tablefunc); + break; case RTE_VALUES: WRITE_NODE_FIELD(values_lists); WRITE_NODE_FIELD(coltypes); @@ -3191,6 +3224,34 @@ _outRangeTableSample(StringInfo str, const RangeTableSample *node) WRITE_LOCATION_FIELD(location); } +static void +_outRangeTableFunc(StringInfo str, const RangeTableFunc *node) +{ + WRITE_NODE_TYPE("RANGETABLEFUNC"); + + WRITE_BOOL_FIELD(lateral); + WRITE_NODE_FIELD(docexpr); + WRITE_NODE_FIELD(rowexpr); + WRITE_NODE_FIELD(namespaces); + WRITE_NODE_FIELD(columns); + WRITE_NODE_FIELD(alias); + WRITE_LOCATION_FIELD(location); +} + +static void +_outRangeTableFuncCol(StringInfo str, const RangeTableFuncCol *node) +{ + WRITE_NODE_TYPE("RANGETABLEFUNCCOL"); + + WRITE_STRING_FIELD(colname); + WRITE_NODE_FIELD(typeName); + WRITE_BOOL_FIELD(for_ordinality); + WRITE_BOOL_FIELD(is_not_null); + WRITE_NODE_FIELD(colexpr); + WRITE_NODE_FIELD(coldefexpr); + WRITE_LOCATION_FIELD(location); +} + static void _outConstraint(StringInfo str, const Constraint *node) { @@ -3440,6 +3501,9 @@ outNode(StringInfo str, const void *obj) case T_FunctionScan: _outFunctionScan(str, obj); break; + case T_TableFuncScan: + _outTableFuncScan(str, obj); + break; case T_ValuesScan: _outValuesScan(str, obj); break; @@ -3512,6 +3576,9 @@ outNode(StringInfo str, const void *obj) case T_RangeVar: _outRangeVar(str, obj); break; + case T_TableFunc: + _outTableFunc(str, obj); + break; case T_IntoClause: _outIntoClause(str, obj); break; @@ -3922,6 +3989,12 @@ outNode(StringInfo str, const void *obj) case T_RangeTableSample: _outRangeTableSample(str, obj); break; + case T_RangeTableFunc: + _outRangeTableFunc(str, obj); + break; + case T_RangeTableFuncCol: + _outRangeTableFuncCol(str, obj); + break; case T_Constraint: _outConstraint(str, obj); break; diff --git a/src/backend/nodes/print.c b/src/backend/nodes/print.c index 926f226f34..dfb8bfa803 100644 --- a/src/backend/nodes/print.c +++ b/src/backend/nodes/print.c @@ -279,6 +279,10 @@ print_rt(const List *rtable) printf("%d\t%s\t[rangefunction]", i, rte->eref->aliasname); break; + case RTE_TABLEFUNC: + printf("%d\t%s\t[table function]", + i, rte->eref->aliasname); + break; case RTE_VALUES: printf("%d\t%s\t[values list]", i, rte->eref->aliasname); diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index 05bf2e93e6..b02d9fa246 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -458,6 +458,31 @@ _readRangeVar(void) READ_DONE(); } +/* + * _readTableFunc + */ +static TableFunc * +_readTableFunc(void) +{ + READ_LOCALS(TableFunc); + + READ_NODE_FIELD(ns_names); + READ_NODE_FIELD(ns_uris); + READ_NODE_FIELD(docexpr); + READ_NODE_FIELD(rowexpr); + READ_NODE_FIELD(colnames); + READ_NODE_FIELD(coltypes); + READ_NODE_FIELD(coltypmods); + READ_NODE_FIELD(colcollations); + READ_NODE_FIELD(colexprs); + READ_NODE_FIELD(coldefexprs); + READ_BITMAPSET_FIELD(notnulls); + READ_INT_FIELD(ordinalitycol); + READ_LOCATION_FIELD(location); + + READ_DONE(); +} + static IntoClause * _readIntoClause(void) { @@ -1313,6 +1338,9 @@ _readRangeTblEntry(void) READ_NODE_FIELD(functions); READ_BOOL_FIELD(funcordinality); break; + case RTE_TABLEFUNC: + READ_NODE_FIELD(tablefunc); + break; case RTE_VALUES: READ_NODE_FIELD(values_lists); READ_NODE_FIELD(coltypes); @@ -1798,6 +1826,21 @@ _readValuesScan(void) READ_DONE(); } +/* + * _readTableFuncScan + */ +static TableFuncScan * +_readTableFuncScan(void) +{ + READ_LOCALS(TableFuncScan); + + ReadCommonScan(&local_node->scan); + + READ_NODE_FIELD(tablefunc); + + READ_DONE(); +} + /* * _readCteScan */ @@ -2356,6 +2399,8 @@ parseNodeString(void) return_value = _readRangeVar(); else if (MATCH("INTOCLAUSE", 10)) return_value = _readIntoClause(); + else if (MATCH("TABLEFUNC", 9)) + return_value = _readTableFunc(); else if (MATCH("VAR", 3)) return_value = _readVar(); else if (MATCH("CONST", 5)) @@ -2498,6 +2543,8 @@ parseNodeString(void) return_value = _readFunctionScan(); else if (MATCH("VALUESSCAN", 10)) return_value = _readValuesScan(); + else if (MATCH("TABLEFUNCSCAN", 13)) + return_value = _readTableFuncScan(); else if (MATCH("CTESCAN", 7)) return_value = _readCteScan(); else if (MATCH("WORKTABLESCAN", 13)) diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index 87a3faff09..932c84c949 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -106,6 +106,8 @@ static void set_function_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte); static void set_values_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte); +static void set_tablefunc_pathlist(PlannerInfo *root, RelOptInfo *rel, + RangeTblEntry *rte); static void set_cte_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte); static void set_worktable_pathlist(PlannerInfo *root, RelOptInfo *rel, @@ -365,6 +367,9 @@ set_rel_size(PlannerInfo *root, RelOptInfo *rel, case RTE_FUNCTION: set_function_size_estimates(root, rel); break; + case RTE_TABLEFUNC: + set_tablefunc_size_estimates(root, rel); + break; case RTE_VALUES: set_values_size_estimates(root, rel); break; @@ -437,6 +442,10 @@ set_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, /* RangeFunction */ set_function_pathlist(root, rel, rte); break; + case RTE_TABLEFUNC: + /* Table Function */ + set_tablefunc_pathlist(root, rel, rte); + break; case RTE_VALUES: /* Values list */ set_values_pathlist(root, rel, rte); @@ -599,6 +608,10 @@ set_rel_consider_parallel(PlannerInfo *root, RelOptInfo *rel, return; break; + case RTE_TABLEFUNC: + /* not parallel safe */ + return; + case RTE_VALUES: /* Check for parallel-restricted functions. */ if (!is_parallel_safe(root, (Node *) rte->values_lists)) @@ -1932,6 +1945,27 @@ set_values_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) add_path(rel, create_valuesscan_path(root, rel, required_outer)); } +/* + * set_tablefunc_pathlist + * Build the (single) access path for a table func RTE + */ +static void +set_tablefunc_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) +{ + Relids required_outer; + + /* + * We don't support pushing join clauses into the quals of a tablefunc + * scan, but it could still have required parameterization due to LATERAL + * refs in the function expression. + */ + required_outer = rel->lateral_relids; + + /* Generate appropriate path */ + add_path(rel, create_tablefuncscan_path(root, rel, + required_outer)); +} + /* * set_cte_pathlist * Build the (single) access path for a non-self-reference CTE RTE @@ -3032,6 +3066,9 @@ print_path(PlannerInfo *root, Path *path, int indent) case T_FunctionScan: ptype = "FunctionScan"; break; + case T_TableFuncScan: + ptype = "TableFuncScan"; + break; case T_ValuesScan: ptype = "ValuesScan"; break; diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index c138f57ebb..3eaed5af7a 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -1277,6 +1277,62 @@ cost_functionscan(Path *path, PlannerInfo *root, path->total_cost = startup_cost + run_cost; } +/* + * cost_tablefuncscan + * Determines and returns the cost of scanning a table function. + * + * 'baserel' is the relation to be scanned + * 'param_info' is the ParamPathInfo if this is a parameterized path, else NULL + */ +void +cost_tablefuncscan(Path *path, PlannerInfo *root, + RelOptInfo *baserel, ParamPathInfo *param_info) +{ + Cost startup_cost = 0; + Cost run_cost = 0; + QualCost qpqual_cost; + Cost cpu_per_tuple; + RangeTblEntry *rte; + QualCost exprcost; + + /* Should only be applied to base relations that are functions */ + Assert(baserel->relid > 0); + rte = planner_rt_fetch(baserel->relid, root); + Assert(rte->rtekind == RTE_TABLEFUNC); + + /* Mark the path with the correct row estimate */ + if (param_info) + path->rows = param_info->ppi_rows; + else + path->rows = baserel->rows; + + /* + * Estimate costs of executing the table func expression(s). + * + * XXX in principle we ought to charge tuplestore spill costs if the + * number of rows is large. However, given how phony our rowcount + * estimates for tablefuncs tend to be, there's not a lot of point in that + * refinement right now. + */ + cost_qual_eval_node(&exprcost, (Node *) rte->tablefunc, root); + + startup_cost += exprcost.startup + exprcost.per_tuple; + + /* Add scanning CPU costs */ + get_restriction_qual_cost(root, baserel, param_info, &qpqual_cost); + + startup_cost += qpqual_cost.startup; + cpu_per_tuple = cpu_tuple_cost + qpqual_cost.per_tuple; + run_cost += cpu_per_tuple * baserel->tuples; + + /* tlist eval costs are paid per output row, not per tuple scanned */ + startup_cost += path->pathtarget->cost.startup; + run_cost += path->pathtarget->cost.per_tuple * path->rows; + + path->startup_cost = startup_cost; + path->total_cost = startup_cost + run_cost; +} + /* * cost_valuesscan * Determines and returns the cost of scanning a VALUES RTE. @@ -4421,6 +4477,31 @@ set_function_size_estimates(PlannerInfo *root, RelOptInfo *rel) set_baserel_size_estimates(root, rel); } +/* + * set_function_size_estimates + * Set the size estimates for a base relation that is a function call. + * + * The rel's targetlist and restrictinfo list must have been constructed + * already. + * + * We set the same fields as set_tablefunc_size_estimates. + */ +void +set_tablefunc_size_estimates(PlannerInfo *root, RelOptInfo *rel) +{ + RangeTblEntry *rte; + + /* Should only be applied to base relations that are functions */ + Assert(rel->relid > 0); + rte = planner_rt_fetch(rel->relid, root); + Assert(rte->rtekind == RTE_TABLEFUNC); + + rel->tuples = 100; + + /* Now estimate number of output rows, etc */ + set_baserel_size_estimates(root, rel); +} + /* * set_values_size_estimates * Set the size estimates for a base relation that is a values list. diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index 1e953b40d6..f1c7f609c0 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -134,6 +134,8 @@ static FunctionScan *create_functionscan_plan(PlannerInfo *root, Path *best_path List *tlist, List *scan_clauses); static ValuesScan *create_valuesscan_plan(PlannerInfo *root, Path *best_path, List *tlist, List *scan_clauses); +static TableFuncScan *create_tablefuncscan_plan(PlannerInfo *root, Path *best_path, + List *tlist, List *scan_clauses); static CteScan *create_ctescan_plan(PlannerInfo *root, Path *best_path, List *tlist, List *scan_clauses); static WorkTableScan *create_worktablescan_plan(PlannerInfo *root, Path *best_path, @@ -190,6 +192,8 @@ static FunctionScan *make_functionscan(List *qptlist, List *qpqual, Index scanrelid, List *functions, bool funcordinality); static ValuesScan *make_valuesscan(List *qptlist, List *qpqual, Index scanrelid, List *values_lists); +static TableFuncScan *make_tablefuncscan(List *qptlist, List *qpqual, + Index scanrelid, TableFunc *tablefunc); static CteScan *make_ctescan(List *qptlist, List *qpqual, Index scanrelid, int ctePlanId, int cteParam); static WorkTableScan *make_worktablescan(List *qptlist, List *qpqual, @@ -355,6 +359,7 @@ create_plan_recurse(PlannerInfo *root, Path *best_path, int flags) case T_TidScan: case T_SubqueryScan: case T_FunctionScan: + case T_TableFuncScan: case T_ValuesScan: case T_CteScan: case T_WorkTableScan: @@ -635,6 +640,13 @@ create_scan_plan(PlannerInfo *root, Path *best_path, int flags) scan_clauses); break; + case T_TableFuncScan: + plan = (Plan *) create_tablefuncscan_plan(root, + best_path, + tlist, + scan_clauses); + break; + case T_ValuesScan: plan = (Plan *) create_valuesscan_plan(root, best_path, @@ -749,11 +761,12 @@ use_physical_tlist(PlannerInfo *root, Path *path, int flags) /* * We can do this for real relation scans, subquery scans, function scans, - * values scans, and CTE scans (but not for, eg, joins). + * tablefunc scans, values scans, and CTE scans (but not for, eg, joins). */ if (rel->rtekind != RTE_RELATION && rel->rtekind != RTE_SUBQUERY && rel->rtekind != RTE_FUNCTION && + rel->rtekind != RTE_TABLEFUNC && rel->rtekind != RTE_VALUES && rel->rtekind != RTE_CTE) return false; @@ -3014,6 +3027,49 @@ create_functionscan_plan(PlannerInfo *root, Path *best_path, return scan_plan; } +/* + * create_tablefuncscan_plan + * Returns a tablefuncscan plan for the base relation scanned by 'best_path' + * with restriction clauses 'scan_clauses' and targetlist 'tlist'. + */ +static TableFuncScan * +create_tablefuncscan_plan(PlannerInfo *root, Path *best_path, + List *tlist, List *scan_clauses) +{ + TableFuncScan *scan_plan; + Index scan_relid = best_path->parent->relid; + RangeTblEntry *rte; + TableFunc *tablefunc; + + /* it should be a function base rel... */ + Assert(scan_relid > 0); + rte = planner_rt_fetch(scan_relid, root); + Assert(rte->rtekind == RTE_TABLEFUNC); + tablefunc = rte->tablefunc; + + /* Sort clauses into best execution order */ + scan_clauses = order_qual_clauses(root, scan_clauses); + + /* Reduce RestrictInfo list to bare expressions; ignore pseudoconstants */ + scan_clauses = extract_actual_clauses(scan_clauses, false); + + /* Replace any outer-relation variables with nestloop params */ + if (best_path->param_info) + { + scan_clauses = (List *) + replace_nestloop_params(root, (Node *) scan_clauses); + /* The function expressions could contain nestloop params, too */ + tablefunc = (TableFunc *) replace_nestloop_params(root, (Node *) tablefunc); + } + + scan_plan = make_tablefuncscan(tlist, scan_clauses, scan_relid, + tablefunc); + + copy_generic_path_info(&scan_plan->scan.plan, best_path); + + return scan_plan; +} + /* * create_valuesscan_plan * Returns a valuesscan plan for the base relation scanned by 'best_path' @@ -4909,6 +4965,25 @@ make_functionscan(List *qptlist, return node; } +static TableFuncScan * +make_tablefuncscan(List *qptlist, + List *qpqual, + Index scanrelid, + TableFunc *tablefunc) +{ + TableFuncScan *node = makeNode(TableFuncScan); + Plan *plan = &node->scan.plan; + + plan->targetlist = qptlist; + plan->qual = qpqual; + plan->lefttree = NULL; + plan->righttree = NULL; + node->scan.scanrelid = scanrelid; + node->tablefunc = tablefunc; + + return node; +} + static ValuesScan * make_valuesscan(List *qptlist, List *qpqual, diff --git a/src/backend/optimizer/plan/initsplan.c b/src/backend/optimizer/plan/initsplan.c index c170e9614f..b4ac224a7a 100644 --- a/src/backend/optimizer/plan/initsplan.c +++ b/src/backend/optimizer/plan/initsplan.c @@ -335,6 +335,8 @@ extract_lateral_references(PlannerInfo *root, RelOptInfo *brel, Index rtindex) vars = pull_vars_of_level((Node *) rte->subquery, 1); else if (rte->rtekind == RTE_FUNCTION) vars = pull_vars_of_level((Node *) rte->functions, 0); + else if (rte->rtekind == RTE_TABLEFUNC) + vars = pull_vars_of_level((Node *) rte->tablefunc, 0); else if (rte->rtekind == RTE_VALUES) vars = pull_vars_of_level((Node *) rte->values_lists, 0); else diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index ca0ae7883e..1636a69dba 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -69,17 +69,19 @@ create_upper_paths_hook_type create_upper_paths_hook = NULL; /* Expression kind codes for preprocess_expression */ -#define EXPRKIND_QUAL 0 -#define EXPRKIND_TARGET 1 -#define EXPRKIND_RTFUNC 2 -#define EXPRKIND_RTFUNC_LATERAL 3 -#define EXPRKIND_VALUES 4 -#define EXPRKIND_VALUES_LATERAL 5 -#define EXPRKIND_LIMIT 6 -#define EXPRKIND_APPINFO 7 -#define EXPRKIND_PHV 8 -#define EXPRKIND_TABLESAMPLE 9 -#define EXPRKIND_ARBITER_ELEM 10 +#define EXPRKIND_QUAL 0 +#define EXPRKIND_TARGET 1 +#define EXPRKIND_RTFUNC 2 +#define EXPRKIND_RTFUNC_LATERAL 3 +#define EXPRKIND_VALUES 4 +#define EXPRKIND_VALUES_LATERAL 5 +#define EXPRKIND_LIMIT 6 +#define EXPRKIND_APPINFO 7 +#define EXPRKIND_PHV 8 +#define EXPRKIND_TABLESAMPLE 9 +#define EXPRKIND_ARBITER_ELEM 10 +#define EXPRKIND_TABLEFUNC 11 +#define EXPRKIND_TABLEFUNC_LATERAL 12 /* Passthrough data for standard_qp_callback */ typedef struct @@ -685,7 +687,15 @@ subquery_planner(PlannerGlobal *glob, Query *parse, { /* Preprocess the function expression(s) fully */ kind = rte->lateral ? EXPRKIND_RTFUNC_LATERAL : EXPRKIND_RTFUNC; - rte->functions = (List *) preprocess_expression(root, (Node *) rte->functions, kind); + rte->functions = (List *) + preprocess_expression(root, (Node *) rte->functions, kind); + } + else if (rte->rtekind == RTE_TABLEFUNC) + { + /* Preprocess the function expression(s) fully */ + kind = rte->lateral ? EXPRKIND_TABLEFUNC_LATERAL : EXPRKIND_TABLEFUNC; + rte->tablefunc = (TableFunc *) + preprocess_expression(root, (Node *) rte->tablefunc, kind); } else if (rte->rtekind == RTE_VALUES) { @@ -844,7 +854,8 @@ preprocess_expression(PlannerInfo *root, Node *expr, int kind) if (root->hasJoinRTEs && !(kind == EXPRKIND_RTFUNC || kind == EXPRKIND_VALUES || - kind == EXPRKIND_TABLESAMPLE)) + kind == EXPRKIND_TABLESAMPLE || + kind == EXPRKIND_TABLEFUNC)) expr = flatten_join_alias_vars(root, expr); /* diff --git a/src/backend/optimizer/plan/setrefs.c b/src/backend/optimizer/plan/setrefs.c index 07ddbcf37e..3d2c12433d 100644 --- a/src/backend/optimizer/plan/setrefs.c +++ b/src/backend/optimizer/plan/setrefs.c @@ -393,6 +393,7 @@ add_rte_to_flat_rtable(PlannerGlobal *glob, RangeTblEntry *rte) newrte->subquery = NULL; newrte->joinaliasvars = NIL; newrte->functions = NIL; + newrte->tablefunc = NULL; newrte->values_lists = NIL; newrte->coltypes = NIL; newrte->coltypmods = NIL; @@ -553,6 +554,19 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) fix_scan_list(root, splan->functions, rtoffset); } break; + case T_TableFuncScan: + { + TableFuncScan *splan = (TableFuncScan *) plan; + + splan->scan.scanrelid += rtoffset; + splan->scan.plan.targetlist = + fix_scan_list(root, splan->scan.plan.targetlist, rtoffset); + splan->scan.plan.qual = + fix_scan_list(root, splan->scan.plan.qual, rtoffset); + splan->tablefunc = (TableFunc *) + fix_scan_expr(root, (Node *) splan->tablefunc, rtoffset); + } + break; case T_ValuesScan: { ValuesScan *splan = (ValuesScan *) plan; diff --git a/src/backend/optimizer/plan/subselect.c b/src/backend/optimizer/plan/subselect.c index 3eb2bb749e..da9a84be3d 100644 --- a/src/backend/optimizer/plan/subselect.c +++ b/src/backend/optimizer/plan/subselect.c @@ -2421,6 +2421,12 @@ finalize_plan(PlannerInfo *root, Plan *plan, Bitmapset *valid_params, } break; + case T_TableFuncScan: + finalize_primnode((Node *) ((TableFuncScan *) plan)->tablefunc, + &context); + context.paramids = bms_add_members(context.paramids, scan_params); + break; + case T_ValuesScan: finalize_primnode((Node *) ((ValuesScan *) plan)->values_lists, &context); diff --git a/src/backend/optimizer/prep/prepjointree.c b/src/backend/optimizer/prep/prepjointree.c index 6c6ac8dc0a..048815d7b0 100644 --- a/src/backend/optimizer/prep/prepjointree.c +++ b/src/backend/optimizer/prep/prepjointree.c @@ -1118,6 +1118,7 @@ pull_up_simple_subquery(PlannerInfo *root, Node *jtnode, RangeTblEntry *rte, case RTE_SUBQUERY: case RTE_FUNCTION: case RTE_VALUES: + case RTE_TABLEFUNC: child_rte->lateral = true; break; case RTE_JOIN: @@ -1964,6 +1965,11 @@ replace_vars_in_jointree(Node *jtnode, pullup_replace_vars((Node *) rte->functions, context); break; + case RTE_TABLEFUNC: + rte->tablefunc = (TableFunc *) + pullup_replace_vars((Node *) rte->tablefunc, + context); + break; case RTE_VALUES: rte->values_lists = (List *) pullup_replace_vars((Node *) rte->values_lists, diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index 324829690d..86aee2f8ec 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -1750,6 +1750,32 @@ create_functionscan_path(PlannerInfo *root, RelOptInfo *rel, return pathnode; } +/* + * create_tablefuncscan_path + * Creates a path corresponding to a sequential scan of a table function, + * returning the pathnode. + */ +Path * +create_tablefuncscan_path(PlannerInfo *root, RelOptInfo *rel, + Relids required_outer) +{ + Path *pathnode = makeNode(Path); + + pathnode->pathtype = T_TableFuncScan; + pathnode->parent = rel; + pathnode->pathtarget = rel->reltarget; + pathnode->param_info = get_baserel_parampathinfo(root, rel, + required_outer); + pathnode->parallel_aware = false; + pathnode->parallel_safe = rel->consider_parallel; + pathnode->parallel_workers = 0; + pathnode->pathkeys = NIL; /* result is always unordered */ + + cost_tablefuncscan(pathnode, root, rel, pathnode->param_info); + + return pathnode; +} + /* * create_valuesscan_path * Creates a path corresponding to a scan of a VALUES list, diff --git a/src/backend/optimizer/util/plancat.c b/src/backend/optimizer/util/plancat.c index 4ed27054a1..463f806467 100644 --- a/src/backend/optimizer/util/plancat.c +++ b/src/backend/optimizer/util/plancat.c @@ -1381,8 +1381,9 @@ relation_excluded_by_constraints(PlannerInfo *root, * dropped cols. * * We also support building a "physical" tlist for subqueries, functions, - * values lists, and CTEs, since the same optimization can occur in - * SubqueryScan, FunctionScan, ValuesScan, CteScan, and WorkTableScan nodes. + * values lists, table expressions and CTEs, since the same optimization can + * occur in SubqueryScan, FunctionScan, ValuesScan, CteScan, TableFunc + * and WorkTableScan nodes. */ List * build_physical_tlist(PlannerInfo *root, RelOptInfo *rel) @@ -1454,6 +1455,7 @@ build_physical_tlist(PlannerInfo *root, RelOptInfo *rel) break; case RTE_FUNCTION: + case RTE_TABLEFUNC: case RTE_VALUES: case RTE_CTE: /* Not all of these can have dropped cols, but share code anyway */ diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index adc1db94f4..caf8291e10 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -150,12 +150,13 @@ build_simple_rel(PlannerInfo *root, int relid, RelOptKind reloptkind) break; case RTE_SUBQUERY: case RTE_FUNCTION: + case RTE_TABLEFUNC: case RTE_VALUES: case RTE_CTE: /* - * Subquery, function, or values list --- set up attr range and - * arrays + * Subquery, function, tablefunc, or values list --- set up attr + * range and arrays * * Note: 0 is included in range to support whole-row Vars */ diff --git a/src/backend/parser/analyze.c b/src/backend/parser/analyze.c index 796b5c9a5f..3571e50aea 100644 --- a/src/backend/parser/analyze.c +++ b/src/backend/parser/analyze.c @@ -2772,6 +2772,15 @@ transformLockingClause(ParseState *pstate, Query *qry, LockingClause *lc, LCS_asString(lc->strength)), parser_errposition(pstate, thisrel->location))); break; + case RTE_TABLEFUNC: + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + /*------ + translator: %s is a SQL row locking clause such as FOR UPDATE */ + errmsg("%s cannot be applied to a table function", + LCS_asString(lc->strength)), + parser_errposition(pstate, thisrel->location))); + break; case RTE_VALUES: ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index bb55e1c95c..e7acc2d9a2 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -464,7 +464,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); %type def_elem reloption_elem old_aggr_elem operator_def_elem %type def_arg columnElem where_clause where_or_current_clause a_expr b_expr c_expr AexprConst indirection_el opt_slice_bound - columnref in_expr having_clause func_table array_expr + columnref in_expr having_clause func_table xmltable array_expr ExclusionWhereClause %type rowsfrom_item rowsfrom_list opt_col_def_list %type opt_ordinality @@ -550,6 +550,11 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); %type xmlexists_argument %type document_or_content %type xml_whitespace_option +%type xmltable_column_list xmltable_column_option_list +%type xmltable_column_el +%type xmltable_column_option_el +%type xml_namespace_list +%type xml_namespace_el %type func_application func_expr_common_subexpr %type func_expr func_expr_windowless @@ -607,7 +612,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); CACHE CALLED CASCADE CASCADED CASE CAST CATALOG_P CHAIN CHAR_P CHARACTER CHARACTERISTICS CHECK CHECKPOINT CLASS CLOSE - CLUSTER COALESCE COLLATE COLLATION COLUMN COMMENT COMMENTS COMMIT + CLUSTER COALESCE COLLATE COLLATION COLUMN COLUMNS COMMENT COMMENTS COMMIT COMMITTED CONCURRENTLY CONFIGURATION CONFLICT CONNECTION CONSTRAINT CONSTRAINTS CONTENT_P CONTINUE_P CONVERSION_P COPY COST CREATE CROSS CSV CUBE CURRENT_P @@ -681,8 +686,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); WHEN WHERE WHITESPACE_P WINDOW WITH WITHIN WITHOUT WORK WRAPPER WRITE - XML_P XMLATTRIBUTES XMLCONCAT XMLELEMENT XMLEXISTS XMLFOREST XMLPARSE - XMLPI XMLROOT XMLSERIALIZE + XML_P XMLATTRIBUTES XMLCONCAT XMLELEMENT XMLEXISTS XMLFOREST XMLNAMESPACES + XMLPARSE XMLPI XMLROOT XMLSERIALIZE XMLTABLE YEAR_P YES_P @@ -11187,6 +11192,19 @@ table_ref: relation_expr opt_alias_clause n->coldeflist = lsecond($3); $$ = (Node *) n; } + | xmltable opt_alias_clause + { + RangeTableFunc *n = (RangeTableFunc *) $1; + n->alias = $2; + $$ = (Node *) n; + } + | LATERAL_P xmltable opt_alias_clause + { + RangeTableFunc *n = (RangeTableFunc *) $2; + n->lateral = true; + n->alias = $3; + $$ = (Node *) n; + } | select_with_parens opt_alias_clause { RangeSubselect *n = makeNode(RangeSubselect); @@ -11626,6 +11644,166 @@ TableFuncElement: ColId Typename opt_collate_clause } ; +/* + * XMLTABLE + */ +xmltable: + XMLTABLE '(' c_expr xmlexists_argument COLUMNS xmltable_column_list ')' + { + RangeTableFunc *n = makeNode(RangeTableFunc); + n->rowexpr = $3; + n->docexpr = $4; + n->columns = $6; + n->namespaces = NIL; + n->location = @1; + $$ = (Node *)n; + } + | XMLTABLE '(' XMLNAMESPACES '(' xml_namespace_list ')' ',' + c_expr xmlexists_argument COLUMNS xmltable_column_list ')' + { + RangeTableFunc *n = makeNode(RangeTableFunc); + n->rowexpr = $8; + n->docexpr = $9; + n->columns = $11; + n->namespaces = $5; + n->location = @1; + $$ = (Node *)n; + } + ; + +xmltable_column_list: xmltable_column_el { $$ = list_make1($1); } + | xmltable_column_list ',' xmltable_column_el { $$ = lappend($1, $3); } + ; + +xmltable_column_el: + ColId Typename + { + RangeTableFuncCol *fc = makeNode(RangeTableFuncCol); + + fc->colname = $1; + fc->for_ordinality = false; + fc->typeName = $2; + fc->is_not_null = false; + fc->colexpr = NULL; + fc->coldefexpr = NULL; + fc->location = @1; + + $$ = (Node *) fc; + } + | ColId Typename xmltable_column_option_list + { + RangeTableFuncCol *fc = makeNode(RangeTableFuncCol); + ListCell *option; + bool nullability_seen = false; + + fc->colname = $1; + fc->typeName = $2; + fc->for_ordinality = false; + fc->is_not_null = false; + fc->colexpr = NULL; + fc->coldefexpr = NULL; + fc->location = @1; + + foreach(option, $3) + { + DefElem *defel = (DefElem *) lfirst(option); + + if (strcmp(defel->defname, "default") == 0) + { + if (fc->coldefexpr != NULL) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("only one DEFAULT value is allowed"), + parser_errposition(defel->location))); + fc->coldefexpr = defel->arg; + } + else if (strcmp(defel->defname, "path") == 0) + { + if (fc->colexpr != NULL) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("only one PATH value per column is allowed"), + parser_errposition(defel->location))); + fc->colexpr = defel->arg; + } + else if (strcmp(defel->defname, "is_not_null") == 0) + { + if (nullability_seen) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant NULL / NOT NULL declarations for column \"%s\"", fc->colname), + parser_errposition(defel->location))); + fc->is_not_null = intVal(defel->arg); + nullability_seen = true; + } + else + { + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("unrecognized column option \"%s\"", + defel->defname), + parser_errposition(defel->location))); + } + } + $$ = (Node *) fc; + } + | ColId FOR ORDINALITY + { + RangeTableFuncCol *fc = makeNode(RangeTableFuncCol); + + fc->colname = $1; + fc->for_ordinality = true; + /* other fields are ignored, initialized by makeNode */ + fc->location = @1; + + $$ = (Node *) fc; + } + ; + +xmltable_column_option_list: + xmltable_column_option_el + { $$ = list_make1($1); } + | xmltable_column_option_list xmltable_column_option_el + { $$ = lappend($1, $2); } + ; + +xmltable_column_option_el: + IDENT b_expr + { $$ = makeDefElem($1, $2, @1); } + | DEFAULT b_expr + { $$ = makeDefElem("default", $2, @1); } + | NOT NULL_P + { $$ = makeDefElem("is_not_null", (Node *) makeInteger(true), @1); } + | NULL_P + { $$ = makeDefElem("is_not_null", (Node *) makeInteger(false), @1); } + ; + +xml_namespace_list: + xml_namespace_el + { $$ = list_make1($1); } + | xml_namespace_list ',' xml_namespace_el + { $$ = lappend($1, $3); } + ; + +xml_namespace_el: + b_expr AS ColLabel + { + $$ = makeNode(ResTarget); + $$->name = $3; + $$->indirection = NIL; + $$->val = $1; + $$->location = @1; + } + | DEFAULT b_expr + { + $$ = makeNode(ResTarget); + $$->name = NULL; + $$->indirection = NIL; + $$->val = $2; + $$->location = @1; + } + ; + /***************************************************************************** * * Type syntax @@ -14205,6 +14383,7 @@ unreserved_keyword: | CLASS | CLOSE | CLUSTER + | COLUMNS | COMMENT | COMMENTS | COMMIT @@ -14510,10 +14689,12 @@ col_name_keyword: | XMLELEMENT | XMLEXISTS | XMLFOREST + | XMLNAMESPACES | XMLPARSE | XMLPI | XMLROOT | XMLSERIALIZE + | XMLTABLE ; /* Type/function identifier --- keywords that can be type or function names. diff --git a/src/backend/parser/parse_clause.c b/src/backend/parser/parse_clause.c index b5eae56006..47ca685b56 100644 --- a/src/backend/parser/parse_clause.c +++ b/src/backend/parser/parse_clause.c @@ -22,6 +22,7 @@ #include "catalog/catalog.h" #include "catalog/heap.h" #include "catalog/pg_am.h" +#include "catalog/pg_collation.h" #include "catalog/pg_constraint_fn.h" #include "catalog/pg_type.h" #include "commands/defrem.h" @@ -65,6 +66,8 @@ static RangeTblEntry *transformRangeSubselect(ParseState *pstate, RangeSubselect *r); static RangeTblEntry *transformRangeFunction(ParseState *pstate, RangeFunction *r); +static RangeTblEntry *transformRangeTableFunc(ParseState *pstate, + RangeTableFunc *t); static TableSampleClause *transformRangeTableSample(ParseState *pstate, RangeTableSample *rts); static Node *transformFromClauseItem(ParseState *pstate, Node *n, @@ -692,6 +695,229 @@ transformRangeFunction(ParseState *pstate, RangeFunction *r) return rte; } +/* + * transformRangeTableFunc - + * Transform a raw RangeTableFunc into TableFunc. + * + * Transform the namespace clauses, the document-generating expression, the + * row-generating expression, the column-generating expressions, and the + * default value expressions. + */ +static RangeTblEntry * +transformRangeTableFunc(ParseState *pstate, RangeTableFunc *rtf) +{ + TableFunc *tf = makeNode(TableFunc); + const char *constructName; + Oid docType; + RangeTblEntry *rte; + bool is_lateral; + ListCell *col; + char **names; + int colno; + + /* Currently only XMLTABLE is supported */ + constructName = "XMLTABLE"; + docType = XMLOID; + + /* + * We make lateral_only names of this level visible, whether or not the + * RangeTableFunc is explicitly marked LATERAL. This is needed for SQL + * spec compliance and seems useful on convenience grounds for all + * functions in FROM. + * + * (LATERAL can't nest within a single pstate level, so we don't need + * save/restore logic here.) + */ + Assert(!pstate->p_lateral_active); + pstate->p_lateral_active = true; + + /* Transform and apply typecast to the row-generating expression ... */ + Assert(rtf->rowexpr != NULL); + tf->rowexpr = coerce_to_specific_type(pstate, + transformExpr(pstate, rtf->rowexpr, EXPR_KIND_FROM_FUNCTION), + TEXTOID, + constructName); + assign_expr_collations(pstate, tf->rowexpr); + + /* ... and to the document itself */ + Assert(rtf->docexpr != NULL); + tf->docexpr = coerce_to_specific_type(pstate, + transformExpr(pstate, rtf->docexpr, EXPR_KIND_FROM_FUNCTION), + docType, + constructName); + assign_expr_collations(pstate, tf->docexpr); + + /* undef ordinality column number */ + tf->ordinalitycol = -1; + + + names = palloc(sizeof(char *) * list_length(rtf->columns)); + + colno = 0; + foreach(col, rtf->columns) + { + RangeTableFuncCol *rawc = (RangeTableFuncCol *) lfirst(col); + Oid typid; + int32 typmod; + Node *colexpr; + Node *coldefexpr; + int j; + + tf->colnames = lappend(tf->colnames, + makeString(pstrdup(rawc->colname))); + + /* + * Determine the type and typmod for the new column. FOR + * ORDINALITY columns are INTEGER per spec; the others are + * user-specified. + */ + if (rawc->for_ordinality) + { + if (tf->ordinalitycol != -1) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("only one FOR ORDINALITY column is allowed"), + parser_errposition(pstate, rawc->location))); + + typid = INT4OID; + typmod = -1; + tf->ordinalitycol = colno; + } + else + { + if (rawc->typeName->setof) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("column \"%s\" cannot be declared SETOF", + rawc->colname), + parser_errposition(pstate, rawc->location))); + + typenameTypeIdAndMod(pstate, rawc->typeName, + &typid, &typmod); + } + + tf->coltypes = lappend_oid(tf->coltypes, typid); + tf->coltypmods = lappend_int(tf->coltypmods, typmod); + tf->colcollations = lappend_oid(tf->colcollations, + type_is_collatable(typid) ? DEFAULT_COLLATION_OID : InvalidOid); + + /* Transform the PATH and DEFAULT expressions */ + if (rawc->colexpr) + { + colexpr = coerce_to_specific_type(pstate, + transformExpr(pstate, rawc->colexpr, + EXPR_KIND_FROM_FUNCTION), + TEXTOID, + constructName); + assign_expr_collations(pstate, colexpr); + } + else + colexpr = NULL; + + if (rawc->coldefexpr) + { + coldefexpr = coerce_to_specific_type_typmod(pstate, + transformExpr(pstate, rawc->coldefexpr, + EXPR_KIND_FROM_FUNCTION), + typid, typmod, + constructName); + assign_expr_collations(pstate, coldefexpr); + } + else + coldefexpr = NULL; + + tf->colexprs = lappend(tf->colexprs, colexpr); + tf->coldefexprs = lappend(tf->coldefexprs, coldefexpr); + + if (rawc->is_not_null) + tf->notnulls = bms_add_member(tf->notnulls, colno); + + /* make sure column names are unique */ + for (j = 0; j < colno; j++) + if (strcmp(names[j], rawc->colname) == 0) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("column name \"%s\" is not unique", + rawc->colname), + parser_errposition(pstate, rawc->location))); + names[colno] = rawc->colname; + + colno++; + } + pfree(names); + + /* Namespaces, if any, also need to be transformed */ + if (rtf->namespaces != NIL) + { + ListCell *ns; + ListCell *lc2; + List *ns_uris = NIL; + List *ns_names = NIL; + bool default_ns_seen = false; + + foreach(ns, rtf->namespaces) + { + ResTarget *r = (ResTarget *) lfirst(ns); + Node *ns_uri; + + Assert(IsA(r, ResTarget)); + ns_uri = transformExpr(pstate, r->val, EXPR_KIND_FROM_FUNCTION); + ns_uri = coerce_to_specific_type(pstate, ns_uri, + TEXTOID, constructName); + assign_expr_collations(pstate, ns_uri); + ns_uris = lappend(ns_uris, ns_uri); + + /* Verify consistency of name list: no dupes, only one DEFAULT */ + if (r->name != NULL) + { + foreach(lc2, ns_names) + { + char *name = strVal(lfirst(lc2)); + + if (name == NULL) + continue; + if (strcmp(name, r->name) == 0) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("namespace name \"%s\" is not unique", + name), + parser_errposition(pstate, r->location))); + } + } + else + { + if (default_ns_seen) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("only one default namespace is allowed"), + parser_errposition(pstate, r->location))); + default_ns_seen = true; + } + + /* Note the string may be NULL */ + ns_names = lappend(ns_names, makeString(r->name)); + } + + tf->ns_uris = ns_uris; + tf->ns_names = ns_names; + } + + tf->location = rtf->location; + + pstate->p_lateral_active = false; + + /* + * Mark the RTE as LATERAL if the user said LATERAL explicitly, or if + * there are any lateral cross-references in it. + */ + is_lateral = rtf->lateral || contain_vars_of_level((Node *) tf, 0); + + rte = addRangeTableEntryForTableFunc(pstate, + tf, rtf->alias, is_lateral, true); + + return rte; +} + /* * transformRangeTableSample --- transform a TABLESAMPLE clause * @@ -795,7 +1021,6 @@ transformRangeTableSample(ParseState *pstate, RangeTableSample *rts) return tablesample; } - /* * transformFromClauseItem - * Transform a FROM-clause item, adding any required entries to the @@ -891,6 +1116,24 @@ transformFromClauseItem(ParseState *pstate, Node *n, rtr->rtindex = rtindex; return (Node *) rtr; } + else if (IsA(n, RangeTableFunc)) + { + /* table function is like a plain relation */ + RangeTblRef *rtr; + RangeTblEntry *rte; + int rtindex; + + rte = transformRangeTableFunc(pstate, (RangeTableFunc *) n); + /* assume new rte is at end */ + rtindex = list_length(pstate->p_rtable); + Assert(rte == rt_fetch(rtindex, pstate->p_rtable)); + *top_rte = rte; + *top_rti = rtindex; + *namespace = list_make1(makeDefaultNSItem(rte)); + rtr = makeNode(RangeTblRef); + rtr->rtindex = rtindex; + return (Node *) rtr; + } else if (IsA(n, RangeTableSample)) { /* TABLESAMPLE clause (wrapping some other valid FROM node) */ diff --git a/src/backend/parser/parse_coerce.c b/src/backend/parser/parse_coerce.c index 2a2ac32157..2c3f3cd9ce 100644 --- a/src/backend/parser/parse_coerce.c +++ b/src/backend/parser/parse_coerce.c @@ -1096,9 +1096,9 @@ coerce_to_boolean(ParseState *pstate, Node *node, } /* - * coerce_to_specific_type() - * Coerce an argument of a construct that requires a specific data type. - * Also check that input is not a set. + * coerce_to_specific_type_typmod() + * Coerce an argument of a construct that requires a specific data type, + * with a specific typmod. Also check that input is not a set. * * Returns the possibly-transformed node tree. * @@ -1106,9 +1106,9 @@ coerce_to_boolean(ParseState *pstate, Node *node, * processing is wanted. */ Node * -coerce_to_specific_type(ParseState *pstate, Node *node, - Oid targetTypeId, - const char *constructName) +coerce_to_specific_type_typmod(ParseState *pstate, Node *node, + Oid targetTypeId, int32 targetTypmod, + const char *constructName) { Oid inputTypeId = exprType(node); @@ -1117,7 +1117,7 @@ coerce_to_specific_type(ParseState *pstate, Node *node, Node *newnode; newnode = coerce_to_target_type(pstate, node, inputTypeId, - targetTypeId, -1, + targetTypeId, targetTypmod, COERCION_ASSIGNMENT, COERCE_IMPLICIT_CAST, -1); @@ -1144,6 +1144,25 @@ coerce_to_specific_type(ParseState *pstate, Node *node, return node; } +/* + * coerce_to_specific_type() + * Coerce an argument of a construct that requires a specific data type. + * Also check that input is not a set. + * + * Returns the possibly-transformed node tree. + * + * As with coerce_type, pstate may be NULL if no special unknown-Param + * processing is wanted. + */ +Node * +coerce_to_specific_type(ParseState *pstate, Node *node, + Oid targetTypeId, + const char *constructName) +{ + return coerce_to_specific_type_typmod(pstate, node, + targetTypeId, -1, + constructName); +} /* * parser_coercion_errposition - report coercion error location, if possible diff --git a/src/backend/parser/parse_relation.c b/src/backend/parser/parse_relation.c index cf69533b53..2eea258d28 100644 --- a/src/backend/parser/parse_relation.c +++ b/src/backend/parser/parse_relation.c @@ -1627,6 +1627,69 @@ addRangeTableEntryForFunction(ParseState *pstate, return rte; } +/* + * Add an entry for a table function to the pstate's range table (p_rtable). + * + * This is much like addRangeTableEntry() except that it makes a tablefunc RTE. + */ +RangeTblEntry * +addRangeTableEntryForTableFunc(ParseState *pstate, + TableFunc *tf, + Alias *alias, + bool lateral, + bool inFromCl) +{ + RangeTblEntry *rte = makeNode(RangeTblEntry); + char *refname = alias ? alias->aliasname : pstrdup("xmltable"); + Alias *eref; + int numaliases; + + Assert(pstate != NULL); + + rte->rtekind = RTE_TABLEFUNC; + rte->relid = InvalidOid; + rte->subquery = NULL; + rte->tablefunc = tf; + rte->coltypes = tf->coltypes; + rte->coltypmods = tf->coltypmods; + rte->colcollations = tf->colcollations; + rte->alias = alias; + + eref = alias ? copyObject(alias) : makeAlias(refname, NIL); + numaliases = list_length(eref->colnames); + + /* fill in any unspecified alias columns */ + if (numaliases < list_length(tf->colnames)) + eref->colnames = list_concat(eref->colnames, + list_copy_tail(tf->colnames, numaliases)); + + rte->eref = eref; + + /* + * Set flags and access permissions. + * + * Tablefuncs are never checked for access rights (at least, not by the + * RTE permissions mechanism). + */ + rte->lateral = lateral; + rte->inh = false; /* never true for tablefunc RTEs */ + rte->inFromCl = inFromCl; + + rte->requiredPerms = 0; + rte->checkAsUser = InvalidOid; + rte->selectedCols = NULL; + rte->insertedCols = NULL; + rte->updatedCols = NULL; + + /* + * Add completed RTE to pstate's range table list, but not to join list + * nor namespace --- caller must do that if appropriate. + */ + pstate->p_rtable = lappend(pstate->p_rtable, rte); + + return rte; +} + /* * Add an entry for a VALUES list to the pstate's range table (p_rtable). * @@ -2226,10 +2289,11 @@ expandRTE(RangeTblEntry *rte, int rtindex, int sublevels_up, } } break; + case RTE_TABLEFUNC: case RTE_VALUES: case RTE_CTE: { - /* Values or CTE RTE */ + /* Tablefunc, Values or CTE RTE */ ListCell *aliasp_item = list_head(rte->eref->colnames); ListCell *lct; ListCell *lcm; @@ -2638,10 +2702,14 @@ get_rte_attribute_type(RangeTblEntry *rte, AttrNumber attnum, *varcollid = exprCollation(aliasvar); } break; + case RTE_TABLEFUNC: case RTE_VALUES: case RTE_CTE: { - /* VALUES or CTE RTE --- get type info from lists in the RTE */ + /* + * tablefunc, VALUES or CTE RTE --- get type info from lists + * in the RTE + */ Assert(attnum > 0 && attnum <= list_length(rte->coltypes)); *vartype = list_nth_oid(rte->coltypes, attnum - 1); *vartypmod = list_nth_int(rte->coltypmods, attnum - 1); @@ -2684,9 +2752,14 @@ get_rte_attribute_is_dropped(RangeTblEntry *rte, AttrNumber attnum) } break; case RTE_SUBQUERY: + case RTE_TABLEFUNC: case RTE_VALUES: case RTE_CTE: - /* Subselect, Values, CTE RTEs never have dropped columns */ + + /* + * Subselect, Table Functions, Values, CTE RTEs never have dropped + * columns + */ result = false; break; case RTE_JOIN: diff --git a/src/backend/parser/parse_target.c b/src/backend/parser/parse_target.c index 2576e31239..3b84140a9b 100644 --- a/src/backend/parser/parse_target.c +++ b/src/backend/parser/parse_target.c @@ -396,6 +396,7 @@ markTargetListOrigin(ParseState *pstate, TargetEntry *tle, break; case RTE_FUNCTION: case RTE_VALUES: + case RTE_TABLEFUNC: /* not a simple relation, leave it unmarked */ break; case RTE_CTE: @@ -1557,6 +1558,12 @@ expandRecordVariable(ParseState *pstate, Var *var, int levelsup) * its result columns as RECORD, which is not allowed. */ break; + case RTE_TABLEFUNC: + + /* + * Table function cannot have columns with RECORD type. + */ + break; case RTE_CTE: /* CTE reference: examine subquery's output expr */ if (!rte->self_reference) diff --git a/src/backend/rewrite/rewriteHandler.c b/src/backend/rewrite/rewriteHandler.c index 20bbb37484..354e5d0462 100644 --- a/src/backend/rewrite/rewriteHandler.c +++ b/src/backend/rewrite/rewriteHandler.c @@ -433,6 +433,10 @@ rewriteRuleAction(Query *parsetree, sub_action->hasSubLinks = checkExprHasSubLink((Node *) rte->functions); break; + case RTE_TABLEFUNC: + sub_action->hasSubLinks = + checkExprHasSubLink((Node *) rte->tablefunc); + break; case RTE_VALUES: sub_action->hasSubLinks = checkExprHasSubLink((Node *) rte->values_lists); diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c index cf79f4126e..3c697f3c0d 100644 --- a/src/backend/utils/adt/ruleutils.c +++ b/src/backend/utils/adt/ruleutils.c @@ -429,6 +429,8 @@ static void get_const_expr(Const *constval, deparse_context *context, static void get_const_collation(Const *constval, deparse_context *context); static void simple_quote_literal(StringInfo buf, const char *val); static void get_sublink_expr(SubLink *sublink, deparse_context *context); +static void get_tablefunc(TableFunc *tf, deparse_context *context, + bool showimplicit); static void get_from_clause(Query *query, const char *prefix, deparse_context *context); static void get_from_clause_item(Node *jtnode, Query *query, @@ -3642,14 +3644,17 @@ set_relation_column_names(deparse_namespace *dpns, RangeTblEntry *rte, * are different from the underlying "real" names. For a function RTE, * always emit a complete column alias list; this is to protect against * possible instability of the default column names (eg, from altering - * parameter names). For other RTE types, print if we changed anything OR - * if there were user-written column aliases (since the latter would be - * part of the underlying "reality"). + * parameter names). For tablefunc RTEs, we never print aliases, because + * the column names are part of the clause itself. For other RTE types, + * print if we changed anything OR if there were user-written column + * aliases (since the latter would be part of the underlying "reality"). */ if (rte->rtekind == RTE_RELATION) colinfo->printaliases = changed_any; else if (rte->rtekind == RTE_FUNCTION) colinfo->printaliases = true; + else if (rte->rtekind == RTE_TABLEFUNC) + colinfo->printaliases = false; else if (rte->alias && rte->alias->colnames != NIL) colinfo->printaliases = true; else @@ -6726,6 +6731,7 @@ get_name_for_var_field(Var *var, int fieldno, /* else fall through to inspect the expression */ break; case RTE_FUNCTION: + case RTE_TABLEFUNC: /* * We couldn't get here unless a function is declared with one of @@ -8562,6 +8568,10 @@ get_rule_expr(Node *node, deparse_context *context, } break; + case T_TableFunc: + get_tablefunc((TableFunc *) node, context, showimplicit); + break; + default: elog(ERROR, "unrecognized node type: %d", (int) nodeTag(node)); break; @@ -9297,6 +9307,120 @@ get_sublink_expr(SubLink *sublink, deparse_context *context) } +/* ---------- + * get_tablefunc - Parse back a table function + * ---------- + */ +static void +get_tablefunc(TableFunc *tf, deparse_context *context, bool showimplicit) +{ + StringInfo buf = context->buf; + + /* XMLTABLE is the only existing implementation. */ + + appendStringInfoString(buf, "XMLTABLE("); + + if (tf->ns_uris != NIL) + { + ListCell *lc1, + *lc2; + bool first = true; + + appendStringInfoString(buf, "XMLNAMESPACES ("); + forboth(lc1, tf->ns_uris, lc2, tf->ns_names) + { + Node *expr = (Node *) lfirst(lc1); + char *name = strVal(lfirst(lc2)); + + if (!first) + appendStringInfoString(buf, ", "); + else + first = false; + + if (name != NULL) + { + get_rule_expr(expr, context, showimplicit); + appendStringInfo(buf, " AS %s", name); + } + else + { + appendStringInfoString(buf, "DEFAULT "); + get_rule_expr(expr, context, showimplicit); + } + } + appendStringInfoString(buf, "), "); + } + + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) tf->rowexpr, context, showimplicit); + appendStringInfoString(buf, ") PASSING ("); + get_rule_expr((Node *) tf->docexpr, context, showimplicit); + appendStringInfoChar(buf, ')'); + + if (tf->colexprs != NIL) + { + ListCell *l1; + ListCell *l2; + ListCell *l3; + ListCell *l4; + ListCell *l5; + int colnum = 0; + + l2 = list_head(tf->coltypes); + l3 = list_head(tf->coltypmods); + l4 = list_head(tf->colexprs); + l5 = list_head(tf->coldefexprs); + + appendStringInfoString(buf, " COLUMNS "); + foreach(l1, tf->colnames) + { + char *colname = strVal(lfirst(l1)); + Oid typid; + int32 typmod; + Node *colexpr; + Node *coldefexpr; + bool ordinality = tf->ordinalitycol == colnum; + bool notnull = bms_is_member(colnum, tf->notnulls); + + typid = lfirst_oid(l2); + l2 = lnext(l2); + typmod = lfirst_int(l3); + l3 = lnext(l3); + colexpr = (Node *) lfirst(l4); + l4 = lnext(l4); + coldefexpr = (Node *) lfirst(l5); + l5 = lnext(l5); + + if (colnum > 0) + appendStringInfoString(buf, ", "); + colnum++; + + appendStringInfo(buf, "%s %s", quote_identifier(colname), + ordinality ? "FOR ORDINALITY" : + format_type_with_typemod(typid, typmod)); + if (ordinality) + continue; + + if (coldefexpr != NULL) + { + appendStringInfoString(buf, " DEFAULT ("); + get_rule_expr((Node *) coldefexpr, context, showimplicit); + appendStringInfoChar(buf, ')'); + } + if (colexpr != NULL) + { + appendStringInfoString(buf, " PATH ("); + get_rule_expr((Node *) colexpr, context, showimplicit); + appendStringInfoChar(buf, ')'); + } + if (notnull) + appendStringInfoString(buf, " NOT NULL"); + } + } + + appendStringInfoChar(buf, ')'); +} + /* ---------- * get_from_clause - Parse back a FROM clause * @@ -9530,6 +9654,9 @@ get_from_clause_item(Node *jtnode, Query *query, deparse_context *context) if (rte->funcordinality) appendStringInfoString(buf, " WITH ORDINALITY"); break; + case RTE_TABLEFUNC: + get_tablefunc(rte->tablefunc, context, true); + break; case RTE_VALUES: /* Values list RTE */ appendStringInfoChar(buf, '('); diff --git a/src/backend/utils/adt/xml.c b/src/backend/utils/adt/xml.c index e8bce3b806..f2e5224fc3 100644 --- a/src/backend/utils/adt/xml.c +++ b/src/backend/utils/adt/xml.c @@ -73,6 +73,7 @@ #include "commands/dbcommands.h" #include "executor/executor.h" #include "executor/spi.h" +#include "executor/tablefunc.h" #include "fmgr.h" #include "lib/stringinfo.h" #include "libpq/pqformat.h" @@ -145,6 +146,7 @@ static text *xml_xmlnodetoxmltype(xmlNodePtr cur, PgXmlErrorContext *xmlerrcxt); static int xml_xpathobjtoxmlarray(xmlXPathObjectPtr xpathobj, ArrayBuildState *astate, PgXmlErrorContext *xmlerrcxt); +static xmlChar *pg_xmlCharStrndup(char *str, size_t len); #endif /* USE_LIBXML */ static StringInfo query_to_xml_internal(const char *query, char *tablename, @@ -165,6 +167,49 @@ static void SPI_sql_row_to_xmlelement(uint64 rownum, StringInfo result, char *tablename, bool nulls, bool tableforest, const char *targetns, bool top_level); +/* XMLTABLE support */ +#ifdef USE_LIBXML +/* random number to identify XmlTableContext */ +#define XMLTABLE_CONTEXT_MAGIC 46922182 +typedef struct XmlTableBuilderData +{ + int magic; + int natts; + long int row_count; + PgXmlErrorContext *xmlerrcxt; + xmlParserCtxtPtr ctxt; + xmlDocPtr doc; + xmlXPathContextPtr xpathcxt; + xmlXPathCompExprPtr xpathcomp; + xmlXPathObjectPtr xpathobj; + xmlXPathCompExprPtr *xpathscomp; +} XmlTableBuilderData; +#endif + +static void XmlTableInitOpaque(struct TableFuncScanState *state, int natts); +static void XmlTableSetDocument(struct TableFuncScanState *state, Datum value); +static void XmlTableSetNamespace(struct TableFuncScanState *state, char *name, + char *uri); +static void XmlTableSetRowFilter(struct TableFuncScanState *state, char *path); +static void XmlTableSetColumnFilter(struct TableFuncScanState *state, + char *path, int colnum); +static bool XmlTableFetchRow(struct TableFuncScanState *state); +static Datum XmlTableGetValue(struct TableFuncScanState *state, int colnum, + Oid typid, int32 typmod, bool *isnull); +static void XmlTableDestroyOpaque(struct TableFuncScanState *state); + +const TableFuncRoutine XmlTableRoutine = +{ + XmlTableInitOpaque, + XmlTableSetDocument, + XmlTableSetNamespace, + XmlTableSetRowFilter, + XmlTableSetColumnFilter, + XmlTableFetchRow, + XmlTableGetValue, + XmlTableDestroyOpaque +}; + #define NO_XML_SUPPORT() \ ereport(ERROR, \ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), \ @@ -1113,6 +1158,19 @@ xml_pnstrdup(const xmlChar *str, size_t len) return result; } +/* Ditto, except input is char* */ +static xmlChar * +pg_xmlCharStrndup(char *str, size_t len) +{ + xmlChar *result; + + result = (xmlChar *) palloc((len + 1) * sizeof(xmlChar)); + memcpy(result, str, len); + result[len] = '\0'; + + return result; +} + /* * str is the null-terminated input string. Remaining arguments are * output arguments; each can be NULL if value is not wanted. @@ -3811,13 +3869,8 @@ xpath_internal(text *xpath_expr_text, xmltype *data, ArrayType *namespaces, (errcode(ERRCODE_DATA_EXCEPTION), errmsg("empty XPath expression"))); - string = (xmlChar *) palloc((len + 1) * sizeof(xmlChar)); - memcpy(string, datastr, len); - string[len] = '\0'; - - xpath_expr = (xmlChar *) palloc((xpath_len + 1) * sizeof(xmlChar)); - memcpy(xpath_expr, VARDATA(xpath_expr_text), xpath_len); - xpath_expr[xpath_len] = '\0'; + string = pg_xmlCharStrndup(datastr, len); + xpath_expr = pg_xmlCharStrndup(VARDATA(xpath_expr_text), xpath_len); xmlerrcxt = pg_xml_init(PG_XML_STRICTNESS_ALL); @@ -4065,3 +4118,494 @@ xml_is_well_formed_content(PG_FUNCTION_ARGS) return 0; #endif /* not USE_LIBXML */ } + +/* + * support functions for XMLTABLE + * + */ +#ifdef USE_LIBXML + +/* + * Returns private data from executor state. Ensure validity by check with + * MAGIC number. + */ +static inline XmlTableBuilderData * +GetXmlTableBuilderPrivateData(TableFuncScanState *state, const char *fname) +{ + XmlTableBuilderData *result; + + if (!IsA(state, TableFuncScanState)) + elog(ERROR, "%s called with invalid TableFuncScanState", fname); + result = (XmlTableBuilderData *) state->opaque; + if (result->magic != XMLTABLE_CONTEXT_MAGIC) + elog(ERROR, "%s called with invalid TableFuncScanState", fname); + + return result; +} +#endif + +/* + * XmlTableInitOpaque + * Fill in TableFuncScanState->opaque for XmlTable processor; initialize + * the XML parser. + * + * Note: Because we call pg_xml_init() here and pg_xml_done() in + * XmlTableDestroyOpaque, it is critical for robustness that no other + * executor nodes run until this node is processed to completion. Caller + * must execute this to completion (probably filling a tuplestore to exhaust + * this node in a single pass) instead of using row-per-call mode. + */ +static void +XmlTableInitOpaque(TableFuncScanState *state, int natts) +{ +#ifdef USE_LIBXML + volatile xmlParserCtxtPtr ctxt = NULL; + XmlTableBuilderData *xtCxt; + PgXmlErrorContext *xmlerrcxt; + + xtCxt = palloc0(sizeof(XmlTableBuilderData)); + xtCxt->magic = XMLTABLE_CONTEXT_MAGIC; + xtCxt->natts = natts; + xtCxt->xpathscomp = palloc0(sizeof(xmlXPathCompExprPtr) * natts); + + xmlerrcxt = pg_xml_init(PG_XML_STRICTNESS_ALL); + + PG_TRY(); + { + xmlInitParser(); + + ctxt = xmlNewParserCtxt(); + if (ctxt == NULL || xmlerrcxt->err_occurred) + xml_ereport(xmlerrcxt, ERROR, ERRCODE_OUT_OF_MEMORY, + "could not allocate parser context"); + } + PG_CATCH(); + { + if (ctxt != NULL) + xmlFreeParserCtxt(ctxt); + + pg_xml_done(xmlerrcxt, true); + + PG_RE_THROW(); + } + PG_END_TRY(); + + xtCxt->xmlerrcxt = xmlerrcxt; + xtCxt->ctxt = ctxt; + + state->opaque = xtCxt; +#else + NO_XML_SUPPORT(); +#endif /* not USE_LIBXML */ +} + +/* + * XmlTableSetDocument + * Install the input document + */ +static void +XmlTableSetDocument(TableFuncScanState *state, Datum value) +{ +#ifdef USE_LIBXML + XmlTableBuilderData *xtCxt; + xmltype *xmlval = DatumGetXmlP(value); + char *str; + xmlChar *xstr; + int length; + volatile xmlDocPtr doc = NULL; + volatile xmlXPathContextPtr xpathcxt = NULL; + + xtCxt = GetXmlTableBuilderPrivateData(state, "XmlTableSetDocument"); + + /* + * Use out function for casting to string (remove encoding property). See + * comment in xml_out. + */ + str = xml_out_internal(xmlval, 0); + + length = strlen(str); + xstr = pg_xmlCharStrndup(str, length); + + PG_TRY(); + { + doc = xmlCtxtReadMemory(xtCxt->ctxt, (char *) xstr, length, NULL, NULL, 0); + if (doc == NULL || xtCxt->xmlerrcxt->err_occurred) + xml_ereport(xtCxt->xmlerrcxt, ERROR, ERRCODE_INVALID_XML_DOCUMENT, + "could not parse XML document"); + xpathcxt = xmlXPathNewContext(doc); + if (xpathcxt == NULL || xtCxt->xmlerrcxt->err_occurred) + xml_ereport(xtCxt->xmlerrcxt, ERROR, ERRCODE_OUT_OF_MEMORY, + "could not allocate XPath context"); + xpathcxt->node = xmlDocGetRootElement(doc); + if (xpathcxt->node == NULL || xtCxt->xmlerrcxt->err_occurred) + xml_ereport(xtCxt->xmlerrcxt, ERROR, ERRCODE_INTERNAL_ERROR, + "could not find root XML element"); + } + PG_CATCH(); + { + if (xpathcxt != NULL) + xmlXPathFreeContext(xpathcxt); + if (doc != NULL) + xmlFreeDoc(doc); + + PG_RE_THROW(); + } + PG_END_TRY(); + + xtCxt->doc = doc; + xtCxt->xpathcxt = xpathcxt; +#else + NO_XML_SUPPORT(); +#endif /* not USE_LIBXML */ +} + +/* + * XmlTableSetNamespace + * Add a namespace declaration + */ +static void +XmlTableSetNamespace(TableFuncScanState *state, char *name, char *uri) +{ +#ifdef USE_LIBXML + XmlTableBuilderData *xtCxt; + + if (name == NULL) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("DEFAULT namespace is not supported"))); + xtCxt = GetXmlTableBuilderPrivateData(state, "XmlTableSetNamespace"); + + if (xmlXPathRegisterNs(xtCxt->xpathcxt, + pg_xmlCharStrndup(name, strlen(name)), + pg_xmlCharStrndup(uri, strlen(uri)))) + xml_ereport(xtCxt->xmlerrcxt, ERROR, ERRCODE_DATA_EXCEPTION, + "could not set XML namespace"); +#else + NO_XML_SUPPORT(); +#endif /* not USE_LIBXML */ +} + +/* + * XmlTableSetRowFilter + * Install the row-filter Xpath expression. + */ +static void +XmlTableSetRowFilter(TableFuncScanState *state, char *path) +{ +#ifdef USE_LIBXML + XmlTableBuilderData *xtCxt; + xmlChar *xstr; + + xtCxt = GetXmlTableBuilderPrivateData(state, "XmlTableSetRowFilter"); + + if (*path == '\0') + ereport(ERROR, + (errcode(ERRCODE_DATA_EXCEPTION), + errmsg("row path filter must not be empty string"))); + + xstr = pg_xmlCharStrndup(path, strlen(path)); + + xtCxt->xpathcomp = xmlXPathCompile(xstr); + if (xtCxt->xpathcomp == NULL || xtCxt->xmlerrcxt->err_occurred) + xml_ereport(xtCxt->xmlerrcxt, ERROR, ERRCODE_SYNTAX_ERROR, + "invalid XPath expression"); +#else + NO_XML_SUPPORT(); +#endif /* not USE_LIBXML */ +} + +/* + * XmlTableSetColumnFilter + * Install the column-filter Xpath expression, for the given column. + */ +static void +XmlTableSetColumnFilter(TableFuncScanState *state, char *path, int colnum) +{ +#ifdef USE_LIBXML + XmlTableBuilderData *xtCxt; + xmlChar *xstr; + + AssertArg(PointerIsValid(path)); + + xtCxt = GetXmlTableBuilderPrivateData(state, "XmlTableSetColumnFilter"); + + if (*path == '\0') + ereport(ERROR, + (errcode(ERRCODE_DATA_EXCEPTION), + errmsg("column path filter must not be empty string"))); + + xstr = pg_xmlCharStrndup(path, strlen(path)); + + xtCxt->xpathscomp[colnum] = xmlXPathCompile(xstr); + if (xtCxt->xpathscomp[colnum] == NULL || xtCxt->xmlerrcxt->err_occurred) + xml_ereport(xtCxt->xmlerrcxt, ERROR, ERRCODE_DATA_EXCEPTION, + "invalid XPath expression"); +#else + NO_XML_SUPPORT(); +#endif /* not USE_LIBXML */ +} + +/* + * XmlTableFetchRow + * Prepare the next "current" tuple for upcoming GetValue calls. + * Returns FALSE if the row-filter expression returned no more rows. + */ +static bool +XmlTableFetchRow(TableFuncScanState *state) +{ +#ifdef USE_LIBXML + XmlTableBuilderData *xtCxt; + + xtCxt = GetXmlTableBuilderPrivateData(state, "XmlTableFetchRow"); + + /* + * XmlTable returns table - set of composite values. The error context, is + * used for producement more values, between two calls, there can be + * created and used another libxml2 error context. It is libxml2 global + * value, so it should be refreshed any time before any libxml2 usage, + * that is finished by returning some value. + */ + xmlSetStructuredErrorFunc((void *) xtCxt->xmlerrcxt, xml_errorHandler); + + if (xtCxt->xpathobj == NULL) + { + xtCxt->xpathobj = xmlXPathCompiledEval(xtCxt->xpathcomp, xtCxt->xpathcxt); + if (xtCxt->xpathobj == NULL || xtCxt->xmlerrcxt->err_occurred) + xml_ereport(xtCxt->xmlerrcxt, ERROR, ERRCODE_INTERNAL_ERROR, + "could not create XPath object"); + + xtCxt->row_count = 0; + } + + if (xtCxt->xpathobj->type == XPATH_NODESET) + { + if (xtCxt->xpathobj->nodesetval != NULL) + { + if (xtCxt->row_count++ < xtCxt->xpathobj->nodesetval->nodeNr) + return true; + } + } + + return false; +#else + NO_XML_SUPPORT(); +#endif /* not USE_LIBXML */ + + return false; +} + +/* + * XmlTableGetValue + * Return the value for column number 'colnum' for the current row. If + * column -1 is requested, return representation of the whole row. + * + * This leaks memory, so be sure to reset often the context in which it's + * called. + */ +static Datum +XmlTableGetValue(TableFuncScanState *state, int colnum, + Oid typid, int32 typmod, bool *isnull) +{ +#ifdef USE_LIBXML + XmlTableBuilderData *xtCxt; + Datum result = (Datum) 0; + xmlNodePtr cur; + char *cstr = NULL; + volatile xmlXPathObjectPtr xpathobj = NULL; + + xtCxt = GetXmlTableBuilderPrivateData(state, "XmlTableGetValue"); + + Assert(xtCxt->xpathobj && + xtCxt->xpathobj->type == XPATH_NODESET && + xtCxt->xpathobj->nodesetval != NULL); + + /* Propagate context related error context to libxml2 */ + xmlSetStructuredErrorFunc((void *) xtCxt->xmlerrcxt, xml_errorHandler); + + *isnull = false; + + cur = xtCxt->xpathobj->nodesetval->nodeTab[xtCxt->row_count - 1]; + + Assert(xtCxt->xpathscomp[colnum] != NULL); + + PG_TRY(); + { + /* Set current node as entry point for XPath evaluation */ + xmlXPathSetContextNode(cur, xtCxt->xpathcxt); + + /* Evaluate column path */ + xpathobj = xmlXPathCompiledEval(xtCxt->xpathscomp[colnum], xtCxt->xpathcxt); + if (xpathobj == NULL || xtCxt->xmlerrcxt->err_occurred) + xml_ereport(xtCxt->xmlerrcxt, ERROR, ERRCODE_INTERNAL_ERROR, + "could not create XPath object"); + + /* + * There are four possible cases, depending on the number of nodes + * returned by the XPath expression and the type of the target column: + * a) XPath returns no nodes. b) One node is returned, and column is + * of type XML. c) One node, column type other than XML. d) Multiple + * nodes are returned. + */ + if (xpathobj->type == XPATH_NODESET) + { + int count = 0; + + if (xpathobj->nodesetval != NULL) + count = xpathobj->nodesetval->nodeNr; + + if (xpathobj->nodesetval == NULL || count == 0) + { + *isnull = true; + } + else if (count == 1 && typid == XMLOID) + { + text *textstr; + + /* simple case, result is one value */ + textstr = xml_xmlnodetoxmltype(xpathobj->nodesetval->nodeTab[0], + xtCxt->xmlerrcxt); + cstr = text_to_cstring(textstr); + } + else if (count == 1) + { + xmlChar *str; + + str = xmlNodeListGetString(xtCxt->doc, + xpathobj->nodesetval->nodeTab[0]->xmlChildrenNode, + 1); + + if (str != NULL) + { + PG_TRY(); + { + cstr = pstrdup((char *) str); + } + PG_CATCH(); + { + xmlFree(str); + PG_RE_THROW(); + } + PG_END_TRY(); + xmlFree(str); + } + else + { + /* + * This line ensure mapping of empty tags to PostgreSQL + * value. Usually we would to map a empty tag to empty + * string. But this mapping can create empty string when + * user doesn't expect it - when empty tag is enforced + * by libxml2 - when user uses a text() function for + * example. + */ + cstr = ""; + } + } + else + { + StringInfoData str; + int i; + + Assert(count > 1); + + /* + * When evaluating the XPath expression returns multiple + * nodes, the result is the concatenation of them all. The + * target type must be XML. + */ + if (typid != XMLOID) + ereport(ERROR, + (errcode(ERRCODE_CARDINALITY_VIOLATION), + errmsg("more than one value returned by column XPath expression"))); + + /* Concatenate serialized values */ + initStringInfo(&str); + for (i = 0; i < count; i++) + { + appendStringInfoText(&str, + xml_xmlnodetoxmltype(xpathobj->nodesetval->nodeTab[i], + xtCxt->xmlerrcxt)); + } + cstr = str.data; + } + } + else if (xpathobj->type == XPATH_STRING) + { + cstr = (char *) xpathobj->stringval; + } + else + elog(ERROR, "unexpected XPath object type %u", xpathobj->type); + + /* + * By here, either cstr contains the result value, or the isnull flag + * has been set. + */ + Assert(cstr || *isnull); + + if (!*isnull) + result = InputFunctionCall(&state->in_functions[colnum], + cstr, + state->typioparams[colnum], + typmod); + } + PG_CATCH(); + { + if (xpathobj != NULL) + xmlXPathFreeObject(xpathobj); + PG_RE_THROW(); + } + PG_END_TRY(); + + xmlXPathFreeObject(xpathobj); + + return result; +#else + NO_XML_SUPPORT(); +#endif /* not USE_LIBXML */ +} + +/* + * XmlTableDestroyOpaque + * Release all libxml2 resources + */ +static void +XmlTableDestroyOpaque(TableFuncScanState *state) +{ +#ifdef USE_LIBXML + XmlTableBuilderData *xtCxt; + + xtCxt = GetXmlTableBuilderPrivateData(state, "XmlTableDestroyOpaque"); + + /* Propagate context related error context to libxml2 */ + xmlSetStructuredErrorFunc((void *) xtCxt->xmlerrcxt, xml_errorHandler); + + if (xtCxt->xpathscomp != NULL) + { + int i; + + for (i = 0; i < xtCxt->natts; i++) + if (xtCxt->xpathscomp[i] != NULL) + xmlXPathFreeCompExpr(xtCxt->xpathscomp[i]); + } + + if (xtCxt->xpathobj != NULL) + xmlXPathFreeObject(xtCxt->xpathobj); + if (xtCxt->xpathcomp != NULL) + xmlXPathFreeCompExpr(xtCxt->xpathcomp); + if (xtCxt->xpathcxt != NULL) + xmlXPathFreeContext(xtCxt->xpathcxt); + if (xtCxt->doc != NULL) + xmlFreeDoc(xtCxt->doc); + if (xtCxt->ctxt != NULL) + xmlFreeParserCtxt(xtCxt->ctxt); + + pg_xml_done(xtCxt->xmlerrcxt, true); + + /* not valid anymore */ + xtCxt->magic = 0; + state->opaque = NULL; + +#else + NO_XML_SUPPORT(); +#endif /* not USE_LIBXML */ +} diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 8d98195576..4c05b30068 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201703061 +#define CATALOG_VERSION_NO 201703081 #endif diff --git a/src/include/executor/nodeTableFuncscan.h b/src/include/executor/nodeTableFuncscan.h new file mode 100644 index 0000000000..529c929993 --- /dev/null +++ b/src/include/executor/nodeTableFuncscan.h @@ -0,0 +1,24 @@ +/*------------------------------------------------------------------------- + * + * nodeTableFuncscan.h + * + * + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/executor/nodeTableFuncscan.h + * + *------------------------------------------------------------------------- + */ +#ifndef NODETABLEFUNCSCAN_H +#define NODETABLEFUNCSCAN_H + +#include "nodes/execnodes.h" + +extern TableFuncScanState *ExecInitTableFuncScan(TableFuncScan *node, EState *estate, int eflags); +extern TupleTableSlot *ExecTableFuncScan(TableFuncScanState *node); +extern void ExecEndTableFuncScan(TableFuncScanState *node); +extern void ExecReScanTableFuncScan(TableFuncScanState *node); + +#endif /* NODETABLEFUNCSCAN_H */ diff --git a/src/include/executor/tablefunc.h b/src/include/executor/tablefunc.h new file mode 100644 index 0000000000..89d6381220 --- /dev/null +++ b/src/include/executor/tablefunc.h @@ -0,0 +1,67 @@ +/*------------------------------------------------------------------------- + * + * tablefunc.h + * interface for TableFunc executor node + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/executor/tablefunc.h + * + *------------------------------------------------------------------------- + */ +#ifndef _TABLEFUNC_H +#define _TABLEFUNC_H + +/* Forward-declare this to avoid including execnodes.h here */ +struct TableFuncScanState; + +/* + * TableFuncRoutine holds function pointers used for generating content of + * table-producer functions, such as XMLTABLE. + * + * InitBuilder initialize table builder private objects. The output tuple + * descriptor, input functions for the columns, and typioparams are passed + * from executor state. + * + * SetDoc is called to define the input document. The table builder may + * apply additional transformations not exposed outside the table builder + * context. + * + * SetNamespace is called to pass namespace declarations from the table + * expression. This function may be NULL if namespaces are not supported by + * the table builder. Namespaces must be given before setting the row and + * column filters. If the name is given as NULL, the entry shall be for the + * default namespace. + * + * SetRowFilter is called do define the row-generating filter, which shall be + * used to extract each row from the input document. + * + * SetColumnFilter is called once for each column, to define the column- + * generating filter for the given column. + * + * FetchRow shall be called repeatedly until it returns that no more rows are + * found in the document. On each invocation it shall set state in the table + * builder context such that each subsequent GetValue call returns the values + * for the indicated column for the row being processed. + * + * DestroyBuilder shall release all resources associated with a table builder + * context. It may be called either because all rows have been consumed, or + * because an error ocurred while processing the table expression. + */ +typedef struct TableFuncRoutine +{ + void (*InitOpaque) (struct TableFuncScanState *state, int natts); + void (*SetDocument) (struct TableFuncScanState *state, Datum value); + void (*SetNamespace) (struct TableFuncScanState *state, char *name, + char *uri); + void (*SetRowFilter) (struct TableFuncScanState *state, char *path); + void (*SetColumnFilter) (struct TableFuncScanState *state, + char *path, int colnum); + bool (*FetchRow) (struct TableFuncScanState *state); + Datum (*GetValue) (struct TableFuncScanState *state, int colnum, + Oid typid, int32 typmod, bool *isnull); + void (*DestroyOpaque) (struct TableFuncScanState *state); +} TableFuncRoutine; + +#endif /* _TABLEFUNC_H */ diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 6332ea0620..2fde67a9c8 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1584,6 +1584,31 @@ typedef struct ValuesScanState int curr_idx; } ValuesScanState; +/* ---------------- + * TableFuncScanState node + * + * Used in table-expression functions like XMLTABLE. + * ---------------- + */ +typedef struct TableFuncScanState +{ + ScanState ss; /* its first field is NodeTag */ + ExprState *docexpr; /* state for document expression */ + ExprState *rowexpr; /* state for row-generating expression */ + List *colexprs; /* state for column-generating expression */ + List *coldefexprs; /* state for column default expressions */ + List *ns_names; /* list of str nodes with namespace names */ + List *ns_uris; /* list of states of namespace uri exprs */ + Bitmapset *notnulls; /* nullability flag for each output column */ + void *opaque; /* table builder private space */ + const struct TableFuncRoutine *routine; /* table builder methods */ + FmgrInfo *in_functions; /* input function for each column */ + Oid *typioparams; /* typioparam for each column */ + int64 ordinal; /* row number to be output next */ + MemoryContext perValueCxt; /* short life context for value evaluation */ + Tuplestorestate *tupstore; /* output tuple store */ +} TableFuncScanState; + /* ---------------- * CteScanState information * diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index ede7ace76b..49fa944755 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -61,6 +61,7 @@ typedef enum NodeTag T_SubqueryScan, T_FunctionScan, T_ValuesScan, + T_TableFuncScan, T_CteScan, T_WorkTableScan, T_ForeignScan, @@ -109,6 +110,7 @@ typedef enum NodeTag T_TidScanState, T_SubqueryScanState, T_FunctionScanState, + T_TableFuncScanState, T_ValuesScanState, T_CteScanState, T_WorkTableScanState, @@ -135,6 +137,7 @@ typedef enum NodeTag */ T_Alias, T_RangeVar, + T_TableFunc, T_Expr, T_Var, T_Const, @@ -439,6 +442,8 @@ typedef enum NodeTag T_RangeSubselect, T_RangeFunction, T_RangeTableSample, + T_RangeTableFunc, + T_RangeTableFuncCol, T_TypeName, T_ColumnDef, T_IndexElem, diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 956f99830c..a44d2178e1 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -554,6 +554,39 @@ typedef struct RangeFunction * of function returning RECORD */ } RangeFunction; +/* + * RangeTableFunc - raw form of "table functions" such as XMLTABLE + */ +typedef struct RangeTableFunc +{ + NodeTag type; + bool lateral; /* does it have LATERAL prefix? */ + Node *docexpr; /* document expression */ + Node *rowexpr; /* row generator expression */ + List *namespaces; /* list of namespaces as ResTarget */ + List *columns; /* list of RangeTableFuncCol */ + Alias *alias; /* table alias & optional column aliases */ + int location; /* token location, or -1 if unknown */ +} RangeTableFunc; + +/* + * RangeTableFuncCol - one column in a RangeTableFunc->columns + * + * If for_ordinality is true (FOR ORDINALITY), then the column is an int4 + * column and the rest of the fields are ignored. + */ +typedef struct RangeTableFuncCol +{ + NodeTag type; + char *colname; /* name of generated column */ + TypeName *typeName; /* type of generated column */ + bool for_ordinality; /* does it have FOR ORDINALITY? */ + bool is_not_null; /* does it have NOT NULL? */ + Node *colexpr; /* column filter expression */ + Node *coldefexpr; /* column default value expression */ + int location; /* token location, or -1 if unknown */ +} RangeTableFuncCol; + /* * RangeTableSample - TABLESAMPLE appearing in a raw FROM clause * @@ -871,6 +904,7 @@ typedef enum RTEKind RTE_SUBQUERY, /* subquery in FROM */ RTE_JOIN, /* join */ RTE_FUNCTION, /* function in FROM */ + RTE_TABLEFUNC, /* TableFunc(.., column list) */ RTE_VALUES, /* VALUES (), (), ... */ RTE_CTE /* common table expr (WITH list element) */ } RTEKind; @@ -931,6 +965,11 @@ typedef struct RangeTblEntry List *functions; /* list of RangeTblFunction nodes */ bool funcordinality; /* is this called WITH ORDINALITY? */ + /* + * Fields valid for a TableFunc RTE (else NULL): + */ + TableFunc *tablefunc; + /* * Fields valid for a values RTE (else NIL): */ diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h index f72f7a8978..e30ed6aa29 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h @@ -495,6 +495,16 @@ typedef struct ValuesScan List *values_lists; /* list of expression lists */ } ValuesScan; +/* ---------------- + * TableFunc scan node + * ---------------- + */ +typedef struct TableFuncScan +{ + Scan scan; + TableFunc *tablefunc; /* table function node */ +} TableFuncScan; + /* ---------------- * CteScan node * ---------------- diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h index 235bc75096..d57b4fab3d 100644 --- a/src/include/nodes/primnodes.h +++ b/src/include/nodes/primnodes.h @@ -18,6 +18,7 @@ #define PRIMNODES_H #include "access/attnum.h" +#include "nodes/bitmapset.h" #include "nodes/pg_list.h" @@ -72,6 +73,27 @@ typedef struct RangeVar int location; /* token location, or -1 if unknown */ } RangeVar; +/* + * TableFunc - node for a table function, such as XMLTABLE. + */ +typedef struct TableFunc +{ + NodeTag type; + List *ns_uris; /* list of namespace uri */ + List *ns_names; /* list of namespace names */ + Node *docexpr; /* input document expression */ + Node *rowexpr; /* row filter expression */ + List *colnames; /* column names (list of String) */ + List *coltypes; /* OID list of column type OIDs */ + List *coltypmods; /* integer list of column typmods */ + List *colcollations; /* OID list of column collation OIDs */ + List *colexprs; /* list of column filter expressions */ + List *coldefexprs; /* list of column default expressions */ + Bitmapset *notnulls; /* nullability flag for each output column */ + int ordinalitycol; /* counts from 0; -1 if none specified */ + int location; /* token location, or -1 if unknown */ +} TableFunc; + /* * IntoClause - target information for SELECT INTO, CREATE TABLE AS, and * CREATE MATERIALIZED VIEW diff --git a/src/include/optimizer/cost.h b/src/include/optimizer/cost.h index 72200fa531..2b386835e3 100644 --- a/src/include/optimizer/cost.h +++ b/src/include/optimizer/cost.h @@ -89,8 +89,12 @@ extern void cost_subqueryscan(SubqueryScanPath *path, PlannerInfo *root, RelOptInfo *baserel, ParamPathInfo *param_info); extern void cost_functionscan(Path *path, PlannerInfo *root, RelOptInfo *baserel, ParamPathInfo *param_info); +extern void cost_tableexprscan(Path *path, PlannerInfo *root, + RelOptInfo *baserel, ParamPathInfo *param_info); extern void cost_valuesscan(Path *path, PlannerInfo *root, RelOptInfo *baserel, ParamPathInfo *param_info); +extern void cost_tablefuncscan(Path *path, PlannerInfo *root, + RelOptInfo *baserel, ParamPathInfo *param_info); extern void cost_ctescan(Path *path, PlannerInfo *root, RelOptInfo *baserel, ParamPathInfo *param_info); extern void cost_recursive_union(Path *runion, Path *nrterm, Path *rterm); @@ -181,6 +185,7 @@ extern void set_function_size_estimates(PlannerInfo *root, RelOptInfo *rel); extern void set_values_size_estimates(PlannerInfo *root, RelOptInfo *rel); extern void set_cte_size_estimates(PlannerInfo *root, RelOptInfo *rel, double cte_rows); +extern void set_tablefunc_size_estimates(PlannerInfo *root, RelOptInfo *rel); extern void set_foreign_size_estimates(PlannerInfo *root, RelOptInfo *rel); extern PathTarget *set_pathtarget_cost_width(PlannerInfo *root, PathTarget *target); extern double compute_bitmap_pages(PlannerInfo *root, RelOptInfo *baserel, diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index 53cad247dc..befe578141 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -82,8 +82,12 @@ extern SubqueryScanPath *create_subqueryscan_path(PlannerInfo *root, List *pathkeys, Relids required_outer); extern Path *create_functionscan_path(PlannerInfo *root, RelOptInfo *rel, List *pathkeys, Relids required_outer); +extern Path *create_tablexprscan_path(PlannerInfo *root, RelOptInfo *rel, + List *pathkeys, Relids required_outer); extern Path *create_valuesscan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer); +extern Path *create_tablefuncscan_path(PlannerInfo *root, RelOptInfo *rel, + Relids required_outer); extern Path *create_ctescan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer); extern Path *create_worktablescan_path(PlannerInfo *root, RelOptInfo *rel, diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h index 985d6505ec..28c4dab258 100644 --- a/src/include/parser/kwlist.h +++ b/src/include/parser/kwlist.h @@ -82,6 +82,7 @@ PG_KEYWORD("coalesce", COALESCE, COL_NAME_KEYWORD) PG_KEYWORD("collate", COLLATE, RESERVED_KEYWORD) PG_KEYWORD("collation", COLLATION, TYPE_FUNC_NAME_KEYWORD) PG_KEYWORD("column", COLUMN, RESERVED_KEYWORD) +PG_KEYWORD("columns", COLUMNS, UNRESERVED_KEYWORD) PG_KEYWORD("comment", COMMENT, UNRESERVED_KEYWORD) PG_KEYWORD("comments", COMMENTS, UNRESERVED_KEYWORD) PG_KEYWORD("commit", COMMIT, UNRESERVED_KEYWORD) @@ -446,10 +447,12 @@ PG_KEYWORD("xmlconcat", XMLCONCAT, COL_NAME_KEYWORD) PG_KEYWORD("xmlelement", XMLELEMENT, COL_NAME_KEYWORD) PG_KEYWORD("xmlexists", XMLEXISTS, COL_NAME_KEYWORD) PG_KEYWORD("xmlforest", XMLFOREST, COL_NAME_KEYWORD) +PG_KEYWORD("xmlnamespaces", XMLNAMESPACES, COL_NAME_KEYWORD) PG_KEYWORD("xmlparse", XMLPARSE, COL_NAME_KEYWORD) PG_KEYWORD("xmlpi", XMLPI, COL_NAME_KEYWORD) PG_KEYWORD("xmlroot", XMLROOT, COL_NAME_KEYWORD) PG_KEYWORD("xmlserialize", XMLSERIALIZE, COL_NAME_KEYWORD) +PG_KEYWORD("xmltable", XMLTABLE, COL_NAME_KEYWORD) PG_KEYWORD("year", YEAR_P, UNRESERVED_KEYWORD) PG_KEYWORD("yes", YES_P, UNRESERVED_KEYWORD) PG_KEYWORD("zone", ZONE, UNRESERVED_KEYWORD) diff --git a/src/include/parser/parse_coerce.h b/src/include/parser/parse_coerce.h index b50992cfd9..3eed81966d 100644 --- a/src/include/parser/parse_coerce.h +++ b/src/include/parser/parse_coerce.h @@ -58,6 +58,10 @@ extern Node *coerce_to_specific_type(ParseState *pstate, Node *node, Oid targetTypeId, const char *constructName); +extern Node *coerce_to_specific_type_typmod(ParseState *pstate, Node *node, + Oid targetTypeId, int32 targetTypmod, + const char *constructName); + extern int parser_coercion_errposition(ParseState *pstate, int coerce_location, Node *input_expr); diff --git a/src/include/parser/parse_relation.h b/src/include/parser/parse_relation.h index cfb2e5b88c..515c06cfef 100644 --- a/src/include/parser/parse_relation.h +++ b/src/include/parser/parse_relation.h @@ -91,6 +91,11 @@ extern RangeTblEntry *addRangeTableEntryForValues(ParseState *pstate, Alias *alias, bool lateral, bool inFromCl); +extern RangeTblEntry *addRangeTableEntryForTableFunc(ParseState *pstate, + TableFunc *tf, + Alias *alias, + bool lateral, + bool inFromCl); extern RangeTblEntry *addRangeTableEntryForJoin(ParseState *pstate, List *colnames, JoinType jointype, diff --git a/src/include/utils/xml.h b/src/include/utils/xml.h index cc1fc390e8..e570b71c04 100644 --- a/src/include/utils/xml.h +++ b/src/include/utils/xml.h @@ -18,6 +18,7 @@ #include "fmgr.h" #include "nodes/execnodes.h" #include "nodes/primnodes.h" +#include "executor/tablefunc.h" typedef struct varlena xmltype; @@ -76,4 +77,6 @@ extern int xmlbinary; /* XmlBinaryType, but int for guc enum */ extern int xmloption; /* XmlOptionType, but int for guc enum */ +extern const TableFuncRoutine XmlTableRoutine; + #endif /* XML_H */ diff --git a/src/test/regress/expected/xml.out b/src/test/regress/expected/xml.out index f21e119f1e..bcc585d427 100644 --- a/src/test/regress/expected/xml.out +++ b/src/test/regress/expected/xml.out @@ -948,3 +948,507 @@ SELECT XMLPARSE(DOCUMENT '  (1 row) +-- XMLPATH tests +CREATE TABLE xmldata(data xml); +INSERT INTO xmldata VALUES(' + + AU + Australia + 3 + + + CN + China + 3 + + + HK + HongKong + 3 + + + IN + India + 3 + + + JP + Japan + 3Sinzo Abe + + + SG + Singapore + 3791 + +'); +-- XMLTABLE with columns +SELECT xmltable.* + FROM (SELECT data FROM xmldata) x, + LATERAL XMLTABLE('/ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + _id FOR ORDINALITY, + country_name text PATH 'COUNTRY_NAME' NOT NULL, + country_id text PATH 'COUNTRY_ID', + region_id int PATH 'REGION_ID', + size float PATH 'SIZE', + unit text PATH 'SIZE/@unit', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); + id | _id | country_name | country_id | region_id | size | unit | premier_name +----+-----+--------------+------------+-----------+------+------+--------------- + 1 | 1 | Australia | AU | 3 | | | not specified + 2 | 2 | China | CN | 3 | | | not specified + 3 | 3 | HongKong | HK | 3 | | | not specified + 4 | 4 | India | IN | 3 | | | not specified + 5 | 5 | Japan | JP | 3 | | | Sinzo Abe + 6 | 6 | Singapore | SG | 3 | 791 | km | not specified +(6 rows) + +CREATE VIEW xmltableview1 AS SELECT xmltable.* + FROM (SELECT data FROM xmldata) x, + LATERAL XMLTABLE('/ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + _id FOR ORDINALITY, + country_name text PATH 'COUNTRY_NAME' NOT NULL, + country_id text PATH 'COUNTRY_ID', + region_id int PATH 'REGION_ID', + size float PATH 'SIZE', + unit text PATH 'SIZE/@unit', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); +SELECT * FROM xmltableview1; + id | _id | country_name | country_id | region_id | size | unit | premier_name +----+-----+--------------+------------+-----------+------+------+--------------- + 1 | 1 | Australia | AU | 3 | | | not specified + 2 | 2 | China | CN | 3 | | | not specified + 3 | 3 | HongKong | HK | 3 | | | not specified + 4 | 4 | India | IN | 3 | | | not specified + 5 | 5 | Japan | JP | 3 | | | Sinzo Abe + 6 | 6 | Singapore | SG | 3 | 791 | km | not specified +(6 rows) + +\sv xmltableview1 +CREATE OR REPLACE VIEW public.xmltableview1 AS + SELECT "xmltable".id, + "xmltable"._id, + "xmltable".country_name, + "xmltable".country_id, + "xmltable".region_id, + "xmltable".size, + "xmltable".unit, + "xmltable".premier_name + FROM ( SELECT xmldata.data + FROM xmldata) x, + LATERAL XMLTABLE(('/ROWS/ROW'::text) PASSING (x.data) COLUMNS id integer PATH ('@id'::text), _id FOR ORDINALITY, country_name text PATH ('COUNTRY_NAME'::text) NOT NULL, country_id text PATH ('COUNTRY_ID'::text), region_id integer PATH ('REGION_ID'::text), size double precision PATH ('SIZE'::text), unit text PATH ('SIZE/@unit'::text), premier_name text DEFAULT ('not specified'::text) PATH ('PREMIER_NAME'::text)) +EXPLAIN (COSTS OFF) SELECT * FROM xmltableview1; + QUERY PLAN +----------------------------------------- + Nested Loop + -> Seq Scan on xmldata + -> Table Function Scan on "xmltable" +(3 rows) + +EXPLAIN (COSTS OFF, VERBOSE) SELECT * FROM xmltableview1; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Nested Loop + Output: "xmltable".id, "xmltable"._id, "xmltable".country_name, "xmltable".country_id, "xmltable".region_id, "xmltable".size, "xmltable".unit, "xmltable".premier_name + -> Seq Scan on public.xmldata + Output: xmldata.data + -> Table Function Scan on "xmltable" + Output: "xmltable".id, "xmltable"._id, "xmltable".country_name, "xmltable".country_id, "xmltable".region_id, "xmltable".size, "xmltable".unit, "xmltable".premier_name + Table Function Call: XMLTABLE(('/ROWS/ROW'::text) PASSING (xmldata.data) COLUMNS id integer PATH ('@id'::text), _id FOR ORDINALITY, country_name text PATH ('COUNTRY_NAME'::text) NOT NULL, country_id text PATH ('COUNTRY_ID'::text), region_id integer PATH ('REGION_ID'::text), size double precision PATH ('SIZE'::text), unit text PATH ('SIZE/@unit'::text), premier_name text DEFAULT ('not specified'::text) PATH ('PREMIER_NAME'::text)) +(7 rows) + +-- XMLNAMESPACES tests +SELECT * FROM XMLTABLE(XMLNAMESPACES('http://x.y' AS zz), + '/zz:rows/zz:row' + PASSING '10' + COLUMNS a int PATH 'zz:a'); + a +---- + 10 +(1 row) + +CREATE VIEW xmltableview2 AS SELECT * FROM XMLTABLE(XMLNAMESPACES('http://x.y' AS zz), + '/zz:rows/zz:row' + PASSING '10' + COLUMNS a int PATH 'zz:a'); +SELECT * FROM xmltableview2; + a +---- + 10 +(1 row) + +SELECT * FROM XMLTABLE(XMLNAMESPACES(DEFAULT 'http://x.y'), + '/rows/row' + PASSING '10' + COLUMNS a int PATH 'a'); +ERROR: DEFAULT namespace is not supported +-- used in prepare statements +PREPARE pp AS +SELECT xmltable.* + FROM (SELECT data FROM xmldata) x, + LATERAL XMLTABLE('/ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + _id FOR ORDINALITY, + country_name text PATH 'COUNTRY_NAME' NOT NULL, + country_id text PATH 'COUNTRY_ID', + region_id int PATH 'REGION_ID', + size float PATH 'SIZE', + unit text PATH 'SIZE/@unit', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); +EXECUTE pp; + id | _id | country_name | country_id | region_id | size | unit | premier_name +----+-----+--------------+------------+-----------+------+------+--------------- + 1 | 1 | Australia | AU | 3 | | | not specified + 2 | 2 | China | CN | 3 | | | not specified + 3 | 3 | HongKong | HK | 3 | | | not specified + 4 | 4 | India | IN | 3 | | | not specified + 5 | 5 | Japan | JP | 3 | | | Sinzo Abe + 6 | 6 | Singapore | SG | 3 | 791 | km | not specified +(6 rows) + +SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS "COUNTRY_NAME" text, "REGION_ID" int); + COUNTRY_NAME | REGION_ID +--------------+----------- + India | 3 + Japan | 3 +(2 rows) + +SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS id FOR ORDINALITY, "COUNTRY_NAME" text, "REGION_ID" int); + id | COUNTRY_NAME | REGION_ID +----+--------------+----------- + 1 | India | 3 + 2 | Japan | 3 +(2 rows) + +SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS id int PATH '@id', "COUNTRY_NAME" text, "REGION_ID" int); + id | COUNTRY_NAME | REGION_ID +----+--------------+----------- + 4 | India | 3 + 5 | Japan | 3 +(2 rows) + +SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS id int PATH '@id'); + id +---- + 4 + 5 +(2 rows) + +SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS id FOR ORDINALITY); + id +---- + 1 + 2 +(2 rows) + +SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS id int PATH '@id', "COUNTRY_NAME" text, "REGION_ID" int, rawdata xml PATH '.'); + id | COUNTRY_NAME | REGION_ID | rawdata +----+--------------+-----------+------------------------------------------------------------------ + 4 | India | 3 | + + | | | IN + + | | | India + + | | | 3 + + | | | + 5 | Japan | 3 | + + | | | JP + + | | | Japan + + | | | 3Sinzo Abe+ + | | | +(2 rows) + +SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS id int PATH '@id', "COUNTRY_NAME" text, "REGION_ID" int, rawdata xml PATH './*'); + id | COUNTRY_NAME | REGION_ID | rawdata +----+--------------+-----------+----------------------------------------------------------------------------------------------------------------------------- + 4 | India | 3 | INIndia3 + 5 | Japan | 3 | JPJapan3Sinzo Abe +(2 rows) + +SELECT * FROM xmltable('/root' passing 'a1aa2a bbbbxxxcccc' COLUMNS element text); + element +------------------- + a1aa2a bbbbcccc +(1 row) + +SELECT * FROM xmltable('/root' passing 'a1aa2a bbbbxxxcccc' COLUMNS element text PATH 'element/text()'); -- should fail +ERROR: more than one value returned by column XPath expression +-- CDATA test +select * from xmltable('r' passing ' &"<>!foo]]>2' columns c text); + c +------------------------- + &"<>!foo + 2 +(2 rows) + +-- XML builtin entities +SELECT * FROM xmltable('/x/a' PASSING ''"&<>' COLUMNS ent text); + ent +----- + ' + " + & + < + > +(5 rows) + +SELECT * FROM xmltable('/x/a' PASSING ''"&<>' COLUMNS ent xml); + ent +------------------ + ' + " + & + < + > +(5 rows) + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT xmltable.* + FROM (SELECT data FROM xmldata) x, + LATERAL XMLTABLE('/ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + _id FOR ORDINALITY, + country_name text PATH 'COUNTRY_NAME' NOT NULL, + country_id text PATH 'COUNTRY_ID', + region_id int PATH 'REGION_ID', + size float PATH 'SIZE', + unit text PATH 'SIZE/@unit', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Nested Loop + Output: "xmltable".id, "xmltable"._id, "xmltable".country_name, "xmltable".country_id, "xmltable".region_id, "xmltable".size, "xmltable".unit, "xmltable".premier_name + -> Seq Scan on public.xmldata + Output: xmldata.data + -> Table Function Scan on "xmltable" + Output: "xmltable".id, "xmltable"._id, "xmltable".country_name, "xmltable".country_id, "xmltable".region_id, "xmltable".size, "xmltable".unit, "xmltable".premier_name + Table Function Call: XMLTABLE(('/ROWS/ROW'::text) PASSING (xmldata.data) COLUMNS id integer PATH ('@id'::text), _id FOR ORDINALITY, country_name text PATH ('COUNTRY_NAME'::text) NOT NULL, country_id text PATH ('COUNTRY_ID'::text), region_id integer PATH ('REGION_ID'::text), size double precision PATH ('SIZE'::text), unit text PATH ('SIZE/@unit'::text), premier_name text DEFAULT ('not specified'::text) PATH ('PREMIER_NAME'::text)) +(7 rows) + +-- test qual +SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS "COUNTRY_NAME" text, "REGION_ID" int) WHERE "COUNTRY_NAME" = 'Japan'; + COUNTRY_NAME | REGION_ID +--------------+----------- + Japan | 3 +(1 row) + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS "COUNTRY_NAME" text, "REGION_ID" int) WHERE "COUNTRY_NAME" = 'Japan'; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Nested Loop + Output: "xmltable"."COUNTRY_NAME", "xmltable"."REGION_ID" + -> Seq Scan on public.xmldata + Output: xmldata.data + -> Table Function Scan on "xmltable" + Output: "xmltable"."COUNTRY_NAME", "xmltable"."REGION_ID" + Table Function Call: XMLTABLE(('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]'::text) PASSING (xmldata.data) COLUMNS "COUNTRY_NAME" text, "REGION_ID" integer) + Filter: ("xmltable"."COUNTRY_NAME" = 'Japan'::text) +(8 rows) + +-- should to work with more data +INSERT INTO xmldata VALUES(' + + CZ + Czech Republic + 2Milos Zeman + + + DE + Germany + 2 + + + FR + France + 2 + +'); +INSERT INTO xmldata VALUES(' + + EG + Egypt + 1 + + + SD + Sudan + 1 + +'); +SELECT xmltable.* + FROM (SELECT data FROM xmldata) x, + LATERAL XMLTABLE('/ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + _id FOR ORDINALITY, + country_name text PATH 'COUNTRY_NAME' NOT NULL, + country_id text PATH 'COUNTRY_ID', + region_id int PATH 'REGION_ID', + size float PATH 'SIZE', + unit text PATH 'SIZE/@unit', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); + id | _id | country_name | country_id | region_id | size | unit | premier_name +----+-----+----------------+------------+-----------+------+------+--------------- + 1 | 1 | Australia | AU | 3 | | | not specified + 2 | 2 | China | CN | 3 | | | not specified + 3 | 3 | HongKong | HK | 3 | | | not specified + 4 | 4 | India | IN | 3 | | | not specified + 5 | 5 | Japan | JP | 3 | | | Sinzo Abe + 6 | 6 | Singapore | SG | 3 | 791 | km | not specified + 10 | 1 | Czech Republic | CZ | 2 | | | Milos Zeman + 11 | 2 | Germany | DE | 2 | | | not specified + 12 | 3 | France | FR | 2 | | | not specified + 20 | 1 | Egypt | EG | 1 | | | not specified + 21 | 2 | Sudan | SD | 1 | | | not specified +(11 rows) + +SELECT xmltable.* + FROM (SELECT data FROM xmldata) x, + LATERAL XMLTABLE('/ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + _id FOR ORDINALITY, + country_name text PATH 'COUNTRY_NAME' NOT NULL, + country_id text PATH 'COUNTRY_ID', + region_id int PATH 'REGION_ID', + size float PATH 'SIZE', + unit text PATH 'SIZE/@unit', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified') + WHERE region_id = 2; + id | _id | country_name | country_id | region_id | size | unit | premier_name +----+-----+----------------+------------+-----------+------+------+--------------- + 10 | 1 | Czech Republic | CZ | 2 | | | Milos Zeman + 11 | 2 | Germany | DE | 2 | | | not specified + 12 | 3 | France | FR | 2 | | | not specified +(3 rows) + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT xmltable.* + FROM (SELECT data FROM xmldata) x, + LATERAL XMLTABLE('/ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + _id FOR ORDINALITY, + country_name text PATH 'COUNTRY_NAME' NOT NULL, + country_id text PATH 'COUNTRY_ID', + region_id int PATH 'REGION_ID', + size float PATH 'SIZE', + unit text PATH 'SIZE/@unit', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified') + WHERE region_id = 2; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Nested Loop + Output: "xmltable".id, "xmltable"._id, "xmltable".country_name, "xmltable".country_id, "xmltable".region_id, "xmltable".size, "xmltable".unit, "xmltable".premier_name + -> Seq Scan on public.xmldata + Output: xmldata.data + -> Table Function Scan on "xmltable" + Output: "xmltable".id, "xmltable"._id, "xmltable".country_name, "xmltable".country_id, "xmltable".region_id, "xmltable".size, "xmltable".unit, "xmltable".premier_name + Table Function Call: XMLTABLE(('/ROWS/ROW'::text) PASSING (xmldata.data) COLUMNS id integer PATH ('@id'::text), _id FOR ORDINALITY, country_name text PATH ('COUNTRY_NAME'::text) NOT NULL, country_id text PATH ('COUNTRY_ID'::text), region_id integer PATH ('REGION_ID'::text), size double precision PATH ('SIZE'::text), unit text PATH ('SIZE/@unit'::text), premier_name text DEFAULT ('not specified'::text) PATH ('PREMIER_NAME'::text)) + Filter: ("xmltable".region_id = 2) +(8 rows) + +-- should fail, NULL value +SELECT xmltable.* + FROM (SELECT data FROM xmldata) x, + LATERAL XMLTABLE('/ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + _id FOR ORDINALITY, + country_name text PATH 'COUNTRY_NAME' NOT NULL, + country_id text PATH 'COUNTRY_ID', + region_id int PATH 'REGION_ID', + size float PATH 'SIZE' NOT NULL, + unit text PATH 'SIZE/@unit', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); +ERROR: null is not allowed in column "size" +-- if all is ok, then result is empty +-- one line xml test +WITH + x AS (SELECT proname, proowner, procost::numeric, pronargs, + array_to_string(proargnames,',') as proargnames, + case when proargtypes <> '' then array_to_string(proargtypes::oid[],',') end as proargtypes + FROM pg_proc WHERE proname = 'f_leak'), + y AS (SELECT xmlelement(name proc, + xmlforest(proname, proowner, + procost, pronargs, + proargnames, proargtypes)) as proc + FROM x), + z AS (SELECT xmltable.* + FROM y, + LATERAL xmltable('/proc' PASSING proc + COLUMNS proname name, + proowner oid, + procost float, + pronargs int, + proargnames text, + proargtypes text)) + SELECT * FROM z + EXCEPT SELECT * FROM x; + proname | proowner | procost | pronargs | proargnames | proargtypes +---------+----------+---------+----------+-------------+------------- +(0 rows) + +-- multi line xml test, result should be empty too +WITH + x AS (SELECT proname, proowner, procost::numeric, pronargs, + array_to_string(proargnames,',') as proargnames, + case when proargtypes <> '' then array_to_string(proargtypes::oid[],',') end as proargtypes + FROM pg_proc), + y AS (SELECT xmlelement(name data, + xmlagg(xmlelement(name proc, + xmlforest(proname, proowner, procost, + pronargs, proargnames, proargtypes)))) as doc + FROM x), + z AS (SELECT xmltable.* + FROM y, + LATERAL xmltable('/data/proc' PASSING doc + COLUMNS proname name, + proowner oid, + procost float, + pronargs int, + proargnames text, + proargtypes text)) + SELECT * FROM z + EXCEPT SELECT * FROM x; + proname | proowner | procost | pronargs | proargnames | proargtypes +---------+----------+---------+----------+-------------+------------- +(0 rows) + +CREATE TABLE xmltest2(x xml, _path text); +INSERT INTO xmltest2 VALUES('1', 'A'); +INSERT INTO xmltest2 VALUES('2', 'B'); +INSERT INTO xmltest2 VALUES('3', 'C'); +INSERT INTO xmltest2 VALUES('2', 'D'); +SELECT xmltable.* FROM xmltest2, LATERAL xmltable('/d/r' PASSING x COLUMNS a int PATH '' || lower(_path) || 'c'); + a +--- + 1 + 2 + 3 + 2 +(4 rows) + +SELECT xmltable.* FROM xmltest2, LATERAL xmltable(('/d/r/' || lower(_path) || 'c') PASSING x COLUMNS a int PATH '.'); + a +--- + 1 + 2 + 3 + 2 +(4 rows) + +SELECT xmltable.* FROM xmltest2, LATERAL xmltable(('/d/r/' || lower(_path) || 'c') PASSING x COLUMNS a int PATH 'x' DEFAULT ascii(_path) - 54); + a +---- + 11 + 12 + 13 + 14 +(4 rows) + diff --git a/src/test/regress/expected/xml_1.out b/src/test/regress/expected/xml_1.out index d7027030c3..d3bd8c91d7 100644 --- a/src/test/regress/expected/xml_1.out +++ b/src/test/regress/expected/xml_1.out @@ -827,3 +827,478 @@ SELECT XMLPARSE(DOCUMENT ' + + AU + Australia + 3 + + + CN + China + 3 + + + HK + HongKong + 3 + + + IN + India + 3 + + + JP + Japan + 3Sinzo Abe + + + SG + Singapore + 3791 + +'); +ERROR: unsupported XML feature +LINE 1: INSERT INTO xmldata VALUES(' + ^ +DETAIL: This functionality requires the server to be built with libxml support. +HINT: You need to rebuild PostgreSQL using --with-libxml. +-- XMLTABLE with columns +SELECT xmltable.* + FROM (SELECT data FROM xmldata) x, + LATERAL XMLTABLE('/ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + _id FOR ORDINALITY, + country_name text PATH 'COUNTRY_NAME' NOT NULL, + country_id text PATH 'COUNTRY_ID', + region_id int PATH 'REGION_ID', + size float PATH 'SIZE', + unit text PATH 'SIZE/@unit', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); + id | _id | country_name | country_id | region_id | size | unit | premier_name +----+-----+--------------+------------+-----------+------+------+-------------- +(0 rows) + +CREATE VIEW xmltableview1 AS SELECT xmltable.* + FROM (SELECT data FROM xmldata) x, + LATERAL XMLTABLE('/ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + _id FOR ORDINALITY, + country_name text PATH 'COUNTRY_NAME' NOT NULL, + country_id text PATH 'COUNTRY_ID', + region_id int PATH 'REGION_ID', + size float PATH 'SIZE', + unit text PATH 'SIZE/@unit', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); +SELECT * FROM xmltableview1; + id | _id | country_name | country_id | region_id | size | unit | premier_name +----+-----+--------------+------------+-----------+------+------+-------------- +(0 rows) + +\sv xmltableview1 +CREATE OR REPLACE VIEW public.xmltableview1 AS + SELECT "xmltable".id, + "xmltable"._id, + "xmltable".country_name, + "xmltable".country_id, + "xmltable".region_id, + "xmltable".size, + "xmltable".unit, + "xmltable".premier_name + FROM ( SELECT xmldata.data + FROM xmldata) x, + LATERAL XMLTABLE(('/ROWS/ROW'::text) PASSING (x.data) COLUMNS id integer PATH ('@id'::text), _id FOR ORDINALITY, country_name text PATH ('COUNTRY_NAME'::text) NOT NULL, country_id text PATH ('COUNTRY_ID'::text), region_id integer PATH ('REGION_ID'::text), size double precision PATH ('SIZE'::text), unit text PATH ('SIZE/@unit'::text), premier_name text DEFAULT ('not specified'::text) PATH ('PREMIER_NAME'::text)) +EXPLAIN (COSTS OFF) SELECT * FROM xmltableview1; + QUERY PLAN +----------------------------------------- + Nested Loop + -> Seq Scan on xmldata + -> Table Function Scan on "xmltable" +(3 rows) + +EXPLAIN (COSTS OFF, VERBOSE) SELECT * FROM xmltableview1; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Nested Loop + Output: "xmltable".id, "xmltable"._id, "xmltable".country_name, "xmltable".country_id, "xmltable".region_id, "xmltable".size, "xmltable".unit, "xmltable".premier_name + -> Seq Scan on public.xmldata + Output: xmldata.data + -> Table Function Scan on "xmltable" + Output: "xmltable".id, "xmltable"._id, "xmltable".country_name, "xmltable".country_id, "xmltable".region_id, "xmltable".size, "xmltable".unit, "xmltable".premier_name + Table Function Call: XMLTABLE(('/ROWS/ROW'::text) PASSING (xmldata.data) COLUMNS id integer PATH ('@id'::text), _id FOR ORDINALITY, country_name text PATH ('COUNTRY_NAME'::text) NOT NULL, country_id text PATH ('COUNTRY_ID'::text), region_id integer PATH ('REGION_ID'::text), size double precision PATH ('SIZE'::text), unit text PATH ('SIZE/@unit'::text), premier_name text DEFAULT ('not specified'::text) PATH ('PREMIER_NAME'::text)) +(7 rows) + +-- XMLNAMESPACES tests +SELECT * FROM XMLTABLE(XMLNAMESPACES('http://x.y' AS zz), + '/zz:rows/zz:row' + PASSING '10' + COLUMNS a int PATH 'zz:a'); +ERROR: unsupported XML feature +LINE 3: PASSING '10' + COLUMNS a int PATH 'zz:a'); +ERROR: unsupported XML feature +LINE 3: PASSING '10' + COLUMNS a int PATH 'a'); +ERROR: unsupported XML feature +LINE 3: PASSING 'a1aa2a bbbbxxxcccc' COLUMNS element text); +ERROR: unsupported XML feature +LINE 1: SELECT * FROM xmltable('/root' passing 'a1aa1aa2a bbbbxxxcccc' COLUMNS element text PATH 'element/text()'); -- should fail +ERROR: unsupported XML feature +LINE 1: SELECT * FROM xmltable('/root' passing 'a1a &"<>!foo]]>2' columns c text); +ERROR: unsupported XML feature +LINE 1: select * from xmltable('r' passing ''"&<>' COLUMNS ent text); +ERROR: unsupported XML feature +LINE 1: SELECT * FROM xmltable('/x/a' PASSING '''"&<>' COLUMNS ent xml); +ERROR: unsupported XML feature +LINE 1: SELECT * FROM xmltable('/x/a' PASSING '' Seq Scan on public.xmldata + Output: xmldata.data + -> Table Function Scan on "xmltable" + Output: "xmltable".id, "xmltable"._id, "xmltable".country_name, "xmltable".country_id, "xmltable".region_id, "xmltable".size, "xmltable".unit, "xmltable".premier_name + Table Function Call: XMLTABLE(('/ROWS/ROW'::text) PASSING (xmldata.data) COLUMNS id integer PATH ('@id'::text), _id FOR ORDINALITY, country_name text PATH ('COUNTRY_NAME'::text) NOT NULL, country_id text PATH ('COUNTRY_ID'::text), region_id integer PATH ('REGION_ID'::text), size double precision PATH ('SIZE'::text), unit text PATH ('SIZE/@unit'::text), premier_name text DEFAULT ('not specified'::text) PATH ('PREMIER_NAME'::text)) +(7 rows) + +-- test qual +SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS "COUNTRY_NAME" text, "REGION_ID" int) WHERE "COUNTRY_NAME" = 'Japan'; + COUNTRY_NAME | REGION_ID +--------------+----------- +(0 rows) + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS "COUNTRY_NAME" text, "REGION_ID" int) WHERE "COUNTRY_NAME" = 'Japan'; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Nested Loop + Output: "xmltable"."COUNTRY_NAME", "xmltable"."REGION_ID" + -> Seq Scan on public.xmldata + Output: xmldata.data + -> Table Function Scan on "xmltable" + Output: "xmltable"."COUNTRY_NAME", "xmltable"."REGION_ID" + Table Function Call: XMLTABLE(('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]'::text) PASSING (xmldata.data) COLUMNS "COUNTRY_NAME" text, "REGION_ID" integer) + Filter: ("xmltable"."COUNTRY_NAME" = 'Japan'::text) +(8 rows) + +-- should to work with more data +INSERT INTO xmldata VALUES(' + + CZ + Czech Republic + 2Milos Zeman + + + DE + Germany + 2 + + + FR + France + 2 + +'); +ERROR: unsupported XML feature +LINE 1: INSERT INTO xmldata VALUES(' + ^ +DETAIL: This functionality requires the server to be built with libxml support. +HINT: You need to rebuild PostgreSQL using --with-libxml. +INSERT INTO xmldata VALUES(' + + EG + Egypt + 1 + + + SD + Sudan + 1 + +'); +ERROR: unsupported XML feature +LINE 1: INSERT INTO xmldata VALUES(' + ^ +DETAIL: This functionality requires the server to be built with libxml support. +HINT: You need to rebuild PostgreSQL using --with-libxml. +SELECT xmltable.* + FROM (SELECT data FROM xmldata) x, + LATERAL XMLTABLE('/ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + _id FOR ORDINALITY, + country_name text PATH 'COUNTRY_NAME' NOT NULL, + country_id text PATH 'COUNTRY_ID', + region_id int PATH 'REGION_ID', + size float PATH 'SIZE', + unit text PATH 'SIZE/@unit', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); + id | _id | country_name | country_id | region_id | size | unit | premier_name +----+-----+--------------+------------+-----------+------+------+-------------- +(0 rows) + +SELECT xmltable.* + FROM (SELECT data FROM xmldata) x, + LATERAL XMLTABLE('/ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + _id FOR ORDINALITY, + country_name text PATH 'COUNTRY_NAME' NOT NULL, + country_id text PATH 'COUNTRY_ID', + region_id int PATH 'REGION_ID', + size float PATH 'SIZE', + unit text PATH 'SIZE/@unit', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified') + WHERE region_id = 2; + id | _id | country_name | country_id | region_id | size | unit | premier_name +----+-----+--------------+------------+-----------+------+------+-------------- +(0 rows) + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT xmltable.* + FROM (SELECT data FROM xmldata) x, + LATERAL XMLTABLE('/ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + _id FOR ORDINALITY, + country_name text PATH 'COUNTRY_NAME' NOT NULL, + country_id text PATH 'COUNTRY_ID', + region_id int PATH 'REGION_ID', + size float PATH 'SIZE', + unit text PATH 'SIZE/@unit', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified') + WHERE region_id = 2; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Nested Loop + Output: "xmltable".id, "xmltable"._id, "xmltable".country_name, "xmltable".country_id, "xmltable".region_id, "xmltable".size, "xmltable".unit, "xmltable".premier_name + -> Seq Scan on public.xmldata + Output: xmldata.data + -> Table Function Scan on "xmltable" + Output: "xmltable".id, "xmltable"._id, "xmltable".country_name, "xmltable".country_id, "xmltable".region_id, "xmltable".size, "xmltable".unit, "xmltable".premier_name + Table Function Call: XMLTABLE(('/ROWS/ROW'::text) PASSING (xmldata.data) COLUMNS id integer PATH ('@id'::text), _id FOR ORDINALITY, country_name text PATH ('COUNTRY_NAME'::text) NOT NULL, country_id text PATH ('COUNTRY_ID'::text), region_id integer PATH ('REGION_ID'::text), size double precision PATH ('SIZE'::text), unit text PATH ('SIZE/@unit'::text), premier_name text DEFAULT ('not specified'::text) PATH ('PREMIER_NAME'::text)) + Filter: ("xmltable".region_id = 2) +(8 rows) + +-- should fail, NULL value +SELECT xmltable.* + FROM (SELECT data FROM xmldata) x, + LATERAL XMLTABLE('/ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + _id FOR ORDINALITY, + country_name text PATH 'COUNTRY_NAME' NOT NULL, + country_id text PATH 'COUNTRY_ID', + region_id int PATH 'REGION_ID', + size float PATH 'SIZE' NOT NULL, + unit text PATH 'SIZE/@unit', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); + id | _id | country_name | country_id | region_id | size | unit | premier_name +----+-----+--------------+------------+-----------+------+------+-------------- +(0 rows) + +-- if all is ok, then result is empty +-- one line xml test +WITH + x AS (SELECT proname, proowner, procost::numeric, pronargs, + array_to_string(proargnames,',') as proargnames, + case when proargtypes <> '' then array_to_string(proargtypes::oid[],',') end as proargtypes + FROM pg_proc WHERE proname = 'f_leak'), + y AS (SELECT xmlelement(name proc, + xmlforest(proname, proowner, + procost, pronargs, + proargnames, proargtypes)) as proc + FROM x), + z AS (SELECT xmltable.* + FROM y, + LATERAL xmltable('/proc' PASSING proc + COLUMNS proname name, + proowner oid, + procost float, + pronargs int, + proargnames text, + proargtypes text)) + SELECT * FROM z + EXCEPT SELECT * FROM x; +ERROR: unsupported XML feature +DETAIL: This functionality requires the server to be built with libxml support. +HINT: You need to rebuild PostgreSQL using --with-libxml. +-- multi line xml test, result should be empty too +WITH + x AS (SELECT proname, proowner, procost::numeric, pronargs, + array_to_string(proargnames,',') as proargnames, + case when proargtypes <> '' then array_to_string(proargtypes::oid[],',') end as proargtypes + FROM pg_proc), + y AS (SELECT xmlelement(name data, + xmlagg(xmlelement(name proc, + xmlforest(proname, proowner, procost, + pronargs, proargnames, proargtypes)))) as doc + FROM x), + z AS (SELECT xmltable.* + FROM y, + LATERAL xmltable('/data/proc' PASSING doc + COLUMNS proname name, + proowner oid, + procost float, + pronargs int, + proargnames text, + proargtypes text)) + SELECT * FROM z + EXCEPT SELECT * FROM x; +ERROR: unsupported XML feature +DETAIL: This functionality requires the server to be built with libxml support. +HINT: You need to rebuild PostgreSQL using --with-libxml. +CREATE TABLE xmltest2(x xml, _path text); +INSERT INTO xmltest2 VALUES('1', 'A'); +ERROR: unsupported XML feature +LINE 1: INSERT INTO xmltest2 VALUES('1', 'A')... + ^ +DETAIL: This functionality requires the server to be built with libxml support. +HINT: You need to rebuild PostgreSQL using --with-libxml. +INSERT INTO xmltest2 VALUES('2', 'B'); +ERROR: unsupported XML feature +LINE 1: INSERT INTO xmltest2 VALUES('2', 'B')... + ^ +DETAIL: This functionality requires the server to be built with libxml support. +HINT: You need to rebuild PostgreSQL using --with-libxml. +INSERT INTO xmltest2 VALUES('3', 'C'); +ERROR: unsupported XML feature +LINE 1: INSERT INTO xmltest2 VALUES('3', 'C')... + ^ +DETAIL: This functionality requires the server to be built with libxml support. +HINT: You need to rebuild PostgreSQL using --with-libxml. +INSERT INTO xmltest2 VALUES('2', 'D'); +ERROR: unsupported XML feature +LINE 1: INSERT INTO xmltest2 VALUES('2', 'D')... + ^ +DETAIL: This functionality requires the server to be built with libxml support. +HINT: You need to rebuild PostgreSQL using --with-libxml. +SELECT xmltable.* FROM xmltest2, LATERAL xmltable('/d/r' PASSING x COLUMNS a int PATH '' || lower(_path) || 'c'); + a +--- +(0 rows) + +SELECT xmltable.* FROM xmltest2, LATERAL xmltable(('/d/r/' || lower(_path) || 'c') PASSING x COLUMNS a int PATH '.'); + a +--- +(0 rows) + +SELECT xmltable.* FROM xmltest2, LATERAL xmltable(('/d/r/' || lower(_path) || 'c') PASSING x COLUMNS a int PATH 'x' DEFAULT ascii(_path) - 54); + a +--- +(0 rows) + diff --git a/src/test/regress/expected/xml_2.out b/src/test/regress/expected/xml_2.out index 530faf5daf..ff77132803 100644 --- a/src/test/regress/expected/xml_2.out +++ b/src/test/regress/expected/xml_2.out @@ -928,3 +928,507 @@ SELECT XMLPARSE(DOCUMENT '  (1 row) +-- XMLPATH tests +CREATE TABLE xmldata(data xml); +INSERT INTO xmldata VALUES(' + + AU + Australia + 3 + + + CN + China + 3 + + + HK + HongKong + 3 + + + IN + India + 3 + + + JP + Japan + 3Sinzo Abe + + + SG + Singapore + 3791 + +'); +-- XMLTABLE with columns +SELECT xmltable.* + FROM (SELECT data FROM xmldata) x, + LATERAL XMLTABLE('/ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + _id FOR ORDINALITY, + country_name text PATH 'COUNTRY_NAME' NOT NULL, + country_id text PATH 'COUNTRY_ID', + region_id int PATH 'REGION_ID', + size float PATH 'SIZE', + unit text PATH 'SIZE/@unit', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); + id | _id | country_name | country_id | region_id | size | unit | premier_name +----+-----+--------------+------------+-----------+------+------+--------------- + 1 | 1 | Australia | AU | 3 | | | not specified + 2 | 2 | China | CN | 3 | | | not specified + 3 | 3 | HongKong | HK | 3 | | | not specified + 4 | 4 | India | IN | 3 | | | not specified + 5 | 5 | Japan | JP | 3 | | | Sinzo Abe + 6 | 6 | Singapore | SG | 3 | 791 | km | not specified +(6 rows) + +CREATE VIEW xmltableview1 AS SELECT xmltable.* + FROM (SELECT data FROM xmldata) x, + LATERAL XMLTABLE('/ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + _id FOR ORDINALITY, + country_name text PATH 'COUNTRY_NAME' NOT NULL, + country_id text PATH 'COUNTRY_ID', + region_id int PATH 'REGION_ID', + size float PATH 'SIZE', + unit text PATH 'SIZE/@unit', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); +SELECT * FROM xmltableview1; + id | _id | country_name | country_id | region_id | size | unit | premier_name +----+-----+--------------+------------+-----------+------+------+--------------- + 1 | 1 | Australia | AU | 3 | | | not specified + 2 | 2 | China | CN | 3 | | | not specified + 3 | 3 | HongKong | HK | 3 | | | not specified + 4 | 4 | India | IN | 3 | | | not specified + 5 | 5 | Japan | JP | 3 | | | Sinzo Abe + 6 | 6 | Singapore | SG | 3 | 791 | km | not specified +(6 rows) + +\sv xmltableview1 +CREATE OR REPLACE VIEW public.xmltableview1 AS + SELECT "xmltable".id, + "xmltable"._id, + "xmltable".country_name, + "xmltable".country_id, + "xmltable".region_id, + "xmltable".size, + "xmltable".unit, + "xmltable".premier_name + FROM ( SELECT xmldata.data + FROM xmldata) x, + LATERAL XMLTABLE(('/ROWS/ROW'::text) PASSING (x.data) COLUMNS id integer PATH ('@id'::text), _id FOR ORDINALITY, country_name text PATH ('COUNTRY_NAME'::text) NOT NULL, country_id text PATH ('COUNTRY_ID'::text), region_id integer PATH ('REGION_ID'::text), size double precision PATH ('SIZE'::text), unit text PATH ('SIZE/@unit'::text), premier_name text DEFAULT ('not specified'::text) PATH ('PREMIER_NAME'::text)) +EXPLAIN (COSTS OFF) SELECT * FROM xmltableview1; + QUERY PLAN +----------------------------------------- + Nested Loop + -> Seq Scan on xmldata + -> Table Function Scan on "xmltable" +(3 rows) + +EXPLAIN (COSTS OFF, VERBOSE) SELECT * FROM xmltableview1; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Nested Loop + Output: "xmltable".id, "xmltable"._id, "xmltable".country_name, "xmltable".country_id, "xmltable".region_id, "xmltable".size, "xmltable".unit, "xmltable".premier_name + -> Seq Scan on public.xmldata + Output: xmldata.data + -> Table Function Scan on "xmltable" + Output: "xmltable".id, "xmltable"._id, "xmltable".country_name, "xmltable".country_id, "xmltable".region_id, "xmltable".size, "xmltable".unit, "xmltable".premier_name + Table Function Call: XMLTABLE(('/ROWS/ROW'::text) PASSING (xmldata.data) COLUMNS id integer PATH ('@id'::text), _id FOR ORDINALITY, country_name text PATH ('COUNTRY_NAME'::text) NOT NULL, country_id text PATH ('COUNTRY_ID'::text), region_id integer PATH ('REGION_ID'::text), size double precision PATH ('SIZE'::text), unit text PATH ('SIZE/@unit'::text), premier_name text DEFAULT ('not specified'::text) PATH ('PREMIER_NAME'::text)) +(7 rows) + +-- XMLNAMESPACES tests +SELECT * FROM XMLTABLE(XMLNAMESPACES('http://x.y' AS zz), + '/zz:rows/zz:row' + PASSING '10' + COLUMNS a int PATH 'zz:a'); + a +---- + 10 +(1 row) + +CREATE VIEW xmltableview2 AS SELECT * FROM XMLTABLE(XMLNAMESPACES('http://x.y' AS zz), + '/zz:rows/zz:row' + PASSING '10' + COLUMNS a int PATH 'zz:a'); +SELECT * FROM xmltableview2; + a +---- + 10 +(1 row) + +SELECT * FROM XMLTABLE(XMLNAMESPACES(DEFAULT 'http://x.y'), + '/rows/row' + PASSING '10' + COLUMNS a int PATH 'a'); +ERROR: DEFAULT namespace is not supported +-- used in prepare statements +PREPARE pp AS +SELECT xmltable.* + FROM (SELECT data FROM xmldata) x, + LATERAL XMLTABLE('/ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + _id FOR ORDINALITY, + country_name text PATH 'COUNTRY_NAME' NOT NULL, + country_id text PATH 'COUNTRY_ID', + region_id int PATH 'REGION_ID', + size float PATH 'SIZE', + unit text PATH 'SIZE/@unit', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); +EXECUTE pp; + id | _id | country_name | country_id | region_id | size | unit | premier_name +----+-----+--------------+------------+-----------+------+------+--------------- + 1 | 1 | Australia | AU | 3 | | | not specified + 2 | 2 | China | CN | 3 | | | not specified + 3 | 3 | HongKong | HK | 3 | | | not specified + 4 | 4 | India | IN | 3 | | | not specified + 5 | 5 | Japan | JP | 3 | | | Sinzo Abe + 6 | 6 | Singapore | SG | 3 | 791 | km | not specified +(6 rows) + +SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS "COUNTRY_NAME" text, "REGION_ID" int); + COUNTRY_NAME | REGION_ID +--------------+----------- + India | 3 + Japan | 3 +(2 rows) + +SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS id FOR ORDINALITY, "COUNTRY_NAME" text, "REGION_ID" int); + id | COUNTRY_NAME | REGION_ID +----+--------------+----------- + 1 | India | 3 + 2 | Japan | 3 +(2 rows) + +SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS id int PATH '@id', "COUNTRY_NAME" text, "REGION_ID" int); + id | COUNTRY_NAME | REGION_ID +----+--------------+----------- + 4 | India | 3 + 5 | Japan | 3 +(2 rows) + +SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS id int PATH '@id'); + id +---- + 4 + 5 +(2 rows) + +SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS id FOR ORDINALITY); + id +---- + 1 + 2 +(2 rows) + +SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS id int PATH '@id', "COUNTRY_NAME" text, "REGION_ID" int, rawdata xml PATH '.'); + id | COUNTRY_NAME | REGION_ID | rawdata +----+--------------+-----------+------------------------------------------------------------------ + 4 | India | 3 | + + | | | IN + + | | | India + + | | | 3 + + | | | + 5 | Japan | 3 | + + | | | JP + + | | | Japan + + | | | 3Sinzo Abe+ + | | | +(2 rows) + +SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS id int PATH '@id', "COUNTRY_NAME" text, "REGION_ID" int, rawdata xml PATH './*'); + id | COUNTRY_NAME | REGION_ID | rawdata +----+--------------+-----------+----------------------------------------------------------------------------------------------------------------------------- + 4 | India | 3 | INIndia3 + 5 | Japan | 3 | JPJapan3Sinzo Abe +(2 rows) + +SELECT * FROM xmltable('/root' passing 'a1aa2a bbbbxxxcccc' COLUMNS element text); + element +------------------- + a1aa2a bbbbcccc +(1 row) + +SELECT * FROM xmltable('/root' passing 'a1aa2a bbbbxxxcccc' COLUMNS element text PATH 'element/text()'); -- should fail +ERROR: more than one value returned by column XPath expression +-- CDATA test +select * from xmltable('r' passing ' &"<>!foo]]>2' columns c text); + c +------------------------- + &"<>!foo + 2 +(2 rows) + +-- XML builtin entities +SELECT * FROM xmltable('/x/a' PASSING ''"&<>' COLUMNS ent text); + ent +----- + ' + " + & + < + > +(5 rows) + +SELECT * FROM xmltable('/x/a' PASSING ''"&<>' COLUMNS ent xml); + ent +------------------ + ' + " + & + < + > +(5 rows) + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT xmltable.* + FROM (SELECT data FROM xmldata) x, + LATERAL XMLTABLE('/ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + _id FOR ORDINALITY, + country_name text PATH 'COUNTRY_NAME' NOT NULL, + country_id text PATH 'COUNTRY_ID', + region_id int PATH 'REGION_ID', + size float PATH 'SIZE', + unit text PATH 'SIZE/@unit', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Nested Loop + Output: "xmltable".id, "xmltable"._id, "xmltable".country_name, "xmltable".country_id, "xmltable".region_id, "xmltable".size, "xmltable".unit, "xmltable".premier_name + -> Seq Scan on public.xmldata + Output: xmldata.data + -> Table Function Scan on "xmltable" + Output: "xmltable".id, "xmltable"._id, "xmltable".country_name, "xmltable".country_id, "xmltable".region_id, "xmltable".size, "xmltable".unit, "xmltable".premier_name + Table Function Call: XMLTABLE(('/ROWS/ROW'::text) PASSING (xmldata.data) COLUMNS id integer PATH ('@id'::text), _id FOR ORDINALITY, country_name text PATH ('COUNTRY_NAME'::text) NOT NULL, country_id text PATH ('COUNTRY_ID'::text), region_id integer PATH ('REGION_ID'::text), size double precision PATH ('SIZE'::text), unit text PATH ('SIZE/@unit'::text), premier_name text DEFAULT ('not specified'::text) PATH ('PREMIER_NAME'::text)) +(7 rows) + +-- test qual +SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS "COUNTRY_NAME" text, "REGION_ID" int) WHERE "COUNTRY_NAME" = 'Japan'; + COUNTRY_NAME | REGION_ID +--------------+----------- + Japan | 3 +(1 row) + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS "COUNTRY_NAME" text, "REGION_ID" int) WHERE "COUNTRY_NAME" = 'Japan'; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Nested Loop + Output: "xmltable"."COUNTRY_NAME", "xmltable"."REGION_ID" + -> Seq Scan on public.xmldata + Output: xmldata.data + -> Table Function Scan on "xmltable" + Output: "xmltable"."COUNTRY_NAME", "xmltable"."REGION_ID" + Table Function Call: XMLTABLE(('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]'::text) PASSING (xmldata.data) COLUMNS "COUNTRY_NAME" text, "REGION_ID" integer) + Filter: ("xmltable"."COUNTRY_NAME" = 'Japan'::text) +(8 rows) + +-- should to work with more data +INSERT INTO xmldata VALUES(' + + CZ + Czech Republic + 2Milos Zeman + + + DE + Germany + 2 + + + FR + France + 2 + +'); +INSERT INTO xmldata VALUES(' + + EG + Egypt + 1 + + + SD + Sudan + 1 + +'); +SELECT xmltable.* + FROM (SELECT data FROM xmldata) x, + LATERAL XMLTABLE('/ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + _id FOR ORDINALITY, + country_name text PATH 'COUNTRY_NAME' NOT NULL, + country_id text PATH 'COUNTRY_ID', + region_id int PATH 'REGION_ID', + size float PATH 'SIZE', + unit text PATH 'SIZE/@unit', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); + id | _id | country_name | country_id | region_id | size | unit | premier_name +----+-----+----------------+------------+-----------+------+------+--------------- + 1 | 1 | Australia | AU | 3 | | | not specified + 2 | 2 | China | CN | 3 | | | not specified + 3 | 3 | HongKong | HK | 3 | | | not specified + 4 | 4 | India | IN | 3 | | | not specified + 5 | 5 | Japan | JP | 3 | | | Sinzo Abe + 6 | 6 | Singapore | SG | 3 | 791 | km | not specified + 10 | 1 | Czech Republic | CZ | 2 | | | Milos Zeman + 11 | 2 | Germany | DE | 2 | | | not specified + 12 | 3 | France | FR | 2 | | | not specified + 20 | 1 | Egypt | EG | 1 | | | not specified + 21 | 2 | Sudan | SD | 1 | | | not specified +(11 rows) + +SELECT xmltable.* + FROM (SELECT data FROM xmldata) x, + LATERAL XMLTABLE('/ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + _id FOR ORDINALITY, + country_name text PATH 'COUNTRY_NAME' NOT NULL, + country_id text PATH 'COUNTRY_ID', + region_id int PATH 'REGION_ID', + size float PATH 'SIZE', + unit text PATH 'SIZE/@unit', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified') + WHERE region_id = 2; + id | _id | country_name | country_id | region_id | size | unit | premier_name +----+-----+----------------+------------+-----------+------+------+--------------- + 10 | 1 | Czech Republic | CZ | 2 | | | Milos Zeman + 11 | 2 | Germany | DE | 2 | | | not specified + 12 | 3 | France | FR | 2 | | | not specified +(3 rows) + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT xmltable.* + FROM (SELECT data FROM xmldata) x, + LATERAL XMLTABLE('/ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + _id FOR ORDINALITY, + country_name text PATH 'COUNTRY_NAME' NOT NULL, + country_id text PATH 'COUNTRY_ID', + region_id int PATH 'REGION_ID', + size float PATH 'SIZE', + unit text PATH 'SIZE/@unit', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified') + WHERE region_id = 2; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Nested Loop + Output: "xmltable".id, "xmltable"._id, "xmltable".country_name, "xmltable".country_id, "xmltable".region_id, "xmltable".size, "xmltable".unit, "xmltable".premier_name + -> Seq Scan on public.xmldata + Output: xmldata.data + -> Table Function Scan on "xmltable" + Output: "xmltable".id, "xmltable"._id, "xmltable".country_name, "xmltable".country_id, "xmltable".region_id, "xmltable".size, "xmltable".unit, "xmltable".premier_name + Table Function Call: XMLTABLE(('/ROWS/ROW'::text) PASSING (xmldata.data) COLUMNS id integer PATH ('@id'::text), _id FOR ORDINALITY, country_name text PATH ('COUNTRY_NAME'::text) NOT NULL, country_id text PATH ('COUNTRY_ID'::text), region_id integer PATH ('REGION_ID'::text), size double precision PATH ('SIZE'::text), unit text PATH ('SIZE/@unit'::text), premier_name text DEFAULT ('not specified'::text) PATH ('PREMIER_NAME'::text)) + Filter: ("xmltable".region_id = 2) +(8 rows) + +-- should fail, NULL value +SELECT xmltable.* + FROM (SELECT data FROM xmldata) x, + LATERAL XMLTABLE('/ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + _id FOR ORDINALITY, + country_name text PATH 'COUNTRY_NAME' NOT NULL, + country_id text PATH 'COUNTRY_ID', + region_id int PATH 'REGION_ID', + size float PATH 'SIZE' NOT NULL, + unit text PATH 'SIZE/@unit', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); +ERROR: null is not allowed in column "size" +-- if all is ok, then result is empty +-- one line xml test +WITH + x AS (SELECT proname, proowner, procost::numeric, pronargs, + array_to_string(proargnames,',') as proargnames, + case when proargtypes <> '' then array_to_string(proargtypes::oid[],',') end as proargtypes + FROM pg_proc WHERE proname = 'f_leak'), + y AS (SELECT xmlelement(name proc, + xmlforest(proname, proowner, + procost, pronargs, + proargnames, proargtypes)) as proc + FROM x), + z AS (SELECT xmltable.* + FROM y, + LATERAL xmltable('/proc' PASSING proc + COLUMNS proname name, + proowner oid, + procost float, + pronargs int, + proargnames text, + proargtypes text)) + SELECT * FROM z + EXCEPT SELECT * FROM x; + proname | proowner | procost | pronargs | proargnames | proargtypes +---------+----------+---------+----------+-------------+------------- +(0 rows) + +-- multi line xml test, result should be empty too +WITH + x AS (SELECT proname, proowner, procost::numeric, pronargs, + array_to_string(proargnames,',') as proargnames, + case when proargtypes <> '' then array_to_string(proargtypes::oid[],',') end as proargtypes + FROM pg_proc), + y AS (SELECT xmlelement(name data, + xmlagg(xmlelement(name proc, + xmlforest(proname, proowner, procost, + pronargs, proargnames, proargtypes)))) as doc + FROM x), + z AS (SELECT xmltable.* + FROM y, + LATERAL xmltable('/data/proc' PASSING doc + COLUMNS proname name, + proowner oid, + procost float, + pronargs int, + proargnames text, + proargtypes text)) + SELECT * FROM z + EXCEPT SELECT * FROM x; + proname | proowner | procost | pronargs | proargnames | proargtypes +---------+----------+---------+----------+-------------+------------- +(0 rows) + +CREATE TABLE xmltest2(x xml, _path text); +INSERT INTO xmltest2 VALUES('1', 'A'); +INSERT INTO xmltest2 VALUES('2', 'B'); +INSERT INTO xmltest2 VALUES('3', 'C'); +INSERT INTO xmltest2 VALUES('2', 'D'); +SELECT xmltable.* FROM xmltest2, LATERAL xmltable('/d/r' PASSING x COLUMNS a int PATH '' || lower(_path) || 'c'); + a +--- + 1 + 2 + 3 + 2 +(4 rows) + +SELECT xmltable.* FROM xmltest2, LATERAL xmltable(('/d/r/' || lower(_path) || 'c') PASSING x COLUMNS a int PATH '.'); + a +--- + 1 + 2 + 3 + 2 +(4 rows) + +SELECT xmltable.* FROM xmltest2, LATERAL xmltable(('/d/r/' || lower(_path) || 'c') PASSING x COLUMNS a int PATH 'x' DEFAULT ascii(_path) - 54); + a +---- + 11 + 12 + 13 + 14 +(4 rows) + diff --git a/src/test/regress/sql/xml.sql b/src/test/regress/sql/xml.sql index 08a0b30067..eb4687fb09 100644 --- a/src/test/regress/sql/xml.sql +++ b/src/test/regress/sql/xml.sql @@ -270,3 +270,291 @@ SELECT XMLPARSE(DOCUMENT ']> SELECT XMLPARSE(DOCUMENT ']>&c;'); -- This might or might not load the requested DTD, but it mustn't throw error. SELECT XMLPARSE(DOCUMENT ' '); + +-- XMLPATH tests +CREATE TABLE xmldata(data xml); +INSERT INTO xmldata VALUES(' + + AU + Australia + 3 + + + CN + China + 3 + + + HK + HongKong + 3 + + + IN + India + 3 + + + JP + Japan + 3Sinzo Abe + + + SG + Singapore + 3791 + +'); + +-- XMLTABLE with columns +SELECT xmltable.* + FROM (SELECT data FROM xmldata) x, + LATERAL XMLTABLE('/ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + _id FOR ORDINALITY, + country_name text PATH 'COUNTRY_NAME' NOT NULL, + country_id text PATH 'COUNTRY_ID', + region_id int PATH 'REGION_ID', + size float PATH 'SIZE', + unit text PATH 'SIZE/@unit', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); + +CREATE VIEW xmltableview1 AS SELECT xmltable.* + FROM (SELECT data FROM xmldata) x, + LATERAL XMLTABLE('/ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + _id FOR ORDINALITY, + country_name text PATH 'COUNTRY_NAME' NOT NULL, + country_id text PATH 'COUNTRY_ID', + region_id int PATH 'REGION_ID', + size float PATH 'SIZE', + unit text PATH 'SIZE/@unit', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); + +SELECT * FROM xmltableview1; + +\sv xmltableview1 + +EXPLAIN (COSTS OFF) SELECT * FROM xmltableview1; +EXPLAIN (COSTS OFF, VERBOSE) SELECT * FROM xmltableview1; + +-- XMLNAMESPACES tests +SELECT * FROM XMLTABLE(XMLNAMESPACES('http://x.y' AS zz), + '/zz:rows/zz:row' + PASSING '10' + COLUMNS a int PATH 'zz:a'); + +CREATE VIEW xmltableview2 AS SELECT * FROM XMLTABLE(XMLNAMESPACES('http://x.y' AS zz), + '/zz:rows/zz:row' + PASSING '10' + COLUMNS a int PATH 'zz:a'); + +SELECT * FROM xmltableview2; + +SELECT * FROM XMLTABLE(XMLNAMESPACES(DEFAULT 'http://x.y'), + '/rows/row' + PASSING '10' + COLUMNS a int PATH 'a'); + +-- used in prepare statements +PREPARE pp AS +SELECT xmltable.* + FROM (SELECT data FROM xmldata) x, + LATERAL XMLTABLE('/ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + _id FOR ORDINALITY, + country_name text PATH 'COUNTRY_NAME' NOT NULL, + country_id text PATH 'COUNTRY_ID', + region_id int PATH 'REGION_ID', + size float PATH 'SIZE', + unit text PATH 'SIZE/@unit', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); + +EXECUTE pp; + +SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS "COUNTRY_NAME" text, "REGION_ID" int); +SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS id FOR ORDINALITY, "COUNTRY_NAME" text, "REGION_ID" int); +SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS id int PATH '@id', "COUNTRY_NAME" text, "REGION_ID" int); +SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS id int PATH '@id'); +SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS id FOR ORDINALITY); +SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS id int PATH '@id', "COUNTRY_NAME" text, "REGION_ID" int, rawdata xml PATH '.'); +SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS id int PATH '@id', "COUNTRY_NAME" text, "REGION_ID" int, rawdata xml PATH './*'); + +SELECT * FROM xmltable('/root' passing 'a1aa2a bbbbxxxcccc' COLUMNS element text); +SELECT * FROM xmltable('/root' passing 'a1aa2a bbbbxxxcccc' COLUMNS element text PATH 'element/text()'); -- should fail + +-- CDATA test +select * from xmltable('r' passing ' &"<>!foo]]>2' columns c text); + +-- XML builtin entities +SELECT * FROM xmltable('/x/a' PASSING ''"&<>' COLUMNS ent text); +SELECT * FROM xmltable('/x/a' PASSING ''"&<>' COLUMNS ent xml); + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT xmltable.* + FROM (SELECT data FROM xmldata) x, + LATERAL XMLTABLE('/ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + _id FOR ORDINALITY, + country_name text PATH 'COUNTRY_NAME' NOT NULL, + country_id text PATH 'COUNTRY_ID', + region_id int PATH 'REGION_ID', + size float PATH 'SIZE', + unit text PATH 'SIZE/@unit', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); + +-- test qual +SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS "COUNTRY_NAME" text, "REGION_ID" int) WHERE "COUNTRY_NAME" = 'Japan'; + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS "COUNTRY_NAME" text, "REGION_ID" int) WHERE "COUNTRY_NAME" = 'Japan'; + +-- should to work with more data +INSERT INTO xmldata VALUES(' + + CZ + Czech Republic + 2Milos Zeman + + + DE + Germany + 2 + + + FR + France + 2 + +'); + +INSERT INTO xmldata VALUES(' + + EG + Egypt + 1 + + + SD + Sudan + 1 + +'); + +SELECT xmltable.* + FROM (SELECT data FROM xmldata) x, + LATERAL XMLTABLE('/ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + _id FOR ORDINALITY, + country_name text PATH 'COUNTRY_NAME' NOT NULL, + country_id text PATH 'COUNTRY_ID', + region_id int PATH 'REGION_ID', + size float PATH 'SIZE', + unit text PATH 'SIZE/@unit', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); + +SELECT xmltable.* + FROM (SELECT data FROM xmldata) x, + LATERAL XMLTABLE('/ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + _id FOR ORDINALITY, + country_name text PATH 'COUNTRY_NAME' NOT NULL, + country_id text PATH 'COUNTRY_ID', + region_id int PATH 'REGION_ID', + size float PATH 'SIZE', + unit text PATH 'SIZE/@unit', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified') + WHERE region_id = 2; + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT xmltable.* + FROM (SELECT data FROM xmldata) x, + LATERAL XMLTABLE('/ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + _id FOR ORDINALITY, + country_name text PATH 'COUNTRY_NAME' NOT NULL, + country_id text PATH 'COUNTRY_ID', + region_id int PATH 'REGION_ID', + size float PATH 'SIZE', + unit text PATH 'SIZE/@unit', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified') + WHERE region_id = 2; + +-- should fail, NULL value +SELECT xmltable.* + FROM (SELECT data FROM xmldata) x, + LATERAL XMLTABLE('/ROWS/ROW' + PASSING data + COLUMNS id int PATH '@id', + _id FOR ORDINALITY, + country_name text PATH 'COUNTRY_NAME' NOT NULL, + country_id text PATH 'COUNTRY_ID', + region_id int PATH 'REGION_ID', + size float PATH 'SIZE' NOT NULL, + unit text PATH 'SIZE/@unit', + premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); + +-- if all is ok, then result is empty +-- one line xml test +WITH + x AS (SELECT proname, proowner, procost::numeric, pronargs, + array_to_string(proargnames,',') as proargnames, + case when proargtypes <> '' then array_to_string(proargtypes::oid[],',') end as proargtypes + FROM pg_proc WHERE proname = 'f_leak'), + y AS (SELECT xmlelement(name proc, + xmlforest(proname, proowner, + procost, pronargs, + proargnames, proargtypes)) as proc + FROM x), + z AS (SELECT xmltable.* + FROM y, + LATERAL xmltable('/proc' PASSING proc + COLUMNS proname name, + proowner oid, + procost float, + pronargs int, + proargnames text, + proargtypes text)) + SELECT * FROM z + EXCEPT SELECT * FROM x; + +-- multi line xml test, result should be empty too +WITH + x AS (SELECT proname, proowner, procost::numeric, pronargs, + array_to_string(proargnames,',') as proargnames, + case when proargtypes <> '' then array_to_string(proargtypes::oid[],',') end as proargtypes + FROM pg_proc), + y AS (SELECT xmlelement(name data, + xmlagg(xmlelement(name proc, + xmlforest(proname, proowner, procost, + pronargs, proargnames, proargtypes)))) as doc + FROM x), + z AS (SELECT xmltable.* + FROM y, + LATERAL xmltable('/data/proc' PASSING doc + COLUMNS proname name, + proowner oid, + procost float, + pronargs int, + proargnames text, + proargtypes text)) + SELECT * FROM z + EXCEPT SELECT * FROM x; + +CREATE TABLE xmltest2(x xml, _path text); + +INSERT INTO xmltest2 VALUES('1', 'A'); +INSERT INTO xmltest2 VALUES('2', 'B'); +INSERT INTO xmltest2 VALUES('3', 'C'); +INSERT INTO xmltest2 VALUES('2', 'D'); + +SELECT xmltable.* FROM xmltest2, LATERAL xmltable('/d/r' PASSING x COLUMNS a int PATH '' || lower(_path) || 'c'); +SELECT xmltable.* FROM xmltest2, LATERAL xmltable(('/d/r/' || lower(_path) || 'c') PASSING x COLUMNS a int PATH '.'); +SELECT xmltable.* FROM xmltest2, LATERAL xmltable(('/d/r/' || lower(_path) || 'c') PASSING x COLUMNS a int PATH 'x' DEFAULT ascii(_path) - 54); -- cgit v1.2.3 From f35742ccb7aa53ee3ed8416bbb378b0c3eeb6bb9 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Wed, 8 Mar 2017 12:05:43 -0500 Subject: Support parallel bitmap heap scans. The index is scanned by a single process, but then all cooperating processes can iterate jointly over the resulting set of heap blocks. In the future, we might also want to support using a parallel bitmap index scan to set up for a parallel bitmap heap scan, but that's a job for another day. Dilip Kumar, with some corrections and cosmetic changes by me. The larger patch set of which this is a part has been reviewed and tested by (at least) Andres Freund, Amit Khandekar, Tushar Ahuja, Rafia Sabih, Haribabu Kommi, Thomas Munro, and me. Discussion: http://postgr.es/m/CAFiTN-uc4=0WxRGfCzs-xfkMYcSEWUC-Fon6thkJGjkh9i=13A@mail.gmail.com --- doc/src/sgml/monitoring.sgml | 6 +- src/backend/access/heap/heapam.c | 16 + src/backend/executor/execParallel.c | 14 + src/backend/executor/nodeBitmapHeapscan.c | 422 +++++++++++++++++++++++--- src/backend/executor/nodeBitmapIndexscan.c | 4 +- src/backend/executor/nodeBitmapOr.c | 4 +- src/backend/nodes/copyfuncs.c | 2 + src/backend/nodes/outfuncs.c | 2 + src/backend/nodes/readfuncs.c | 2 + src/backend/optimizer/path/allpaths.c | 24 ++ src/backend/optimizer/path/costsize.c | 16 +- src/backend/optimizer/path/indxpath.c | 19 +- src/backend/optimizer/plan/createplan.c | 22 ++ src/backend/optimizer/util/pathnode.c | 9 +- src/backend/postmaster/pgstat.c | 3 + src/include/access/heapam.h | 1 + src/include/executor/nodeBitmapHeapscan.h | 7 + src/include/nodes/execnodes.h | 57 ++++ src/include/nodes/plannodes.h | 2 + src/include/optimizer/pathnode.h | 3 +- src/include/optimizer/paths.h | 2 + src/include/pgstat.h | 1 + src/test/regress/expected/select_parallel.out | 19 ++ src/test/regress/sql/select_parallel.sql | 10 + 24 files changed, 612 insertions(+), 55 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 27ed35f0a7..4d03531cc1 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -1211,7 +1211,7 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser Waiting in an extension. - IPC + IPC BgWorkerShutdown Waiting for background worker to shut down. @@ -1247,6 +1247,10 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser ParallelFinish Waiting for parallel workers to finish computing. + + ParallelBitmapPopulate + Waiting for the leader to populate the TidBitmap. + SafeSnapshot Waiting for a snapshot for a READ ONLY DEFERRABLE transaction. diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index af258366a2..bffc971d68 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -1753,6 +1753,22 @@ retry: return page; } +/* ---------------- + * heap_update_snapshot + * + * Update snapshot info in heap scan descriptor. + * ---------------- + */ +void +heap_update_snapshot(HeapScanDesc scan, Snapshot snapshot) +{ + Assert(IsMVCCSnapshot(snapshot)); + + RegisterSnapshot(snapshot); + scan->rs_snapshot = snapshot; + scan->rs_temp_snap = true; +} + /* ---------------- * heap_getnext - retrieve next tuple in scan * diff --git a/src/backend/executor/execParallel.c b/src/backend/executor/execParallel.c index de0e2bafe6..a1289e5f12 100644 --- a/src/backend/executor/execParallel.c +++ b/src/backend/executor/execParallel.c @@ -25,6 +25,7 @@ #include "executor/execParallel.h" #include "executor/executor.h" +#include "executor/nodeBitmapHeapscan.h" #include "executor/nodeCustom.h" #include "executor/nodeForeignscan.h" #include "executor/nodeSeqscan.h" @@ -217,6 +218,10 @@ ExecParallelEstimate(PlanState *planstate, ExecParallelEstimateContext *e) ExecCustomScanEstimate((CustomScanState *) planstate, e->pcxt); break; + case T_BitmapHeapScanState: + ExecBitmapHeapEstimate((BitmapHeapScanState *) planstate, + e->pcxt); + break; default: break; } @@ -277,6 +282,11 @@ ExecParallelInitializeDSM(PlanState *planstate, ExecCustomScanInitializeDSM((CustomScanState *) planstate, d->pcxt); break; + case T_BitmapHeapScanState: + ExecBitmapHeapInitializeDSM((BitmapHeapScanState *) planstate, + d->pcxt); + break; + default: break; } @@ -775,6 +785,10 @@ ExecParallelInitializeWorker(PlanState *planstate, shm_toc *toc) ExecCustomScanInitializeWorker((CustomScanState *) planstate, toc); break; + case T_BitmapHeapScanState: + ExecBitmapHeapInitializeWorker( + (BitmapHeapScanState *) planstate, toc); + break; default: break; } diff --git a/src/backend/executor/nodeBitmapHeapscan.c b/src/backend/executor/nodeBitmapHeapscan.c index c1aa9f13bd..833a93e1b7 100644 --- a/src/backend/executor/nodeBitmapHeapscan.c +++ b/src/backend/executor/nodeBitmapHeapscan.c @@ -53,11 +53,15 @@ static TupleTableSlot *BitmapHeapNext(BitmapHeapScanState *node); static void bitgetpage(HeapScanDesc scan, TBMIterateResult *tbmres); +static inline void BitmapDoneInitializingSharedState( + ParallelBitmapHeapState *pstate); static inline void BitmapAdjustPrefetchIterator(BitmapHeapScanState *node, TBMIterateResult *tbmres); static inline void BitmapAdjustPrefetchTarget(BitmapHeapScanState *node); static inline void BitmapPrefetch(BitmapHeapScanState *node, HeapScanDesc scan); +static bool BitmapShouldInitializeSharedState( + ParallelBitmapHeapState *pstate); /* ---------------------------------------------------------------- @@ -73,9 +77,12 @@ BitmapHeapNext(BitmapHeapScanState *node) HeapScanDesc scan; TIDBitmap *tbm; TBMIterator *tbmiterator; + TBMSharedIterator *shared_tbmiterator; TBMIterateResult *tbmres; OffsetNumber targoffset; TupleTableSlot *slot; + ParallelBitmapHeapState *pstate = node->pstate; + dsa_area *dsa = node->ss.ps.state->es_query_dsa; /* * extract necessary information from index scan node @@ -84,7 +91,10 @@ BitmapHeapNext(BitmapHeapScanState *node) slot = node->ss.ss_ScanTupleSlot; scan = node->ss.ss_currentScanDesc; tbm = node->tbm; - tbmiterator = node->tbmiterator; + if (pstate == NULL) + tbmiterator = node->tbmiterator; + else + shared_tbmiterator = node->shared_tbmiterator; tbmres = node->tbmres; /* @@ -99,25 +109,82 @@ BitmapHeapNext(BitmapHeapScanState *node) * node->prefetch_maximum. This is to avoid doing a lot of prefetching in * a scan that stops after a few tuples because of a LIMIT. */ - if (tbm == NULL) + if (!node->initialized) { - tbm = (TIDBitmap *) MultiExecProcNode(outerPlanState(node)); + if (!pstate) + { + tbm = (TIDBitmap *) MultiExecProcNode(outerPlanState(node)); - if (!tbm || !IsA(tbm, TIDBitmap)) - elog(ERROR, "unrecognized result from subplan"); + if (!tbm || !IsA(tbm, TIDBitmap)) + elog(ERROR, "unrecognized result from subplan"); - node->tbm = tbm; - node->tbmiterator = tbmiterator = tbm_begin_iterate(tbm); - node->tbmres = tbmres = NULL; + node->tbm = tbm; + node->tbmiterator = tbmiterator = tbm_begin_iterate(tbm); + node->tbmres = tbmres = NULL; #ifdef USE_PREFETCH - if (node->prefetch_maximum > 0) - { - node->prefetch_iterator = tbm_begin_iterate(tbm); - node->prefetch_pages = 0; - node->prefetch_target = -1; + if (node->prefetch_maximum > 0) + { + node->prefetch_iterator = tbm_begin_iterate(tbm); + node->prefetch_pages = 0; + node->prefetch_target = -1; + } +#endif /* USE_PREFETCH */ } + else + { + /* + * The leader will immediately come out of the function, but + * others will be blocked until leader populates the TBM and wakes + * them up. + */ + if (BitmapShouldInitializeSharedState(pstate)) + { + tbm = (TIDBitmap *) MultiExecProcNode(outerPlanState(node)); + if (!tbm || !IsA(tbm, TIDBitmap)) + elog(ERROR, "unrecognized result from subplan"); + + node->tbm = tbm; + + /* + * Prepare to iterate over the TBM. This will return the + * dsa_pointer of the iterator state which will be used by + * multiple processes to iterate jointly. + */ + pstate->tbmiterator = tbm_prepare_shared_iterate(tbm); +#ifdef USE_PREFETCH + if (node->prefetch_maximum > 0) + { + pstate->prefetch_iterator = + tbm_prepare_shared_iterate(tbm); + + /* + * We don't need the mutex here as we haven't yet woke up + * others. + */ + pstate->prefetch_pages = 0; + pstate->prefetch_target = -1; + } +#endif + + /* We have initialized the shared state so wake up others. */ + BitmapDoneInitializingSharedState(pstate); + } + + /* Allocate a private iterator and attach the shared state to it */ + node->shared_tbmiterator = shared_tbmiterator = + tbm_attach_shared_iterate(dsa, pstate->tbmiterator); + node->tbmres = tbmres = NULL; + +#ifdef USE_PREFETCH + if (node->prefetch_maximum > 0) + { + node->shared_prefetch_iterator = + tbm_attach_shared_iterate(dsa, pstate->prefetch_iterator); + } #endif /* USE_PREFETCH */ + } + node->initialized = true; } for (;;) @@ -130,7 +197,10 @@ BitmapHeapNext(BitmapHeapScanState *node) */ if (tbmres == NULL) { - node->tbmres = tbmres = tbm_iterate(tbmiterator); + if (!pstate) + node->tbmres = tbmres = tbm_iterate(tbmiterator); + else + node->tbmres = tbmres = tbm_shared_iterate(shared_tbmiterator); if (tbmres == NULL) { /* no more entries in the bitmap */ @@ -182,8 +252,19 @@ BitmapHeapNext(BitmapHeapScanState *node) * Try to prefetch at least a few pages even before we get to the * second page if we don't stop reading after the first tuple. */ - if (node->prefetch_target < node->prefetch_maximum) - node->prefetch_target++; + if (!pstate) + { + if (node->prefetch_target < node->prefetch_maximum) + node->prefetch_target++; + } + else if (pstate->prefetch_target < node->prefetch_maximum) + { + /* take spinlock while updating shared state */ + SpinLockAcquire(&pstate->mutex); + if (pstate->prefetch_target < node->prefetch_maximum) + pstate->prefetch_target++; + SpinLockRelease(&pstate->mutex); + } #endif /* USE_PREFETCH */ } @@ -361,6 +442,21 @@ bitgetpage(HeapScanDesc scan, TBMIterateResult *tbmres) scan->rs_ntuples = ntup; } +/* + * BitmapDoneInitializingSharedState - Shared state is initialized + * + * By this time the leader has already populated the TBM and initialized the + * shared state so wake up other processes. + */ +static inline void +BitmapDoneInitializingSharedState(ParallelBitmapHeapState *pstate) +{ + SpinLockAcquire(&pstate->mutex); + pstate->state = BM_FINISHED; + SpinLockRelease(&pstate->mutex); + ConditionVariableBroadcast(&pstate->cv); +} + /* * BitmapAdjustPrefetchIterator - Adjust the prefetch iterator */ @@ -369,20 +465,53 @@ BitmapAdjustPrefetchIterator(BitmapHeapScanState *node, TBMIterateResult *tbmres) { #ifdef USE_PREFETCH - TBMIterator *prefetch_iterator = node->prefetch_iterator; + ParallelBitmapHeapState *pstate = node->pstate; - if (node->prefetch_pages > 0) + if (pstate == NULL) { - /* The main iterator has closed the distance by one page */ - node->prefetch_pages--; + TBMIterator *prefetch_iterator = node->prefetch_iterator; + + if (node->prefetch_pages > 0) + { + /* The main iterator has closed the distance by one page */ + node->prefetch_pages--; + } + else if (prefetch_iterator) + { + /* Do not let the prefetch iterator get behind the main one */ + TBMIterateResult *tbmpre = tbm_iterate(prefetch_iterator); + + if (tbmpre == NULL || tbmpre->blockno != tbmres->blockno) + elog(ERROR, "prefetch and main iterators are out of sync"); + } + return; } - else if (prefetch_iterator) + + if (node->prefetch_maximum > 0) { - /* Do not let the prefetch iterator get behind the main one */ - TBMIterateResult *tbmpre = tbm_iterate(prefetch_iterator); + TBMSharedIterator *prefetch_iterator = node->shared_prefetch_iterator; + + SpinLockAcquire(&pstate->mutex); + if (pstate->prefetch_pages > 0) + { + node->prefetch_pages--; + SpinLockRelease(&pstate->mutex); + } + else + { + /* Release the mutex before iterating */ + SpinLockRelease(&pstate->mutex); - if (tbmpre == NULL || tbmpre->blockno != tbmres->blockno) - elog(ERROR, "prefetch and main iterators are out of sync"); + /* + * In case of shared mode, we can not ensure that the current + * blockno of the main iterator and that of the prefetch iterator + * are same. It's possible that whatever blockno we are + * prefetching will be processed by another process. Therefore, we + * don't validate the blockno here as we do in non-parallel case. + */ + if (prefetch_iterator) + tbm_shared_iterate(prefetch_iterator); + } } #endif /* USE_PREFETCH */ } @@ -399,14 +528,35 @@ static inline void BitmapAdjustPrefetchTarget(BitmapHeapScanState *node) { #ifdef USE_PREFETCH - if (node->prefetch_target >= node->prefetch_maximum) - /* don't increase any further */ ; - else if (node->prefetch_target >= node->prefetch_maximum / 2) - node->prefetch_target = node->prefetch_maximum; - else if (node->prefetch_target > 0) - node->prefetch_target *= 2; - else - node->prefetch_target++; + ParallelBitmapHeapState *pstate = node->pstate; + + if (pstate == NULL) + { + if (node->prefetch_target >= node->prefetch_maximum) + /* don't increase any further */ ; + else if (node->prefetch_target >= node->prefetch_maximum / 2) + node->prefetch_target = node->prefetch_maximum; + else if (node->prefetch_target > 0) + node->prefetch_target *= 2; + else + node->prefetch_target++; + return; + } + + /* Do an unlocked check first to save spinlock acquisitions. */ + if (pstate->prefetch_target < node->prefetch_maximum) + { + SpinLockAcquire(&pstate->mutex); + if (pstate->prefetch_target >= node->prefetch_maximum) + /* don't increase any further */ ; + else if (pstate->prefetch_target >= node->prefetch_maximum / 2) + pstate->prefetch_target = node->prefetch_maximum; + else if (pstate->prefetch_target > 0) + pstate->prefetch_target *= 2; + else + pstate->prefetch_target++; + SpinLockRelease(&pstate->mutex); + } #endif /* USE_PREFETCH */ } @@ -417,23 +567,70 @@ static inline void BitmapPrefetch(BitmapHeapScanState *node, HeapScanDesc scan) { #ifdef USE_PREFETCH - TBMIterator *prefetch_iterator = node->prefetch_iterator; + ParallelBitmapHeapState *pstate = node->pstate; - if (prefetch_iterator) + if (pstate == NULL) { - while (node->prefetch_pages < node->prefetch_target) + TBMIterator *prefetch_iterator = node->prefetch_iterator; + + if (prefetch_iterator) { - TBMIterateResult *tbmpre = tbm_iterate(prefetch_iterator); + while (node->prefetch_pages < node->prefetch_target) + { + TBMIterateResult *tbmpre = tbm_iterate(prefetch_iterator); + + if (tbmpre == NULL) + { + /* No more pages to prefetch */ + tbm_end_iterate(prefetch_iterator); + node->prefetch_iterator = NULL; + break; + } + node->prefetch_pages++; + PrefetchBuffer(scan->rs_rd, MAIN_FORKNUM, tbmpre->blockno); + } + } + + return; + } - if (tbmpre == NULL) + if (pstate->prefetch_pages < pstate->prefetch_target) + { + TBMSharedIterator *prefetch_iterator = node->shared_prefetch_iterator; + + if (prefetch_iterator) + { + while (1) { - /* No more pages to prefetch */ - tbm_end_iterate(prefetch_iterator); - node->prefetch_iterator = NULL; - break; + TBMIterateResult *tbmpre; + bool do_prefetch = false; + + /* + * Recheck under the mutex. If some other process has already + * done enough prefetching then we need not to do anything. + */ + SpinLockAcquire(&pstate->mutex); + if (pstate->prefetch_pages < pstate->prefetch_target) + { + pstate->prefetch_pages++; + do_prefetch = true; + } + SpinLockRelease(&pstate->mutex); + + if (!do_prefetch) + return; + + tbmpre = tbm_shared_iterate(prefetch_iterator); + if (tbmpre == NULL) + { + /* No more pages to prefetch */ + tbm_end_shared_iterate(prefetch_iterator); + node->shared_prefetch_iterator = NULL; + break; + } + + PrefetchBuffer(scan->rs_rd, MAIN_FORKNUM, tbmpre->blockno); } - node->prefetch_pages++; - PrefetchBuffer(scan->rs_rd, MAIN_FORKNUM, tbmpre->blockno); } } #endif /* USE_PREFETCH */ @@ -488,12 +685,36 @@ ExecReScanBitmapHeapScan(BitmapHeapScanState *node) tbm_end_iterate(node->tbmiterator); if (node->prefetch_iterator) tbm_end_iterate(node->prefetch_iterator); + if (node->shared_tbmiterator) + tbm_end_shared_iterate(node->shared_tbmiterator); + if (node->shared_prefetch_iterator) + tbm_end_shared_iterate(node->shared_prefetch_iterator); if (node->tbm) tbm_free(node->tbm); node->tbm = NULL; node->tbmiterator = NULL; node->tbmres = NULL; node->prefetch_iterator = NULL; + node->initialized = false; + node->shared_tbmiterator = NULL; + node->shared_prefetch_iterator = NULL; + + /* Reset parallel bitmap state, if present */ + if (node->pstate) + { + dsa_area *dsa = node->ss.ps.state->es_query_dsa; + + node->pstate->state = BM_INITIAL; + + if (DsaPointerIsValid(node->pstate->tbmiterator)) + tbm_free_shared_area(dsa, node->pstate->tbmiterator); + + if (DsaPointerIsValid(node->pstate->prefetch_iterator)) + tbm_free_shared_area(dsa, node->pstate->prefetch_iterator); + + node->pstate->tbmiterator = InvalidDsaPointer; + node->pstate->prefetch_iterator = InvalidDsaPointer; + } ExecScanReScan(&node->ss); @@ -546,6 +767,10 @@ ExecEndBitmapHeapScan(BitmapHeapScanState *node) tbm_end_iterate(node->prefetch_iterator); if (node->tbm) tbm_free(node->tbm); + if (node->shared_tbmiterator) + tbm_end_shared_iterate(node->shared_tbmiterator); + if (node->shared_prefetch_iterator) + tbm_end_shared_iterate(node->shared_prefetch_iterator); /* * close heap scan @@ -597,6 +822,10 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->prefetch_target = 0; /* may be updated below */ scanstate->prefetch_maximum = target_prefetch_pages; + scanstate->pscan_len = 0; + scanstate->initialized = false; + scanstate->shared_tbmiterator = NULL; + scanstate->pstate = NULL; /* * Miscellaneous initialization @@ -681,3 +910,108 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) */ return scanstate; } + +/*---------------- + * BitmapShouldInitializeSharedState + * + * The first process to come here and see the state to the BM_INITIAL + * will become the leader for the parallel bitmap scan and will be + * responsible for populating the TIDBitmap. The other processes will + * be blocked by the condition variable until the leader wakes them up. + * --------------- + */ +static bool +BitmapShouldInitializeSharedState(ParallelBitmapHeapState *pstate) +{ + SharedBitmapState state; + + while (1) + { + SpinLockAcquire(&pstate->mutex); + state = pstate->state; + if (pstate->state == BM_INITIAL) + pstate->state = BM_INPROGRESS; + SpinLockRelease(&pstate->mutex); + + /* Exit if bitmap is done, or if we're the leader. */ + if (state != BM_INPROGRESS) + break; + + /* Wait for the leader to wake us up. */ + ConditionVariableSleep(&pstate->cv, WAIT_EVENT_PARALLEL_BITMAP_SCAN); + } + + ConditionVariableCancelSleep(); + + return (state == BM_INITIAL); +} + +/* ---------------------------------------------------------------- + * ExecBitmapHeapEstimate + * + * estimates the space required to serialize bitmap scan node. + * ---------------------------------------------------------------- + */ +void +ExecBitmapHeapEstimate(BitmapHeapScanState *node, + ParallelContext *pcxt) +{ + EState *estate = node->ss.ps.state; + + node->pscan_len = add_size(offsetof(ParallelBitmapHeapState, + phs_snapshot_data), + EstimateSnapshotSpace(estate->es_snapshot)); + + shm_toc_estimate_chunk(&pcxt->estimator, node->pscan_len); + shm_toc_estimate_keys(&pcxt->estimator, 1); +} + +/* ---------------------------------------------------------------- + * ExecBitmapHeapInitializeDSM + * + * Set up a parallel bitmap heap scan descriptor. + * ---------------------------------------------------------------- + */ +void +ExecBitmapHeapInitializeDSM(BitmapHeapScanState *node, + ParallelContext *pcxt) +{ + ParallelBitmapHeapState *pstate; + EState *estate = node->ss.ps.state; + + pstate = shm_toc_allocate(pcxt->toc, node->pscan_len); + + pstate->tbmiterator = 0; + pstate->prefetch_iterator = 0; + + /* Initialize the mutex */ + SpinLockInit(&pstate->mutex); + pstate->prefetch_pages = 0; + pstate->prefetch_target = 0; + pstate->state = BM_INITIAL; + + ConditionVariableInit(&pstate->cv); + SerializeSnapshot(estate->es_snapshot, pstate->phs_snapshot_data); + + shm_toc_insert(pcxt->toc, node->ss.ps.plan->plan_node_id, pstate); + node->pstate = pstate; +} + +/* ---------------------------------------------------------------- + * ExecBitmapHeapInitializeWorker + * + * Copy relevant information from TOC into planstate. + * ---------------------------------------------------------------- + */ +void +ExecBitmapHeapInitializeWorker(BitmapHeapScanState *node, shm_toc *toc) +{ + ParallelBitmapHeapState *pstate; + Snapshot snapshot; + + pstate = shm_toc_lookup(toc, node->ss.ps.plan->plan_node_id); + node->pstate = pstate; + + snapshot = RestoreSnapshot(pstate->phs_snapshot_data); + heap_update_snapshot(node->ss.ss_currentScanDesc, snapshot); +} diff --git a/src/backend/executor/nodeBitmapIndexscan.c b/src/backend/executor/nodeBitmapIndexscan.c index 94bb289f1b..ce2f3210a4 100644 --- a/src/backend/executor/nodeBitmapIndexscan.c +++ b/src/backend/executor/nodeBitmapIndexscan.c @@ -78,7 +78,9 @@ MultiExecBitmapIndexScan(BitmapIndexScanState *node) else { /* XXX should we use less than work_mem for this? */ - tbm = tbm_create(work_mem * 1024L, NULL); + tbm = tbm_create(work_mem * 1024L, + ((BitmapIndexScan *) node->ss.ps.plan)->isshared ? + node->ss.ps.state->es_query_dsa : NULL); } /* diff --git a/src/backend/executor/nodeBitmapOr.c b/src/backend/executor/nodeBitmapOr.c index 1d280beddc..c0f261407b 100644 --- a/src/backend/executor/nodeBitmapOr.c +++ b/src/backend/executor/nodeBitmapOr.c @@ -129,7 +129,9 @@ MultiExecBitmapOr(BitmapOrState *node) if (result == NULL) /* first subplan */ { /* XXX should we use less than work_mem for this? */ - result = tbm_create(work_mem * 1024L, NULL); + result = tbm_create(work_mem * 1024L, + ((BitmapOr *) node->ps.plan)->isshared ? + node->ps.state->es_query_dsa : NULL); } ((BitmapIndexScanState *) subnode)->biss_result = result; diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index b3eac06c50..ac8e50ef1d 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -331,6 +331,7 @@ _copyBitmapOr(const BitmapOr *from) /* * copy remainder of node */ + COPY_SCALAR_FIELD(isshared); COPY_NODE_FIELD(bitmapplans); return newnode; @@ -496,6 +497,7 @@ _copyBitmapIndexScan(const BitmapIndexScan *from) * copy remainder of node */ COPY_SCALAR_FIELD(indexid); + COPY_SCALAR_FIELD(isshared); COPY_NODE_FIELD(indexqual); COPY_NODE_FIELD(indexqualorig); diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index d4297d11b1..825a7b283a 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -441,6 +441,7 @@ _outBitmapOr(StringInfo str, const BitmapOr *node) _outPlanInfo(str, (const Plan *) node); + WRITE_BOOL_FIELD(isshared); WRITE_NODE_FIELD(bitmapplans); } @@ -520,6 +521,7 @@ _outBitmapIndexScan(StringInfo str, const BitmapIndexScan *node) _outScanInfo(str, (const Scan *) node); WRITE_OID_FIELD(indexid); + WRITE_BOOL_FIELD(isshared); WRITE_NODE_FIELD(indexqual); WRITE_NODE_FIELD(indexqualorig); } diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index b02d9fa246..8f39d93a12 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -1633,6 +1633,7 @@ _readBitmapOr(void) ReadCommonPlan(&local_node->plan); + READ_BOOL_FIELD(isshared); READ_NODE_FIELD(bitmapplans); READ_DONE(); @@ -1744,6 +1745,7 @@ _readBitmapIndexScan(void) ReadCommonScan(&local_node->scan); READ_OID_FIELD(indexid); + READ_BOOL_FIELD(isshared); READ_NODE_FIELD(indexqual); READ_NODE_FIELD(indexqualorig); diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index 932c84c949..fbb2cda9d7 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -2911,6 +2911,30 @@ remove_unused_subquery_outputs(Query *subquery, RelOptInfo *rel) } } +/* + * create_partial_bitmap_paths + * Build partial bitmap heap path for the relation + */ +void +create_partial_bitmap_paths(PlannerInfo *root, RelOptInfo *rel, + Path *bitmapqual) +{ + int parallel_workers; + double pages_fetched; + + /* Compute heap pages for bitmap heap scan */ + pages_fetched = compute_bitmap_pages(root, rel, bitmapqual, 1.0, + NULL, NULL); + + parallel_workers = compute_parallel_worker(rel, pages_fetched, 0); + + if (parallel_workers <= 0) + return; + + add_partial_path(rel, (Path *) create_bitmap_heap_path(root, rel, + bitmapqual, rel->lateral_relids, 1.0, parallel_workers)); +} + /* * Compute the number of parallel workers that should be used to scan a * relation. We compute the parallel workers based on the size of the heap to diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index 3eaed5af7a..627e3f1b95 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -860,6 +860,7 @@ cost_bitmap_heap_scan(Path *path, PlannerInfo *root, RelOptInfo *baserel, QualCost qpqual_cost; Cost cpu_per_tuple; Cost cost_per_page; + Cost cpu_run_cost; double tuples_fetched; double pages_fetched; double spc_seq_page_cost, @@ -921,8 +922,21 @@ cost_bitmap_heap_scan(Path *path, PlannerInfo *root, RelOptInfo *baserel, startup_cost += qpqual_cost.startup; cpu_per_tuple = cpu_tuple_cost + qpqual_cost.per_tuple; + cpu_run_cost = cpu_per_tuple * tuples_fetched; + + /* Adjust costing for parallelism, if used. */ + if (path->parallel_workers > 0) + { + double parallel_divisor = get_parallel_divisor(path); + + /* The CPU cost is divided among all the workers. */ + cpu_run_cost /= parallel_divisor; - run_cost += cpu_per_tuple * tuples_fetched; + path->rows = clamp_row_est(path->rows / parallel_divisor); + } + + + run_cost += cpu_run_cost; /* tlist eval costs are paid per output row, not per tuple scanned */ startup_cost += path->pathtarget->cost.startup; diff --git a/src/backend/optimizer/path/indxpath.c b/src/backend/optimizer/path/indxpath.c index d8e5b81126..c2b72d410a 100644 --- a/src/backend/optimizer/path/indxpath.c +++ b/src/backend/optimizer/path/indxpath.c @@ -337,8 +337,12 @@ create_index_paths(PlannerInfo *root, RelOptInfo *rel) bitmapqual = choose_bitmap_and(root, rel, bitindexpaths); bpath = create_bitmap_heap_path(root, rel, bitmapqual, - rel->lateral_relids, 1.0); + rel->lateral_relids, 1.0, 0); add_path(rel, (Path *) bpath); + + /* create a partial bitmap heap path */ + if (rel->consider_parallel && rel->lateral_relids == NULL) + create_partial_bitmap_paths(root, rel, bitmapqual); } /* @@ -410,7 +414,7 @@ create_index_paths(PlannerInfo *root, RelOptInfo *rel) required_outer = get_bitmap_tree_required_outer(bitmapqual); loop_count = get_loop_count(root, rel->relid, required_outer); bpath = create_bitmap_heap_path(root, rel, bitmapqual, - required_outer, loop_count); + required_outer, loop_count, 0); add_path(rel, (Path *) bpath); } } @@ -1617,6 +1621,11 @@ bitmap_scan_cost_est(PlannerInfo *root, RelOptInfo *rel, Path *ipath) bpath.path.pathkeys = NIL; bpath.bitmapqual = ipath; + /* + * Check the cost of temporary path without considering parallelism. + * Parallel bitmap heap path will be considered at later stage. + */ + bpath.path.parallel_workers = 0; cost_bitmap_heap_scan(&bpath.path, root, rel, bpath.path.param_info, ipath, @@ -1659,6 +1668,12 @@ bitmap_and_cost_est(PlannerInfo *root, RelOptInfo *rel, List *paths) bpath.path.pathkeys = NIL; bpath.bitmapqual = (Path *) &apath; + /* + * Check the cost of temporary path without considering parallelism. + * Parallel bitmap heap path will be considered at later stage. + */ + bpath.path.parallel_workers = 0; + /* Now we can do cost_bitmap_heap_scan */ cost_bitmap_heap_scan(&bpath.path, root, rel, bpath.path.param_info, diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index f1c7f609c0..8f8663c1e1 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -125,6 +125,7 @@ static BitmapHeapScan *create_bitmap_scan_plan(PlannerInfo *root, List *tlist, List *scan_clauses); static Plan *create_bitmap_subplan(PlannerInfo *root, Path *bitmapqual, List **qual, List **indexqual, List **indexECs); +static void bitmap_subplan_mark_shared(Plan *plan); static TidScan *create_tidscan_plan(PlannerInfo *root, TidPath *best_path, List *tlist, List *scan_clauses); static SubqueryScan *create_subqueryscan_plan(PlannerInfo *root, @@ -2590,6 +2591,9 @@ create_bitmap_scan_plan(PlannerInfo *root, &bitmapqualorig, &indexquals, &indexECs); + if (best_path->path.parallel_aware) + bitmap_subplan_mark_shared(bitmapqualplan); + /* * The qpqual list must contain all restrictions not automatically handled * by the index, other than pseudoconstant clauses which will be handled @@ -4756,6 +4760,24 @@ label_sort_with_costsize(PlannerInfo *root, Sort *plan, double limit_tuples) plan->plan.parallel_aware = false; } +/* + * bitmap_subplan_mark_shared + * Set isshared flag in bitmap subplan so that it will be created in + * shared memory. + */ +static void +bitmap_subplan_mark_shared(Plan *plan) +{ + if (IsA(plan, BitmapAnd)) + bitmap_subplan_mark_shared( + linitial(((BitmapAnd *) plan)->bitmapplans)); + else if (IsA(plan, BitmapOr)) + ((BitmapOr *) plan)->isshared = true; + else if (IsA(plan, BitmapIndexScan)) + ((BitmapIndexScan *) plan)->isshared = true; + else + elog(ERROR, "unrecognized node type: %d", nodeTag(plan)); +} /***************************************************************************** * diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index 86aee2f8ec..0d925c6fcb 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -1068,7 +1068,8 @@ create_bitmap_heap_path(PlannerInfo *root, RelOptInfo *rel, Path *bitmapqual, Relids required_outer, - double loop_count) + double loop_count, + int parallel_degree) { BitmapHeapPath *pathnode = makeNode(BitmapHeapPath); @@ -1077,9 +1078,9 @@ create_bitmap_heap_path(PlannerInfo *root, pathnode->path.pathtarget = rel->reltarget; pathnode->path.param_info = get_baserel_parampathinfo(root, rel, required_outer); - pathnode->path.parallel_aware = false; + pathnode->path.parallel_aware = parallel_degree > 0 ? true : false; pathnode->path.parallel_safe = rel->consider_parallel; - pathnode->path.parallel_workers = 0; + pathnode->path.parallel_workers = parallel_degree; pathnode->path.pathkeys = NIL; /* always unordered */ pathnode->bitmapqual = bitmapqual; @@ -3281,7 +3282,7 @@ reparameterize_path(PlannerInfo *root, Path *path, rel, bpath->bitmapqual, required_outer, - loop_count); + loop_count, 0); } case T_SubqueryScan: { diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index 2fb9a8bf58..7cacb1e9b2 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -3395,6 +3395,9 @@ pgstat_get_wait_ipc(WaitEventIPC w) case WAIT_EVENT_PARALLEL_FINISH: event_name = "ParallelFinish"; break; + case WAIT_EVENT_PARALLEL_BITMAP_SCAN: + event_name = "ParallelBitmapScan"; + break; case WAIT_EVENT_SAFE_SNAPSHOT: event_name = "SafeSnapshot"; break; diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h index a864f7860d..7e85510d2f 100644 --- a/src/include/access/heapam.h +++ b/src/include/access/heapam.h @@ -179,6 +179,7 @@ extern void simple_heap_update(Relation relation, ItemPointer otid, HeapTuple tup); extern void heap_sync(Relation relation); +extern void heap_update_snapshot(HeapScanDesc scan, Snapshot snapshot); /* in heap/pruneheap.c */ extern void heap_page_prune_opt(Relation relation, Buffer buffer); diff --git a/src/include/executor/nodeBitmapHeapscan.h b/src/include/executor/nodeBitmapHeapscan.h index d7659b94e6..465c58e6ee 100644 --- a/src/include/executor/nodeBitmapHeapscan.h +++ b/src/include/executor/nodeBitmapHeapscan.h @@ -15,10 +15,17 @@ #define NODEBITMAPHEAPSCAN_H #include "nodes/execnodes.h" +#include "access/parallel.h" extern BitmapHeapScanState *ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags); extern TupleTableSlot *ExecBitmapHeapScan(BitmapHeapScanState *node); extern void ExecEndBitmapHeapScan(BitmapHeapScanState *node); extern void ExecReScanBitmapHeapScan(BitmapHeapScanState *node); +extern void ExecBitmapHeapEstimate(BitmapHeapScanState *node, + ParallelContext *pcxt); +extern void ExecBitmapHeapInitializeDSM(BitmapHeapScanState *node, + ParallelContext *pcxt); +extern void ExecBitmapHeapInitializeWorker(BitmapHeapScanState *node, + shm_toc *toc); #endif /* NODEBITMAPHEAPSCAN_H */ diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 2fde67a9c8..6a0d590ef2 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -26,6 +26,8 @@ #include "utils/sortsupport.h" #include "utils/tuplestore.h" #include "utils/tuplesort.h" +#include "nodes/tidbitmap.h" +#include "storage/condition_variable.h" /* ---------------- @@ -1464,6 +1466,51 @@ typedef struct BitmapIndexScanState IndexScanDesc biss_ScanDesc; } BitmapIndexScanState; +/* ---------------- + * SharedBitmapState information + * + * BM_INITIAL TIDBitmap creation is not yet started, so first worker + * to see this state will set the state to BM_INPROGRESS + * and that process will be responsible for creating + * TIDBitmap. + * BM_INPROGRESS TIDBitmap creation is in progress; workers need to + * sleep until it's finished. + * BM_FINISHED TIDBitmap creation is done, so now all workers can + * proceed to iterate over TIDBitmap. + * ---------------- + */ +typedef enum +{ + BM_INITIAL, + BM_INPROGRESS, + BM_FINISHED +} SharedBitmapState; + +/* ---------------- + * ParallelBitmapHeapState information + * tbmiterator iterator for scanning current pages + * prefetch_iterator iterator for prefetching ahead of current page + * mutex mutual exclusion for the prefetching variable + * and state + * prefetch_pages # pages prefetch iterator is ahead of current + * prefetch_target current target prefetch distance + * state current state of the TIDBitmap + * cv conditional wait variable + * phs_snapshot_data snapshot data shared to workers + * ---------------- + */ +typedef struct ParallelBitmapHeapState +{ + dsa_pointer tbmiterator; + dsa_pointer prefetch_iterator; + slock_t mutex; + int prefetch_pages; + int prefetch_target; + SharedBitmapState state; + ConditionVariable cv; + char phs_snapshot_data[FLEXIBLE_ARRAY_MEMBER]; +} ParallelBitmapHeapState; + /* ---------------- * BitmapHeapScanState information * @@ -1477,6 +1524,11 @@ typedef struct BitmapIndexScanState * prefetch_pages # pages prefetch iterator is ahead of current * prefetch_target current target prefetch distance * prefetch_maximum maximum value for prefetch_target + * pscan_len size of the shared memory for parallel bitmap + * initialized is node is ready to iterate + * shared_tbmiterator shared iterator + * shared_prefetch_iterator shared iterator for prefetching + * pstate shared state for parallel bitmap scan * ---------------- */ typedef struct BitmapHeapScanState @@ -1492,6 +1544,11 @@ typedef struct BitmapHeapScanState int prefetch_pages; int prefetch_target; int prefetch_maximum; + Size pscan_len; + bool initialized; + TBMSharedIterator *shared_tbmiterator; + TBMSharedIterator *shared_prefetch_iterator; + ParallelBitmapHeapState *pstate; } BitmapHeapScanState; /* ---------------- diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h index e30ed6aa29..7fbb0c2c77 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h @@ -292,6 +292,7 @@ typedef struct BitmapAnd typedef struct BitmapOr { Plan plan; + bool isshared; List *bitmapplans; } BitmapOr; @@ -420,6 +421,7 @@ typedef struct BitmapIndexScan { Scan scan; Oid indexid; /* OID of index to scan */ + bool isshared; /* Create shared bitmap if set */ List *indexqual; /* list of index quals (OpExprs) */ List *indexqualorig; /* the same in original form */ } BitmapIndexScan; diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index befe578141..f0fe830722 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -53,7 +53,8 @@ extern BitmapHeapPath *create_bitmap_heap_path(PlannerInfo *root, RelOptInfo *rel, Path *bitmapqual, Relids required_outer, - double loop_count); + double loop_count, + int parallel_degree); extern BitmapAndPath *create_bitmap_and_path(PlannerInfo *root, RelOptInfo *rel, List *bitmapquals); diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index bc0dcf4468..247fd11879 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -56,6 +56,8 @@ extern RelOptInfo *standard_join_search(PlannerInfo *root, int levels_needed, extern void generate_gather_paths(PlannerInfo *root, RelOptInfo *rel); extern int compute_parallel_worker(RelOptInfo *rel, BlockNumber heap_pages, BlockNumber index_pages); +extern void create_partial_bitmap_paths(PlannerInfo *root, RelOptInfo *rel, + Path *bitmapqual); #ifdef OPTIMIZER_DEBUG extern void debug_print_rel(PlannerInfo *root, RelOptInfo *rel); diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 0062fb8af2..60c78d118f 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -787,6 +787,7 @@ typedef enum WAIT_EVENT_MQ_RECEIVE, WAIT_EVENT_MQ_SEND, WAIT_EVENT_PARALLEL_FINISH, + WAIT_EVENT_PARALLEL_BITMAP_SCAN, WAIT_EVENT_SAFE_SNAPSHOT, WAIT_EVENT_SYNC_REP } WaitEventIPC; diff --git a/src/test/regress/expected/select_parallel.out b/src/test/regress/expected/select_parallel.out index 75558d05e0..290b735b6b 100644 --- a/src/test/regress/expected/select_parallel.out +++ b/src/test/regress/expected/select_parallel.out @@ -169,6 +169,25 @@ select count(*) from tenk1 where thousand > 95; reset enable_seqscan; reset enable_bitmapscan; +-- test parallel bitmap heap scan. +set enable_seqscan to off; +set enable_indexscan to off; +explain (costs off) + select count((unique1)) from tenk1 where hundred > 1; + QUERY PLAN +------------------------------------------------------------ + Finalize Aggregate + -> Gather + Workers Planned: 4 + -> Partial Aggregate + -> Parallel Bitmap Heap Scan on tenk1 + Recheck Cond: (hundred > 1) + -> Bitmap Index Scan on tenk1_hundred + Index Cond: (hundred > 1) +(8 rows) + +reset enable_seqscan; +reset enable_indexscan; -- test parallel merge join path. set enable_hashjoin to off; set enable_nestloop to off; diff --git a/src/test/regress/sql/select_parallel.sql b/src/test/regress/sql/select_parallel.sql index ebdae7e939..80412b990d 100644 --- a/src/test/regress/sql/select_parallel.sql +++ b/src/test/regress/sql/select_parallel.sql @@ -64,6 +64,16 @@ select count(*) from tenk1 where thousand > 95; reset enable_seqscan; reset enable_bitmapscan; +-- test parallel bitmap heap scan. +set enable_seqscan to off; +set enable_indexscan to off; + +explain (costs off) + select count((unique1)) from tenk1 where hundred > 1; + +reset enable_seqscan; +reset enable_indexscan; + -- test parallel merge join path. set enable_hashjoin to off; set enable_nestloop to off; -- cgit v1.2.3 From f9b1a0dd403ec0931213c66d5f979a3d3e8e7e30 Mon Sep 17 00:00:00 2001 From: Stephen Frost Date: Wed, 8 Mar 2017 15:14:03 -0500 Subject: Expose explain's SUMMARY option This exposes the existing explain summary option to users to allow them to choose if they wish to have the planning time and totalled run time included in the EXPLAIN result. The existing default behavior is retained if SUMMARY is not specified- running explain without analyze will not print the summary lines (just the planning time, currently) while running explain with analyze will include the summary lines (both the planning time and the totalled execution time). Users who wish to see the summary information for plain explain can now use: EXPLAIN (SUMMARY ON) query; Users who do not want to have the summary printed for an analyze run can use: EXPLAIN (ANALYZE ON, SUMMARY OFF) query; With this, we can now also have EXPLAIN ANALYZE queries included in our regression tests by using: EXPLAIN (ANALYZE ON, TIMING OFF, SUMMARY off) query; I went ahead and added an example of this, which will hopefully not make the buildfarm complain. Author: Ashutosh Bapat Discussion: https://postgr.es/m/CAFjFpReE5z2h98U2Vuia8hcEkpRRwrauRjHmyE44hNv8-xk+XA@mail.gmail.com --- doc/src/sgml/ref/explain.sgml | 16 ++++++++++++++++ src/backend/commands/explain.c | 18 +++++++++++++++--- src/backend/commands/prepare.c | 9 ++++++++- src/test/regress/expected/select.out | 10 ++++++++++ src/test/regress/sql/select.sql | 3 +++ 5 files changed, 52 insertions(+), 4 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/explain.sgml b/doc/src/sgml/ref/explain.sgml index f14a58dfc6..6d909f1a97 100644 --- a/doc/src/sgml/ref/explain.sgml +++ b/doc/src/sgml/ref/explain.sgml @@ -41,6 +41,7 @@ EXPLAIN [ ANALYZE ] [ VERBOSE ] statementboolean ] BUFFERS [ boolean ] TIMING [ boolean ] + SUMMARY [ boolean ] FORMAT { TEXT | XML | JSON | YAML } @@ -196,6 +197,21 @@ ROLLBACK; + + SUMMARY + + + Include summary information (eg: totalled timing information) after the + query plan. Summary information is included by default when + ANALYZE is used but otherwise is not included by + default, but can be enabled using this option. Planning time in + EXPLAIN EXECUTE includes the time required to fetch + the plan from the cache and the time required for re-planning, if + necessary. + + + + FORMAT diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index 7a8d36c8db..6fd82e9d52 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -149,6 +149,7 @@ ExplainQuery(ParseState *pstate, ExplainStmt *stmt, const char *queryString, List *rewritten; ListCell *lc; bool timing_set = false; + bool summary_set = false; /* Parse options list. */ foreach(lc, stmt->options) @@ -168,6 +169,11 @@ ExplainQuery(ParseState *pstate, ExplainStmt *stmt, const char *queryString, timing_set = true; es->timing = defGetBoolean(opt); } + else if (strcmp(opt->defname, "summary") == 0) + { + summary_set = true; + es->summary = defGetBoolean(opt); + } else if (strcmp(opt->defname, "format") == 0) { char *p = defGetString(opt); @@ -209,8 +215,8 @@ ExplainQuery(ParseState *pstate, ExplainStmt *stmt, const char *queryString, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("EXPLAIN option TIMING requires ANALYZE"))); - /* currently, summary option is not exposed to users; just set it */ - es->summary = es->analyze; + /* if the summary was not set explicitly, set default value */ + es->summary = (summary_set) ? es->summary : es->analyze; /* * Parse analysis was done already, but we still have to run the rule @@ -571,7 +577,13 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es, totaltime += elapsed_time(&starttime); - if (es->summary) + /* + * We only report execution time if we actually ran the query (that is, + * the user specified ANALYZE), and if summary reporting is enabled (the + * user can set SUMMARY OFF to not have the timing information included in + * the output). By default, ANALYZE sets SUMMARY to true. + */ + if (es->summary && es->analyze) { if (es->format == EXPLAIN_FORMAT_TEXT) appendStringInfo(es->str, "Execution time: %.3f ms\n", diff --git a/src/backend/commands/prepare.c b/src/backend/commands/prepare.c index 116ed67547..1cf0d2b971 100644 --- a/src/backend/commands/prepare.c +++ b/src/backend/commands/prepare.c @@ -638,6 +638,10 @@ ExplainExecuteQuery(ExecuteStmt *execstmt, IntoClause *into, ExplainState *es, ListCell *p; ParamListInfo paramLI = NULL; EState *estate = NULL; + instr_time planstart; + instr_time planduration; + + INSTR_TIME_SET_CURRENT(planstart); /* Look it up in the hash table */ entry = FetchPreparedStatement(execstmt->name, true); @@ -666,6 +670,9 @@ ExplainExecuteQuery(ExecuteStmt *execstmt, IntoClause *into, ExplainState *es, /* Replan if needed, and acquire a transient refcount */ cplan = GetCachedPlan(entry->plansource, paramLI, true); + INSTR_TIME_SET_CURRENT(planduration); + INSTR_TIME_SUBTRACT(planduration, planstart); + plan_list = cplan->stmt_list; /* Explain each query */ @@ -674,7 +681,7 @@ ExplainExecuteQuery(ExecuteStmt *execstmt, IntoClause *into, ExplainState *es, PlannedStmt *pstmt = castNode(PlannedStmt, lfirst(p)); if (pstmt->commandType != CMD_UTILITY) - ExplainOnePlan(pstmt, into, es, query_string, paramLI, NULL); + ExplainOnePlan(pstmt, into, es, query_string, paramLI, &planduration); else ExplainOneUtility(pstmt->utilityStmt, into, es, query_string, paramLI); diff --git a/src/test/regress/expected/select.out b/src/test/regress/expected/select.out index f84f8ac767..1fab5136d2 100644 --- a/src/test/regress/expected/select.out +++ b/src/test/regress/expected/select.out @@ -752,6 +752,16 @@ select * from onek2 where unique2 = 11 and stringu1 = 'ATAAAA'; 494 | 11 | 0 | 2 | 4 | 14 | 4 | 94 | 94 | 494 | 494 | 8 | 9 | ATAAAA | LAAAAA | VVVVxx (1 row) +-- actually run the query with an analyze to use the partial index +explain (costs off, analyze on, timing off, summary off) +select * from onek2 where unique2 = 11 and stringu1 = 'ATAAAA'; + QUERY PLAN +----------------------------------------------------------------- + Index Scan using onek2_u2_prtl on onek2 (actual rows=1 loops=1) + Index Cond: (unique2 = 11) + Filter: (stringu1 = 'ATAAAA'::name) +(3 rows) + explain (costs off) select unique2 from onek2 where unique2 = 11 and stringu1 = 'ATAAAA'; QUERY PLAN diff --git a/src/test/regress/sql/select.sql b/src/test/regress/sql/select.sql index abdd785a77..c80429e7d0 100644 --- a/src/test/regress/sql/select.sql +++ b/src/test/regress/sql/select.sql @@ -195,6 +195,9 @@ SELECT * FROM foo ORDER BY f1 DESC NULLS LAST; explain (costs off) select * from onek2 where unique2 = 11 and stringu1 = 'ATAAAA'; select * from onek2 where unique2 = 11 and stringu1 = 'ATAAAA'; +-- actually run the query with an analyze to use the partial index +explain (costs off, analyze on, timing off, summary off) +select * from onek2 where unique2 = 11 and stringu1 = 'ATAAAA'; explain (costs off) select unique2 from onek2 where unique2 = 11 and stringu1 = 'ATAAAA'; select unique2 from onek2 where unique2 = 11 and stringu1 = 'ATAAAA'; -- cgit v1.2.3 From 355d3993c53ed62c5b53d020648e4fbcfbf5f155 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Thu, 9 Mar 2017 07:40:36 -0500 Subject: Add a Gather Merge executor node. Like Gather, we spawn multiple workers and run the same plan in each one; however, Gather Merge is used when each worker produces the same output ordering and we want to preserve that output ordering while merging together the streams of tuples from various workers. (In a way, Gather Merge is like a hybrid of Gather and MergeAppend.) This works out to a win if it saves us from having to perform an expensive Sort. In cases where only a small amount of data would need to be sorted, it may actually be faster to use a regular Gather node and then sort the results afterward, because Gather Merge sometimes needs to wait synchronously for tuples whereas a pure Gather generally doesn't. But if this avoids an expensive sort then it's a win. Rushabh Lathia, reviewed and tested by Amit Kapila, Thomas Munro, and Neha Sharma, and reviewed and revised by me. Discussion: http://postgr.es/m/CAGPqQf09oPX-cQRpBKS0Gq49Z+m6KBxgxd_p9gX8CKk_d75HoQ@mail.gmail.com --- doc/src/sgml/config.sgml | 14 + src/backend/commands/explain.c | 23 + src/backend/executor/Makefile | 2 +- src/backend/executor/execProcnode.c | 17 + src/backend/executor/nodeGatherMerge.c | 687 ++++++++++++++++++++++++++ src/backend/nodes/copyfuncs.c | 28 ++ src/backend/nodes/outfuncs.c | 46 ++ src/backend/nodes/readfuncs.c | 22 + src/backend/optimizer/path/allpaths.c | 36 +- src/backend/optimizer/path/costsize.c | 68 +++ src/backend/optimizer/plan/createplan.c | 86 ++++ src/backend/optimizer/plan/planner.c | 127 ++++- src/backend/optimizer/plan/setrefs.c | 1 + src/backend/optimizer/plan/subselect.c | 1 + src/backend/optimizer/util/pathnode.c | 60 +++ src/backend/utils/misc/guc.c | 9 + src/include/executor/nodeGatherMerge.h | 27 + src/include/nodes/execnodes.h | 29 ++ src/include/nodes/nodes.h | 3 + src/include/nodes/plannodes.h | 16 + src/include/nodes/relation.h | 13 + src/include/optimizer/cost.h | 5 + src/include/optimizer/pathnode.h | 7 + src/test/regress/expected/select_parallel.out | 27 + src/test/regress/expected/sysviews.out | 3 +- src/test/regress/sql/select_parallel.sql | 11 + src/tools/pgindent/typedefs.list | 3 + 27 files changed, 1355 insertions(+), 16 deletions(-) create mode 100644 src/backend/executor/nodeGatherMerge.c create mode 100644 src/include/executor/nodeGatherMerge.h (limited to 'doc/src') diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 1881236726..69844e5b29 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3497,6 +3497,20 @@ ANY num_sync ( + enable_gathermerge (boolean) + + enable_gathermerge configuration parameter + + + + + Enables or disables the query planner's use of gather + merge plan types. The default is on. + + + + enable_hashagg (boolean) diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index 6fd82e9d52..c9b55ead3d 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -918,6 +918,9 @@ ExplainNode(PlanState *planstate, List *ancestors, case T_Gather: pname = sname = "Gather"; break; + case T_GatherMerge: + pname = sname = "Gather Merge"; + break; case T_IndexScan: pname = sname = "Index Scan"; break; @@ -1411,6 +1414,26 @@ ExplainNode(PlanState *planstate, List *ancestors, ExplainPropertyBool("Single Copy", gather->single_copy, es); } break; + case T_GatherMerge: + { + GatherMerge *gm = (GatherMerge *) plan; + + show_scan_qual(plan->qual, "Filter", planstate, ancestors, es); + if (plan->qual) + show_instrumentation_count("Rows Removed by Filter", 1, + planstate, es); + ExplainPropertyInteger("Workers Planned", + gm->num_workers, es); + if (es->analyze) + { + int nworkers; + + nworkers = ((GatherMergeState *) planstate)->nworkers_launched; + ExplainPropertyInteger("Workers Launched", + nworkers, es); + } + } + break; case T_FunctionScan: if (es->verbose) { diff --git a/src/backend/executor/Makefile b/src/backend/executor/Makefile index a9893c2b22..d281906cd5 100644 --- a/src/backend/executor/Makefile +++ b/src/backend/executor/Makefile @@ -20,7 +20,7 @@ OBJS = execAmi.o execCurrent.o execGrouping.o execIndexing.o execJunk.o \ nodeBitmapHeapscan.o nodeBitmapIndexscan.o \ nodeCustom.o nodeFunctionscan.o nodeGather.o \ nodeHash.o nodeHashjoin.o nodeIndexscan.o nodeIndexonlyscan.o \ - nodeLimit.o nodeLockRows.o \ + nodeLimit.o nodeLockRows.o nodeGatherMerge.o \ nodeMaterial.o nodeMergeAppend.o nodeMergejoin.o nodeModifyTable.o \ nodeNestloop.o nodeProjectSet.o nodeRecursiveunion.o nodeResult.o \ nodeSamplescan.o nodeSeqscan.o nodeSetOp.o nodeSort.o nodeUnique.o \ diff --git a/src/backend/executor/execProcnode.c b/src/backend/executor/execProcnode.c index 468f50e6a6..80c77addb8 100644 --- a/src/backend/executor/execProcnode.c +++ b/src/backend/executor/execProcnode.c @@ -89,6 +89,7 @@ #include "executor/nodeForeignscan.h" #include "executor/nodeFunctionscan.h" #include "executor/nodeGather.h" +#include "executor/nodeGatherMerge.h" #include "executor/nodeGroup.h" #include "executor/nodeHash.h" #include "executor/nodeHashjoin.h" @@ -326,6 +327,11 @@ ExecInitNode(Plan *node, EState *estate, int eflags) estate, eflags); break; + case T_GatherMerge: + result = (PlanState *) ExecInitGatherMerge((GatherMerge *) node, + estate, eflags); + break; + case T_Hash: result = (PlanState *) ExecInitHash((Hash *) node, estate, eflags); @@ -535,6 +541,10 @@ ExecProcNode(PlanState *node) result = ExecGather((GatherState *) node); break; + case T_GatherMergeState: + result = ExecGatherMerge((GatherMergeState *) node); + break; + case T_HashState: result = ExecHash((HashState *) node); break; @@ -697,6 +707,10 @@ ExecEndNode(PlanState *node) ExecEndGather((GatherState *) node); break; + case T_GatherMergeState: + ExecEndGatherMerge((GatherMergeState *) node); + break; + case T_IndexScanState: ExecEndIndexScan((IndexScanState *) node); break; @@ -842,6 +856,9 @@ ExecShutdownNode(PlanState *node) case T_CustomScanState: ExecShutdownCustomScan((CustomScanState *) node); break; + case T_GatherMergeState: + ExecShutdownGatherMerge((GatherMergeState *) node); + break; default: break; } diff --git a/src/backend/executor/nodeGatherMerge.c b/src/backend/executor/nodeGatherMerge.c new file mode 100644 index 0000000000..62a6b1866d --- /dev/null +++ b/src/backend/executor/nodeGatherMerge.c @@ -0,0 +1,687 @@ +/*------------------------------------------------------------------------- + * + * nodeGatherMerge.c + * Scan a plan in multiple workers, and do order-preserving merge. + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/executor/nodeGatherMerge.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "access/relscan.h" +#include "access/xact.h" +#include "executor/execdebug.h" +#include "executor/execParallel.h" +#include "executor/nodeGatherMerge.h" +#include "executor/nodeSubplan.h" +#include "executor/tqueue.h" +#include "lib/binaryheap.h" +#include "miscadmin.h" +#include "utils/memutils.h" +#include "utils/rel.h" + +/* + * Tuple array for each worker + */ +typedef struct GMReaderTupleBuffer +{ + HeapTuple *tuple; + int readCounter; + int nTuples; + bool done; +} GMReaderTupleBuffer; + +/* + * When we read tuples from workers, it's a good idea to read several at once + * for efficiency when possible: this minimizes context-switching overhead. + * But reading too many at a time wastes memory without improving performance. + */ +#define MAX_TUPLE_STORE 10 + +static int32 heap_compare_slots(Datum a, Datum b, void *arg); +static TupleTableSlot *gather_merge_getnext(GatherMergeState *gm_state); +static HeapTuple gm_readnext_tuple(GatherMergeState *gm_state, int nreader, + bool nowait, bool *done); +static void gather_merge_init(GatherMergeState *gm_state); +static void ExecShutdownGatherMergeWorkers(GatherMergeState *node); +static bool gather_merge_readnext(GatherMergeState *gm_state, int reader, + bool nowait); +static void form_tuple_array(GatherMergeState *gm_state, int reader); + +/* ---------------------------------------------------------------- + * ExecInitGather + * ---------------------------------------------------------------- + */ +GatherMergeState * +ExecInitGatherMerge(GatherMerge *node, EState *estate, int eflags) +{ + GatherMergeState *gm_state; + Plan *outerNode; + bool hasoid; + TupleDesc tupDesc; + + /* Gather merge node doesn't have innerPlan node. */ + Assert(innerPlan(node) == NULL); + + /* + * create state structure + */ + gm_state = makeNode(GatherMergeState); + gm_state->ps.plan = (Plan *) node; + gm_state->ps.state = estate; + + /* + * Miscellaneous initialization + * + * create expression context for node + */ + ExecAssignExprContext(estate, &gm_state->ps); + + /* + * initialize child expressions + */ + gm_state->ps.targetlist = (List *) + ExecInitExpr((Expr *) node->plan.targetlist, + (PlanState *) gm_state); + gm_state->ps.qual = (List *) + ExecInitExpr((Expr *) node->plan.qual, + (PlanState *) gm_state); + + /* + * tuple table initialization + */ + ExecInitResultTupleSlot(estate, &gm_state->ps); + + /* + * now initialize outer plan + */ + outerNode = outerPlan(node); + outerPlanState(gm_state) = ExecInitNode(outerNode, estate, eflags); + + /* + * Initialize result tuple type and projection info. + */ + ExecAssignResultTypeFromTL(&gm_state->ps); + ExecAssignProjectionInfo(&gm_state->ps, NULL); + + gm_state->gm_initialized = false; + + /* + * initialize sort-key information + */ + if (node->numCols) + { + int i; + + gm_state->gm_nkeys = node->numCols; + gm_state->gm_sortkeys = + palloc0(sizeof(SortSupportData) * node->numCols); + + for (i = 0; i < node->numCols; i++) + { + SortSupport sortKey = gm_state->gm_sortkeys + i; + + sortKey->ssup_cxt = CurrentMemoryContext; + sortKey->ssup_collation = node->collations[i]; + sortKey->ssup_nulls_first = node->nullsFirst[i]; + sortKey->ssup_attno = node->sortColIdx[i]; + + /* + * We don't perform abbreviated key conversion here, for the same + * reasons that it isn't used in MergeAppend + */ + sortKey->abbreviate = false; + + PrepareSortSupportFromOrderingOp(node->sortOperators[i], sortKey); + } + } + + /* + * store the tuple descriptor into gather merge state, so we can use it + * later while initializing the gather merge slots. + */ + if (!ExecContextForcesOids(&gm_state->ps, &hasoid)) + hasoid = false; + tupDesc = ExecTypeFromTL(outerNode->targetlist, hasoid); + gm_state->tupDesc = tupDesc; + + return gm_state; +} + +/* ---------------------------------------------------------------- + * ExecGatherMerge(node) + * + * Scans the relation via multiple workers and returns + * the next qualifying tuple. + * ---------------------------------------------------------------- + */ +TupleTableSlot * +ExecGatherMerge(GatherMergeState *node) +{ + TupleTableSlot *slot; + ExprContext *econtext; + int i; + + /* + * As with Gather, we don't launch workers until this node is actually + * executed. + */ + if (!node->initialized) + { + EState *estate = node->ps.state; + GatherMerge *gm = (GatherMerge *) node->ps.plan; + + /* + * Sometimes we might have to run without parallelism; but if parallel + * mode is active then we can try to fire up some workers. + */ + if (gm->num_workers > 0 && IsInParallelMode()) + { + ParallelContext *pcxt; + + /* Initialize data structures for workers. */ + if (!node->pei) + node->pei = ExecInitParallelPlan(node->ps.lefttree, + estate, + gm->num_workers); + + /* Try to launch workers. */ + pcxt = node->pei->pcxt; + LaunchParallelWorkers(pcxt); + node->nworkers_launched = pcxt->nworkers_launched; + + /* Set up tuple queue readers to read the results. */ + if (pcxt->nworkers_launched > 0) + { + node->nreaders = 0; + node->reader = palloc(pcxt->nworkers_launched * + sizeof(TupleQueueReader *)); + + Assert(gm->numCols); + + for (i = 0; i < pcxt->nworkers_launched; ++i) + { + shm_mq_set_handle(node->pei->tqueue[i], + pcxt->worker[i].bgwhandle); + node->reader[node->nreaders++] = + CreateTupleQueueReader(node->pei->tqueue[i], + node->tupDesc); + } + } + else + { + /* No workers? Then never mind. */ + ExecShutdownGatherMergeWorkers(node); + } + } + + /* always allow leader to participate */ + node->need_to_scan_locally = true; + node->initialized = true; + } + + /* + * Reset per-tuple memory context to free any expression evaluation + * storage allocated in the previous tuple cycle. + */ + econtext = node->ps.ps_ExprContext; + ResetExprContext(econtext); + + /* + * Get next tuple, either from one of our workers, or by running the + * plan ourselves. + */ + slot = gather_merge_getnext(node); + if (TupIsNull(slot)) + return NULL; + + /* + * form the result tuple using ExecProject(), and return it --- unless + * the projection produces an empty set, in which case we must loop + * back around for another tuple + */ + econtext->ecxt_outertuple = slot; + return ExecProject(node->ps.ps_ProjInfo); +} + +/* ---------------------------------------------------------------- + * ExecEndGatherMerge + * + * frees any storage allocated through C routines. + * ---------------------------------------------------------------- + */ +void +ExecEndGatherMerge(GatherMergeState *node) +{ + ExecEndNode(outerPlanState(node)); /* let children clean up first */ + ExecShutdownGatherMerge(node); + ExecFreeExprContext(&node->ps); + ExecClearTuple(node->ps.ps_ResultTupleSlot); +} + +/* ---------------------------------------------------------------- + * ExecShutdownGatherMerge + * + * Destroy the setup for parallel workers including parallel context. + * Collect all the stats after workers are stopped, else some work + * done by workers won't be accounted. + * ---------------------------------------------------------------- + */ +void +ExecShutdownGatherMerge(GatherMergeState *node) +{ + ExecShutdownGatherMergeWorkers(node); + + /* Now destroy the parallel context. */ + if (node->pei != NULL) + { + ExecParallelCleanup(node->pei); + node->pei = NULL; + } +} + +/* ---------------------------------------------------------------- + * ExecShutdownGatherMergeWorkers + * + * Destroy the parallel workers. Collect all the stats after + * workers are stopped, else some work done by workers won't be + * accounted. + * ---------------------------------------------------------------- + */ +static void +ExecShutdownGatherMergeWorkers(GatherMergeState *node) +{ + /* Shut down tuple queue readers before shutting down workers. */ + if (node->reader != NULL) + { + int i; + + for (i = 0; i < node->nreaders; ++i) + if (node->reader[i]) + DestroyTupleQueueReader(node->reader[i]); + + pfree(node->reader); + node->reader = NULL; + } + + /* Now shut down the workers. */ + if (node->pei != NULL) + ExecParallelFinish(node->pei); +} + +/* ---------------------------------------------------------------- + * ExecReScanGatherMerge + * + * Re-initialize the workers and rescans a relation via them. + * ---------------------------------------------------------------- + */ +void +ExecReScanGatherMerge(GatherMergeState *node) +{ + /* + * Re-initialize the parallel workers to perform rescan of relation. We + * want to gracefully shutdown all the workers so that they should be able + * to propagate any error or other information to master backend before + * dying. Parallel context will be reused for rescan. + */ + ExecShutdownGatherMergeWorkers(node); + + node->initialized = false; + + if (node->pei) + ExecParallelReinitialize(node->pei); + + ExecReScan(node->ps.lefttree); +} + +/* + * Initialize the Gather merge tuple read. + * + * Pull at least a single tuple from each worker + leader and set up the heap. + */ +static void +gather_merge_init(GatherMergeState *gm_state) +{ + int nreaders = gm_state->nreaders; + bool initialize = true; + int i; + + /* + * Allocate gm_slots for the number of worker + one more slot for leader. + * Last slot is always for leader. Leader always calls ExecProcNode() to + * read the tuple which will return the TupleTableSlot. Later it will + * directly get assigned to gm_slot. So just initialize leader gm_slot + * with NULL. For other slots below code will call + * ExecInitExtraTupleSlot() which will do the initialization of worker + * slots. + */ + gm_state->gm_slots = + palloc((gm_state->nreaders + 1) * sizeof(TupleTableSlot *)); + gm_state->gm_slots[gm_state->nreaders] = NULL; + + /* Initialize the tuple slot and tuple array for each worker */ + gm_state->gm_tuple_buffers = + (GMReaderTupleBuffer *) palloc0(sizeof(GMReaderTupleBuffer) * + (gm_state->nreaders + 1)); + for (i = 0; i < gm_state->nreaders; i++) + { + /* Allocate the tuple array with MAX_TUPLE_STORE size */ + gm_state->gm_tuple_buffers[i].tuple = + (HeapTuple *) palloc0(sizeof(HeapTuple) * MAX_TUPLE_STORE); + + /* Initialize slot for worker */ + gm_state->gm_slots[i] = ExecInitExtraTupleSlot(gm_state->ps.state); + ExecSetSlotDescriptor(gm_state->gm_slots[i], + gm_state->tupDesc); + } + + /* Allocate the resources for the merge */ + gm_state->gm_heap = binaryheap_allocate(gm_state->nreaders + 1, + heap_compare_slots, + gm_state); + + /* + * First, try to read a tuple from each worker (including leader) in + * nowait mode, so that we initialize read from each worker as well as + * leader. After this, if all active workers are unable to produce a + * tuple, then re-read and this time use wait mode. For workers that were + * able to produce a tuple in the earlier loop and are still active, just + * try to fill the tuple array if more tuples are avaiable. + */ +reread: + for (i = 0; i < nreaders + 1; i++) + { + if (!gm_state->gm_tuple_buffers[i].done && + (TupIsNull(gm_state->gm_slots[i]) || + gm_state->gm_slots[i]->tts_isempty)) + { + if (gather_merge_readnext(gm_state, i, initialize)) + { + binaryheap_add_unordered(gm_state->gm_heap, + Int32GetDatum(i)); + } + } + else + form_tuple_array(gm_state, i); + } + initialize = false; + + for (i = 0; i < nreaders; i++) + if (!gm_state->gm_tuple_buffers[i].done && + (TupIsNull(gm_state->gm_slots[i]) || + gm_state->gm_slots[i]->tts_isempty)) + goto reread; + + binaryheap_build(gm_state->gm_heap); + gm_state->gm_initialized = true; +} + +/* + * Clear out a slot in the tuple table for each gather merge + * slot and return the clear cleared slot. + */ +static TupleTableSlot * +gather_merge_clear_slots(GatherMergeState *gm_state) +{ + int i; + + for (i = 0; i < gm_state->nreaders; i++) + { + pfree(gm_state->gm_tuple_buffers[i].tuple); + gm_state->gm_slots[i] = ExecClearTuple(gm_state->gm_slots[i]); + } + + /* Free tuple array as we don't need it any more */ + pfree(gm_state->gm_tuple_buffers); + /* Free the binaryheap, which was created for sort */ + binaryheap_free(gm_state->gm_heap); + + /* return any clear slot */ + return gm_state->gm_slots[0]; +} + +/* + * Read the next tuple for gather merge. + * + * Fetch the sorted tuple out of the heap. + */ +static TupleTableSlot * +gather_merge_getnext(GatherMergeState *gm_state) +{ + int i; + + /* + * First time through: pull the first tuple from each participate, and set + * up the heap. + */ + if (gm_state->gm_initialized == false) + gather_merge_init(gm_state); + else + { + /* + * Otherwise, pull the next tuple from whichever participant we + * returned from last time, and reinsert the index into the heap, + * because it might now compare differently against the existing + * elements of the heap. + */ + i = DatumGetInt32(binaryheap_first(gm_state->gm_heap)); + + if (gather_merge_readnext(gm_state, i, false)) + binaryheap_replace_first(gm_state->gm_heap, Int32GetDatum(i)); + else + (void) binaryheap_remove_first(gm_state->gm_heap); + } + + if (binaryheap_empty(gm_state->gm_heap)) + { + /* All the queues are exhausted, and so is the heap */ + return gather_merge_clear_slots(gm_state); + } + else + { + i = DatumGetInt32(binaryheap_first(gm_state->gm_heap)); + return gm_state->gm_slots[i]; + } + + return gather_merge_clear_slots(gm_state); +} + +/* + * Read the tuple for given reader in nowait mode, and form the tuple array. + */ +static void +form_tuple_array(GatherMergeState *gm_state, int reader) +{ + GMReaderTupleBuffer *tuple_buffer = &gm_state->gm_tuple_buffers[reader]; + int i; + + /* Last slot is for leader and we don't build tuple array for leader */ + if (reader == gm_state->nreaders) + return; + + /* + * We here because we already read all the tuples from the tuple array, so + * initialize the counter to zero. + */ + if (tuple_buffer->nTuples == tuple_buffer->readCounter) + tuple_buffer->nTuples = tuple_buffer->readCounter = 0; + + /* Tuple array is already full? */ + if (tuple_buffer->nTuples == MAX_TUPLE_STORE) + return; + + for (i = tuple_buffer->nTuples; i < MAX_TUPLE_STORE; i++) + { + tuple_buffer->tuple[i] = heap_copytuple(gm_readnext_tuple(gm_state, + reader, + false, + &tuple_buffer->done)); + if (!HeapTupleIsValid(tuple_buffer->tuple[i])) + break; + tuple_buffer->nTuples++; + } +} + +/* + * Store the next tuple for a given reader into the appropriate slot. + * + * Returns false if the reader is exhausted, and true otherwise. + */ +static bool +gather_merge_readnext(GatherMergeState *gm_state, int reader, bool nowait) +{ + GMReaderTupleBuffer *tuple_buffer; + HeapTuple tup = NULL; + + /* + * If we're being asked to generate a tuple from the leader, then we + * just call ExecProcNode as normal to produce one. + */ + if (gm_state->nreaders == reader) + { + if (gm_state->need_to_scan_locally) + { + PlanState *outerPlan = outerPlanState(gm_state); + TupleTableSlot *outerTupleSlot; + + outerTupleSlot = ExecProcNode(outerPlan); + + if (!TupIsNull(outerTupleSlot)) + { + gm_state->gm_slots[reader] = outerTupleSlot; + return true; + } + gm_state->gm_tuple_buffers[reader].done = true; + gm_state->need_to_scan_locally = false; + } + return false; + } + + /* Otherwise, check the state of the relevant tuple buffer. */ + tuple_buffer = &gm_state->gm_tuple_buffers[reader]; + + if (tuple_buffer->nTuples > tuple_buffer->readCounter) + { + /* Return any tuple previously read that is still buffered. */ + tuple_buffer = &gm_state->gm_tuple_buffers[reader]; + tup = tuple_buffer->tuple[tuple_buffer->readCounter++]; + } + else if (tuple_buffer->done) + { + /* Reader is known to be exhausted. */ + DestroyTupleQueueReader(gm_state->reader[reader]); + gm_state->reader[reader] = NULL; + return false; + } + else + { + /* Read and buffer next tuple. */ + tup = heap_copytuple(gm_readnext_tuple(gm_state, + reader, + nowait, + &tuple_buffer->done)); + + /* + * Attempt to read more tuples in nowait mode and store them in + * the tuple array. + */ + if (HeapTupleIsValid(tup)) + form_tuple_array(gm_state, reader); + else + return false; + } + + Assert(HeapTupleIsValid(tup)); + + /* Build the TupleTableSlot for the given tuple */ + ExecStoreTuple(tup, /* tuple to store */ + gm_state->gm_slots[reader], /* slot in which to store the + * tuple */ + InvalidBuffer, /* buffer associated with this tuple */ + true); /* pfree this pointer if not from heap */ + + return true; +} + +/* + * Attempt to read a tuple from given reader. + */ +static HeapTuple +gm_readnext_tuple(GatherMergeState *gm_state, int nreader, bool nowait, + bool *done) +{ + TupleQueueReader *reader; + HeapTuple tup = NULL; + MemoryContext oldContext; + MemoryContext tupleContext; + + tupleContext = gm_state->ps.ps_ExprContext->ecxt_per_tuple_memory; + + if (done != NULL) + *done = false; + + /* Check for async events, particularly messages from workers. */ + CHECK_FOR_INTERRUPTS(); + + /* Attempt to read a tuple. */ + reader = gm_state->reader[nreader]; + + /* Run TupleQueueReaders in per-tuple context */ + oldContext = MemoryContextSwitchTo(tupleContext); + tup = TupleQueueReaderNext(reader, nowait, done); + MemoryContextSwitchTo(oldContext); + + return tup; +} + +/* + * We have one slot for each item in the heap array. We use SlotNumber + * to store slot indexes. This doesn't actually provide any formal + * type-safety, but it makes the code more self-documenting. + */ +typedef int32 SlotNumber; + +/* + * Compare the tuples in the two given slots. + */ +static int32 +heap_compare_slots(Datum a, Datum b, void *arg) +{ + GatherMergeState *node = (GatherMergeState *) arg; + SlotNumber slot1 = DatumGetInt32(a); + SlotNumber slot2 = DatumGetInt32(b); + + TupleTableSlot *s1 = node->gm_slots[slot1]; + TupleTableSlot *s2 = node->gm_slots[slot2]; + int nkey; + + Assert(!TupIsNull(s1)); + Assert(!TupIsNull(s2)); + + for (nkey = 0; nkey < node->gm_nkeys; nkey++) + { + SortSupport sortKey = node->gm_sortkeys + nkey; + AttrNumber attno = sortKey->ssup_attno; + Datum datum1, + datum2; + bool isNull1, + isNull2; + int compare; + + datum1 = slot_getattr(s1, attno, &isNull1); + datum2 = slot_getattr(s2, attno, &isNull2); + + compare = ApplySortComparator(datum1, isNull1, + datum2, isNull2, + sortKey); + if (compare != 0) + return -compare; + } + return 0; +} diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index ac8e50ef1d..bfc2ac1716 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -360,6 +360,31 @@ _copyGather(const Gather *from) return newnode; } +/* + * _copyGatherMerge + */ +static GatherMerge * +_copyGatherMerge(const GatherMerge *from) +{ + GatherMerge *newnode = makeNode(GatherMerge); + + /* + * copy node superclass fields + */ + CopyPlanFields((const Plan *) from, (Plan *) newnode); + + /* + * copy remainder of node + */ + COPY_SCALAR_FIELD(num_workers); + COPY_SCALAR_FIELD(numCols); + COPY_POINTER_FIELD(sortColIdx, from->numCols * sizeof(AttrNumber)); + COPY_POINTER_FIELD(sortOperators, from->numCols * sizeof(Oid)); + COPY_POINTER_FIELD(collations, from->numCols * sizeof(Oid)); + COPY_POINTER_FIELD(nullsFirst, from->numCols * sizeof(bool)); + + return newnode; +} /* * CopyScanFields @@ -4594,6 +4619,9 @@ copyObject(const void *from) case T_Gather: retval = _copyGather(from); break; + case T_GatherMerge: + retval = _copyGatherMerge(from); + break; case T_SeqScan: retval = _copySeqScan(from); break; diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 825a7b283a..7418fbeded 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -457,6 +457,35 @@ _outGather(StringInfo str, const Gather *node) WRITE_BOOL_FIELD(invisible); } +static void +_outGatherMerge(StringInfo str, const GatherMerge *node) +{ + int i; + + WRITE_NODE_TYPE("GATHERMERGE"); + + _outPlanInfo(str, (const Plan *) node); + + WRITE_INT_FIELD(num_workers); + WRITE_INT_FIELD(numCols); + + appendStringInfoString(str, " :sortColIdx"); + for (i = 0; i < node->numCols; i++) + appendStringInfo(str, " %d", node->sortColIdx[i]); + + appendStringInfoString(str, " :sortOperators"); + for (i = 0; i < node->numCols; i++) + appendStringInfo(str, " %u", node->sortOperators[i]); + + appendStringInfoString(str, " :collations"); + for (i = 0; i < node->numCols; i++) + appendStringInfo(str, " %u", node->collations[i]); + + appendStringInfoString(str, " :nullsFirst"); + for (i = 0; i < node->numCols; i++) + appendStringInfo(str, " %s", booltostr(node->nullsFirst[i])); +} + static void _outScan(StringInfo str, const Scan *node) { @@ -2016,6 +2045,17 @@ _outLimitPath(StringInfo str, const LimitPath *node) WRITE_NODE_FIELD(limitCount); } +static void +_outGatherMergePath(StringInfo str, const GatherMergePath *node) +{ + WRITE_NODE_TYPE("GATHERMERGEPATH"); + + _outPathInfo(str, (const Path *) node); + + WRITE_NODE_FIELD(subpath); + WRITE_INT_FIELD(num_workers); +} + static void _outNestPath(StringInfo str, const NestPath *node) { @@ -3473,6 +3513,9 @@ outNode(StringInfo str, const void *obj) case T_Gather: _outGather(str, obj); break; + case T_GatherMerge: + _outGatherMerge(str, obj); + break; case T_Scan: _outScan(str, obj); break; @@ -3809,6 +3852,9 @@ outNode(StringInfo str, const void *obj) case T_LimitPath: _outLimitPath(str, obj); break; + case T_GatherMergePath: + _outGatherMergePath(str, obj); + break; case T_NestPath: _outNestPath(str, obj); break; diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index 8f39d93a12..d3bbc02f24 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -2137,6 +2137,26 @@ _readGather(void) READ_DONE(); } +/* + * _readGatherMerge + */ +static GatherMerge * +_readGatherMerge(void) +{ + READ_LOCALS(GatherMerge); + + ReadCommonPlan(&local_node->plan); + + READ_INT_FIELD(num_workers); + READ_INT_FIELD(numCols); + READ_ATTRNUMBER_ARRAY(sortColIdx, local_node->numCols); + READ_OID_ARRAY(sortOperators, local_node->numCols); + READ_OID_ARRAY(collations, local_node->numCols); + READ_BOOL_ARRAY(nullsFirst, local_node->numCols); + + READ_DONE(); +} + /* * _readHash */ @@ -2577,6 +2597,8 @@ parseNodeString(void) return_value = _readUnique(); else if (MATCH("GATHER", 6)) return_value = _readGather(); + else if (MATCH("GATHERMERGE", 11)) + return_value = _readGatherMerge(); else if (MATCH("HASH", 4)) return_value = _readHash(); else if (MATCH("SETOP", 5)) diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index fbb2cda9d7..b263359fde 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -2084,39 +2084,51 @@ set_worktable_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) /* * generate_gather_paths - * Generate parallel access paths for a relation by pushing a Gather on - * top of a partial path. + * Generate parallel access paths for a relation by pushing a Gather or + * Gather Merge on top of a partial path. * * This must not be called until after we're done creating all partial paths * for the specified relation. (Otherwise, add_partial_path might delete a - * path that some GatherPath has a reference to.) + * path that some GatherPath or GatherMergePath has a reference to.) */ void generate_gather_paths(PlannerInfo *root, RelOptInfo *rel) { Path *cheapest_partial_path; Path *simple_gather_path; + ListCell *lc; /* If there are no partial paths, there's nothing to do here. */ if (rel->partial_pathlist == NIL) return; /* - * The output of Gather is currently always unsorted, so there's only one - * partial path of interest: the cheapest one. That will be the one at - * the front of partial_pathlist because of the way add_partial_path - * works. - * - * Eventually, we should have a Gather Merge operation that can merge - * multiple tuple streams together while preserving their ordering. We - * could usefully generate such a path from each partial path that has - * non-NIL pathkeys. + * The output of Gather is always unsorted, so there's only one partial + * path of interest: the cheapest one. That will be the one at the front + * of partial_pathlist because of the way add_partial_path works. */ cheapest_partial_path = linitial(rel->partial_pathlist); simple_gather_path = (Path *) create_gather_path(root, rel, cheapest_partial_path, rel->reltarget, NULL, NULL); add_path(rel, simple_gather_path); + + /* + * For each useful ordering, we can consider an order-preserving Gather + * Merge. + */ + foreach (lc, rel->partial_pathlist) + { + Path *subpath = (Path *) lfirst(lc); + GatherMergePath *path; + + if (subpath->pathkeys == NIL) + continue; + + path = create_gather_merge_path(root, rel, subpath, rel->reltarget, + subpath->pathkeys, NULL, NULL); + add_path(rel, &path->path); + } } /* diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index 627e3f1b95..e78f3a84c5 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -126,6 +126,7 @@ bool enable_nestloop = true; bool enable_material = true; bool enable_mergejoin = true; bool enable_hashjoin = true; +bool enable_gathermerge = true; typedef struct { @@ -372,6 +373,73 @@ cost_gather(GatherPath *path, PlannerInfo *root, path->path.total_cost = (startup_cost + run_cost); } +/* + * cost_gather_merge + * Determines and returns the cost of gather merge path. + * + * GatherMerge merges several pre-sorted input streams, using a heap that at + * any given instant holds the next tuple from each stream. If there are N + * streams, we need about N*log2(N) tuple comparisons to construct the heap at + * startup, and then for each output tuple, about log2(N) comparisons to + * replace the top heap entry with the next tuple from the same stream. + */ +void +cost_gather_merge(GatherMergePath *path, PlannerInfo *root, + RelOptInfo *rel, ParamPathInfo *param_info, + Cost input_startup_cost, Cost input_total_cost, + double *rows) +{ + Cost startup_cost = 0; + Cost run_cost = 0; + Cost comparison_cost; + double N; + double logN; + + /* Mark the path with the correct row estimate */ + if (rows) + path->path.rows = *rows; + else if (param_info) + path->path.rows = param_info->ppi_rows; + else + path->path.rows = rel->rows; + + if (!enable_gathermerge) + startup_cost += disable_cost; + + /* + * Add one to the number of workers to account for the leader. This might + * be overgenerous since the leader will do less work than other workers + * in typical cases, but we'll go with it for now. + */ + Assert(path->num_workers > 0); + N = (double) path->num_workers + 1; + logN = LOG2(N); + + /* Assumed cost per tuple comparison */ + comparison_cost = 2.0 * cpu_operator_cost; + + /* Heap creation cost */ + startup_cost += comparison_cost * N * logN; + + /* Per-tuple heap maintenance cost */ + run_cost += path->path.rows * comparison_cost * logN; + + /* small cost for heap management, like cost_merge_append */ + run_cost += cpu_operator_cost * path->path.rows; + + /* + * Parallel setup and communication cost. Since Gather Merge, unlike + * Gather, requires us to block until a tuple is available from every + * worker, we bump the IPC cost up a little bit as compared with Gather. + * For lack of a better idea, charge an extra 5%. + */ + startup_cost += parallel_setup_cost; + run_cost += parallel_tuple_cost * path->path.rows * 1.05; + + path->path.startup_cost = startup_cost + input_startup_cost; + path->path.total_cost = (startup_cost + run_cost + input_total_cost); +} + /* * cost_index * Determines and returns the cost of scanning a relation using an index. diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index 8f8663c1e1..e18c634a7b 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -277,6 +277,8 @@ static ModifyTable *make_modifytable(PlannerInfo *root, List *resultRelations, List *subplans, List *withCheckOptionLists, List *returningLists, List *rowMarks, OnConflictExpr *onconflict, int epqParam); +static GatherMerge *create_gather_merge_plan(PlannerInfo *root, + GatherMergePath *best_path); /* @@ -475,6 +477,10 @@ create_plan_recurse(PlannerInfo *root, Path *best_path, int flags) (LimitPath *) best_path, flags); break; + case T_GatherMerge: + plan = (Plan *) create_gather_merge_plan(root, + (GatherMergePath *) best_path); + break; default: elog(ERROR, "unrecognized node type: %d", (int) best_path->pathtype); @@ -1451,6 +1457,86 @@ create_gather_plan(PlannerInfo *root, GatherPath *best_path) return gather_plan; } +/* + * create_gather_merge_plan + * + * Create a Gather Merge plan for 'best_path' and (recursively) + * plans for its subpaths. + */ +static GatherMerge * +create_gather_merge_plan(PlannerInfo *root, GatherMergePath *best_path) +{ + GatherMerge *gm_plan; + Plan *subplan; + List *pathkeys = best_path->path.pathkeys; + int numsortkeys; + AttrNumber *sortColIdx; + Oid *sortOperators; + Oid *collations; + bool *nullsFirst; + + /* As with Gather, it's best to project away columns in the workers. */ + subplan = create_plan_recurse(root, best_path->subpath, CP_EXACT_TLIST); + + /* See create_merge_append_plan for why there's no make_xxx function */ + gm_plan = makeNode(GatherMerge); + gm_plan->plan.targetlist = subplan->targetlist; + gm_plan->num_workers = best_path->num_workers; + copy_generic_path_info(&gm_plan->plan, &best_path->path); + + /* Gather Merge is pointless with no pathkeys; use Gather instead. */ + Assert(pathkeys != NIL); + + /* Compute sort column info, and adjust GatherMerge tlist as needed */ + (void) prepare_sort_from_pathkeys(&gm_plan->plan, pathkeys, + best_path->path.parent->relids, + NULL, + true, + &gm_plan->numCols, + &gm_plan->sortColIdx, + &gm_plan->sortOperators, + &gm_plan->collations, + &gm_plan->nullsFirst); + + + /* Compute sort column info, and adjust subplan's tlist as needed */ + subplan = prepare_sort_from_pathkeys(subplan, pathkeys, + best_path->subpath->parent->relids, + gm_plan->sortColIdx, + false, + &numsortkeys, + &sortColIdx, + &sortOperators, + &collations, + &nullsFirst); + + /* As for MergeAppend, check that we got the same sort key information. */ + Assert(numsortkeys == gm_plan->numCols); + if (memcmp(sortColIdx, gm_plan->sortColIdx, + numsortkeys * sizeof(AttrNumber)) != 0) + elog(ERROR, "GatherMerge child's targetlist doesn't match GatherMerge"); + Assert(memcmp(sortOperators, gm_plan->sortOperators, + numsortkeys * sizeof(Oid)) == 0); + Assert(memcmp(collations, gm_plan->collations, + numsortkeys * sizeof(Oid)) == 0); + Assert(memcmp(nullsFirst, gm_plan->nullsFirst, + numsortkeys * sizeof(bool)) == 0); + + /* Now, insert a Sort node if subplan isn't sufficiently ordered */ + if (!pathkeys_contained_in(pathkeys, best_path->subpath->pathkeys)) + subplan = (Plan *) make_sort(subplan, numsortkeys, + sortColIdx, sortOperators, + collations, nullsFirst); + + /* Now insert the subplan under GatherMerge. */ + gm_plan->plan.lefttree = subplan; + + /* use parallel mode for parallel plans. */ + root->glob->parallelModeNeeded = true; + + return gm_plan; +} + /* * create_projection_plan * diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index 1636a69dba..209f769632 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -3663,8 +3663,7 @@ create_grouping_paths(PlannerInfo *root, /* * Now generate a complete GroupAgg Path atop of the cheapest partial - * path. We need only bother with the cheapest path here, as the - * output of Gather is never sorted. + * path. We can do this using either Gather or Gather Merge. */ if (grouped_rel->partial_pathlist) { @@ -3711,6 +3710,70 @@ create_grouping_paths(PlannerInfo *root, parse->groupClause, (List *) parse->havingQual, dNumGroups)); + + /* + * The point of using Gather Merge rather than Gather is that it + * can preserve the ordering of the input path, so there's no + * reason to try it unless (1) it's possible to produce more than + * one output row and (2) we want the output path to be ordered. + */ + if (parse->groupClause != NIL && root->group_pathkeys != NIL) + { + foreach(lc, grouped_rel->partial_pathlist) + { + Path *subpath = (Path *) lfirst(lc); + Path *gmpath; + double total_groups; + + /* + * It's useful to consider paths that are already properly + * ordered for Gather Merge, because those don't need a + * sort. It's also useful to consider the cheapest path, + * because sorting it in parallel and then doing Gather + * Merge may be better than doing an unordered Gather + * followed by a sort. But there's no point in + * considering non-cheapest paths that aren't already + * sorted correctly. + */ + if (path != subpath && + !pathkeys_contained_in(root->group_pathkeys, + subpath->pathkeys)) + continue; + + total_groups = subpath->rows * subpath->parallel_workers; + + gmpath = (Path *) + create_gather_merge_path(root, + grouped_rel, + subpath, + NULL, + root->group_pathkeys, + NULL, + &total_groups); + + if (parse->hasAggs) + add_path(grouped_rel, (Path *) + create_agg_path(root, + grouped_rel, + gmpath, + target, + parse->groupClause ? AGG_SORTED : AGG_PLAIN, + AGGSPLIT_FINAL_DESERIAL, + parse->groupClause, + (List *) parse->havingQual, + &agg_final_costs, + dNumGroups)); + else + add_path(grouped_rel, (Path *) + create_group_path(root, + grouped_rel, + gmpath, + target, + parse->groupClause, + (List *) parse->havingQual, + dNumGroups)); + } + } } } @@ -3808,6 +3871,16 @@ create_grouping_paths(PlannerInfo *root, /* Now choose the best path(s) */ set_cheapest(grouped_rel); + /* + * We've been using the partial pathlist for the grouped relation to hold + * partially aggregated paths, but that's actually a little bit bogus + * because it's unsafe for later planning stages -- like ordered_rel --- + * to get the idea that they can use these partial paths as if they didn't + * need a FinalizeAggregate step. Zap the partial pathlist at this stage + * so we don't get confused. + */ + grouped_rel->partial_pathlist = NIL; + return grouped_rel; } @@ -4275,6 +4348,56 @@ create_ordered_paths(PlannerInfo *root, } } + /* + * generate_gather_paths() will have already generated a simple Gather + * path for the best parallel path, if any, and the loop above will have + * considered sorting it. Similarly, generate_gather_paths() will also + * have generated order-preserving Gather Merge plans which can be used + * without sorting if they happen to match the sort_pathkeys, and the loop + * above will have handled those as well. However, there's one more + * possibility: it may make sense to sort the cheapest partial path + * according to the required output order and then use Gather Merge. + */ + if (ordered_rel->consider_parallel && root->sort_pathkeys != NIL && + input_rel->partial_pathlist != NIL) + { + Path *cheapest_partial_path; + + cheapest_partial_path = linitial(input_rel->partial_pathlist); + + /* + * If cheapest partial path doesn't need a sort, this is redundant + * with what's already been tried. + */ + if (!pathkeys_contained_in(root->sort_pathkeys, + cheapest_partial_path->pathkeys)) + { + Path *path; + double total_groups; + + path = (Path *) create_sort_path(root, + ordered_rel, + cheapest_partial_path, + root->sort_pathkeys, + limit_tuples); + + total_groups = cheapest_partial_path->rows * + cheapest_partial_path->parallel_workers; + path = (Path *) + create_gather_merge_path(root, ordered_rel, + path, + target, root->sort_pathkeys, NULL, + &total_groups); + + /* Add projection step if needed */ + if (path->pathtarget != target) + path = apply_projection_to_path(root, ordered_rel, + path, target); + + add_path(ordered_rel, path); + } + } + /* * If there is an FDW that's responsible for all baserels of the query, * let it consider adding ForeignPaths. diff --git a/src/backend/optimizer/plan/setrefs.c b/src/backend/optimizer/plan/setrefs.c index 3d2c12433d..5f3027e96f 100644 --- a/src/backend/optimizer/plan/setrefs.c +++ b/src/backend/optimizer/plan/setrefs.c @@ -616,6 +616,7 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) break; case T_Gather: + case T_GatherMerge: set_upper_references(root, plan, rtoffset); break; diff --git a/src/backend/optimizer/plan/subselect.c b/src/backend/optimizer/plan/subselect.c index da9a84be3d..6fa6540662 100644 --- a/src/backend/optimizer/plan/subselect.c +++ b/src/backend/optimizer/plan/subselect.c @@ -2700,6 +2700,7 @@ finalize_plan(PlannerInfo *root, Plan *plan, Bitmapset *valid_params, case T_Sort: case T_Unique: case T_Gather: + case T_GatherMerge: case T_SetOp: case T_Group: /* no node-type-specific fields need fixing */ diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index 0d925c6fcb..8ce772d274 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -1627,6 +1627,66 @@ create_unique_path(PlannerInfo *root, RelOptInfo *rel, Path *subpath, return pathnode; } +/* + * create_gather_merge_path + * + * Creates a path corresponding to a gather merge scan, returning + * the pathnode. + */ +GatherMergePath * +create_gather_merge_path(PlannerInfo *root, RelOptInfo *rel, Path *subpath, + PathTarget *target, List *pathkeys, + Relids required_outer, double *rows) +{ + GatherMergePath *pathnode = makeNode(GatherMergePath); + Cost input_startup_cost = 0; + Cost input_total_cost = 0; + + Assert(subpath->parallel_safe); + Assert(pathkeys); + + pathnode->path.pathtype = T_GatherMerge; + pathnode->path.parent = rel; + pathnode->path.param_info = get_baserel_parampathinfo(root, rel, + required_outer); + pathnode->path.parallel_aware = false; + + pathnode->subpath = subpath; + pathnode->num_workers = subpath->parallel_workers; + pathnode->path.pathkeys = pathkeys; + pathnode->path.pathtarget = target ? target : rel->reltarget; + pathnode->path.rows += subpath->rows; + + if (pathkeys_contained_in(pathkeys, subpath->pathkeys)) + { + /* Subpath is adequately ordered, we won't need to sort it */ + input_startup_cost += subpath->startup_cost; + input_total_cost += subpath->total_cost; + } + else + { + /* We'll need to insert a Sort node, so include cost for that */ + Path sort_path; /* dummy for result of cost_sort */ + + cost_sort(&sort_path, + root, + pathkeys, + subpath->total_cost, + subpath->rows, + subpath->pathtarget->width, + 0.0, + work_mem, + -1); + input_startup_cost += sort_path.startup_cost; + input_total_cost += sort_path.total_cost; + } + + cost_gather_merge(pathnode, root, rel, pathnode->path.param_info, + input_startup_cost, input_total_cost, rows); + + return pathnode; +} + /* * translate_sub_tlist - get subquery column numbers represented by tlist * diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index f8b073d8a9..811ea5153b 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -902,6 +902,15 @@ static struct config_bool ConfigureNamesBool[] = true, NULL, NULL, NULL }, + { + {"enable_gathermerge", PGC_USERSET, QUERY_TUNING_METHOD, + gettext_noop("Enables the planner's use of gather merge plans."), + NULL + }, + &enable_gathermerge, + true, + NULL, NULL, NULL + }, { {"geqo", PGC_USERSET, QUERY_TUNING_GEQO, diff --git a/src/include/executor/nodeGatherMerge.h b/src/include/executor/nodeGatherMerge.h new file mode 100644 index 0000000000..3c8b42b6e5 --- /dev/null +++ b/src/include/executor/nodeGatherMerge.h @@ -0,0 +1,27 @@ +/*------------------------------------------------------------------------- + * + * nodeGatherMerge.h + * prototypes for nodeGatherMerge.c + * + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/executor/nodeGatherMerge.h + * + *------------------------------------------------------------------------- + */ +#ifndef NODEGATHERMERGE_H +#define NODEGATHERMERGE_H + +#include "nodes/execnodes.h" + +extern GatherMergeState *ExecInitGatherMerge(GatherMerge * node, + EState *estate, + int eflags); +extern TupleTableSlot *ExecGatherMerge(GatherMergeState * node); +extern void ExecEndGatherMerge(GatherMergeState * node); +extern void ExecReScanGatherMerge(GatherMergeState * node); +extern void ExecShutdownGatherMerge(GatherMergeState * node); + +#endif /* NODEGATHERMERGE_H */ diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 6a0d590ef2..f856f6036f 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -2094,6 +2094,35 @@ typedef struct GatherState bool need_to_scan_locally; } GatherState; +/* ---------------- + * GatherMergeState information + * + * Gather merge nodes launch 1 or more parallel workers, run a + * subplan which produces sorted output in each worker, and then + * merge the results into a single sorted stream. + * ---------------- + */ +struct GMReaderTuple; + +typedef struct GatherMergeState +{ + PlanState ps; /* its first field is NodeTag */ + bool initialized; + struct ParallelExecutorInfo *pei; + int nreaders; + int nworkers_launched; + struct TupleQueueReader **reader; + TupleDesc tupDesc; + TupleTableSlot **gm_slots; + struct binaryheap *gm_heap; /* binary heap of slot indices */ + bool gm_initialized; /* gather merge initilized ? */ + bool need_to_scan_locally; + int gm_nkeys; + SortSupport gm_sortkeys; /* array of length ms_nkeys */ + struct GMReaderTupleBuffer *gm_tuple_buffers; /* tuple buffer per + * reader */ +} GatherMergeState; + /* ---------------- * HashState information * ---------------- diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index 49fa944755..2bc7a5df11 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -77,6 +77,7 @@ typedef enum NodeTag T_WindowAgg, T_Unique, T_Gather, + T_GatherMerge, T_Hash, T_SetOp, T_LockRows, @@ -127,6 +128,7 @@ typedef enum NodeTag T_WindowAggState, T_UniqueState, T_GatherState, + T_GatherMergeState, T_HashState, T_SetOpState, T_LockRowsState, @@ -249,6 +251,7 @@ typedef enum NodeTag T_MaterialPath, T_UniquePath, T_GatherPath, + T_GatherMergePath, T_ProjectionPath, T_ProjectSetPath, T_SortPath, diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h index 7fbb0c2c77..b880dc16cf 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h @@ -797,6 +797,22 @@ typedef struct Gather bool invisible; /* suppress EXPLAIN display (for testing)? */ } Gather; +/* ------------ + * gather merge node + * ------------ + */ +typedef struct GatherMerge +{ + Plan plan; + int num_workers; + /* remaining fields are just like the sort-key info in struct Sort */ + int numCols; /* number of sort-key columns */ + AttrNumber *sortColIdx; /* their indexes in the target list */ + Oid *sortOperators; /* OIDs of operators to sort them by */ + Oid *collations; /* OIDs of collations */ + bool *nullsFirst; /* NULLS FIRST/LAST directions */ +} GatherMerge; + /* ---------------- * hash build node * diff --git a/src/include/nodes/relation.h b/src/include/nodes/relation.h index f7ac6f600f..05d6f07aea 100644 --- a/src/include/nodes/relation.h +++ b/src/include/nodes/relation.h @@ -1203,6 +1203,19 @@ typedef struct GatherPath bool single_copy; /* don't execute path more than once */ } GatherPath; +/* + * GatherMergePath runs several copies of a plan in parallel and + * collects the results. For gather merge parallel leader always execute the + * plan. + */ +typedef struct GatherMergePath +{ + Path path; + Path *subpath; /* path for each worker */ + int num_workers; /* number of workers sought to help */ +} GatherMergePath; + + /* * All join-type paths share these fields. */ diff --git a/src/include/optimizer/cost.h b/src/include/optimizer/cost.h index 2b386835e3..d9a9b12a06 100644 --- a/src/include/optimizer/cost.h +++ b/src/include/optimizer/cost.h @@ -66,6 +66,7 @@ extern bool enable_nestloop; extern bool enable_material; extern bool enable_mergejoin; extern bool enable_hashjoin; +extern bool enable_gathermerge; extern int constraint_exclusion; extern double clamp_row_est(double nrows); @@ -205,5 +206,9 @@ extern Selectivity clause_selectivity(PlannerInfo *root, int varRelid, JoinType jointype, SpecialJoinInfo *sjinfo); +extern void cost_gather_merge(GatherMergePath *path, PlannerInfo *root, + RelOptInfo *rel, ParamPathInfo *param_info, + Cost input_startup_cost, Cost input_total_cost, + double *rows); #endif /* COST_H */ diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index f0fe830722..373c7221a8 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -78,6 +78,13 @@ extern UniquePath *create_unique_path(PlannerInfo *root, RelOptInfo *rel, extern GatherPath *create_gather_path(PlannerInfo *root, RelOptInfo *rel, Path *subpath, PathTarget *target, Relids required_outer, double *rows); +extern GatherMergePath *create_gather_merge_path(PlannerInfo *root, + RelOptInfo *rel, + Path *subpath, + PathTarget *target, + List *pathkeys, + Relids required_outer, + double *rows); extern SubqueryScanPath *create_subqueryscan_path(PlannerInfo *root, RelOptInfo *rel, Path *subpath, List *pathkeys, Relids required_outer); diff --git a/src/test/regress/expected/select_parallel.out b/src/test/regress/expected/select_parallel.out index 290b735b6b..038a62efd7 100644 --- a/src/test/regress/expected/select_parallel.out +++ b/src/test/regress/expected/select_parallel.out @@ -213,6 +213,33 @@ select count(*) from tenk1, tenk2 where tenk1.unique1 = tenk2.unique1; reset enable_hashjoin; reset enable_nestloop; +--test gather merge +set enable_hashagg to off; +explain (costs off) + select string4, count((unique2)) from tenk1 group by string4 order by string4; + QUERY PLAN +---------------------------------------------------- + Finalize GroupAggregate + Group Key: string4 + -> Gather Merge + Workers Planned: 4 + -> Partial GroupAggregate + Group Key: string4 + -> Sort + Sort Key: string4 + -> Parallel Seq Scan on tenk1 +(9 rows) + +select string4, count((unique2)) from tenk1 group by string4 order by string4; + string4 | count +---------+------- + AAAAxx | 2500 + HHHHxx | 2500 + OOOOxx | 2500 + VVVVxx | 2500 +(4 rows) + +reset enable_hashagg; set force_parallel_mode=1; explain (costs off) select stringu1::int2 from tenk1 where unique1 = 1; diff --git a/src/test/regress/expected/sysviews.out b/src/test/regress/expected/sysviews.out index d48abd7e09..568b783f5e 100644 --- a/src/test/regress/expected/sysviews.out +++ b/src/test/regress/expected/sysviews.out @@ -73,6 +73,7 @@ select name, setting from pg_settings where name like 'enable%'; name | setting ----------------------+--------- enable_bitmapscan | on + enable_gathermerge | on enable_hashagg | on enable_hashjoin | on enable_indexonlyscan | on @@ -83,7 +84,7 @@ select name, setting from pg_settings where name like 'enable%'; enable_seqscan | on enable_sort | on enable_tidscan | on -(11 rows) +(12 rows) -- Test that the pg_timezone_names and pg_timezone_abbrevs views are -- more-or-less working. We can't test their contents in any great detail diff --git a/src/test/regress/sql/select_parallel.sql b/src/test/regress/sql/select_parallel.sql index 80412b990d..9311a775af 100644 --- a/src/test/regress/sql/select_parallel.sql +++ b/src/test/regress/sql/select_parallel.sql @@ -84,6 +84,17 @@ select count(*) from tenk1, tenk2 where tenk1.unique1 = tenk2.unique1; reset enable_hashjoin; reset enable_nestloop; + +--test gather merge +set enable_hashagg to off; + +explain (costs off) + select string4, count((unique2)) from tenk1 group by string4 order by string4; + +select string4, count((unique2)) from tenk1 group by string4 order by string4; + +reset enable_hashagg; + set force_parallel_mode=1; explain (costs off) diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 3155ec6d5b..296552e394 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -779,6 +779,9 @@ GV Gather GatherPath GatherState +GatherMerge +GatherMergePath +GatherMergeState Gene GenericCosts GenericExprState -- cgit v1.2.3 From be37c2120a2a88e5ba852d42952c77b6bf5d5271 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 9 Mar 2017 08:27:16 -0500 Subject: Enable replication connections by default in pg_hba.conf initdb now initializes a pg_hba.conf that allows replication connections from the local host, same as it does for regular connections. The connecting user still needs to have the REPLICATION attribute or be a superuser. The intent is to allow pg_basebackup from the local host to succeed without requiring additional configuration. Michael Paquier and me --- doc/src/sgml/ref/initdb.sgml | 16 +++++++++++----- src/backend/libpq/pg_hba.conf.sample | 6 +++--- src/bin/initdb/initdb.c | 5 ----- src/bin/pg_basebackup/t/010_pg_basebackup.pl | 7 ++----- src/test/perl/PostgresNode.pm | 19 ++----------------- 5 files changed, 18 insertions(+), 35 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/initdb.sgml b/doc/src/sgml/ref/initdb.sgml index 1aaa4901af..d9faa96021 100644 --- a/doc/src/sgml/ref/initdb.sgml +++ b/doc/src/sgml/ref/initdb.sgml @@ -120,11 +120,17 @@ PostgreSQL documentation - This option specifies the authentication method for local users used - in pg_hba.conf (host - and local lines). Do not use trust - unless you trust all local users on your system. trust is - the default for ease of installation. + This option specifies the default authentication method for local + users used in pg_hba.conf (host + and local lines). initdb will + prepopulate pg_hba.conf entries using the + specified authentication method for non-replication as well as + replication connections. + + + + Do not use trust unless you trust all local users on your + system. trust is the default for ease of installation. diff --git a/src/backend/libpq/pg_hba.conf.sample b/src/backend/libpq/pg_hba.conf.sample index 73f7973ea2..6b1778a721 100644 --- a/src/backend/libpq/pg_hba.conf.sample +++ b/src/backend/libpq/pg_hba.conf.sample @@ -84,6 +84,6 @@ host all all 127.0.0.1/32 @authmethodhost@ host all all ::1/128 @authmethodhost@ # Allow replication connections from localhost, by a user with the # replication privilege. -@remove-line-for-nolocal@#local replication @default_username@ @authmethodlocal@ -#host replication @default_username@ 127.0.0.1/32 @authmethodhost@ -#host replication @default_username@ ::1/128 @authmethodhost@ +@remove-line-for-nolocal@local replication all @authmethodlocal@ +host replication all 127.0.0.1/32 @authmethodhost@ +host replication all ::1/128 @authmethodhost@ diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index 4968fc783e..da40d7ab67 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -1235,11 +1235,6 @@ setup_config(void) "@authcomment@", (strcmp(authmethodlocal, "trust") == 0 || strcmp(authmethodhost, "trust") == 0) ? AUTHTRUST_WARNING : ""); - /* Replace username for replication */ - conflines = replace_token(conflines, - "@default_username@", - username); - snprintf(path, sizeof(path), "%s/pg_hba.conf", pg_data); writefile(path, conflines); diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/010_pg_basebackup.pl index aafb138fd5..14bd813896 100644 --- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl +++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl @@ -4,7 +4,7 @@ use Cwd; use Config; use PostgresNode; use TestLib; -use Test::More tests => 73; +use Test::More tests => 72; program_help_ok('pg_basebackup'); program_version_ok('pg_basebackup'); @@ -15,15 +15,12 @@ my $tempdir = TestLib::tempdir; my $node = get_new_node('main'); # Initialize node without replication settings -$node->init(hba_permit_replication => 0); +$node->init; $node->start; my $pgdata = $node->data_dir; $node->command_fails(['pg_basebackup'], 'pg_basebackup needs target directory specified'); -$node->command_fails( - [ 'pg_basebackup', '-D', "$tempdir/backup" ], - 'pg_basebackup fails because of hba'); # Some Windows ANSI code pages may reject this filename, in which case we # quietly proceed without this bit of test coverage. diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm index e5cb348f4c..7e530676b2 100644 --- a/src/test/perl/PostgresNode.pm +++ b/src/test/perl/PostgresNode.pm @@ -349,11 +349,7 @@ sub set_replication_conf open my $hba, ">>$pgdata/pg_hba.conf"; print $hba "\n# Allow replication (set up by PostgresNode.pm)\n"; - if (!$TestLib::windows_os) - { - print $hba "local replication all trust\n"; - } - else + if ($TestLib::windows_os) { print $hba "host replication all $test_localhost/32 sspi include_realm=1 map=regress\n"; @@ -373,9 +369,6 @@ a directory that's only accessible to the current user to ensure that. On Windows, we use SSPI authentication to ensure the same (by pg_regress --config-auth). -pg_hba.conf is configured to allow replication connections. Pass the keyword -parameter hba_permit_replication => 0 to disable this. - WAL archiving can be enabled on this node by passing the keyword parameter has_archiving => 1. This is disabled by default. @@ -396,8 +389,6 @@ sub init my $pgdata = $self->data_dir; my $host = $self->host; - $params{hba_permit_replication} = 1 - unless defined $params{hba_permit_replication}; $params{allows_streaming} = 0 unless defined $params{allows_streaming}; $params{has_archiving} = 0 unless defined $params{has_archiving}; @@ -451,7 +442,7 @@ sub init } close $conf; - $self->set_replication_conf if $params{hba_permit_replication}; + $self->set_replication_conf if $params{allows_streaming}; $self->enable_archiving if $params{has_archiving}; } @@ -591,9 +582,6 @@ Does not start the node after initializing it. A recovery.conf is not created. -pg_hba.conf is configured to allow replication connections. Pass the keyword -parameter hba_permit_replication => 0 to disable this. - Streaming replication can be enabled on this node by passing the keyword parameter has_streaming => 1. This is disabled by default. @@ -615,8 +603,6 @@ sub init_from_backup my $root_name = $root_node->name; $params{has_streaming} = 0 unless defined $params{has_streaming}; - $params{hba_permit_replication} = 1 - unless defined $params{hba_permit_replication}; $params{has_restoring} = 0 unless defined $params{has_restoring}; print @@ -638,7 +624,6 @@ sub init_from_backup qq( port = $port )); - $self->set_replication_conf if $params{hba_permit_replication}; $self->enable_streaming($root_node) if $params{has_streaming}; $self->enable_restoring($root_node) if $params{has_restoring}; } -- cgit v1.2.3 From 054637d2e08cda6a096f48cc99696136a06f4ef5 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Thu, 9 Mar 2017 13:02:34 -0500 Subject: Document some new parallel query capabilities. This updates the text for parallel index scan, parallel index-only scan, parallel bitmap heap scan, and parallel merge join. It also expands the discussion of parallel joins slightly. Discussion: http://postgr.es/m/CA+TgmoZnCUoM31w3w7JSakVQJQOtcuTyX=HLUr-X1rto2=2bjw@mail.gmail.com --- doc/src/sgml/parallel.sgml | 73 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 57 insertions(+), 16 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/parallel.sgml b/doc/src/sgml/parallel.sgml index e8624fcab6..2ea5c34ba2 100644 --- a/doc/src/sgml/parallel.sgml +++ b/doc/src/sgml/parallel.sgml @@ -268,14 +268,43 @@ EXPLAIN SELECT * FROM pgbench_accounts WHERE filler LIKE '%x%'; Parallel Scans - Currently, the only type of scan which has been modified to work with - parallel query is a sequential scan. Therefore, the driving table in - a parallel plan will always be scanned using a - Parallel Seq Scan. The relation's blocks will be divided - among the cooperating processes. Blocks are handed out one at a - time, so that access to the relation remains sequential. Each process - will visit every tuple on the page assigned to it before requesting a new - page. + The following types of parallel-aware table scans are currently supported. + + + + + In a parallel sequential scan, the table's blocks will + be divided among the cooperating processes. Blocks are handed out one + at a time, so that access to the table remains sequential. + + + + + In a parallel bitmap heap scan, one process is chosen + as the leader. That process performs a scan of one or more indexes + and builds a bitmap indicating which table blocks need to be visited. + These blocks are then divided among the cooperating processes as in + a parallel sequential scan. In other words, the heap scan is performed + in parallel, but the underlying index scan is not. + + + + + In a parallel index scan or parallel index-only + scan, the cooperating processes take turns reading data from the + index. Currently, parallel index scans are supported only for + btree indexes. Each process will claim a single index block and will + scan and return all tuples referenced by that block; other process can + at the same time be returning tuples from a different index block. + The results of a parallel btree scan are returned in sorted order + within each worker process. + + + + + Only the scan types listed above may be used for a scan on the driving + table within a parallel plan. Other scan types, such as parallel scans of + non-btree indexes, may be supported in the future. @@ -283,14 +312,26 @@ EXPLAIN SELECT * FROM pgbench_accounts WHERE filler LIKE '%x%'; Parallel Joins - The driving table may be joined to one or more other tables using nested - loops or hash joins. The inner side of the join may be any kind of - non-parallel plan that is otherwise supported by the planner provided that - it is safe to run within a parallel worker. For example, it may be an - index scan which looks up a value taken from the outer side of the join. - Each worker will execute the inner side of the join in full, which for - hash join means that an identical hash table is built in each worker - process. + Just as in a non-parallel plan, the driving table may be joined to one or + more other tables using a nested loop, hash join, or merge join. The + inner side of the join may be any kind of non-parallel plan that is + otherwise supported by the planner provided that it is safe to run within + a parallel worker. For example, if a nested loop join is chosen, the + inner plan may be an index scan which looks up a value taken from the outer + side of the join. + + + + Each worker will execute the inner side of the join in full. This is + typically not a problem for nested loops, but may be inefficient for + cases involving hash or merge joins. For example, for a hash join, this + restriction means that an identical hash table is built in each worker + process, which works fine for joins against small tables but may not be + efficient when the inner table is large. For a merge join, it might mean + that each worker performs a separate sort of the inner relation, which + could be slow. Of course, in cases where a parallel plan of this type + would be inefficient, the query planner will normally choose some other + plan (possibly one which does not use parallelism) instead. -- cgit v1.2.3 From b54aad8e34bd6299093e965c50f4a23da96d7cc3 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Thu, 9 Mar 2017 13:09:48 -0500 Subject: Document lack of validation when attaching foreign partitions. Ashutosh Bapat, revised a bit by me. Discussion: http://postgr.es/m/CAFjFpRdLaCa-1wJase0=YWG5o3cJnbuUt_vrqm2TDBKM_vQ_oA@mail.gmail.com --- doc/src/sgml/ref/alter_table.sgml | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/alter_table.sgml b/doc/src/sgml/ref/alter_table.sgml index 8b251f9e5d..077c00373d 100644 --- a/doc/src/sgml/ref/alter_table.sgml +++ b/doc/src/sgml/ref/alter_table.sgml @@ -743,18 +743,25 @@ ALTER TABLE [ IF EXISTS ] name - A full table scan is performed on the table being attached to check that - no existing row in the table violates the partition constraint. It is - possible to avoid this scan by adding a valid CHECK - constraint to the table that would allow only the rows satisfying the - desired partition constraint before running this command. It will be - determined using such a constraint that the table need not be scanned - to validate the partition constraint. This does not work, however, if - any of the partition keys is an expression and the partition does not - accept NULL values. If attaching a list partition - that will not accept NULL values, also add - NOT NULL constraint to the partition key column, - unless it's an expression. + If the new partition is a regular table, a full table scan is performed + to check that no existing row in the table violates the partition + constraint. It is possible to avoid this scan by adding a valid + CHECK constraint to the table that would allow only + the rows satisfying the desired partition constraint before running this + command. It will be determined using such a constraint that the table + need not be scanned to validate the partition constraint. This does not + work, however, if any of the partition keys is an expression and the + partition does not accept NULL values. If attaching + a list partition that will not accept NULL values, + also add NOT NULL constraint to the partition key + column, unless it's an expression. + + + + If the new partition is a foreign table, nothing is done to verify + that all the rows in the foreign table obey the partition constraint. + (See the discussion in about + constraints on the foreign table.) -- cgit v1.2.3 From 3717dc149ecf44b8be95350a68605ba7299474fd Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Thu, 9 Mar 2017 15:50:40 -0800 Subject: Add amcheck extension to contrib. This is the beginning of a collection of SQL-callable functions to verify the integrity of data files. For now it only contains code to verify B-Tree indexes. This adds two SQL-callable functions, validating B-Tree consistency to a varying degree. Check the, extensive, docs for details. The goal is to later extend the coverage of the module to further access methods, possibly including the heap. Once checks for additional access methods exist, we'll likely add some "dispatch" functions that cover multiple access methods. Author: Peter Geoghegan, editorialized by Andres Freund Reviewed-By: Andres Freund, Tomas Vondra, Thomas Munro, Anastasia Lubennikova, Robert Haas, Amit Langote Discussion: CAM3SWZQzLMhMwmBqjzK+pRKXrNUZ4w90wYMUWfkeV8mZ3Debvw@mail.gmail.com --- contrib/Makefile | 1 + contrib/amcheck/Makefile | 21 + contrib/amcheck/amcheck--1.0.sql | 24 + contrib/amcheck/amcheck.control | 5 + contrib/amcheck/expected/check.out | 1 + contrib/amcheck/expected/check_btree.out | 90 +++ contrib/amcheck/sql/check.sql | 1 + contrib/amcheck/sql/check_btree.sql | 59 ++ contrib/amcheck/verify_nbtree.c | 1249 ++++++++++++++++++++++++++++++ doc/src/sgml/amcheck.sgml | 273 +++++++ doc/src/sgml/contrib.sgml | 1 + doc/src/sgml/filelist.sgml | 1 + src/tools/pgindent/typedefs.list | 2 + 13 files changed, 1728 insertions(+) create mode 100644 contrib/amcheck/Makefile create mode 100644 contrib/amcheck/amcheck--1.0.sql create mode 100644 contrib/amcheck/amcheck.control create mode 100644 contrib/amcheck/expected/check.out create mode 100644 contrib/amcheck/expected/check_btree.out create mode 100644 contrib/amcheck/sql/check.sql create mode 100644 contrib/amcheck/sql/check_btree.sql create mode 100644 contrib/amcheck/verify_nbtree.c create mode 100644 doc/src/sgml/amcheck.sgml (limited to 'doc/src') diff --git a/contrib/Makefile b/contrib/Makefile index 9a74e1b99f..e84eb67008 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -6,6 +6,7 @@ include $(top_builddir)/src/Makefile.global SUBDIRS = \ adminpack \ + amcheck \ auth_delay \ auto_explain \ bloom \ diff --git a/contrib/amcheck/Makefile b/contrib/amcheck/Makefile new file mode 100644 index 0000000000..43bed919ae --- /dev/null +++ b/contrib/amcheck/Makefile @@ -0,0 +1,21 @@ +# contrib/amcheck/Makefile + +MODULE_big = amcheck +OBJS = verify_nbtree.o $(WIN32RES) + +EXTENSION = amcheck +DATA = amcheck--1.0.sql +PGFILEDESC = "amcheck - function for verifying relation integrity" + +REGRESS = check check_btree + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = contrib/amcheck +top_builddir = ../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/contrib/amcheck/amcheck--1.0.sql b/contrib/amcheck/amcheck--1.0.sql new file mode 100644 index 0000000000..a6612d130c --- /dev/null +++ b/contrib/amcheck/amcheck--1.0.sql @@ -0,0 +1,24 @@ +/* contrib/amcheck/amcheck--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION amcheck" to load this file. \quit + +-- +-- bt_index_check() +-- +CREATE FUNCTION bt_index_check(index regclass) +RETURNS VOID +AS 'MODULE_PATHNAME', 'bt_index_check' +LANGUAGE C STRICT PARALLEL RESTRICTED; + +-- +-- bt_index_parent_check() +-- +CREATE FUNCTION bt_index_parent_check(index regclass) +RETURNS VOID +AS 'MODULE_PATHNAME', 'bt_index_parent_check' +LANGUAGE C STRICT PARALLEL RESTRICTED; + +-- Don't want these to be available to public +REVOKE ALL ON FUNCTION bt_index_check(regclass) FROM PUBLIC; +REVOKE ALL ON FUNCTION bt_index_parent_check(regclass) FROM PUBLIC; diff --git a/contrib/amcheck/amcheck.control b/contrib/amcheck/amcheck.control new file mode 100644 index 0000000000..05e2861d7a --- /dev/null +++ b/contrib/amcheck/amcheck.control @@ -0,0 +1,5 @@ +# amcheck extension +comment = 'functions for verifying relation integrity' +default_version = '1.0' +module_pathname = '$libdir/amcheck' +relocatable = true diff --git a/contrib/amcheck/expected/check.out b/contrib/amcheck/expected/check.out new file mode 100644 index 0000000000..5b3e6d530c --- /dev/null +++ b/contrib/amcheck/expected/check.out @@ -0,0 +1 @@ +CREATE EXTENSION amcheck; diff --git a/contrib/amcheck/expected/check_btree.out b/contrib/amcheck/expected/check_btree.out new file mode 100644 index 0000000000..612ce7799d --- /dev/null +++ b/contrib/amcheck/expected/check_btree.out @@ -0,0 +1,90 @@ +-- minimal test, basically just verifying that amcheck +CREATE TABLE bttest_a(id int8); +CREATE TABLE bttest_b(id int8); +INSERT INTO bttest_a SELECT * FROM generate_series(1, 100000); +INSERT INTO bttest_b SELECT * FROM generate_series(100000, 1, -1); +CREATE INDEX bttest_a_idx ON bttest_a USING btree (id); +CREATE INDEX bttest_b_idx ON bttest_b USING btree (id); +CREATE ROLE bttest_role; +-- verify permissions are checked (error due to function not callable) +SET ROLE bttest_role; +SELECT bt_index_check('bttest_a_idx'::regclass); +ERROR: permission denied for function bt_index_check +SELECT bt_index_parent_check('bttest_a_idx'::regclass); +ERROR: permission denied for function bt_index_parent_check +RESET ROLE; +-- we, intentionally, don't check relation permissions - it's useful +-- to run this cluster-wide with a restricted account, and as tested +-- above explicit permission has to be granted for that. +GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO bttest_role; +GRANT EXECUTE ON FUNCTION bt_index_parent_check(regclass) TO bttest_role; +SET ROLE bttest_role; +SELECT bt_index_check('bttest_a_idx'); + bt_index_check +---------------- + +(1 row) + +SELECT bt_index_parent_check('bttest_a_idx'); + bt_index_parent_check +----------------------- + +(1 row) + +RESET ROLE; +-- verify plain tables are rejected (error) +SELECT bt_index_check('bttest_a'); +ERROR: "bttest_a" is not an index +SELECT bt_index_parent_check('bttest_a'); +ERROR: "bttest_a" is not an index +-- verify non-existing indexes are rejected (error) +SELECT bt_index_check(17); +ERROR: could not open relation with OID 17 +SELECT bt_index_parent_check(17); +ERROR: could not open relation with OID 17 +-- verify wrong index types are rejected (error) +BEGIN; +CREATE INDEX bttest_a_brin_idx ON bttest_a USING brin(id); +SELECT bt_index_parent_check('bttest_a_brin_idx'); +ERROR: only B-Tree indexes are supported as targets for verification +DETAIL: Relation "bttest_a_brin_idx" is not a B-Tree index. +ROLLBACK; +-- normal check outside of xact +SELECT bt_index_check('bttest_a_idx'); + bt_index_check +---------------- + +(1 row) + +-- more expansive test +SELECT bt_index_parent_check('bttest_b_idx'); + bt_index_parent_check +----------------------- + +(1 row) + +BEGIN; +SELECT bt_index_check('bttest_a_idx'); + bt_index_check +---------------- + +(1 row) + +SELECT bt_index_parent_check('bttest_b_idx'); + bt_index_parent_check +----------------------- + +(1 row) + +-- make sure we don't have any leftover locks +SELECT * FROM pg_locks WHERE relation IN ('bttest_a_idx'::regclass, 'bttest_b_idx'::regclass); + locktype | database | relation | page | tuple | virtualxid | transactionid | classid | objid | objsubid | virtualtransaction | pid | mode | granted | fastpath +----------+----------+----------+------+-------+------------+---------------+---------+-------+----------+--------------------+-----+------+---------+---------- +(0 rows) + +COMMIT; +-- cleanup +DROP TABLE bttest_a; +DROP TABLE bttest_b; +DROP OWNED BY bttest_role; -- permissions +DROP ROLE bttest_role; diff --git a/contrib/amcheck/sql/check.sql b/contrib/amcheck/sql/check.sql new file mode 100644 index 0000000000..5b3e6d530c --- /dev/null +++ b/contrib/amcheck/sql/check.sql @@ -0,0 +1 @@ +CREATE EXTENSION amcheck; diff --git a/contrib/amcheck/sql/check_btree.sql b/contrib/amcheck/sql/check_btree.sql new file mode 100644 index 0000000000..783fb635e7 --- /dev/null +++ b/contrib/amcheck/sql/check_btree.sql @@ -0,0 +1,59 @@ +-- minimal test, basically just verifying that amcheck +CREATE TABLE bttest_a(id int8); +CREATE TABLE bttest_b(id int8); + +INSERT INTO bttest_a SELECT * FROM generate_series(1, 100000); +INSERT INTO bttest_b SELECT * FROM generate_series(100000, 1, -1); + +CREATE INDEX bttest_a_idx ON bttest_a USING btree (id); +CREATE INDEX bttest_b_idx ON bttest_b USING btree (id); + +CREATE ROLE bttest_role; + +-- verify permissions are checked (error due to function not callable) +SET ROLE bttest_role; +SELECT bt_index_check('bttest_a_idx'::regclass); +SELECT bt_index_parent_check('bttest_a_idx'::regclass); +RESET ROLE; + +-- we, intentionally, don't check relation permissions - it's useful +-- to run this cluster-wide with a restricted account, and as tested +-- above explicit permission has to be granted for that. +GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO bttest_role; +GRANT EXECUTE ON FUNCTION bt_index_parent_check(regclass) TO bttest_role; +SET ROLE bttest_role; +SELECT bt_index_check('bttest_a_idx'); +SELECT bt_index_parent_check('bttest_a_idx'); +RESET ROLE; + +-- verify plain tables are rejected (error) +SELECT bt_index_check('bttest_a'); +SELECT bt_index_parent_check('bttest_a'); + +-- verify non-existing indexes are rejected (error) +SELECT bt_index_check(17); +SELECT bt_index_parent_check(17); + +-- verify wrong index types are rejected (error) +BEGIN; +CREATE INDEX bttest_a_brin_idx ON bttest_a USING brin(id); +SELECT bt_index_parent_check('bttest_a_brin_idx'); +ROLLBACK; + +-- normal check outside of xact +SELECT bt_index_check('bttest_a_idx'); +-- more expansive test +SELECT bt_index_parent_check('bttest_b_idx'); + +BEGIN; +SELECT bt_index_check('bttest_a_idx'); +SELECT bt_index_parent_check('bttest_b_idx'); +-- make sure we don't have any leftover locks +SELECT * FROM pg_locks WHERE relation IN ('bttest_a_idx'::regclass, 'bttest_b_idx'::regclass); +COMMIT; + +-- cleanup +DROP TABLE bttest_a; +DROP TABLE bttest_b; +DROP OWNED BY bttest_role; -- permissions +DROP ROLE bttest_role; diff --git a/contrib/amcheck/verify_nbtree.c b/contrib/amcheck/verify_nbtree.c new file mode 100644 index 0000000000..5724aa6be3 --- /dev/null +++ b/contrib/amcheck/verify_nbtree.c @@ -0,0 +1,1249 @@ +/*------------------------------------------------------------------------- + * + * verify_nbtree.c + * Verifies the integrity of nbtree indexes based on invariants. + * + * For B-Tree indexes, verification includes checking that each page in the + * target index has items in logical order as reported by an insertion scankey + * (the insertion scankey sort-wise NULL semantics are needed for + * verification). + * + * + * Copyright (c) 2017, PostgreSQL Global Development Group + * + * IDENTIFICATION + * contrib/amcheck/verify_nbtree.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/nbtree.h" +#include "access/transam.h" +#include "catalog/index.h" +#include "catalog/pg_am.h" +#include "commands/tablecmds.h" +#include "miscadmin.h" +#include "storage/lmgr.h" +#include "utils/memutils.h" +#include "utils/snapmgr.h" + + +PG_MODULE_MAGIC; + +/* + * A B-Tree cannot possibly have this many levels, since there must be one + * block per level, which is bound by the range of BlockNumber: + */ +#define InvalidBtreeLevel ((uint32) InvalidBlockNumber) + +/* + * State associated with verifying a B-Tree index + * + * target is the point of reference for a verification operation. + * + * Other B-Tree pages may be allocated, but those are always auxiliary (e.g., + * they are current target's child pages). Conceptually, problems are only + * ever found in the current target page. Each page found by verification's + * left/right, top/bottom scan becomes the target exactly once. + */ +typedef struct BtreeCheckState +{ + /* + * Unchanging state, established at start of verification: + */ + + /* B-Tree Index Relation */ + Relation rel; + /* ExclusiveLock held? */ + bool exclusivelylocked; + /* Per-page context */ + MemoryContext targetcontext; + /* Buffer access strategy */ + BufferAccessStrategy checkstrategy; + + /* + * Mutable state, for verification of particular page: + */ + + /* Current target page */ + Page target; + /* Target block number */ + BlockNumber targetblock; + /* Target page's LSN */ + XLogRecPtr targetlsn; +} BtreeCheckState; + +/* + * Starting point for verifying an entire B-Tree index level + */ +typedef struct BtreeLevel +{ + /* Level number (0 is leaf page level). */ + uint32 level; + + /* Left most block on level. Scan of level begins here. */ + BlockNumber leftmost; + + /* Is this level reported as "true" root level by meta page? */ + bool istruerootlevel; +} BtreeLevel; + +PG_FUNCTION_INFO_V1(bt_index_check); +PG_FUNCTION_INFO_V1(bt_index_parent_check); + +static void bt_index_check_internal(Oid indrelid, bool parentcheck); +static inline void btree_index_checkable(Relation rel); +static void bt_check_every_level(Relation rel, bool exclusivelylocked); +static BtreeLevel bt_check_level_from_leftmost(BtreeCheckState *state, + BtreeLevel level); +static void bt_target_page_check(BtreeCheckState *state); +static ScanKey bt_right_page_check_scankey(BtreeCheckState *state); +static void bt_downlink_check(BtreeCheckState *state, BlockNumber childblock, + ScanKey targetkey); +static inline bool offset_is_negative_infinity(BTPageOpaque opaque, + OffsetNumber offset); +static inline bool invariant_leq_offset(BtreeCheckState *state, + ScanKey key, + OffsetNumber upperbound); +static inline bool invariant_geq_offset(BtreeCheckState *state, + ScanKey key, + OffsetNumber lowerbound); +static inline bool invariant_leq_nontarget_offset(BtreeCheckState *state, + Page other, + ScanKey key, + OffsetNumber upperbound); +static Page palloc_btree_page(BtreeCheckState *state, BlockNumber blocknum); + +/* + * bt_index_check(index regclass) + * + * Verify integrity of B-Tree index. + * + * Acquires AccessShareLock on heap & index relations. Does not consider + * invariants that exist between parent/child pages. + */ +Datum +bt_index_check(PG_FUNCTION_ARGS) +{ + Oid indrelid = PG_GETARG_OID(0); + + bt_index_check_internal(indrelid, false); + + PG_RETURN_VOID(); +} + +/* + * bt_index_parent_check(index regclass) + * + * Verify integrity of B-Tree index. + * + * Acquires ShareLock on heap & index relations. Verifies that downlinks in + * parent pages are valid lower bounds on child pages. + */ +Datum +bt_index_parent_check(PG_FUNCTION_ARGS) +{ + Oid indrelid = PG_GETARG_OID(0); + + bt_index_check_internal(indrelid, true); + + PG_RETURN_VOID(); +} + +/* + * Helper for bt_index_[parent_]check, coordinating the bulk of the work. + */ +static void +bt_index_check_internal(Oid indrelid, bool parentcheck) +{ + Oid heapid; + Relation indrel; + Relation heaprel; + LOCKMODE lockmode; + + if (parentcheck) + lockmode = ShareLock; + else + lockmode = AccessShareLock; + + /* + * We must lock table before index to avoid deadlocks. However, if the + * passed indrelid isn't an index then IndexGetRelation() will fail. + * Rather than emitting a not-very-helpful error message, postpone + * complaining, expecting that the is-it-an-index test below will fail. + * + * In hot standby mode this will raise an error when parentcheck is true. + */ + heapid = IndexGetRelation(indrelid, true); + if (OidIsValid(heapid)) + heaprel = heap_open(heapid, lockmode); + else + heaprel = NULL; + + /* + * Open the target index relations separately (like relation_openrv(), but + * with heap relation locked first to prevent deadlocking). In hot + * standby mode this will raise an error when parentcheck is true. + */ + indrel = index_open(indrelid, lockmode); + + /* + * Since we did the IndexGetRelation call above without any lock, it's + * barely possible that a race against an index drop/recreation could have + * netted us the wrong table. Although the table itself won't actually be + * examined during verification currently, a recheck still seems like a + * good idea. + */ + if (heaprel == NULL || heapid != IndexGetRelation(indrelid, false)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_TABLE), + errmsg("could not open parent table of index %s", + RelationGetRelationName(indrel)))); + + /* Relation suitable for checking as B-Tree? */ + btree_index_checkable(indrel); + + /* Check index */ + bt_check_every_level(indrel, parentcheck); + + /* + * Release locks early. That's ok here because nothing in the called + * routines will trigger shared cache invalidations to be sent, so we can + * relax the usual pattern of only releasing locks after commit. + */ + index_close(indrel, lockmode); + if (heaprel) + heap_close(heaprel, lockmode); +} + +/* + * Basic checks about the suitability of a relation for checking as a B-Tree + * index. + * + * NB: Intentionally not checking permissions, the function is normally not + * callable by non-superusers. If granted, it's useful to be able to check a + * whole cluster. + */ +static inline void +btree_index_checkable(Relation rel) +{ + if (rel->rd_rel->relkind != RELKIND_INDEX || + rel->rd_rel->relam != BTREE_AM_OID) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("only B-Tree indexes are supported as targets for verification"), + errdetail("Relation \"%s\" is not a B-Tree index.", + RelationGetRelationName(rel)))); + + if (RELATION_IS_OTHER_TEMP(rel)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot access temporary tables of other sessions"), + errdetail("Index \"%s\" is associated with temporary relation.", + RelationGetRelationName(rel)))); + + if (!IndexIsValid(rel->rd_index)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot check index \"%s\"", + RelationGetRelationName(rel)), + errdetail("Index is not valid"))); +} + +/* + * Main entry point for B-Tree SQL-callable functions. Walks the B-Tree in + * logical order, verifying invariants as it goes. + * + * It is the caller's responsibility to acquire appropriate heavyweight lock on + * the index relation, and advise us if extra checks are safe when an + * ExclusiveLock is held. + * + * An ExclusiveLock is generally assumed to prevent any kind of physical + * modification to the index structure, including modifications that VACUUM may + * make. This does not include setting of the LP_DEAD bit by concurrent index + * scans, although that is just metadata that is not able to directly affect + * any check performed here. Any concurrent process that might act on the + * LP_DEAD bit being set (recycle space) requires a heavyweight lock that + * cannot be held while we hold an ExclusiveLock. (Besides, even if that could + * happen, the ad-hoc recycling when a page might otherwise split is performed + * per-page, and requires an exclusive buffer lock, which wouldn't cause us + * trouble. _bt_delitems_vacuum() may only delete leaf items, and so the extra + * parent/child check cannot be affected.) + */ +static void +bt_check_every_level(Relation rel, bool exclusivelylocked) +{ + BtreeCheckState *state; + Page metapage; + BTMetaPageData *metad; + uint32 previouslevel; + BtreeLevel current; + + /* + * RecentGlobalXmin assertion matches index_getnext_tid(). See note on + * RecentGlobalXmin/B-Tree page deletion. + */ + Assert(TransactionIdIsValid(RecentGlobalXmin)); + + /* + * Initialize state for entire verification operation + */ + state = palloc(sizeof(BtreeCheckState)); + state->rel = rel; + state->exclusivelylocked = exclusivelylocked; + /* Create context for page */ + state->targetcontext = AllocSetContextCreate(CurrentMemoryContext, + "amcheck context", + ALLOCSET_DEFAULT_MINSIZE, + ALLOCSET_DEFAULT_INITSIZE, + ALLOCSET_DEFAULT_MAXSIZE); + state->checkstrategy = GetAccessStrategy(BAS_BULKREAD); + + /* Get true root block from meta-page */ + metapage = palloc_btree_page(state, BTREE_METAPAGE); + metad = BTPageGetMeta(metapage); + + /* + * Certain deletion patterns can result in "skinny" B-Tree indexes, where + * the fast root and true root differ. + * + * Start from the true root, not the fast root, unlike conventional index + * scans. This approach is more thorough, and removes the risk of + * following a stale fast root from the meta page. + */ + if (metad->btm_fastroot != metad->btm_root) + ereport(DEBUG1, + (errcode(ERRCODE_NO_DATA), + errmsg("harmless fast root mismatch in index %s", + RelationGetRelationName(rel)), + errdetail_internal("Fast root block %u (level %u) differs from true root block %u (level %u).", + metad->btm_fastroot, metad->btm_fastlevel, + metad->btm_root, metad->btm_level))); + + /* + * Starting at the root, verify every level. Move left to right, top to + * bottom. Note that there may be no pages other than the meta page (meta + * page can indicate that root is P_NONE when the index is totally empty). + */ + previouslevel = InvalidBtreeLevel; + current.level = metad->btm_level; + current.leftmost = metad->btm_root; + current.istruerootlevel = true; + while (current.leftmost != P_NONE) + { + /* + * Verify this level, and get left most page for next level down, if + * not at leaf level + */ + current = bt_check_level_from_leftmost(state, current); + + if (current.leftmost == InvalidBlockNumber) + ereport(ERROR, + (errcode(ERRCODE_INDEX_CORRUPTED), + errmsg("index \"%s\" has no valid pages on level below %u or first level", + RelationGetRelationName(rel), previouslevel))); + + previouslevel = current.level; + } + + /* Be tidy: */ + MemoryContextDelete(state->targetcontext); +} + +/* + * Given a left-most block at some level, move right, verifying each page + * individually (with more verification across pages for "exclusivelylocked" + * callers). Caller should pass the true root page as the leftmost initially, + * working their way down by passing what is returned for the last call here + * until level 0 (leaf page level) was reached. + * + * Returns state for next call, if any. This includes left-most block number + * one level lower that should be passed on next level/call, which is set to + * P_NONE on last call here (when leaf level is verified). Level numbers + * follow the nbtree convention: higher levels have higher numbers, because new + * levels are added only due to a root page split. Note that prior to the + * first root page split, the root is also a leaf page, so there is always a + * level 0 (leaf level), and it's always the last level processed. + * + * Note on memory management: State's per-page context is reset here, between + * each call to bt_target_page_check(). + */ +static BtreeLevel +bt_check_level_from_leftmost(BtreeCheckState *state, BtreeLevel level) +{ + /* State to establish early, concerning entire level */ + BTPageOpaque opaque; + MemoryContext oldcontext; + BtreeLevel nextleveldown; + + /* Variables for iterating across level using right links */ + BlockNumber leftcurrent = P_NONE; + BlockNumber current = level.leftmost; + + /* Initialize return state */ + nextleveldown.leftmost = InvalidBlockNumber; + nextleveldown.level = InvalidBtreeLevel; + nextleveldown.istruerootlevel = false; + + /* Use page-level context for duration of this call */ + oldcontext = MemoryContextSwitchTo(state->targetcontext); + + elog(DEBUG2, "verifying level %u%s", level.level, + level.istruerootlevel ? + " (true root level)" : level.level == 0 ? " (leaf level)" : ""); + + do + { + /* Don't rely on CHECK_FOR_INTERRUPTS() calls at lower level */ + CHECK_FOR_INTERRUPTS(); + + /* Initialize state for this iteration */ + state->targetblock = current; + state->target = palloc_btree_page(state, state->targetblock); + state->targetlsn = PageGetLSN(state->target); + + opaque = (BTPageOpaque) PageGetSpecialPointer(state->target); + + if (P_IGNORE(opaque)) + { + if (P_RIGHTMOST(opaque)) + ereport(ERROR, + (errcode(ERRCODE_INDEX_CORRUPTED), + errmsg("block %u fell off the end of index \"%s\"", + current, RelationGetRelationName(state->rel)))); + else + ereport(DEBUG1, + (errcode(ERRCODE_NO_DATA), + errmsg("block %u of index \"%s\" ignored", + current, RelationGetRelationName(state->rel)))); + goto nextpage; + } + else if (nextleveldown.leftmost == InvalidBlockNumber) + { + /* + * A concurrent page split could make the caller supplied leftmost + * block no longer contain the leftmost page, or no longer be the + * true root, but where that isn't possible due to heavyweight + * locking, check that the first valid page meets caller's + * expectations. + */ + if (state->exclusivelylocked) + { + if (!P_LEFTMOST(opaque)) + ereport(ERROR, + (errcode(ERRCODE_INDEX_CORRUPTED), + errmsg("block %u is not leftmost in index \"%s\"", + current, RelationGetRelationName(state->rel)))); + + if (level.istruerootlevel && !P_ISROOT(opaque)) + ereport(ERROR, + (errcode(ERRCODE_INDEX_CORRUPTED), + errmsg("block %u is not true root in index \"%s\"", + current, RelationGetRelationName(state->rel)))); + } + + /* + * Before beginning any non-trivial examination of level, prepare + * state for next bt_check_level_from_leftmost() invocation for + * the next level for the next level down (if any). + * + * There should be at least one non-ignorable page per level, + * unless this is the leaf level, which is assumed by caller to be + * final level. + */ + if (!P_ISLEAF(opaque)) + { + IndexTuple itup; + ItemId itemid; + + /* Internal page -- downlink gets leftmost on next level */ + itemid = PageGetItemId(state->target, P_FIRSTDATAKEY(opaque)); + itup = (IndexTuple) PageGetItem(state->target, itemid); + nextleveldown.leftmost = ItemPointerGetBlockNumber(&(itup->t_tid)); + nextleveldown.level = opaque->btpo.level - 1; + } + else + { + /* + * Leaf page -- final level caller must process. + * + * Note that this could also be the root page, if there has + * been no root page split yet. + */ + nextleveldown.leftmost = P_NONE; + nextleveldown.level = InvalidBtreeLevel; + } + + /* + * Finished setting up state for this call/level. Control will + * never end up back here in any future loop iteration for this + * level. + */ + } + + if (state->exclusivelylocked && opaque->btpo_prev != leftcurrent) + ereport(ERROR, + (errcode(ERRCODE_INDEX_CORRUPTED), + errmsg("left link/right link pair in index \"%s\" not in agreement", + RelationGetRelationName(state->rel)), + errdetail_internal("Block=%u left block=%u left link from block=%u.", + current, leftcurrent, opaque->btpo_prev))); + + /* Check level, which must be valid for non-ignorable page */ + if (level.level != opaque->btpo.level) + ereport(ERROR, + (errcode(ERRCODE_INDEX_CORRUPTED), + errmsg("leftmost down link for level points to block in index \"%s\" whose level is not one level down", + RelationGetRelationName(state->rel)), + errdetail_internal("Block pointed to=%u expected level=%u level in pointed to block=%u.", + current, level.level, opaque->btpo.level))); + + /* Verify invariants for page -- all important checks occur here */ + bt_target_page_check(state); + +nextpage: + + /* Try to detect circular links */ + if (current == leftcurrent || current == opaque->btpo_prev) + ereport(ERROR, + (errcode(ERRCODE_INDEX_CORRUPTED), + errmsg("circular link chain found in block %u of index \"%s\"", + current, RelationGetRelationName(state->rel)))); + + leftcurrent = current; + current = opaque->btpo_next; + + /* Free page and associated memory for this iteration */ + MemoryContextReset(state->targetcontext); + } + while (current != P_NONE); + + /* Don't change context for caller */ + MemoryContextSwitchTo(oldcontext); + + return nextleveldown; +} + +/* + * Function performs the following checks on target page, or pages ancillary to + * target page: + * + * - That every "real" data item is less than or equal to the high key, which + * is an upper bound on the items on the pages (where there is a high key at + * all -- pages that are rightmost lack one). + * + * - That within the page, every "real" item is less than or equal to the item + * immediately to its right, if any (i.e., that the items are in order within + * the page, so that the binary searches performed by index scans are sane). + * + * - That the last item stored on the page is less than or equal to the first + * "real" data item on the page to the right (if such a first item is + * available). + * + * Furthermore, when state passed shows ExclusiveLock held, and target page is + * internal page, function also checks: + * + * - That all child pages respect downlinks lower bound. + * + * Note: Memory allocated in this routine is expected to be released by caller + * resetting state->targetcontext. + */ +static void +bt_target_page_check(BtreeCheckState *state) +{ + OffsetNumber offset; + OffsetNumber max; + BTPageOpaque topaque; + + topaque = (BTPageOpaque) PageGetSpecialPointer(state->target); + max = PageGetMaxOffsetNumber(state->target); + + elog(DEBUG2, "verifying %u items on %s block %u", max, + P_ISLEAF(topaque) ? "leaf" : "internal", state->targetblock); + + /* + * Loop over page items, starting from first non-highkey item, not high + * key (if any). Also, immediately skip "negative infinity" real item (if + * any). + */ + for (offset = P_FIRSTDATAKEY(topaque); + offset <= max; + offset = OffsetNumberNext(offset)) + { + ItemId itemid; + IndexTuple itup; + ScanKey skey; + + CHECK_FOR_INTERRUPTS(); + + /* + * Don't try to generate scankey using "negative infinity" garbage + * data + */ + if (offset_is_negative_infinity(topaque, offset)) + continue; + + /* Build insertion scankey for current page offset */ + itemid = PageGetItemId(state->target, offset); + itup = (IndexTuple) PageGetItem(state->target, itemid); + skey = _bt_mkscankey(state->rel, itup); + + /* + * * High key check * + * + * If there is a high key (if this is not the rightmost page on its + * entire level), check that high key actually is upper bound on all + * page items. + * + * We prefer to check all items against high key rather than checking + * just the first and trusting that the operator class obeys the + * transitive law (which implies that all subsequent items also + * respected the high key invariant if they pass the item order + * check). + * + * Ideally, we'd compare every item in the index against every other + * item in the index, and not trust opclass obedience of the + * transitive law to bridge the gap between children and their + * grandparents (as well as great-grandparents, and so on). We don't + * go to those lengths because that would be prohibitively expensive, + * and probably not markedly more effective in practice. + */ + if (!P_RIGHTMOST(topaque) && + !invariant_leq_offset(state, skey, P_HIKEY)) + { + char *itid, + *htid; + + itid = psprintf("(%u,%u)", state->targetblock, offset); + htid = psprintf("(%u,%u)", + ItemPointerGetBlockNumber(&(itup->t_tid)), + ItemPointerGetOffsetNumber(&(itup->t_tid))); + + ereport(ERROR, + (errcode(ERRCODE_INDEX_CORRUPTED), + errmsg("high key invariant violated for index \"%s\"", + RelationGetRelationName(state->rel)), + errdetail_internal("Index tid=%s points to %s tid=%s page lsn=%X/%X.", + itid, + P_ISLEAF(topaque) ? "heap" : "index", + htid, + (uint32) (state->targetlsn >> 32), + (uint32) state->targetlsn))); + } + + /* + * * Item order check * + * + * Check that items are stored on page in logical order, by checking + * current item is less than or equal to next item (if any). + */ + if (OffsetNumberNext(offset) <= max && + !invariant_leq_offset(state, skey, + OffsetNumberNext(offset))) + { + char *itid, + *htid, + *nitid, + *nhtid; + + itid = psprintf("(%u,%u)", state->targetblock, offset); + htid = psprintf("(%u,%u)", + ItemPointerGetBlockNumber(&(itup->t_tid)), + ItemPointerGetOffsetNumber(&(itup->t_tid))); + nitid = psprintf("(%u,%u)", state->targetblock, + OffsetNumberNext(offset)); + + /* Reuse itup to get pointed-to heap location of second item */ + itemid = PageGetItemId(state->target, OffsetNumberNext(offset)); + itup = (IndexTuple) PageGetItem(state->target, itemid); + nhtid = psprintf("(%u,%u)", + ItemPointerGetBlockNumber(&(itup->t_tid)), + ItemPointerGetOffsetNumber(&(itup->t_tid))); + + ereport(ERROR, + (errcode(ERRCODE_INDEX_CORRUPTED), + errmsg("item order invariant violated for index \"%s\"", + RelationGetRelationName(state->rel)), + errdetail_internal("Lower index tid=%s (points to %s tid=%s) " + "higher index tid=%s (points to %s tid=%s) " + "page lsn=%X/%X.", + itid, + P_ISLEAF(topaque) ? "heap" : "index", + htid, + nitid, + P_ISLEAF(topaque) ? "heap" : "index", + nhtid, + (uint32) (state->targetlsn >> 32), + (uint32) state->targetlsn))); + } + + /* + * * Last item check * + * + * Check last item against next/right page's first data item's when + * last item on page is reached. This additional check can detect + * transposed pages. + * + * This check is similar to the item order check that will have + * already been performed for every other "real" item on target page + * when last item is checked. The difference is that the next item + * (the item that is compared to target's last item) needs to come + * from the next/sibling page. There may not be such an item + * available from sibling for various reasons, though (e.g., target is + * the rightmost page on level). + */ + else if (offset == max) + { + ScanKey rightkey; + + /* Get item in next/right page */ + rightkey = bt_right_page_check_scankey(state); + + if (rightkey && + !invariant_geq_offset(state, rightkey, max)) + { + /* + * As explained at length in bt_right_page_check_scankey(), + * there is a known !exclusivelylocked race that could account + * for apparent violation of invariant, which we must check + * for before actually proceeding with raising error. Our + * canary condition is that target page was deleted. + */ + if (!state->exclusivelylocked) + { + /* Get fresh copy of target page */ + state->target = palloc_btree_page(state, state->targetblock); + /* Note that we deliberately do not update target LSN */ + topaque = (BTPageOpaque) PageGetSpecialPointer(state->target); + + /* + * All !exclusivelylocked checks now performed; just + * return + */ + if (P_IGNORE(topaque)) + return; + } + + ereport(ERROR, + (errcode(ERRCODE_INDEX_CORRUPTED), + errmsg("cross page item order invariant violated for index \"%s\"", + RelationGetRelationName(state->rel)), + errdetail_internal("Last item on page tid=(%u,%u) page lsn=%X/%X.", + state->targetblock, offset, + (uint32) (state->targetlsn >> 32), + (uint32) state->targetlsn))); + } + } + + /* + * * Downlink check * + * + * Additional check of child items iff this is an internal page and + * caller holds an ExclusiveLock. This happens for every downlink + * (item) in target excluding the negative-infinity downlink (again, + * this is because it has no useful value to compare). + */ + if (!P_ISLEAF(topaque) && state->exclusivelylocked) + { + BlockNumber childblock = ItemPointerGetBlockNumber(&(itup->t_tid)); + + bt_downlink_check(state, childblock, skey); + } + } +} + +/* + * Return a scankey for an item on page to right of current target (or the + * first non-ignorable page), sufficient to check ordering invariant on last + * item in current target page. Returned scankey relies on local memory + * allocated for the child page, which caller cannot pfree(). Caller's memory + * context should be reset between calls here. + * + * This is the first data item, and so all adjacent items are checked against + * their immediate sibling item (which may be on a sibling page, or even a + * "cousin" page at parent boundaries where target's rightlink points to page + * with different parent page). If no such valid item is available, return + * NULL instead. + * + * Note that !exclusivelylocked callers must reverify that target page has not + * been concurrently deleted. + */ +static ScanKey +bt_right_page_check_scankey(BtreeCheckState *state) +{ + BTPageOpaque opaque; + ItemId rightitem; + BlockNumber targetnext; + Page rightpage; + OffsetNumber nline; + + /* Determine target's next block number */ + opaque = (BTPageOpaque) PageGetSpecialPointer(state->target); + + /* If target is already rightmost, no right sibling; nothing to do here */ + if (P_RIGHTMOST(opaque)) + return NULL; + + /* + * General notes on concurrent page splits and page deletion: + * + * Routines like _bt_search() don't require *any* page split interlock + * when descending the tree, including something very light like a buffer + * pin. That's why it's okay that we don't either. This avoidance of any + * need to "couple" buffer locks is the raison d' etre of the Lehman & Yao + * algorithm, in fact. + * + * That leaves deletion. A deleted page won't actually be recycled by + * VACUUM early enough for us to fail to at least follow its right link + * (or left link, or downlink) and find its sibling, because recycling + * does not occur until no possible index scan could land on the page. + * Index scans can follow links with nothing more than their snapshot as + * an interlock and be sure of at least that much. (See page + * recycling/RecentGlobalXmin notes in nbtree README.) + * + * Furthermore, it's okay if we follow a rightlink and find a half-dead or + * dead (ignorable) page one or more times. There will either be a + * further right link to follow that leads to a live page before too long + * (before passing by parent's rightmost child), or we will find the end + * of the entire level instead (possible when parent page is itself the + * rightmost on its level). + */ + targetnext = opaque->btpo_next; + for (;;) + { + CHECK_FOR_INTERRUPTS(); + + rightpage = palloc_btree_page(state, targetnext); + opaque = (BTPageOpaque) PageGetSpecialPointer(rightpage); + + if (!P_IGNORE(opaque) || P_RIGHTMOST(opaque)) + break; + + /* We landed on a deleted page, so step right to find a live page */ + targetnext = opaque->btpo_next; + ereport(DEBUG1, + (errcode(ERRCODE_NO_DATA), + errmsg("level %u leftmost page of index \"%s\" was found deleted or half dead", + opaque->btpo.level, RelationGetRelationName(state->rel)), + errdetail_internal("Deleted page found when building scankey from right sibling."))); + + /* Be slightly more pro-active in freeing this memory, just in case */ + pfree(rightpage); + } + + /* + * No ExclusiveLock held case -- why it's safe to proceed. + * + * Problem: + * + * We must avoid false positive reports of corruption when caller treats + * item returned here as an upper bound on target's last item. In + * general, false positives are disallowed. Avoiding them here when + * caller is !exclusivelylocked is subtle. + * + * A concurrent page deletion by VACUUM of the target page can result in + * the insertion of items on to this right sibling page that would + * previously have been inserted on our target page. There might have + * been insertions that followed the target's downlink after it was made + * to point to right sibling instead of target by page deletion's first + * phase. The inserters insert items that would belong on target page. + * This race is very tight, but it's possible. This is our only problem. + * + * Non-problems: + * + * We are not hindered by a concurrent page split of the target; we'll + * never land on the second half of the page anyway. A concurrent split + * of the right page will also not matter, because the first data item + * remains the same within the left half, which we'll reliably land on. If + * we had to skip over ignorable/deleted pages, it cannot matter because + * their key space has already been atomically merged with the first + * non-ignorable page we eventually find (doesn't matter whether the page + * we eventually find is a true sibling or a cousin of target, which we go + * into below). + * + * Solution: + * + * Caller knows that it should reverify that target is not ignorable + * (half-dead or deleted) when cross-page sibling item comparison appears + * to indicate corruption (invariant fails). This detects the single race + * condition that exists for caller. This is correct because the + * continued existence of target block as non-ignorable (not half-dead or + * deleted) implies that target page was not merged into from the right by + * deletion; the key space at or after target never moved left. Target's + * parent either has the same downlink to target as before, or a <= + * downlink due to deletion at the left of target. Target either has the + * same highkey as before, or a highkey <= before when there is a page + * split. (The rightmost concurrently-split-from-target-page page will + * still have the same highkey as target was originally found to have, + * which for our purposes is equivalent to target's highkey itself never + * changing, since we reliably skip over + * concurrently-split-from-target-page pages.) + * + * In simpler terms, we allow that the key space of the target may expand + * left (the key space can move left on the left side of target only), but + * the target key space cannot expand right and get ahead of us without + * our detecting it. The key space of the target cannot shrink, unless it + * shrinks to zero due to the deletion of the original page, our canary + * condition. (To be very precise, we're a bit stricter than that because + * it might just have been that the target page split and only the + * original target page was deleted. We can be more strict, just not more + * lax.) + * + * Top level tree walk caller moves on to next page (makes it the new + * target) following recovery from this race. (cf. The rationale for + * child/downlink verification needing an ExclusiveLock within + * bt_downlink_check(), where page deletion is also the main source of + * trouble.) + * + * Note that it doesn't matter if right sibling page here is actually a + * cousin page, because in order for the key space to be readjusted in a + * way that causes us issues in next level up (guiding problematic + * concurrent insertions to the cousin from the grandparent rather than to + * the sibling from the parent), there'd have to be page deletion of + * target's parent page (affecting target's parent's downlink in target's + * grandparent page). Internal page deletion only occurs when there are + * no child pages (they were all fully deleted), and caller is checking + * that the target's parent has at least one non-deleted (so + * non-ignorable) child: the target page. (Note that the first phase of + * deletion atomically marks the page to be deleted half-dead/ignorable at + * the same time downlink in its parent is removed, so caller will + * definitely not fail to detect that this happened.) + * + * This trick is inspired by the method backward scans use for dealing + * with concurrent page splits; concurrent page deletion is a problem that + * similarly receives special consideration sometimes (it's possible that + * the backwards scan will re-read its "original" block after failing to + * find a right-link to it, having already moved in the opposite direction + * (right/"forwards") a few times to try to locate one). Just like us, + * that happens only to determine if there was a concurrent page deletion + * of a reference page, and just like us if there was a page deletion of + * that reference page it means we can move on from caring about the + * reference page. See the nbtree README for a full description of how + * that works. + */ + nline = PageGetMaxOffsetNumber(rightpage); + + /* + * Get first data item, if any + */ + if (P_ISLEAF(opaque) && nline >= P_FIRSTDATAKEY(opaque)) + { + /* Return first data item (if any) */ + rightitem = PageGetItemId(rightpage, P_FIRSTDATAKEY(opaque)); + } + else if (!P_ISLEAF(opaque) && + nline >= OffsetNumberNext(P_FIRSTDATAKEY(opaque))) + { + /* + * Return first item after the internal page's "negative infinity" + * item + */ + rightitem = PageGetItemId(rightpage, + OffsetNumberNext(P_FIRSTDATAKEY(opaque))); + } + else + { + /* + * No first item. Page is probably empty leaf page, but it's also + * possible that it's an internal page with only a negative infinity + * item. + */ + ereport(DEBUG1, + (errcode(ERRCODE_NO_DATA), + errmsg("%s block %u of index \"%s\" has no first data item", + P_ISLEAF(opaque) ? "leaf" : "internal", targetnext, + RelationGetRelationName(state->rel)))); + return NULL; + } + + /* + * Return first real item scankey. Note that this relies on right page + * memory remaining allocated. + */ + return _bt_mkscankey(state->rel, + (IndexTuple) PageGetItem(rightpage, rightitem)); +} + +/* + * Checks one of target's downlink against its child page. + * + * Conceptually, the target page continues to be what is checked here. The + * target block is still blamed in the event of finding an invariant violation. + * The downlink insertion into the target is probably where any problem raised + * here arises, and there is no such thing as a parent link, so doing the + * verification this way around is much more practical. + */ +static void +bt_downlink_check(BtreeCheckState *state, BlockNumber childblock, + ScanKey targetkey) +{ + OffsetNumber offset; + OffsetNumber maxoffset; + Page child; + BTPageOpaque copaque; + + /* + * Caller must have ExclusiveLock on target relation, because of + * considerations around page deletion by VACUUM. + * + * NB: In general, page deletion deletes the right sibling's downlink, not + * the downlink of the page being deleted; the deleted page's downlink is + * reused for its sibling. The key space is thereby consolidated between + * the deleted page and its right sibling. (We cannot delete a parent + * page's rightmost child unless it is the last child page, and we intend + * to also delete the parent itself.) + * + * If this verification happened without an ExclusiveLock, the following + * race condition could cause false positives: + * + * In general, concurrent page deletion might occur, including deletion of + * the left sibling of the child page that is examined here. If such a + * page deletion were to occur, closely followed by an insertion into the + * newly expanded key space of the child, a window for the false positive + * opens up: the stale parent/target downlink originally followed to get + * to the child legitimately ceases to be a lower bound on all items in + * the page, since the key space was concurrently expanded "left". + * (Insertion followed the "new" downlink for the child, not our now-stale + * downlink, which was concurrently physically removed in target/parent as + * part of deletion's first phase.) + * + * Note that while the cross-page-same-level last item check uses a trick + * that allows it to perform verification for !exclusivelylocked callers, + * a similar trick seems difficult here. The trick that that other check + * uses is, in essence, to lock down race conditions to those that occur + * due to concurrent page deletion of the target; that's a race that can + * be reliably detected before actually reporting corruption. + * + * On the other hand, we'd need to lock down race conditions involving + * deletion of child's left page, for long enough to read the child page + * into memory (in other words, a scheme with concurrently held buffer + * locks on both child and left-of-child pages). That's unacceptable for + * amcheck functions on general principle, though. + */ + Assert(state->exclusivelylocked); + + /* + * Verify child page has the downlink key from target page (its parent) as + * a lower bound. + * + * Check all items, rather than checking just the first and trusting that + * the operator class obeys the transitive law. + */ + child = palloc_btree_page(state, childblock); + copaque = (BTPageOpaque) PageGetSpecialPointer(child); + maxoffset = PageGetMaxOffsetNumber(child); + + for (offset = P_FIRSTDATAKEY(copaque); + offset <= maxoffset; + offset = OffsetNumberNext(offset)) + { + /* + * Skip comparison of target page key against "negative infinity" + * item, if any. Checking it would indicate that it's not an upper + * bound, but that's only because of the hard-coding within + * _bt_compare(). + */ + if (offset_is_negative_infinity(copaque, offset)) + continue; + + if (!invariant_leq_nontarget_offset(state, child, + targetkey, offset)) + ereport(ERROR, + (errcode(ERRCODE_INDEX_CORRUPTED), + errmsg("down-link lower bound invariant violated for index \"%s\"", + RelationGetRelationName(state->rel)), + errdetail_internal("Parent block=%u child index tid=(%u,%u) parent page lsn=%X/%X.", + state->targetblock, childblock, offset, + (uint32) (state->targetlsn >> 32), + (uint32) state->targetlsn))); + } + + pfree(child); +} + +/* + * Is particular offset within page (whose special state is passed by caller) + * the page negative-infinity item? + * + * As noted in comments above _bt_compare(), there is special handling of the + * first data item as a "negative infinity" item. The hard-coding within + * _bt_compare() makes comparing this item for the purposes of verification + * pointless at best, since the IndexTuple only contains a valid TID (a + * reference TID to child page). + */ +static inline bool +offset_is_negative_infinity(BTPageOpaque opaque, OffsetNumber offset) +{ + /* + * For internal pages only, the first item after high key, if any, is + * negative infinity item. Internal pages always have a negative infinity + * item, whereas leaf pages never have one. This implies that negative + * infinity item is either first or second line item, or there is none + * within page. + * + * Right-most pages don't have a high key, but could be said to + * conceptually have a "positive infinity" high key. Thus, there is a + * symmetry between down link items in parent pages, and high keys in + * children. Together, they represent the part of the key space that + * belongs to each page in the index. For example, all children of the + * root page will have negative infinity as a lower bound from root + * negative infinity downlink, and positive infinity as an upper bound + * (implicitly, from "imaginary" positive infinity high key in root). + */ + return !P_ISLEAF(opaque) && offset == P_FIRSTDATAKEY(opaque); +} + +/* + * Does the invariant hold that the key is less than or equal to a given upper + * bound offset item? + * + * If this function returns false, convention is that caller throws error due + * to corruption. + */ +static inline bool +invariant_leq_offset(BtreeCheckState *state, ScanKey key, + OffsetNumber upperbound) +{ + int16 natts = state->rel->rd_rel->relnatts; + int32 cmp; + + cmp = _bt_compare(state->rel, natts, key, state->target, upperbound); + + return cmp <= 0; +} + +/* + * Does the invariant hold that the key is greater than or equal to a given + * lower bound offset item? + * + * If this function returns false, convention is that caller throws error due + * to corruption. + */ +static inline bool +invariant_geq_offset(BtreeCheckState *state, ScanKey key, + OffsetNumber lowerbound) +{ + int16 natts = state->rel->rd_rel->relnatts; + int32 cmp; + + cmp = _bt_compare(state->rel, natts, key, state->target, lowerbound); + + return cmp >= 0; +} + +/* + * Does the invariant hold that the key is less than or equal to a given upper + * bound offset item, with the offset relating to a caller-supplied page that + * is not the current target page? Caller's non-target page is typically a + * child page of the target, checked as part of checking a property of the + * target page (i.e. the key comes from the target). + * + * If this function returns false, convention is that caller throws error due + * to corruption. + */ +static inline bool +invariant_leq_nontarget_offset(BtreeCheckState *state, + Page nontarget, ScanKey key, + OffsetNumber upperbound) +{ + int16 natts = state->rel->rd_rel->relnatts; + int32 cmp; + + cmp = _bt_compare(state->rel, natts, key, nontarget, upperbound); + + return cmp <= 0; +} + +/* + * Given a block number of a B-Tree page, return page in palloc()'d memory. + * While at it, perform some basic checks of the page. + * + * There is never an attempt to get a consistent view of multiple pages using + * multiple concurrent buffer locks; in general, we only acquire a single pin + * and buffer lock at a time, which is often all that the nbtree code requires. + * + * Operating on a copy of the page is useful because it prevents control + * getting stuck in an uninterruptible state when an underlying operator class + * misbehaves. + */ +static Page +palloc_btree_page(BtreeCheckState *state, BlockNumber blocknum) +{ + Buffer buffer; + Page page; + BTPageOpaque opaque; + + page = palloc(BLCKSZ); + + /* + * We copy the page into local storage to avoid holding pin on the buffer + * longer than we must. + */ + buffer = ReadBufferExtended(state->rel, MAIN_FORKNUM, blocknum, RBM_NORMAL, + state->checkstrategy); + LockBuffer(buffer, BT_READ); + + /* + * Perform the same basic sanity checking that nbtree itself performs for + * every page: + */ + _bt_checkpage(state->rel, buffer); + + /* Only use copy of page in palloc()'d memory */ + memcpy(page, BufferGetPage(buffer), BLCKSZ); + UnlockReleaseBuffer(buffer); + + opaque = (BTPageOpaque) PageGetSpecialPointer(page); + + if (opaque->btpo_flags & BTP_META && blocknum != BTREE_METAPAGE) + ereport(ERROR, + (errcode(ERRCODE_INDEX_CORRUPTED), + errmsg("invalid meta page found at block %u in index \"%s\"", + blocknum, RelationGetRelationName(state->rel)))); + + /* Check page from block that ought to be meta page */ + if (blocknum == BTREE_METAPAGE) + { + BTMetaPageData *metad = BTPageGetMeta(page); + + if (!(opaque->btpo_flags & BTP_META) || + metad->btm_magic != BTREE_MAGIC) + ereport(ERROR, + (errcode(ERRCODE_INDEX_CORRUPTED), + errmsg("index \"%s\" meta page is corrupt", + RelationGetRelationName(state->rel)))); + + if (metad->btm_version != BTREE_VERSION) + ereport(ERROR, + (errcode(ERRCODE_INDEX_CORRUPTED), + errmsg("version mismatch in index \"%s\": file version %d, code version %d", + RelationGetRelationName(state->rel), + metad->btm_version, BTREE_VERSION))); + } + + /* + * Deleted pages have no sane "level" field, so can only check non-deleted + * page level + */ + if (P_ISLEAF(opaque) && !P_ISDELETED(opaque) && opaque->btpo.level != 0) + ereport(ERROR, + (errcode(ERRCODE_INDEX_CORRUPTED), + errmsg("invalid leaf page level %u for block %u in index \"%s\"", + opaque->btpo.level, blocknum, RelationGetRelationName(state->rel)))); + + if (blocknum != BTREE_METAPAGE && !P_ISLEAF(opaque) && + !P_ISDELETED(opaque) && opaque->btpo.level == 0) + ereport(ERROR, + (errcode(ERRCODE_INDEX_CORRUPTED), + errmsg("invalid internal page level 0 for block %u in index \"%s\"", + opaque->btpo.level, RelationGetRelationName(state->rel)))); + + if (!P_ISLEAF(opaque) && P_HAS_GARBAGE(opaque)) + ereport(ERROR, + (errcode(ERRCODE_INDEX_CORRUPTED), + errmsg("internal page block %u in index \"%s\" has garbage items", + blocknum, RelationGetRelationName(state->rel)))); + + return page; +} diff --git a/doc/src/sgml/amcheck.sgml b/doc/src/sgml/amcheck.sgml new file mode 100644 index 0000000000..893a5b41d9 --- /dev/null +++ b/doc/src/sgml/amcheck.sgml @@ -0,0 +1,273 @@ + + + + amcheck + + + amcheck + + + + The amcheck module provides functions that allow you to + verify the logical consistency of the structure of indexes. If the + structure appears to be valid, no error is raised. + + + + The functions verify various invariants in the + structure of the representation of particular indexes. The + correctness of the access method functions behind index scans and + other important operations relies on these invariants always + holding. For example, certain functions verify, among other things, + that all B-Tree pages have items in logical order (e.g., + for B-Tree indexes on text, index tuples should be in + collated lexical order). If that particular invariant somehow fails + to hold, we can expect binary searches on the affected page to + incorrectly guide index scans, resulting in wrong answers to SQL + queries. + + + Verification is performed using the same procedures as those used by + index scans themselves, which may be user-defined operator class + code. For example, B-Tree index verification relies on comparisons + made with one or more B-Tree support function 1 routines. See for details of operator class support + functions. + + + amcheck functions may be used only by superusers. + + + + Functions + + + + + bt_index_check(index regclass) returns void + + bt_index_check + + + + + + bt_index_check tests that its target, a + B-Tree index, respects a variety of invariants. Example usage: + +test=# SELECT bt_index_check(c.oid), c.relname, c.relpages +FROM pg_index i +JOIN pg_opclass op ON i.indclass[0] = op.oid +JOIN pg_am am ON op.opcmethod = am.oid +JOIN pg_class c ON i.indexrelid = c.oid +JOIN pg_namespace n ON c.relnamespace = n.oid +WHERE am.amname = 'btree' AND n.nspname = 'pg_catalog' +-- Don't check temp tables, which may be from another session: +AND c.relpersistence != 't' +-- Function may throw an error when this is omitted: +AND i.indisready AND i.indisvalid +ORDER BY c.relpages DESC LIMIT 10; + bt_index_check | relname | relpages +----------------+---------------------------------+---------- + | pg_depend_reference_index | 43 + | pg_depend_depender_index | 40 + | pg_proc_proname_args_nsp_index | 31 + | pg_description_o_c_o_index | 21 + | pg_attribute_relid_attnam_index | 14 + | pg_proc_oid_index | 10 + | pg_attribute_relid_attnum_index | 9 + | pg_amproc_fam_proc_index | 5 + | pg_amop_opr_fam_index | 5 + | pg_amop_fam_strat_index | 5 +(10 rows) + + This example shows a session that performs verification of every + catalog index in the database test. Details of just + the 10 largest indexes verified are displayed. Since no error + is raised, all indexes tested appear to be logically consistent. + Naturally, this query could easily be changed to call + bt_index_check for every index in the + database where verification is supported. + + + bt_index_check acquires an AccessShareLock + on the target index and the heap relation it belongs to. This lock mode + is the same lock mode acquired on relations by simple + SELECT statements. + bt_index_check does not verify invariants + that span child/parent relationships, nor does it verify that + the target index is consistent with its heap relation. When a + routine, lightweight test for corruption is required in a live + production environment, using + bt_index_check often provides the best + trade-off between thoroughness of verification and limiting the + impact on application performance and availability. + + + + + + + bt_index_parent_check(index regclass) returns void + + bt_index_parent_check + + + + + + bt_index_parent_check tests that its + target, a B-Tree index, respects a variety of invariants. The + checks performed by bt_index_parent_check + are a superset of the checks performed by + bt_index_check. + bt_index_parent_check can be thought of as + a more thorough variant of bt_index_check: + unlike bt_index_check, + bt_index_parent_check also checks + invariants that span parent/child relationships. However, it + does not verify that the target index is consistent with its + heap relation. bt_index_parent_check + follows the general convention of raising an error if it finds a + logical inconsistency or other problem. + + + A ShareLock is required on the target index by + bt_index_parent_check (a + ShareLock is also acquired on the heap relation). + These locks prevent concurrent data modification from + INSERT, UPDATE, and DELETE + commands. The locks also prevent the underlying relation from + being concurrently processed by VACUUM, as well as + all other utility commands. Note that the function holds locks + only while running, not for the entire transaction. + + + bt_index_parent_check's additional + verification is more likely to detect various pathological + cases. These cases may involve an incorrectly implemented + B-Tree operator class used by the index that is checked, or, + hypothetically, undiscovered bugs in the underlying B-Tree index + access method code. Note that + bt_index_parent_check cannot be used when + Hot Standby mode is enabled (i.e., on read-only physical + replicas), unlike bt_index_check. + + + + + + + + Using <filename>amcheck</> effectively + + + amcheck can be effective at detecting various types of + failure modes that data page + checksums will always fail to catch. These include: + + + + + Structural inconsistencies caused by incorrect operator class + implementations. + + + This includes issues caused by the comparison rules of operating + system collations changing. Comparisons of datums of a collatable + type like text must be immutable (just as all + comparisons used for B-Tree index scans must be immutable), which + implies that operating system collation rules must never change. + Though rare, updates to operating system collation rules can + cause these issues. More commonly, an inconsistency in the + collation order between a master server and a standby server is + implicated, possibly because the major operating + system version in use is inconsistent. Such inconsistencies will + generally only arise on standby servers, and so can generally + only be detected on standby servers. + + + If a problem like this arises, it may not affect each individual + index that is ordered using an affected collation, simply because + indexed values might happen to have the same + absolute ordering regardless of the behavioral inconsistency. See + and for + further details about how PostgreSQL uses + operating system locales and collations. + + + + + Corruption caused by hypothetical undiscovered bugs in the + underlying PostgreSQL access method code or sort + code. + + + Automatic verification of the structural integrity of indexes + plays a role in the general testing of new or proposed + PostgreSQL features that could plausibly allow a + logical inconsistency to be introduced. One obvious testing + strategy is to call amcheck functions continuously + when running the standard regression tests. See for details on running the tests. + + + + + Filesystem or storage subsystem faults where checksums happen to + simply not be enabled. + + + Note that amcheck examines a page as represented in some + shared memory buffer at the time of verification if there is only a + shared buffer hit when accessing the block. Consequently, + amcheck does not necessarily examine data read from the + filesystem at the time of verification. Note that when checksums are + enabled, amcheck may raise an error due to a checksum + failure when a corrupt block is read into a buffer. + + + + + Corruption caused by faulty RAM, and the broader memory subsystem + and operating system. + + + PostgreSQL does not protect against correctable + memory errors and it is assumed you will operate using RAM that + uses industry standard Error Correcting Codes (ECC) or better + protection. However, ECC memory is typically only immune to + single-bit errors, and should not be assumed to provide + absolute protection against failures that + result in memory corruption. + + + + In general, amcheck can only prove the presence of + corruption; it cannot prove its absence. + + + + + Repairing corruption + + No error concerning corruption raised by amcheck should + ever be a false positive. In practice, amcheck is more + likely to find software bugs than problems with hardware. + amcheck raises errors in the event of conditions that, + by definition, should never happen, and so careful analysis of + amcheck errors is often required. + + + There is no general method of repairing problems that + amcheck detects. An explanation for the root cause of + an invariant violation should be sought. may play a useful role in diagnosing + corruption that amcheck detects. A REINDEX + may not be effective in repairing corruption. + + + + + diff --git a/doc/src/sgml/contrib.sgml b/doc/src/sgml/contrib.sgml index 03e5889839..eaaa36cb87 100644 --- a/doc/src/sgml/contrib.sgml +++ b/doc/src/sgml/contrib.sgml @@ -103,6 +103,7 @@ CREATE EXTENSION module_name FROM unpackaged; &adminpack; + &amcheck; &auth-delay; &auto-explain; &bloom; diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml index e7aa92f914..6782f07aea 100644 --- a/doc/src/sgml/filelist.sgml +++ b/doc/src/sgml/filelist.sgml @@ -107,6 +107,7 @@ + diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 296552e394..3487f7becb 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -244,6 +244,8 @@ BrinRevmap BrinSpecialSpace BrinTuple BrinValues +BtreeCheckState +BtreeLevel Bucket BufFile Buffer -- cgit v1.2.3 From 07a61e16703b5f90650d509e29a724c68efd79ab Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 10 Mar 2017 10:16:04 -0500 Subject: Improve gitignore file One file was listed under a wrong comment. --- doc/src/sgml/.gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/.gitignore b/doc/src/sgml/.gitignore index cdeace2991..8197c0140d 100644 --- a/doc/src/sgml/.gitignore +++ b/doc/src/sgml/.gitignore @@ -7,7 +7,6 @@ /man-stamp # Other popular build targets /INSTALL -/INSTALL.xml /postgres-US.pdf /postgres-A4.pdf /postgres.html @@ -22,6 +21,7 @@ # Assorted byproducts from building the above /postgres.xml /INSTALL.html +/INSTALL.xml /postgres-US.aux /postgres-US.log /postgres-US.out -- cgit v1.2.3 From 8b358b42f8eb6156a82ac9a41fc4e8335c8dc37a Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 10 Mar 2017 13:15:47 -0500 Subject: Change the relkind for partitioned tables from 'P' to 'p'. Seven of the eight other relkind codes are lower-case, so it wasn't consistent for this one to be upper-case. Fix it while we still can. Historical notes: the reason for the lone exception, i.e. sequences being 'S', is that 's' was once used for "special" relations. Also, at one time the partitioned-tables patch used both 'P' and 'p', but that got changed, leaving only a surprising choice behind. This also fixes a couple little bits of technical debt, such as type_sanity.sql not knowing that 'm' is a legal value for relkind. Discussion: https://postgr.es/m/27899.1488909319@sss.pgh.pa.us --- doc/src/sgml/catalogs.sgml | 13 +++++---- src/backend/catalog/information_schema.sql | 44 ++++++++++++++++-------------- src/backend/catalog/system_views.sql | 4 +-- src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_class.h | 11 ++++---- src/test/regress/expected/create_table.out | 2 +- src/test/regress/expected/rules.out | 4 +-- src/test/regress/expected/sanity_check.out | 2 +- src/test/regress/expected/type_sanity.out | 2 +- src/test/regress/sql/sanity_check.sql | 2 +- src/test/regress/sql/type_sanity.sql | 2 +- 11 files changed, 47 insertions(+), 41 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 28cdabe6fe..2c2da2ad8a 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -1758,12 +1758,15 @@ char - r = ordinary table, P = partitioned table, - i = index - S = sequence, v = view, + r = ordinary table, + i = index, + S = sequence, + t = TOAST table, + v = view, m = materialized view, - c = composite type, t = TOAST table, - f = foreign table + c = composite type, + f = foreign table, + p = partitioned table diff --git a/src/backend/catalog/information_schema.sql b/src/backend/catalog/information_schema.sql index 51795cd6de..fa2a88fc5c 100644 --- a/src/backend/catalog/information_schema.sql +++ b/src/backend/catalog/information_schema.sql @@ -365,7 +365,7 @@ CREATE VIEW attributes AS ON a.attcollation = co.oid AND (nco.nspname, co.collname) <> ('pg_catalog', 'default') WHERE a.attnum > 0 AND NOT a.attisdropped - AND c.relkind in ('c') + AND c.relkind IN ('c') AND (pg_has_role(c.relowner, 'USAGE') OR has_type_privilege(c.reltype, 'USAGE')); @@ -453,7 +453,7 @@ CREATE VIEW check_constraints AS AND a.attnum > 0 AND NOT a.attisdropped AND a.attnotnull - AND r.relkind IN ('r', 'P') + AND r.relkind IN ('r', 'p') AND pg_has_role(r.relowner, 'USAGE'); GRANT SELECT ON check_constraints TO PUBLIC; @@ -525,7 +525,7 @@ CREATE VIEW column_domain_usage AS AND a.attrelid = c.oid AND a.atttypid = t.oid AND t.typtype = 'd' - AND c.relkind IN ('r', 'v', 'f', 'P') + AND c.relkind IN ('r', 'v', 'f', 'p') AND a.attnum > 0 AND NOT a.attisdropped AND pg_has_role(t.typowner, 'USAGE'); @@ -564,7 +564,7 @@ CREATE VIEW column_privileges AS pr_c.relowner FROM (SELECT oid, relname, relnamespace, relowner, (aclexplode(coalesce(relacl, acldefault('r', relowner)))).* FROM pg_class - WHERE relkind IN ('r', 'v', 'f', 'P') + WHERE relkind IN ('r', 'v', 'f', 'p') ) pr_c (oid, relname, relnamespace, relowner, grantor, grantee, prtype, grantable), pg_attribute a WHERE a.attrelid = pr_c.oid @@ -586,7 +586,7 @@ CREATE VIEW column_privileges AS ) pr_a (attrelid, attname, grantor, grantee, prtype, grantable), pg_class c WHERE pr_a.attrelid = c.oid - AND relkind IN ('r', 'v', 'f', 'P') + AND relkind IN ('r', 'v', 'f', 'p') ) x, pg_namespace nc, pg_authid u_grantor, @@ -629,7 +629,8 @@ CREATE VIEW column_udt_usage AS WHERE a.attrelid = c.oid AND a.atttypid = t.oid AND nc.oid = c.relnamespace - AND a.attnum > 0 AND NOT a.attisdropped AND c.relkind in ('r', 'v', 'f', 'P') + AND a.attnum > 0 AND NOT a.attisdropped + AND c.relkind in ('r', 'v', 'f', 'p') AND pg_has_role(coalesce(bt.typowner, t.typowner), 'USAGE'); GRANT SELECT ON column_udt_usage TO PUBLIC; @@ -738,7 +739,7 @@ CREATE VIEW columns AS CAST('NEVER' AS character_data) AS is_generated, CAST(null AS character_data) AS generation_expression, - CAST(CASE WHEN c.relkind IN ('r', 'P') OR + CAST(CASE WHEN c.relkind IN ('r', 'p') OR (c.relkind IN ('v', 'f') AND pg_column_is_updatable(c.oid, a.attnum, false)) THEN 'YES' ELSE 'NO' END AS yes_or_no) AS is_updatable @@ -753,7 +754,8 @@ CREATE VIEW columns AS WHERE (NOT pg_is_other_temp_schema(nc.oid)) - AND a.attnum > 0 AND NOT a.attisdropped AND c.relkind in ('r', 'v', 'f', 'P') + AND a.attnum > 0 AND NOT a.attisdropped + AND c.relkind IN ('r', 'v', 'f', 'p') AND (pg_has_role(c.relowner, 'USAGE') OR has_column_privilege(c.oid, a.attnum, @@ -789,7 +791,7 @@ CREATE VIEW constraint_column_usage AS AND d.objid = c.oid AND c.connamespace = nc.oid AND c.contype = 'c' - AND r.relkind IN ('r', 'P') + AND r.relkind IN ('r', 'p') AND NOT a.attisdropped UNION ALL @@ -805,7 +807,7 @@ CREATE VIEW constraint_column_usage AS AND a.attnum = ANY (CASE c.contype WHEN 'f' THEN c.confkey ELSE c.conkey END) AND NOT a.attisdropped AND c.contype IN ('p', 'u', 'f') - AND r.relkind IN ('r', 'P') + AND r.relkind IN ('r', 'p') ) AS x (tblschema, tblname, tblowner, colname, cstrschema, cstrname) @@ -841,7 +843,7 @@ CREATE VIEW constraint_table_usage AS WHERE c.connamespace = nc.oid AND r.relnamespace = nr.oid AND ( (c.contype = 'f' AND c.confrelid = r.oid) OR (c.contype IN ('p', 'u') AND c.conrelid = r.oid) ) - AND r.relkind IN ('r', 'P') + AND r.relkind IN ('r', 'p') AND pg_has_role(r.relowner, 'USAGE'); GRANT SELECT ON constraint_table_usage TO PUBLIC; @@ -1058,7 +1060,7 @@ CREATE VIEW key_column_usage AS AND r.oid = c.conrelid AND nc.oid = c.connamespace AND c.contype IN ('p', 'u', 'f') - AND r.relkind IN ('r', 'P') + AND r.relkind IN ('r', 'p') AND (NOT pg_is_other_temp_schema(nr.oid)) ) AS ss WHERE ss.roid = a.attrelid AND a.attnum = (ss.x).x @@ -1774,7 +1776,7 @@ CREATE VIEW table_constraints AS WHERE nc.oid = c.connamespace AND nr.oid = r.relnamespace AND c.conrelid = r.oid AND c.contype NOT IN ('t', 'x') -- ignore nonstandard constraints - AND r.relkind IN ('r', 'P') + AND r.relkind IN ('r', 'p') AND (NOT pg_is_other_temp_schema(nr.oid)) AND (pg_has_role(r.relowner, 'USAGE') -- SELECT privilege omitted, per SQL standard @@ -1804,7 +1806,7 @@ CREATE VIEW table_constraints AS AND a.attnotnull AND a.attnum > 0 AND NOT a.attisdropped - AND r.relkind IN ('r', 'P') + AND r.relkind IN ('r', 'p') AND (NOT pg_is_other_temp_schema(nr.oid)) AND (pg_has_role(r.relowner, 'USAGE') -- SELECT privilege omitted, per SQL standard @@ -1854,7 +1856,7 @@ CREATE VIEW table_privileges AS ) AS grantee (oid, rolname) WHERE c.relnamespace = nc.oid - AND c.relkind IN ('r', 'v', 'P') + AND c.relkind IN ('r', 'v', 'p') AND c.grantee = grantee.oid AND c.grantor = u_grantor.oid AND c.prtype IN ('INSERT', 'SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'REFERENCES', 'TRIGGER') @@ -1898,7 +1900,7 @@ CREATE VIEW tables AS CAST( CASE WHEN nc.oid = pg_my_temp_schema() THEN 'LOCAL TEMPORARY' - WHEN c.relkind IN ('r', 'P') THEN 'BASE TABLE' + WHEN c.relkind IN ('r', 'p') THEN 'BASE TABLE' WHEN c.relkind = 'v' THEN 'VIEW' WHEN c.relkind = 'f' THEN 'FOREIGN TABLE' ELSE null END @@ -1911,7 +1913,7 @@ CREATE VIEW tables AS CAST(nt.nspname AS sql_identifier) AS user_defined_type_schema, CAST(t.typname AS sql_identifier) AS user_defined_type_name, - CAST(CASE WHEN c.relkind IN ('r', 'P') OR + CAST(CASE WHEN c.relkind IN ('r', 'p') OR (c.relkind IN ('v', 'f') AND -- 1 << CMD_INSERT pg_relation_is_updatable(c.oid, false) & 8 = 8) @@ -1923,7 +1925,7 @@ CREATE VIEW tables AS FROM pg_namespace nc JOIN pg_class c ON (nc.oid = c.relnamespace) LEFT JOIN (pg_type t JOIN pg_namespace nt ON (t.typnamespace = nt.oid)) ON (c.reloftype = t.oid) - WHERE c.relkind IN ('r', 'v', 'f', 'P') + WHERE c.relkind IN ('r', 'v', 'f', 'p') AND (NOT pg_is_other_temp_schema(nc.oid)) AND (pg_has_role(c.relowner, 'USAGE') OR has_table_privilege(c.oid, 'SELECT, INSERT, UPDATE, DELETE, TRUNCATE, REFERENCES, TRIGGER') @@ -2442,7 +2444,7 @@ CREATE VIEW view_column_usage AS AND dt.refclassid = 'pg_catalog.pg_class'::regclass AND dt.refobjid = t.oid AND t.relnamespace = nt.oid - AND t.relkind IN ('r', 'v', 'f', 'P') + AND t.relkind IN ('r', 'v', 'f', 'p') AND t.oid = a.attrelid AND dt.refobjsubid = a.attnum AND pg_has_role(t.relowner, 'USAGE'); @@ -2520,7 +2522,7 @@ CREATE VIEW view_table_usage AS AND dt.refclassid = 'pg_catalog.pg_class'::regclass AND dt.refobjid = t.oid AND t.relnamespace = nt.oid - AND t.relkind IN ('r', 'v', 'f', 'P') + AND t.relkind IN ('r', 'v', 'f', 'p') AND pg_has_role(t.relowner, 'USAGE'); GRANT SELECT ON view_table_usage TO PUBLIC; @@ -2673,7 +2675,7 @@ CREATE VIEW element_types AS a.attnum, a.atttypid, a.attcollation FROM pg_class c, pg_attribute a WHERE c.oid = a.attrelid - AND c.relkind IN ('r', 'v', 'f', 'c', 'P') + AND c.relkind IN ('r', 'v', 'f', 'c', 'p') AND attnum > 0 AND NOT attisdropped UNION ALL diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index ba980de86b..0bce20914e 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -136,7 +136,7 @@ CREATE VIEW pg_tables AS C.relrowsecurity AS rowsecurity FROM pg_class C LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace) LEFT JOIN pg_tablespace T ON (T.oid = C.reltablespace) - WHERE C.relkind IN ('r', 'P'); + WHERE C.relkind IN ('r', 'p'); CREATE VIEW pg_matviews AS SELECT @@ -294,7 +294,7 @@ CREATE VIEW pg_prepared_statements AS CREATE VIEW pg_seclabels AS SELECT l.objoid, l.classoid, l.objsubid, - CASE WHEN rel.relkind IN ('r', 'P') THEN 'table'::text + CASE WHEN rel.relkind IN ('r', 'p') THEN 'table'::text WHEN rel.relkind = 'v' THEN 'view'::text WHEN rel.relkind = 'm' THEN 'materialized view'::text WHEN rel.relkind = 'S' THEN 'sequence'::text diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 4c05b30068..9f5e302061 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201703081 +#define CATALOG_VERSION_NO 201703101 #endif diff --git a/src/include/catalog/pg_class.h b/src/include/catalog/pg_class.h index 3f96611f74..d1d493ee05 100644 --- a/src/include/catalog/pg_class.h +++ b/src/include/catalog/pg_class.h @@ -162,10 +162,10 @@ DESCR(""); #define RELKIND_SEQUENCE 'S' /* sequence object */ #define RELKIND_TOASTVALUE 't' /* for out-of-line values */ #define RELKIND_VIEW 'v' /* view */ +#define RELKIND_MATVIEW 'm' /* materialized view */ #define RELKIND_COMPOSITE_TYPE 'c' /* composite type */ #define RELKIND_FOREIGN_TABLE 'f' /* foreign table */ -#define RELKIND_MATVIEW 'm' /* materialized view */ -#define RELKIND_PARTITIONED_TABLE 'P' /* partitioned table */ +#define RELKIND_PARTITIONED_TABLE 'p' /* partitioned table */ #define RELPERSISTENCE_PERMANENT 'p' /* regular table */ #define RELPERSISTENCE_UNLOGGED 'u' /* unlogged permanent table */ @@ -178,9 +178,10 @@ DESCR(""); /* all columns are logged as replica identity */ #define REPLICA_IDENTITY_FULL 'f' /* - * an explicitly chosen candidate key's columns are used as identity; - * will still be set if the index has been dropped, in that case it - * has the same meaning as 'd' + * an explicitly chosen candidate key's columns are used as replica identity. + * Note this will still be set if the index has been dropped; in that case it + * has the same meaning as 'd'. */ #define REPLICA_IDENTITY_INDEX 'i' + #endif /* PG_CLASS_H */ diff --git a/src/test/regress/expected/create_table.out b/src/test/regress/expected/create_table.out index c07a474b3d..6f8645ddbd 100644 --- a/src/test/regress/expected/create_table.out +++ b/src/test/regress/expected/create_table.out @@ -404,7 +404,7 @@ CREATE TABLE partitioned ( SELECT relkind FROM pg_class WHERE relname = 'partitioned'; relkind --------- - P + p (1 row) -- check that range partition key columns are marked NOT NULL diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index c661f1d962..bd13ae6010 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1481,7 +1481,7 @@ pg_seclabels| SELECT l.objoid, l.classoid, l.objsubid, CASE - WHEN (rel.relkind = ANY (ARRAY['r'::"char", 'P'::"char"])) THEN 'table'::text + WHEN (rel.relkind = ANY (ARRAY['r'::"char", 'p'::"char"])) THEN 'table'::text WHEN (rel.relkind = 'v'::"char") THEN 'view'::text WHEN (rel.relkind = 'm'::"char") THEN 'materialized view'::text WHEN (rel.relkind = 'S'::"char") THEN 'sequence'::text @@ -2171,7 +2171,7 @@ pg_tables| SELECT n.nspname AS schemaname, FROM ((pg_class c LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace))) LEFT JOIN pg_tablespace t ON ((t.oid = c.reltablespace))) - WHERE (c.relkind = ANY (ARRAY['r'::"char", 'P'::"char"])); + WHERE (c.relkind = ANY (ARRAY['r'::"char", 'p'::"char"])); pg_timezone_abbrevs| SELECT pg_timezone_abbrevs.abbrev, pg_timezone_abbrevs.utc_offset, pg_timezone_abbrevs.is_dst diff --git a/src/test/regress/expected/sanity_check.out b/src/test/regress/expected/sanity_check.out index bdbb39180f..b5eff55e9b 100644 --- a/src/test/regress/expected/sanity_check.out +++ b/src/test/regress/expected/sanity_check.out @@ -9,7 +9,7 @@ VACUUM; \a\t SELECT relname, relhasindex FROM pg_class c LEFT JOIN pg_namespace n ON n.oid = relnamespace - WHERE relkind IN ('r', 'P') AND (nspname ~ '^pg_temp_') IS NOT TRUE + WHERE relkind IN ('r', 'p') AND (nspname ~ '^pg_temp_') IS NOT TRUE ORDER BY relname; a|f a_star|f diff --git a/src/test/regress/expected/type_sanity.out b/src/test/regress/expected/type_sanity.out index 312d290e73..8d75bbfab3 100644 --- a/src/test/regress/expected/type_sanity.out +++ b/src/test/regress/expected/type_sanity.out @@ -464,7 +464,7 @@ ORDER BY 1; -- Look for illegal values in pg_class fields SELECT p1.oid, p1.relname FROM pg_class as p1 -WHERE relkind NOT IN ('r', 'i', 's', 'S', 'c', 't', 'v', 'f') OR +WHERE relkind NOT IN ('r', 'i', 'S', 't', 'v', 'm', 'c', 'f', 'p') OR relpersistence NOT IN ('p', 'u', 't') OR relreplident NOT IN ('d', 'n', 'f', 'i'); oid | relname diff --git a/src/test/regress/sql/sanity_check.sql b/src/test/regress/sql/sanity_check.sql index fa3a90ff11..04aee457dd 100644 --- a/src/test/regress/sql/sanity_check.sql +++ b/src/test/regress/sql/sanity_check.sql @@ -12,7 +12,7 @@ VACUUM; SELECT relname, relhasindex FROM pg_class c LEFT JOIN pg_namespace n ON n.oid = relnamespace - WHERE relkind IN ('r', 'P') AND (nspname ~ '^pg_temp_') IS NOT TRUE + WHERE relkind IN ('r', 'p') AND (nspname ~ '^pg_temp_') IS NOT TRUE ORDER BY relname; -- restore normal output mode diff --git a/src/test/regress/sql/type_sanity.sql b/src/test/regress/sql/type_sanity.sql index 0282f84d2e..0a31249f5d 100644 --- a/src/test/regress/sql/type_sanity.sql +++ b/src/test/regress/sql/type_sanity.sql @@ -339,7 +339,7 @@ ORDER BY 1; SELECT p1.oid, p1.relname FROM pg_class as p1 -WHERE relkind NOT IN ('r', 'i', 's', 'S', 'c', 't', 'v', 'f') OR +WHERE relkind NOT IN ('r', 'i', 'S', 't', 'v', 'm', 'c', 'f', 'p') OR relpersistence NOT IN ('p', 'u', 't') OR relreplident NOT IN ('d', 'n', 'f', 'i'); -- cgit v1.2.3 From f9dfa5c9776649f769d537dd0923003b35f128de Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 10 Mar 2017 16:32:18 -0500 Subject: Improve postmaster's logging of listen socket creation. When one of the kernel calls in the socket()/bind()/listen() sequence fails, include the specific address we're trying to bind to in the log message. This greatly eases debugging of network misconfigurations. Also, after successfully setting up a listen socket, report its address in the log, to ease verification that the expected addresses were bound. There was some debate about whether to print this message at LOG level or only DEBUG1, but the majority of votes were for the former. Discussion: https://postgr.es/m/9564.1489091245@sss.pgh.pa.us --- doc/src/sgml/runtime.sgml | 8 ++++---- src/backend/libpq/pqcomm.c | 51 +++++++++++++++++++++++++++++++++++----------- 2 files changed, 43 insertions(+), 16 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/runtime.sgml b/doc/src/sgml/runtime.sgml index 5e3d783c6a..01153f9a37 100644 --- a/doc/src/sgml/runtime.sgml +++ b/doc/src/sgml/runtime.sgml @@ -464,9 +464,9 @@ su - postgres -c "/usr/local/pgsql/bin/pg_ctl start -l logfile -D /usr/local/pgs -LOG: could not bind IPv4 socket: Address already in use +LOG: could not bind IPv4 address "127.0.0.1": Address already in use HINT: Is another postmaster already running on port 5432? If not, wait a few seconds and retry. -FATAL: could not create TCP/IP listen socket +FATAL: could not create any TCP/IP sockets This usually means just what it suggests: you tried to start another server on the same port where one is already running. @@ -476,9 +476,9 @@ FATAL: could not create TCP/IP listen socket on a reserved port number might draw something like: $ postgres -p 666 -LOG: could not bind IPv4 socket: Permission denied +LOG: could not bind IPv4 address "127.0.0.1": Permission denied HINT: Is another postmaster already running on port 666? If not, wait a few seconds and retry. -FATAL: could not create TCP/IP listen socket +FATAL: could not create any TCP/IP sockets diff --git a/src/backend/libpq/pqcomm.c b/src/backend/libpq/pqcomm.c index 7939b1f544..aa934569da 100644 --- a/src/backend/libpq/pqcomm.c +++ b/src/backend/libpq/pqcomm.c @@ -319,6 +319,8 @@ StreamServerPort(int family, char *hostName, unsigned short portNumber, char portNumberStr[32]; const char *familyDesc; char familyDescBuf[64]; + const char *addrDesc; + char addrBuf[NI_MAXHOST]; char *service; struct addrinfo *addrs = NULL, *addr; @@ -407,7 +409,7 @@ StreamServerPort(int family, char *hostName, unsigned short portNumber, break; } - /* set up family name for possible error messages */ + /* set up address family name for log messages */ switch (addr->ai_family) { case AF_INET: @@ -431,13 +433,28 @@ StreamServerPort(int family, char *hostName, unsigned short portNumber, break; } + /* set up text form of address for log messages */ +#ifdef HAVE_UNIX_SOCKETS + if (addr->ai_family == AF_UNIX) + addrDesc = unixSocketPath; + else +#endif + { + pg_getnameinfo_all((const struct sockaddr_storage *) addr->ai_addr, + addr->ai_addrlen, + addrBuf, sizeof(addrBuf), + NULL, 0, + NI_NUMERICHOST); + addrDesc = addrBuf; + } + if ((fd = socket(addr->ai_family, SOCK_STREAM, 0)) == PGINVALID_SOCKET) { ereport(LOG, (errcode_for_socket_access(), - /* translator: %s is IPv4, IPv6, or Unix */ - errmsg("could not create %s socket: %m", - familyDesc))); + /* translator: first %s is IPv4, IPv6, or Unix */ + errmsg("could not create %s socket for address \"%s\": %m", + familyDesc, addrDesc))); continue; } @@ -461,7 +478,9 @@ StreamServerPort(int family, char *hostName, unsigned short portNumber, { ereport(LOG, (errcode_for_socket_access(), - errmsg("setsockopt(SO_REUSEADDR) failed: %m"))); + /* translator: first %s is IPv4, IPv6, or Unix */ + errmsg("setsockopt(SO_REUSEADDR) failed for %s address \"%s\": %m", + familyDesc, addrDesc))); closesocket(fd); continue; } @@ -476,7 +495,9 @@ StreamServerPort(int family, char *hostName, unsigned short portNumber, { ereport(LOG, (errcode_for_socket_access(), - errmsg("setsockopt(IPV6_V6ONLY) failed: %m"))); + /* translator: first %s is IPv4, IPv6, or Unix */ + errmsg("setsockopt(IPV6_V6ONLY) failed for %s address \"%s\": %m", + familyDesc, addrDesc))); closesocket(fd); continue; } @@ -494,9 +515,9 @@ StreamServerPort(int family, char *hostName, unsigned short portNumber, { ereport(LOG, (errcode_for_socket_access(), - /* translator: %s is IPv4, IPv6, or Unix */ - errmsg("could not bind %s socket: %m", - familyDesc), + /* translator: first %s is IPv4, IPv6, or Unix */ + errmsg("could not bind %s address \"%s\": %m", + familyDesc, addrDesc), (IS_AF_UNIX(addr->ai_family)) ? errhint("Is another postmaster already running on port %d?" " If not, remove socket file \"%s\" and retry.", @@ -533,12 +554,18 @@ StreamServerPort(int family, char *hostName, unsigned short portNumber, { ereport(LOG, (errcode_for_socket_access(), - /* translator: %s is IPv4, IPv6, or Unix */ - errmsg("could not listen on %s socket: %m", - familyDesc))); + /* translator: first %s is IPv4, IPv6, or Unix */ + errmsg("could not listen on %s address \"%s\": %m", + familyDesc, addrDesc))); closesocket(fd); continue; } + + ereport(LOG, + /* translator: first %s is IPv4, IPv6, or Unix */ + (errmsg("listening on %s address \"%s\"", + familyDesc, addrDesc))); + ListenSocket[listen_index] = fd; added++; } -- cgit v1.2.3 From b58fd4a9cab21e9d937a4e369bab31b3a5d24710 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 11 Mar 2017 14:37:05 -0500 Subject: Add a "subtransaction" command to PL/Tcl. This allows rolling back the effects of some SPI commands without having to fail the entire PL/Tcl function. Victor Wagner, reviewed by Pavel Stehule Discussion: https://postgr.es/m/20170108205750.2dab04a1@wagner.wagner.home --- doc/src/sgml/pltcl.sgml | 112 ++++++++++++++++++++++++-- src/pl/tcl/Makefile | 2 +- src/pl/tcl/expected/pltcl_subxact.out | 143 ++++++++++++++++++++++++++++++++++ src/pl/tcl/pltcl.c | 53 +++++++++++++ src/pl/tcl/sql/pltcl_subxact.sql | 95 ++++++++++++++++++++++ 5 files changed, 398 insertions(+), 7 deletions(-) create mode 100644 src/pl/tcl/expected/pltcl_subxact.out create mode 100644 src/pl/tcl/sql/pltcl_subxact.sql (limited to 'doc/src') diff --git a/doc/src/sgml/pltcl.sgml b/doc/src/sgml/pltcl.sgml index ad216dd5b7..ed745a7481 100644 --- a/doc/src/sgml/pltcl.sgml +++ b/doc/src/sgml/pltcl.sgml @@ -476,6 +476,20 @@ $$ LANGUAGE pltcl; + + subtransaction command + + + The Tcl script contained in command is + executed within a SQL subtransaction. If the script returns an + error, that entire subtransaction is rolled back before returning the + error out to the surrounding Tcl code. + See for more details and an + example. + + + + quote string @@ -844,18 +858,22 @@ CREATE EVENT TRIGGER tcl_a_snitch ON ddl_command_start EXECUTE PROCEDURE tclsnit either by executing some invalid operation or by generating an error using the Tcl error command or PL/Tcl's elog command. Such errors can be caught - within Tcl using the Tcl catch command. If they - are not caught but are allowed to propagate out to the top level of - execution of the PL/Tcl function, they turn into database errors. + within Tcl using the Tcl catch command. If an + error is not caught but is allowed to propagate out to the top level of + execution of the PL/Tcl function, it is reported as a SQL error in the + function's calling query. - Conversely, database errors that occur within PL/Tcl's + Conversely, SQL errors that occur within PL/Tcl's spi_exec, spi_prepare, and spi_execp commands are reported as Tcl errors, so they are catchable by Tcl's catch command. - Again, if they propagate out to the top level without being caught, - they turn back into database errors. + (Each of these PL/Tcl commands runs its SQL operation in a + subtransaction, which is rolled back on error, so that any + partially-completed operation is automatically cleaned up.) + Again, if an error propagates out to the top level without being caught, + it turns back into a SQL error. @@ -902,6 +920,88 @@ if {[catch { spi_exec $sql_command }]} { + + Explicit Subtransactions in PL/Tcl + + + subtransactions + in PL/Tcl + + + + Recovering from errors caused by database access as described in + can lead to an undesirable + situation where some operations succeed before one of them fails, + and after recovering from that error the data is left in an + inconsistent state. PL/Tcl offers a solution to this problem in + the form of explicit subtransactions. + + + + Consider a function that implements a transfer between two accounts: + +CREATE FUNCTION transfer_funds() RETURNS void AS $$ + if [catch { + spi_exec "UPDATE accounts SET balance = balance - 100 WHERE account_name = 'joe'" + spi_exec "UPDATE accounts SET balance = balance + 100 WHERE account_name = 'mary'" + } errormsg] { + set result [format "error transferring funds: %s" $errormsg] + } else { + set result "funds transferred successfully" + } + spi_exec "INSERT INTO operations (result) VALUES ('[quote $result]')" +$$ LANGUAGE pltcl; + + If the second UPDATE statement results in an + exception being raised, this function will log the failure, but + the result of the first UPDATE will + nevertheless be committed. In other words, the funds will be + withdrawn from Joe's account, but will not be transferred to + Mary's account. This happens because each spi_exec + is a separate subtransaction, and only one of those subtransactions + got rolled back. + + + + To handle such cases, you can wrap multiple database operations in an + explicit subtransaction, which will succeed or roll back as a whole. + PL/Tcl provides a subtransaction command to manage + this. We can rewrite our function as: + +CREATE FUNCTION transfer_funds2() RETURNS void AS $$ + if [catch { + subtransaction { + spi_exec "UPDATE accounts SET balance = balance - 100 WHERE account_name = 'joe'" + spi_exec "UPDATE accounts SET balance = balance + 100 WHERE account_name = 'mary'" + } + } errormsg] { + set result [format "error transferring funds: %s" $errormsg] + } else { + set result "funds transferred successfully" + } + spi_exec "INSERT INTO operations (result) VALUES ('[quote $result]')" +$$ LANGUAGE pltcl; + + Note that use of catch is still required for this + purpose. Otherwise the error would propagate to the top level of the + function, preventing the desired insertion into + the operations table. + The subtransaction command does not trap errors, it + only assures that all database operations executed inside its scope will + be rolled back together when an error is reported. + + + + A rollback of an explicit subtransaction occurs on any error reported + by the contained Tcl code, not only errors originating from database + access. Thus a regular Tcl exception raised inside + a subtransaction command will also cause the + subtransaction to be rolled back. However, non-error exits out of the + contained Tcl code (for instance, due to return) do + not cause a rollback. + + + PL/Tcl Configuration diff --git a/src/pl/tcl/Makefile b/src/pl/tcl/Makefile index 1096c4faf0..0275c125b1 100644 --- a/src/pl/tcl/Makefile +++ b/src/pl/tcl/Makefile @@ -28,7 +28,7 @@ DATA = pltcl.control pltcl--1.0.sql pltcl--unpackaged--1.0.sql \ pltclu.control pltclu--1.0.sql pltclu--unpackaged--1.0.sql REGRESS_OPTS = --dbname=$(PL_TESTDB) --load-extension=pltcl -REGRESS = pltcl_setup pltcl_queries pltcl_start_proc pltcl_unicode +REGRESS = pltcl_setup pltcl_queries pltcl_start_proc pltcl_subxact pltcl_unicode # Tcl on win32 ships with import libraries only for Microsoft Visual C++, # which are not compatible with mingw gcc. Therefore we need to build a diff --git a/src/pl/tcl/expected/pltcl_subxact.out b/src/pl/tcl/expected/pltcl_subxact.out new file mode 100644 index 0000000000..4393f4acf6 --- /dev/null +++ b/src/pl/tcl/expected/pltcl_subxact.out @@ -0,0 +1,143 @@ +-- +-- Test explicit subtransactions +-- +CREATE TABLE subtransaction_tbl ( + i integer +); +-- +-- We use this wrapper to catch errors and return errormsg only, +-- because values of $::errorinfo variable contain procedure name which +-- includes OID, so it's not stable +-- +CREATE FUNCTION pltcl_wrapper(statement text) RETURNS text +AS $$ + if [catch {spi_exec $1} msg] { + return "ERROR: $msg" + } else { + return "SUCCESS: $msg" + } +$$ LANGUAGE pltcl; +-- Test subtransaction successfully committed +CREATE FUNCTION subtransaction_ctx_success() RETURNS void +AS $$ + spi_exec "INSERT INTO subtransaction_tbl VALUES(1)" + subtransaction { + spi_exec "INSERT INTO subtransaction_tbl VALUES(2)" + } +$$ LANGUAGE pltcl; +BEGIN; +INSERT INTO subtransaction_tbl VALUES(0); +SELECT subtransaction_ctx_success(); + subtransaction_ctx_success +---------------------------- + +(1 row) + +COMMIT; +SELECT * FROM subtransaction_tbl; + i +--- + 0 + 1 + 2 +(3 rows) + +TRUNCATE subtransaction_tbl; +-- Test subtransaction rollback +CREATE FUNCTION subtransaction_ctx_test(what_error text = NULL) RETURNS void +AS $$ + spi_exec "INSERT INTO subtransaction_tbl VALUES (1)" + subtransaction { + spi_exec "INSERT INTO subtransaction_tbl VALUES (2)" + if {$1 == "SPI"} { + spi_exec "INSERT INTO subtransaction_tbl VALUES ('oops')" + } elseif { $1 == "Tcl"} { + elog ERROR "Tcl error" + } + } +$$ LANGUAGE pltcl; +SELECT pltcl_wrapper('SELECT subtransaction_ctx_test()'); + pltcl_wrapper +--------------- + SUCCESS: 1 +(1 row) + +SELECT * FROM subtransaction_tbl; + i +--- + 1 + 2 +(2 rows) + +TRUNCATE subtransaction_tbl; +SELECT pltcl_wrapper('SELECT subtransaction_ctx_test(''SPI'')'); + pltcl_wrapper +------------------------------------------------- + ERROR: invalid input syntax for integer: "oops" +(1 row) + +SELECT * FROM subtransaction_tbl; + i +--- +(0 rows) + +TRUNCATE subtransaction_tbl; +SELECT pltcl_wrapper('SELECT subtransaction_ctx_test(''Tcl'')'); + pltcl_wrapper +------------------ + ERROR: Tcl error +(1 row) + +SELECT * FROM subtransaction_tbl; + i +--- +(0 rows) + +TRUNCATE subtransaction_tbl; +-- Nested subtransactions +CREATE FUNCTION subtransaction_nested_test(swallow boolean = 'f') RETURNS text +AS $$ +spi_exec "INSERT INTO subtransaction_tbl VALUES (1)" +subtransaction { + spi_exec "INSERT INTO subtransaction_tbl VALUES (2)" + if [catch { + subtransaction { + spi_exec "INSERT INTO subtransaction_tbl VALUES (3)" + spi_exec "error" + } + } errormsg] { + if {$1 != "t"} { + error $errormsg $::errorInfo $::errorCode + } + elog NOTICE "Swallowed $errormsg" + } +} +return "ok" +$$ LANGUAGE pltcl; +SELECT pltcl_wrapper('SELECT subtransaction_nested_test()'); + pltcl_wrapper +---------------------------------------- + ERROR: syntax error at or near "error" +(1 row) + +SELECT * FROM subtransaction_tbl; + i +--- +(0 rows) + +TRUNCATE subtransaction_tbl; +SELECT pltcl_wrapper('SELECT subtransaction_nested_test(''t'')'); +NOTICE: Swallowed syntax error at or near "error" + pltcl_wrapper +--------------- + SUCCESS: 1 +(1 row) + +SELECT * FROM subtransaction_tbl; + i +--- + 1 + 2 +(2 rows) + +TRUNCATE subtransaction_tbl; diff --git a/src/pl/tcl/pltcl.c b/src/pl/tcl/pltcl.c index 2cf7e6619b..b8fcf0673d 100644 --- a/src/pl/tcl/pltcl.c +++ b/src/pl/tcl/pltcl.c @@ -306,6 +306,8 @@ static int pltcl_SPI_execute_plan(ClientData cdata, Tcl_Interp *interp, int objc, Tcl_Obj *const objv[]); static int pltcl_SPI_lastoid(ClientData cdata, Tcl_Interp *interp, int objc, Tcl_Obj *const objv[]); +static int pltcl_subtransaction(ClientData cdata, Tcl_Interp *interp, + int objc, Tcl_Obj *const objv[]); static void pltcl_subtrans_begin(MemoryContext oldcontext, ResourceOwner oldowner); @@ -516,6 +518,8 @@ pltcl_init_interp(pltcl_interp_desc *interp_desc, Oid prolang, bool pltrusted) pltcl_SPI_execute_plan, NULL, NULL); Tcl_CreateObjCommand(interp, "spi_lastoid", pltcl_SPI_lastoid, NULL, NULL); + Tcl_CreateObjCommand(interp, "subtransaction", + pltcl_subtransaction, NULL, NULL); /************************************************************ * Call the appropriate start_proc, if there is one. @@ -2850,6 +2854,55 @@ pltcl_SPI_lastoid(ClientData cdata, Tcl_Interp *interp, } +/********************************************************************** + * pltcl_subtransaction() - Execute some Tcl code in a subtransaction + * + * The subtransaction is aborted if the Tcl code fragment returns TCL_ERROR, + * otherwise it's subcommitted. + **********************************************************************/ +static int +pltcl_subtransaction(ClientData cdata, Tcl_Interp *interp, + int objc, Tcl_Obj *const objv[]) +{ + MemoryContext oldcontext = CurrentMemoryContext; + ResourceOwner oldowner = CurrentResourceOwner; + int retcode; + + if (objc != 2) + { + Tcl_WrongNumArgs(interp, 1, objv, "command"); + return TCL_ERROR; + } + + /* + * Note: we don't use pltcl_subtrans_begin and friends here because we + * don't want the error handling in pltcl_subtrans_abort. But otherwise + * the processing should be about the same as in those functions. + */ + BeginInternalSubTransaction(NULL); + MemoryContextSwitchTo(oldcontext); + + retcode = Tcl_EvalObjEx(interp, objv[1], 0); + + if (retcode == TCL_ERROR) + { + /* Rollback the subtransaction */ + RollbackAndReleaseCurrentSubTransaction(); + } + else + { + /* Commit the subtransaction */ + ReleaseCurrentSubTransaction(); + } + + /* In either case, restore previous memory context and resource owner */ + MemoryContextSwitchTo(oldcontext); + CurrentResourceOwner = oldowner; + + return retcode; +} + + /********************************************************************** * pltcl_set_tuple_values() - Set variables for all attributes * of a given tuple diff --git a/src/pl/tcl/sql/pltcl_subxact.sql b/src/pl/tcl/sql/pltcl_subxact.sql new file mode 100644 index 0000000000..0625736ea4 --- /dev/null +++ b/src/pl/tcl/sql/pltcl_subxact.sql @@ -0,0 +1,95 @@ +-- +-- Test explicit subtransactions +-- + +CREATE TABLE subtransaction_tbl ( + i integer +); + +-- +-- We use this wrapper to catch errors and return errormsg only, +-- because values of $::errorinfo variable contain procedure name which +-- includes OID, so it's not stable +-- +CREATE FUNCTION pltcl_wrapper(statement text) RETURNS text +AS $$ + if [catch {spi_exec $1} msg] { + return "ERROR: $msg" + } else { + return "SUCCESS: $msg" + } +$$ LANGUAGE pltcl; + +-- Test subtransaction successfully committed + +CREATE FUNCTION subtransaction_ctx_success() RETURNS void +AS $$ + spi_exec "INSERT INTO subtransaction_tbl VALUES(1)" + subtransaction { + spi_exec "INSERT INTO subtransaction_tbl VALUES(2)" + } +$$ LANGUAGE pltcl; + +BEGIN; +INSERT INTO subtransaction_tbl VALUES(0); +SELECT subtransaction_ctx_success(); +COMMIT; +SELECT * FROM subtransaction_tbl; +TRUNCATE subtransaction_tbl; + +-- Test subtransaction rollback + +CREATE FUNCTION subtransaction_ctx_test(what_error text = NULL) RETURNS void +AS $$ + spi_exec "INSERT INTO subtransaction_tbl VALUES (1)" + subtransaction { + spi_exec "INSERT INTO subtransaction_tbl VALUES (2)" + if {$1 == "SPI"} { + spi_exec "INSERT INTO subtransaction_tbl VALUES ('oops')" + } elseif { $1 == "Tcl"} { + elog ERROR "Tcl error" + } + } +$$ LANGUAGE pltcl; + +SELECT pltcl_wrapper('SELECT subtransaction_ctx_test()'); +SELECT * FROM subtransaction_tbl; +TRUNCATE subtransaction_tbl; + +SELECT pltcl_wrapper('SELECT subtransaction_ctx_test(''SPI'')'); +SELECT * FROM subtransaction_tbl; +TRUNCATE subtransaction_tbl; + +SELECT pltcl_wrapper('SELECT subtransaction_ctx_test(''Tcl'')'); +SELECT * FROM subtransaction_tbl; +TRUNCATE subtransaction_tbl; + +-- Nested subtransactions + +CREATE FUNCTION subtransaction_nested_test(swallow boolean = 'f') RETURNS text +AS $$ +spi_exec "INSERT INTO subtransaction_tbl VALUES (1)" +subtransaction { + spi_exec "INSERT INTO subtransaction_tbl VALUES (2)" + if [catch { + subtransaction { + spi_exec "INSERT INTO subtransaction_tbl VALUES (3)" + spi_exec "error" + } + } errormsg] { + if {$1 != "t"} { + error $errormsg $::errorInfo $::errorCode + } + elog NOTICE "Swallowed $errormsg" + } +} +return "ok" +$$ LANGUAGE pltcl; + +SELECT pltcl_wrapper('SELECT subtransaction_nested_test()'); +SELECT * FROM subtransaction_tbl; +TRUNCATE subtransaction_tbl; + +SELECT pltcl_wrapper('SELECT subtransaction_nested_test(''t'')'); +SELECT * FROM subtransaction_tbl; +TRUNCATE subtransaction_tbl; -- cgit v1.2.3 From 9d7726c2ba06b932f791f2d0cc5acf73cc0b4dca Mon Sep 17 00:00:00 2001 From: Noah Misch Date: Sun, 12 Mar 2017 19:35:33 -0400 Subject: Recommend wrappers of PG_DETOAST_DATUM_PACKED(). When commit 3e23b68dac006e8deb0afa327e855258df8de064 introduced single-byte varlena headers, its fmgr.h changes presented PG_GETARG_TEXT_PP() and PG_GETARG_TEXT_P() as equals. Its postgres.h changes presented PG_DETOAST_DATUM_PACKED() and VARDATA_ANY() as the exceptional case. Now, instead, firmly recommend PG_GETARG_TEXT_PP() over PG_GETARG_TEXT_P(); likewise for other ...PP() macros. This shaves cycles and invites consistency of style. --- doc/src/sgml/xfunc.sgml | 32 +++++++++++++++++++------------- src/include/c.h | 9 +++++---- src/include/fmgr.h | 34 +++++++++++++++++++++------------- src/include/postgres.h | 26 ++++++++++++-------------- src/include/utils/inet.h | 5 +++-- src/tutorial/funcs_new.c | 29 ++++++++++++++++------------- 6 files changed, 76 insertions(+), 59 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/xfunc.sgml b/doc/src/sgml/xfunc.sgml index 255bfddad7..94a7ad747d 100644 --- a/doc/src/sgml/xfunc.sgml +++ b/doc/src/sgml/xfunc.sgml @@ -2388,18 +2388,23 @@ PG_FUNCTION_INFO_V1(copytext); Datum copytext(PG_FUNCTION_ARGS) { - text *t = PG_GETARG_TEXT_P(0); + text *t = PG_GETARG_TEXT_PP(0); + /* - * VARSIZE is the total size of the struct in bytes. + * VARSIZE_ANY_EXHDR is the size of the struct in bytes, minus the + * VARHDRSZ or VARHDRSZ_SHORT of its header. Construct the copy with a + * full-length header. */ - text *new_t = (text *) palloc(VARSIZE(t)); - SET_VARSIZE(new_t, VARSIZE(t)); + text *new_t = (text *) palloc(VARSIZE_ANY_EXHDR(t) + VARHDRSZ); + SET_VARSIZE(new_t, VARSIZE_ANY_EXHDR(t) + VARHDRSZ); + /* - * VARDATA is a pointer to the data region of the struct. + * VARDATA is a pointer to the data region of the new struct. The source + * could be a short datum, so retrieve its data through VARDATA_ANY. */ memcpy((void *) VARDATA(new_t), /* destination */ - (void *) VARDATA(t), /* source */ - VARSIZE(t) - VARHDRSZ); /* how many bytes */ + (void *) VARDATA_ANY(t), /* source */ + VARSIZE_ANY_EXHDR(t)); /* how many bytes */ PG_RETURN_TEXT_P(new_t); } @@ -2408,15 +2413,16 @@ PG_FUNCTION_INFO_V1(concat_text); Datum concat_text(PG_FUNCTION_ARGS) { - text *arg1 = PG_GETARG_TEXT_P(0); - text *arg2 = PG_GETARG_TEXT_P(1); - int32 new_text_size = VARSIZE(arg1) + VARSIZE(arg2) - VARHDRSZ; + text *arg1 = PG_GETARG_TEXT_PP(0); + text *arg2 = PG_GETARG_TEXT_PP(1); + int32 arg1_size = VARSIZE_ANY_EXHDR(arg1); + int32 arg2_size = VARSIZE_ANY_EXHDR(arg2); + int32 new_text_size = arg1_size + arg2_size + VARHDRSZ; text *new_text = (text *) palloc(new_text_size); SET_VARSIZE(new_text, new_text_size); - memcpy(VARDATA(new_text), VARDATA(arg1), VARSIZE(arg1) - VARHDRSZ); - memcpy(VARDATA(new_text) + (VARSIZE(arg1) - VARHDRSZ), - VARDATA(arg2), VARSIZE(arg2) - VARHDRSZ); + memcpy(VARDATA(new_text), VARDATA_ANY(arg1), arg1_size); + memcpy(VARDATA(new_text) + arg1_size, VARDATA_ANY(arg2), arg2_size); PG_RETURN_TEXT_P(new_text); } ]]> diff --git a/src/include/c.h b/src/include/c.h index 492648cd8a..fba07c651f 100644 --- a/src/include/c.h +++ b/src/include/c.h @@ -429,10 +429,11 @@ typedef struct * may be compressed or moved out-of-line. However datatype-specific routines * are mostly content to deal with de-TOASTed values only, and of course * client-side routines should never see a TOASTed value. But even in a - * de-TOASTed value, beware of touching vl_len_ directly, as its representation - * is no longer convenient. It's recommended that code always use the VARDATA, - * VARSIZE, and SET_VARSIZE macros instead of relying on direct mentions of - * the struct fields. See postgres.h for details of the TOASTed form. + * de-TOASTed value, beware of touching vl_len_ directly, as its + * representation is no longer convenient. It's recommended that code always + * use macros VARDATA_ANY, VARSIZE_ANY, VARSIZE_ANY_EXHDR, VARDATA, VARSIZE, + * and SET_VARSIZE instead of relying on direct mentions of the struct fields. + * See postgres.h for details of the TOASTed form. * ---------------- */ struct varlena diff --git a/src/include/fmgr.h b/src/include/fmgr.h index a671480004..4d400d2924 100644 --- a/src/include/fmgr.h +++ b/src/include/fmgr.h @@ -178,11 +178,12 @@ extern void fmgr_info_copy(FmgrInfo *dstinfo, FmgrInfo *srcinfo, * The resulting datum can be accessed using VARSIZE_ANY() and VARDATA_ANY() * (beware of multiple evaluations in those macros!) * - * WARNING: It is only safe to use pg_detoast_datum_packed() and - * VARDATA_ANY() if you really don't care about the alignment. Either because - * you're working with something like text where the alignment doesn't matter - * or because you're not going to access its constituent parts and just use - * things like memcpy on it anyways. + * In consumers oblivious to data alignment, call PG_DETOAST_DATUM_PACKED(), + * VARDATA_ANY(), VARSIZE_ANY() and VARSIZE_ANY_EXHDR(). Elsewhere, call + * PG_DETOAST_DATUM(), VARDATA() and VARSIZE(). Directly fetching an int16, + * int32 or wider field in the struct representing the datum layout requires + * aligned data. memcpy() is alignment-oblivious, as are most operations on + * datatypes, such as text, whose layout struct contains only char fields. * * Note: it'd be nice if these could be macros, but I see no way to do that * without evaluating the arguments multiple times, which is NOT acceptable. @@ -243,13 +244,9 @@ extern struct varlena *pg_detoast_datum_packed(struct varlena * datum); /* and this if you can handle 1-byte-header datums: */ #define PG_GETARG_VARLENA_PP(n) PG_DETOAST_DATUM_PACKED(PG_GETARG_DATUM(n)) /* DatumGetFoo macros for varlena types will typically look like this: */ -#define DatumGetByteaP(X) ((bytea *) PG_DETOAST_DATUM(X)) #define DatumGetByteaPP(X) ((bytea *) PG_DETOAST_DATUM_PACKED(X)) -#define DatumGetTextP(X) ((text *) PG_DETOAST_DATUM(X)) #define DatumGetTextPP(X) ((text *) PG_DETOAST_DATUM_PACKED(X)) -#define DatumGetBpCharP(X) ((BpChar *) PG_DETOAST_DATUM(X)) #define DatumGetBpCharPP(X) ((BpChar *) PG_DETOAST_DATUM_PACKED(X)) -#define DatumGetVarCharP(X) ((VarChar *) PG_DETOAST_DATUM(X)) #define DatumGetVarCharPP(X) ((VarChar *) PG_DETOAST_DATUM_PACKED(X)) #define DatumGetHeapTupleHeader(X) ((HeapTupleHeader) PG_DETOAST_DATUM(X)) /* And we also offer variants that return an OK-to-write copy */ @@ -264,13 +261,9 @@ extern struct varlena *pg_detoast_datum_packed(struct varlena * datum); #define DatumGetBpCharPSlice(X,m,n) ((BpChar *) PG_DETOAST_DATUM_SLICE(X,m,n)) #define DatumGetVarCharPSlice(X,m,n) ((VarChar *) PG_DETOAST_DATUM_SLICE(X,m,n)) /* GETARG macros for varlena types will typically look like this: */ -#define PG_GETARG_BYTEA_P(n) DatumGetByteaP(PG_GETARG_DATUM(n)) #define PG_GETARG_BYTEA_PP(n) DatumGetByteaPP(PG_GETARG_DATUM(n)) -#define PG_GETARG_TEXT_P(n) DatumGetTextP(PG_GETARG_DATUM(n)) #define PG_GETARG_TEXT_PP(n) DatumGetTextPP(PG_GETARG_DATUM(n)) -#define PG_GETARG_BPCHAR_P(n) DatumGetBpCharP(PG_GETARG_DATUM(n)) #define PG_GETARG_BPCHAR_PP(n) DatumGetBpCharPP(PG_GETARG_DATUM(n)) -#define PG_GETARG_VARCHAR_P(n) DatumGetVarCharP(PG_GETARG_DATUM(n)) #define PG_GETARG_VARCHAR_PP(n) DatumGetVarCharPP(PG_GETARG_DATUM(n)) #define PG_GETARG_HEAPTUPLEHEADER(n) DatumGetHeapTupleHeader(PG_GETARG_DATUM(n)) /* And we also offer variants that return an OK-to-write copy */ @@ -284,6 +277,21 @@ extern struct varlena *pg_detoast_datum_packed(struct varlena * datum); #define PG_GETARG_TEXT_P_SLICE(n,a,b) DatumGetTextPSlice(PG_GETARG_DATUM(n),a,b) #define PG_GETARG_BPCHAR_P_SLICE(n,a,b) DatumGetBpCharPSlice(PG_GETARG_DATUM(n),a,b) #define PG_GETARG_VARCHAR_P_SLICE(n,a,b) DatumGetVarCharPSlice(PG_GETARG_DATUM(n),a,b) +/* + * Obsolescent variants that guarantee INT alignment for the return value. + * Few operations on these particular types need alignment, mainly operations + * that cast the VARDATA pointer to a type like int16[]. Most code should use + * the ...PP(X) counterpart. Nonetheless, these appear frequently in code + * predating the PostgreSQL 8.3 introduction of the ...PP(X) variants. + */ +#define DatumGetByteaP(X) ((bytea *) PG_DETOAST_DATUM(X)) +#define DatumGetTextP(X) ((text *) PG_DETOAST_DATUM(X)) +#define DatumGetBpCharP(X) ((BpChar *) PG_DETOAST_DATUM(X)) +#define DatumGetVarCharP(X) ((VarChar *) PG_DETOAST_DATUM(X)) +#define PG_GETARG_BYTEA_P(n) DatumGetByteaP(PG_GETARG_DATUM(n)) +#define PG_GETARG_TEXT_P(n) DatumGetTextP(PG_GETARG_DATUM(n)) +#define PG_GETARG_BPCHAR_P(n) DatumGetBpCharP(PG_GETARG_DATUM(n)) +#define PG_GETARG_VARCHAR_P(n) DatumGetVarCharP(PG_GETARG_DATUM(n)) /* To return a NULL do this: */ #define PG_RETURN_NULL() \ diff --git a/src/include/postgres.h b/src/include/postgres.h index ff2c5c051e..f3582d5523 100644 --- a/src/include/postgres.h +++ b/src/include/postgres.h @@ -287,20 +287,18 @@ typedef struct /* Externally visible macros */ /* - * VARDATA, VARSIZE, and SET_VARSIZE are the recommended API for most code - * for varlena datatypes. Note that they only work on untoasted, - * 4-byte-header Datums! - * - * Code that wants to use 1-byte-header values without detoasting should - * use VARSIZE_ANY/VARSIZE_ANY_EXHDR/VARDATA_ANY. The other macros here - * should usually be used only by tuple assembly/disassembly code and - * code that specifically wants to work with still-toasted Datums. - * - * WARNING: It is only safe to use VARDATA_ANY() -- typically with - * PG_DETOAST_DATUM_PACKED() -- if you really don't care about the alignment. - * Either because you're working with something like text where the alignment - * doesn't matter or because you're not going to access its constituent parts - * and just use things like memcpy on it anyways. + * In consumers oblivious to data alignment, call PG_DETOAST_DATUM_PACKED(), + * VARDATA_ANY(), VARSIZE_ANY() and VARSIZE_ANY_EXHDR(). Elsewhere, call + * PG_DETOAST_DATUM(), VARDATA() and VARSIZE(). Directly fetching an int16, + * int32 or wider field in the struct representing the datum layout requires + * aligned data. memcpy() is alignment-oblivious, as are most operations on + * datatypes, such as text, whose layout struct contains only char fields. + * + * Code assembling a new datum should call VARDATA() and SET_VARSIZE(). + * (Datums begin life untoasted.) + * + * Other macros here should usually be used only by tuple assembly/disassembly + * code and code that specifically wants to work with still-toasted Datums. */ #define VARDATA(PTR) VARDATA_4B(PTR) #define VARSIZE(PTR) VARSIZE_4B(PTR) diff --git a/src/include/utils/inet.h b/src/include/utils/inet.h index 577b34dbf9..b4d7359f19 100644 --- a/src/include/utils/inet.h +++ b/src/include/utils/inet.h @@ -104,12 +104,13 @@ typedef struct macaddr /* * fmgr interface macros */ -#define DatumGetInetP(X) ((inet *) PG_DETOAST_DATUM(X)) #define DatumGetInetPP(X) ((inet *) PG_DETOAST_DATUM_PACKED(X)) #define InetPGetDatum(X) PointerGetDatum(X) -#define PG_GETARG_INET_P(n) DatumGetInetP(PG_GETARG_DATUM(n)) #define PG_GETARG_INET_PP(n) DatumGetInetPP(PG_GETARG_DATUM(n)) #define PG_RETURN_INET_P(x) return InetPGetDatum(x) +/* obsolescent variants */ +#define DatumGetInetP(X) ((inet *) PG_DETOAST_DATUM(X)) +#define PG_GETARG_INET_P(n) DatumGetInetP(PG_GETARG_DATUM(n)) /* macaddr is a fixed-length pass-by-reference datatype */ #define DatumGetMacaddrP(X) ((macaddr *) DatumGetPointer(X)) #define MacaddrPGetDatum(X) PointerGetDatum(X) diff --git a/src/tutorial/funcs_new.c b/src/tutorial/funcs_new.c index f668d281bb..2e09f8de6e 100644 --- a/src/tutorial/funcs_new.c +++ b/src/tutorial/funcs_new.c @@ -66,21 +66,24 @@ PG_FUNCTION_INFO_V1(copytext); Datum copytext(PG_FUNCTION_ARGS) { - text *t = PG_GETARG_TEXT_P(0); + text *t = PG_GETARG_TEXT_PP(0); /* - * VARSIZE is the total size of the struct in bytes. + * VARSIZE_ANY_EXHDR is the size of the struct in bytes, minus the + * VARHDRSZ or VARHDRSZ_SHORT of its header. Construct the copy with a + * full-length header. */ - text *new_t = (text *) palloc(VARSIZE(t)); + text *new_t = (text *) palloc(VARSIZE_ANY_EXHDR(t) + VARHDRSZ); - SET_VARSIZE(new_t, VARSIZE(t)); + SET_VARSIZE(new_t, VARSIZE_ANY_EXHDR(t) + VARHDRSZ); /* - * VARDATA is a pointer to the data region of the struct. + * VARDATA is a pointer to the data region of the new struct. The source + * could be a short datum, so retrieve its data through VARDATA_ANY. */ memcpy((void *) VARDATA(new_t), /* destination */ - (void *) VARDATA(t), /* source */ - VARSIZE(t) - VARHDRSZ); /* how many bytes */ + (void *) VARDATA_ANY(t), /* source */ + VARSIZE_ANY_EXHDR(t)); /* how many bytes */ PG_RETURN_TEXT_P(new_t); } @@ -89,16 +92,16 @@ PG_FUNCTION_INFO_V1(concat_text); Datum concat_text(PG_FUNCTION_ARGS) { - text *arg1 = PG_GETARG_TEXT_P(0); - text *arg2 = PG_GETARG_TEXT_P(1); - int32 arg1_size = VARSIZE(arg1) - VARHDRSZ; - int32 arg2_size = VARSIZE(arg2) - VARHDRSZ; + text *arg1 = PG_GETARG_TEXT_PP(0); + text *arg2 = PG_GETARG_TEXT_PP(1); + int32 arg1_size = VARSIZE_ANY_EXHDR(arg1); + int32 arg2_size = VARSIZE_ANY_EXHDR(arg2); int32 new_text_size = arg1_size + arg2_size + VARHDRSZ; text *new_text = (text *) palloc(new_text_size); SET_VARSIZE(new_text, new_text_size); - memcpy(VARDATA(new_text), VARDATA(arg1), arg1_size); - memcpy(VARDATA(new_text) + arg1_size, VARDATA(arg2), arg2_size); + memcpy(VARDATA(new_text), VARDATA_ANY(arg1), arg1_size); + memcpy(VARDATA(new_text) + arg1_size, VARDATA_ANY(arg2), arg2_size); PG_RETURN_TEXT_P(new_text); } -- cgit v1.2.3 From 9ca5c8721dfa2413e910531373395c6e18062ddf Mon Sep 17 00:00:00 2001 From: Michael Meskes Date: Mon, 13 Mar 2017 20:48:29 +0100 Subject: Document two phase commit commands in ecpg docu. Patch by Masahiko Sawada --- doc/src/sgml/ecpg.sgml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) (limited to 'doc/src') diff --git a/doc/src/sgml/ecpg.sgml b/doc/src/sgml/ecpg.sgml index b8021cbe5b..bc4dd68620 100644 --- a/doc/src/sgml/ecpg.sgml +++ b/doc/src/sgml/ecpg.sgml @@ -507,6 +507,35 @@ EXEC SQL COMMIT; + + + EXEC SQL PREPARE TRANSACTION transaction_id + + + Prepare the current transaction for two-phase commit. + + + + + + + EXEC SQL COMMIT PREPARED transaction_id + + + Commit a transaction that is in prepared state. + + + + + + EXEC SQL ROLLBACK PREPARED transaction_id + + + Roll back a transaction that is in prepared state. + + + + EXEC SQL SET AUTOCOMMIT TO ON -- cgit v1.2.3 From 0c87cd003d9966fcb19d6998ccf90d3276b08e0c Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 13 Mar 2017 19:06:28 -0400 Subject: Fix busted markup. Oversight in commit 9ca5c8721. Per buildfarm. --- doc/src/sgml/ecpg.sgml | 2 -- 1 file changed, 2 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ecpg.sgml b/doc/src/sgml/ecpg.sgml index bc4dd68620..508406823d 100644 --- a/doc/src/sgml/ecpg.sgml +++ b/doc/src/sgml/ecpg.sgml @@ -507,7 +507,6 @@ EXEC SQL COMMIT; - EXEC SQL PREPARE TRANSACTION transaction_id @@ -517,7 +516,6 @@ EXEC SQL COMMIT; - EXEC SQL COMMIT PREPARED transaction_id -- cgit v1.2.3 From 37289ffdbfc7319a3cdb271b11227bf682f76d90 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 14 Mar 2017 11:19:12 -0400 Subject: doc: Fix synopsis --- doc/src/sgml/ref/drop_subscription.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/drop_subscription.sgml b/doc/src/sgml/ref/drop_subscription.sgml index 4228f1a253..f5e449a9a5 100644 --- a/doc/src/sgml/ref/drop_subscription.sgml +++ b/doc/src/sgml/ref/drop_subscription.sgml @@ -21,7 +21,7 @@ PostgreSQL documentation -DROP SUBSCRIPTION [ IF EXISTS ] name [ DROP SLOT | NODROP SLOT ] +DROP SUBSCRIPTION [ IF EXISTS ] name [ DROP SLOT | NODROP SLOT ] -- cgit v1.2.3 From a47b38c9ee4960556b0edb76a42171bd281cc1cf Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 14 Mar 2017 12:57:10 -0400 Subject: Spelling fixes From: Josh Soref --- contrib/ltree/ltxtquery_io.c | 2 +- doc/src/sgml/biblio.sgml | 2 +- doc/src/sgml/monitoring.sgml | 2 +- doc/src/sgml/protocol.sgml | 6 +++--- doc/src/sgml/ref/create_trigger.sgml | 2 +- doc/src/sgml/release-8.4.sgml | 2 +- doc/src/sgml/release-9.0.sgml | 2 +- doc/src/sgml/release-9.1.sgml | 2 +- doc/src/sgml/release-9.2.sgml | 2 +- doc/src/sgml/release-9.3.sgml | 2 +- doc/src/sgml/release-9.4.sgml | 2 +- doc/src/sgml/release-9.6.sgml | 2 +- doc/src/sgml/release-old.sgml | 6 +++--- src/backend/access/transam/xlog.c | 2 +- src/backend/executor/nodeAgg.c | 2 +- src/backend/optimizer/geqo/geqo_erx.c | 2 +- src/backend/port/dynloader/linux.c | 2 +- src/backend/replication/logical/worker.c | 2 +- src/backend/replication/walreceiverfuncs.c | 8 ++++---- src/bin/pg_basebackup/t/030_pg_recvlogical.pl | 2 +- src/include/access/xlog_internal.h | 2 +- src/interfaces/libpq/bcc32.mak | 2 +- src/test/regress/expected/tsdicts.out | 8 ++++---- src/test/regress/sql/tsdicts.sql | 2 +- 24 files changed, 34 insertions(+), 34 deletions(-) (limited to 'doc/src') diff --git a/contrib/ltree/ltxtquery_io.c b/contrib/ltree/ltxtquery_io.c index 32d9046258..9ca1994249 100644 --- a/contrib/ltree/ltxtquery_io.c +++ b/contrib/ltree/ltxtquery_io.c @@ -96,7 +96,7 @@ gettoken_query(QPRS_STATE *state, int32 *val, int32 *lenval, char **strval, uint if (*flag) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("modificators syntax error"))); + errmsg("modifiers syntax error"))); *lenval += charlen; } else if (charlen == 1 && t_iseq(state->buf, '%')) diff --git a/doc/src/sgml/biblio.sgml b/doc/src/sgml/biblio.sgml index e2cd69d278..0da3a83fae 100644 --- a/doc/src/sgml/biblio.sgml +++ b/doc/src/sgml/biblio.sgml @@ -295,7 +295,7 @@ ssimkovi@ag.or.at April, 1990 University of California -
    Berkely, California
    +
    Berkeley, California
     diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 4d03531cc1..9eaf43adb1 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -1233,7 +1233,7 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser MessageQueuePutMessage - Waiting to write a protoocol message to a shared message queue. + Waiting to write a protocol message to a shared message queue. MessageQueueReceive diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index 3d6e8eed43..67f3f455db 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -6019,7 +6019,7 @@ TupleData - Idenfifies the data as NULL value. + Identifies the data as NULL value.     @@ -6032,7 +6032,7 @@ TupleData - Idenfifies unchanged TOASTed value (the actual value is not + Identifies unchanged TOASTed value (the actual value is not sent). @@ -6046,7 +6046,7 @@ TupleData - Idenfifies the data as text formatted value. + Identifies the data as text formatted value. diff --git a/doc/src/sgml/ref/create_trigger.sgml b/doc/src/sgml/ref/create_trigger.sgml index 8590e226e3..c9a9fa6ba2 100644 --- a/doc/src/sgml/ref/create_trigger.sgml +++ b/doc/src/sgml/ref/create_trigger.sgml @@ -295,7 +295,7 @@ UPDATE OF column_name1 [, column_name2REFERENCING - This immediately preceeds the declaration of one or two relations which + This immediately precedes the declaration of one or two relations which can be used to read the before and/or after images of all rows directly affected by the triggering statement. An AFTER EACH ROW trigger is allowed to use both these transition relation names and the diff --git a/doc/src/sgml/release-8.4.sgml b/doc/src/sgml/release-8.4.sgml index 0d0478855e..f43b0958bc 100644 --- a/doc/src/sgml/release-8.4.sgml +++ b/doc/src/sgml/release-8.4.sgml @@ -381,7 +381,7 @@ This prevents scenarios wherein a pathological regular expression - could lock up a server process uninterruptably for a long time. + could lock up a server process uninterruptibly for a long time. diff --git a/doc/src/sgml/release-9.0.sgml b/doc/src/sgml/release-9.0.sgml index 2238b53745..08bc339e2e 100644 --- a/doc/src/sgml/release-9.0.sgml +++ b/doc/src/sgml/release-9.0.sgml @@ -2250,7 +2250,7 @@ This prevents scenarios wherein a pathological regular expression - could lock up a server process uninterruptably for a long time. + could lock up a server process uninterruptibly for a long time. diff --git a/doc/src/sgml/release-9.1.sgml b/doc/src/sgml/release-9.1.sgml index edacfbf355..eb2d6ac278 100644 --- a/doc/src/sgml/release-9.1.sgml +++ b/doc/src/sgml/release-9.1.sgml @@ -3941,7 +3941,7 @@ Branch: REL9_0_STABLE [9d6af7367] 2015-08-15 11:02:34 -0400 This prevents scenarios wherein a pathological regular expression - could lock up a server process uninterruptably for a long time. + could lock up a server process uninterruptibly for a long time. diff --git a/doc/src/sgml/release-9.2.sgml b/doc/src/sgml/release-9.2.sgml index 5c80517eaf..b27176bdd6 100644 --- a/doc/src/sgml/release-9.2.sgml +++ b/doc/src/sgml/release-9.2.sgml @@ -4774,7 +4774,7 @@ Branch: REL9_2_STABLE [6b700301c] 2015-02-17 16:03:00 +0100 This prevents scenarios wherein a pathological regular expression - could lock up a server process uninterruptably for a long time. + could lock up a server process uninterruptibly for a long time. diff --git a/doc/src/sgml/release-9.3.sgml b/doc/src/sgml/release-9.3.sgml index 7f4adcd8c3..df36947930 100644 --- a/doc/src/sgml/release-9.3.sgml +++ b/doc/src/sgml/release-9.3.sgml @@ -6968,7 +6968,7 @@ Branch: REL8_4_STABLE [b6e143458] 2014-03-01 15:21:13 -0500 This prevents scenarios wherein a pathological regular expression - could lock up a server process uninterruptably for a long time. + could lock up a server process uninterruptibly for a long time. diff --git a/doc/src/sgml/release-9.4.sgml b/doc/src/sgml/release-9.4.sgml index 4b7e41b67f..3d5e4cf52a 100644 --- a/doc/src/sgml/release-9.4.sgml +++ b/doc/src/sgml/release-9.4.sgml @@ -8899,7 +8899,7 @@ Branch: REL9_4_STABLE [c2b06ab17] 2015-01-30 22:45:58 -0500 Add option diff --git a/doc/src/sgml/release-9.6.sgml b/doc/src/sgml/release-9.6.sgml index 02cc8c9003..7a37f3f44b 100644 --- a/doc/src/sgml/release-9.6.sgml +++ b/doc/src/sgml/release-9.6.sgml @@ -3116,7 +3116,7 @@ and many others in the same vein This view exposes the same information available from - the pg_config comand-line utility, + the pg_config command-line utility, namely assorted compile-time configuration information for PostgreSQL. diff --git a/doc/src/sgml/release-old.sgml b/doc/src/sgml/release-old.sgml index 266d8719b6..a8622188eb 100644 --- a/doc/src/sgml/release-old.sgml +++ b/doc/src/sgml/release-old.sgml @@ -5737,8 +5737,8 @@ fix rtree for use in inner scan (Vadim) fix gist for use in inner scan, cleanups (Vadim, Andrea) avoid unnecessary local buffers allocation (Vadim, Massimo) fix local buffers leak in transaction aborts (Vadim) -fix file manager memmory leaks, cleanups (Vadim, Massimo) -fix storage manager memmory leaks (Vadim) +fix file manager memory leaks, cleanups (Vadim, Massimo) +fix storage manager memory leaks (Vadim) fix btree duplicates handling (Vadim) fix deleted rows reincarnation caused by vacuum (Vadim) fix SELECT varchar()/char() INTO TABLE made zero-length fields(Bruce) @@ -5904,7 +5904,7 @@ European date format now set when postmaster is started Execute lowercase function names if not found with exact case Fixes for aggregate/GROUP processing, allow 'select sum(func(x),sum(x+y) from z' Gist now included in the distribution(Marc) -Idend authentication of local users(Bryan) +Ident authentication of local users(Bryan) Implement BETWEEN qualifier(Bruce) Implement IN qualifier(Bruce) libpq has PQgetisnull()(Bruce) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index c8c2dd8cca..64335f909e 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -859,7 +859,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, bool find_free, XLogSegNo max_segno, bool use_lock); static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, - int source, bool notexistOk); + int source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source); static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *readBuf, diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c index aa08152350..3207ee460c 100644 --- a/src/backend/executor/nodeAgg.c +++ b/src/backend/executor/nodeAgg.c @@ -486,7 +486,7 @@ static void agg_fill_hash_table(AggState *aggstate); static TupleTableSlot *agg_retrieve_hash_table(AggState *aggstate); static Datum GetAggInitVal(Datum textInitVal, Oid transtype); static void build_pertrans_for_aggref(AggStatePerTrans pertrans, - AggState *aggsate, EState *estate, + AggState *aggstate, EState *estate, Aggref *aggref, Oid aggtransfn, Oid aggtranstype, Oid aggserialfn, Oid aggdeserialfn, Datum initValue, bool initValueIsNull, diff --git a/src/backend/optimizer/geqo/geqo_erx.c b/src/backend/optimizer/geqo/geqo_erx.c index 023abf70e2..133fe32348 100644 --- a/src/backend/optimizer/geqo/geqo_erx.c +++ b/src/backend/optimizer/geqo/geqo_erx.c @@ -458,7 +458,7 @@ edge_failure(PlannerInfo *root, Gene *gene, int index, Edge *edge_table, int num if (edge_table[i].unused_edges >= 0) return (Gene) i; - elog(LOG, "no edge found via looking for the last ununsed point"); + elog(LOG, "no edge found via looking for the last unused point"); } diff --git a/src/backend/port/dynloader/linux.c b/src/backend/port/dynloader/linux.c index 5d8a76af59..368bf0006f 100644 --- a/src/backend/port/dynloader/linux.c +++ b/src/backend/port/dynloader/linux.c @@ -124,7 +124,7 @@ char * pg_dlerror(void) { #ifndef HAVE_DLD_H - return "dynaloader unspported"; + return "dynaloader unsupported"; #else return dld_strerror(dld_errno); #endif diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 535aa2df1b..c3e54af259 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -575,7 +575,7 @@ check_relation_updatable(LogicalRepRelMapEntry *rel) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("logical replication target relation \"%s.%s\" has " - "neither REPLICA IDENTIY index nor PRIMARY " + "neither REPLICA IDENTITY index nor PRIMARY " "KEY and published relation does not have " "REPLICA IDENTITY FULL", rel->remoterel.nspname, rel->remoterel.relname))); diff --git a/src/backend/replication/walreceiverfuncs.c b/src/backend/replication/walreceiverfuncs.c index 8bf1fe8565..8ed7254b5c 100644 --- a/src/backend/replication/walreceiverfuncs.c +++ b/src/backend/replication/walreceiverfuncs.c @@ -321,7 +321,7 @@ GetReplicationApplyDelay(void) long secs; int usecs; - TimestampTz chunckReplayStartTime; + TimestampTz chunkReplayStartTime; SpinLockAcquire(&walrcv->mutex); receivePtr = walrcv->receivedUpto; @@ -332,12 +332,12 @@ GetReplicationApplyDelay(void) if (receivePtr == replayPtr) return 0; - chunckReplayStartTime = GetCurrentChunkReplayStartTime(); + chunkReplayStartTime = GetCurrentChunkReplayStartTime(); - if (chunckReplayStartTime == 0) + if (chunkReplayStartTime == 0) return -1; - TimestampDifference(chunckReplayStartTime, + TimestampDifference(chunkReplayStartTime, GetCurrentTimestamp(), &secs, &usecs); diff --git a/src/bin/pg_basebackup/t/030_pg_recvlogical.pl b/src/bin/pg_basebackup/t/030_pg_recvlogical.pl index b43176fdc6..3e66f70fce 100644 --- a/src/bin/pg_basebackup/t/030_pg_recvlogical.pl +++ b/src/bin/pg_basebackup/t/030_pg_recvlogical.pl @@ -29,7 +29,7 @@ $node->command_fails(['pg_recvlogical', '-S', 'test'], $node->command_fails(['pg_recvlogical', '-S', 'test', '-d', 'postgres'], 'pg_recvlogical needs an action'); $node->command_fails(['pg_recvlogical', '-S', 'test', '-d', $node->connstr('postgres'), '--start'], - 'no destionation file'); + 'no destination file'); $node->command_ok(['pg_recvlogical', '-S', 'test', '-d', $node->connstr('postgres'), '--create-slot'], 'slot created'); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index 578bff593c..b8b15f1875 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -288,7 +288,7 @@ extern const RmgrData RmgrTable[]; * Exported to support xlog switching from checkpointer */ extern pg_time_t GetLastSegSwitchData(XLogRecPtr *lastSwitchLSN); -extern XLogRecPtr RequestXLogSwitch(bool mark_uninmportant); +extern XLogRecPtr RequestXLogSwitch(bool mark_unimportant); extern void GetOldestRestartPoint(XLogRecPtr *oldrecptr, TimeLineID *oldtli); diff --git a/src/interfaces/libpq/bcc32.mak b/src/interfaces/libpq/bcc32.mak index 78102fafd4..f541fa8ee6 100644 --- a/src/interfaces/libpq/bcc32.mak +++ b/src/interfaces/libpq/bcc32.mak @@ -8,7 +8,7 @@ !IF "$(BCB)" == "" !MESSAGE You must edit bcc32.mak and define BCB at the top -!ERROR misssing BCB +!ERROR missing BCB !ENDIF !IF "$(__NMAKE__)" == "" diff --git a/src/test/regress/expected/tsdicts.out b/src/test/regress/expected/tsdicts.out index 493a25587c..0744ef803b 100644 --- a/src/test/regress/expected/tsdicts.out +++ b/src/test/regress/expected/tsdicts.out @@ -568,10 +568,10 @@ SELECT to_tsvector('thesaurus_tst', 'one postgres one two one two three one'); '1':1,5 '12':3 '123':4 'pgsql':2 (1 row) -SELECT to_tsvector('thesaurus_tst', 'Supernovae star is very new star and usually called supernovae (abbrevation SN)'); - to_tsvector -------------------------------------------------------------- - 'abbrev':10 'call':8 'new':4 'sn':1,9,11 'star':5 'usual':7 +SELECT to_tsvector('thesaurus_tst', 'Supernovae star is very new star and usually called supernovae (abbreviation SN)'); + to_tsvector +-------------------------------------------------------------- + 'abbrevi':10 'call':8 'new':4 'sn':1,9,11 'star':5 'usual':7 (1 row) SELECT to_tsvector('thesaurus_tst', 'Booking tickets is looking like a booking a tickets'); diff --git a/src/test/regress/sql/tsdicts.sql b/src/test/regress/sql/tsdicts.sql index ed2cbe1fec..a5a569e1ad 100644 --- a/src/test/regress/sql/tsdicts.sql +++ b/src/test/regress/sql/tsdicts.sql @@ -186,5 +186,5 @@ ALTER TEXT SEARCH CONFIGURATION thesaurus_tst ALTER MAPPING FOR WITH synonym, thesaurus, english_stem; SELECT to_tsvector('thesaurus_tst', 'one postgres one two one two three one'); -SELECT to_tsvector('thesaurus_tst', 'Supernovae star is very new star and usually called supernovae (abbrevation SN)'); +SELECT to_tsvector('thesaurus_tst', 'Supernovae star is very new star and usually called supernovae (abbreviation SN)'); SELECT to_tsvector('thesaurus_tst', 'Booking tickets is looking like a booking a tickets'); -- cgit v1.2.3 From c11453ce0aeaa377cbbcc9a3fc418acb94629330 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Tue, 14 Mar 2017 13:27:02 -0400 Subject: hash: Add write-ahead logging support. The warning about hash indexes not being write-ahead logged and their use being discouraged has been removed. "snapshot too old" is now supported for tables with hash indexes. Most importantly, barring bugs, hash indexes will now be crash-safe and usable on standbys. This commit doesn't yet add WAL consistency checking for hash indexes, as we now have for other index types; a separate patch has been submitted to cure that lack. Amit Kapila, reviewed and slightly modified by me. The larger patch series of which this is a part has been reviewed and tested by Álvaro Herrera, Ashutosh Sharma, Mark Kirkwood, Jeff Janes, and Jesper Pedersen. Discussion: http://postgr.es/m/CAA4eK1JOBX=YU33631Qh-XivYXtPSALh514+jR8XeD7v+K3r_Q@mail.gmail.com --- contrib/pageinspect/expected/hash.out | 1 - contrib/pgstattuple/expected/pgstattuple.out | 2 - doc/src/sgml/backup.sgml | 13 - doc/src/sgml/config.sgml | 7 +- doc/src/sgml/high-availability.sgml | 6 - doc/src/sgml/indices.sgml | 12 - doc/src/sgml/ref/create_index.sgml | 13 - src/backend/access/hash/Makefile | 2 +- src/backend/access/hash/README | 138 +++- src/backend/access/hash/hash.c | 81 ++- src/backend/access/hash/hash_xlog.c | 963 +++++++++++++++++++++++++ src/backend/access/hash/hashinsert.c | 59 +- src/backend/access/hash/hashovfl.c | 209 +++++- src/backend/access/hash/hashpage.c | 236 +++++- src/backend/access/hash/hashsearch.c | 5 + src/backend/access/rmgrdesc/hashdesc.c | 134 +++- src/backend/commands/indexcmds.c | 5 - src/backend/utils/cache/relcache.c | 12 +- src/include/access/hash_xlog.h | 232 ++++++ src/test/regress/expected/create_index.out | 5 - src/test/regress/expected/enum.out | 1 - src/test/regress/expected/hash_index.out | 2 - src/test/regress/expected/macaddr.out | 1 - src/test/regress/expected/replica_identity.out | 1 - src/test/regress/expected/uuid.out | 1 - 25 files changed, 2001 insertions(+), 140 deletions(-) create mode 100644 src/backend/access/hash/hash_xlog.c (limited to 'doc/src') diff --git a/contrib/pageinspect/expected/hash.out b/contrib/pageinspect/expected/hash.out index 8ed60bccc0..3ba01f6ca3 100644 --- a/contrib/pageinspect/expected/hash.out +++ b/contrib/pageinspect/expected/hash.out @@ -1,7 +1,6 @@ CREATE TABLE test_hash (a int, b text); INSERT INTO test_hash VALUES (1, 'one'); CREATE INDEX test_hash_a_idx ON test_hash USING hash (a); -WARNING: hash indexes are not WAL-logged and their use is discouraged \x SELECT hash_page_type(get_raw_page('test_hash_a_idx', 0)); -[ RECORD 1 ]--+--------- diff --git a/contrib/pgstattuple/expected/pgstattuple.out b/contrib/pgstattuple/expected/pgstattuple.out index baee2bd96e..2c3515b4ef 100644 --- a/contrib/pgstattuple/expected/pgstattuple.out +++ b/contrib/pgstattuple/expected/pgstattuple.out @@ -131,7 +131,6 @@ select * from pgstatginindex('test_ginidx'); (1 row) create index test_hashidx on test using hash (b); -WARNING: hash indexes are not WAL-logged and their use is discouraged select * from pgstathashindex('test_hashidx'); version | bucket_pages | overflow_pages | bitmap_pages | zero_pages | live_items | dead_items | free_percent ---------+--------------+----------------+--------------+------------+------------+------------+-------------- @@ -226,7 +225,6 @@ ERROR: "test_partition" is not an index -- an actual index of a partitioned table should work though create index test_partition_idx on test_partition(a); create index test_partition_hash_idx on test_partition using hash (a); -WARNING: hash indexes are not WAL-logged and their use is discouraged -- these should work select pgstatindex('test_partition_idx'); pgstatindex diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml index 12f2a14a5c..69c599e180 100644 --- a/doc/src/sgml/backup.sgml +++ b/doc/src/sgml/backup.sgml @@ -1536,19 +1536,6 @@ archive_command = 'local_backup_script.sh "%p" "%f"' technique. These will probably be fixed in future releases: - - - Operations on hash indexes are not presently WAL-logged, so - replay will not update these indexes. This will mean that any new inserts - will be ignored by the index, updated rows will apparently disappear and - deleted rows will still retain pointers. In other words, if you modify a - table with a hash index on it then you will get incorrect query results - on a standby server. When recovery completes it is recommended that you - manually - each such index after completing a recovery operation. - - - If a diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 69844e5b29..eadbfcd22f 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -2153,10 +2153,9 @@ include_dir 'conf.d' has materialized a result set, no error will be generated even if the underlying rows in the referenced table have been vacuumed away. Some tables cannot safely be vacuumed early, and so will not be - affected by this setting. Examples include system catalogs and any - table which has a hash index. For such tables this setting will - neither reduce bloat nor create a possibility of a snapshot - too old error on scanning. + affected by this setting, such as system catalogs. For such tables + this setting will neither reduce bloat nor create a possibility + of a snapshot too old error on scanning. diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml index 48de2ced23..0e619912d8 100644 --- a/doc/src/sgml/high-availability.sgml +++ b/doc/src/sgml/high-availability.sgml @@ -2351,12 +2351,6 @@ LOG: database system is ready to accept read only connections These can and probably will be fixed in future releases: - - - Operations on hash indexes are not presently WAL-logged, so - replay will not update these indexes. - - Full knowledge of running transactions is required before snapshots diff --git a/doc/src/sgml/indices.sgml b/doc/src/sgml/indices.sgml index 271c135519..e40750e8ec 100644 --- a/doc/src/sgml/indices.sgml +++ b/doc/src/sgml/indices.sgml @@ -193,18 +193,6 @@ CREATE INDEX name ON table - - - Hash index operations are not presently WAL-logged, - so hash indexes might need to be rebuilt with REINDEX - after a database crash if there were unwritten changes. - Also, changes to hash indexes are not replicated over streaming or - file-based replication after the initial base backup, so they - give wrong answers to queries that subsequently use them. - For these reasons, hash index use is presently discouraged. - - - index diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index fcb7a60ce3..7163b032b1 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -510,19 +510,6 @@ Indexes: they can be useful. - - - Hash index operations are not presently WAL-logged, - so hash indexes might need to be rebuilt with REINDEX - after a database crash if there were unwritten changes. - Also, changes to hash indexes are not replicated over streaming or - file-based replication after the initial base backup, so they - give wrong answers to queries that subsequently use them. - Hash indexes are also not properly restored during point-in-time - recovery. For these reasons, hash index use is presently discouraged. - - - Currently, only the B-tree, GiST, GIN, and BRIN index methods support multicolumn indexes. Up to 32 fields can be specified by default. diff --git a/src/backend/access/hash/Makefile b/src/backend/access/hash/Makefile index e2e7e91493..b154569b46 100644 --- a/src/backend/access/hash/Makefile +++ b/src/backend/access/hash/Makefile @@ -13,6 +13,6 @@ top_builddir = ../../../.. include $(top_builddir)/src/Makefile.global OBJS = hash.o hashfunc.o hashinsert.o hashovfl.o hashpage.o hashsearch.o \ - hashsort.o hashutil.o hashvalidate.o + hashsort.o hashutil.o hashvalidate.o hash_xlog.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/access/hash/README b/src/backend/access/hash/README index 703ae98207..00beb86ffa 100644 --- a/src/backend/access/hash/README +++ b/src/backend/access/hash/README @@ -213,7 +213,7 @@ this flag must be clear before splitting a bucket; thus, a bucket can't be split again until the previous split is totally complete. The moved-by-split flag on a tuple indicates that tuple is moved from old to -new bucket. Concurrent scans can skip such tuples till the split operation +new bucket. Concurrent scans will skip such tuples until the split operation is finished. Once the tuple is marked as moved-by-split, it will remain so forever but that does no harm. We have intentionally not cleared it as that can generate an additional I/O which is not necessary. @@ -287,13 +287,17 @@ The insertion algorithm is rather similar: if current page is full, release lock but not pin, read/exclusive-lock next page; repeat as needed >> see below if no space in any page of bucket + take buffer content lock in exclusive mode on metapage insert tuple at appropriate place in page - mark current page dirty and release buffer content lock and pin - if the current page is not a bucket page, release the pin on bucket page - pin meta page and take buffer content lock in exclusive mode + mark current page dirty increment tuple count, decide if split needed - mark meta page dirty and release buffer content lock and pin - done if no split needed, else enter Split algorithm below + mark meta page dirty + write WAL for insertion of tuple + release the buffer content lock on metapage + release buffer content lock on current page + if current page is not a bucket page, release the pin on bucket page + if split is needed, enter Split algorithm below + release the pin on metapage To speed searches, the index entries within any individual index page are kept sorted by hash code; the insertion code must take care to insert new @@ -328,12 +332,17 @@ existing bucket in two, thereby lowering the fill ratio: try to finish the split and the cleanup work if that succeeds, start over; if it fails, give up mark the old and new buckets indicating split is in progress + mark both old and new buckets as dirty + write WAL for allocation of new page for split copy the tuples that belongs to new bucket from old bucket, marking them as moved-by-split + write WAL record for moving tuples to new page once the new page is full + or all the pages of old bucket are finished release lock but not pin for primary bucket page of old bucket, read/shared-lock next page; repeat as needed clear the bucket-being-split and bucket-being-populated flags mark the old bucket indicating split-cleanup + write WAL for changing the flags on both old and new buckets The split operation's attempt to acquire cleanup-lock on the old bucket number could fail if another process holds any lock or pin on it. We do not want to @@ -369,6 +378,8 @@ The fourth operation is garbage collection (bulk deletion): acquire cleanup lock on primary bucket page loop: scan and remove tuples + mark the target page dirty + write WAL for deleting tuples from target page if this is the last bucket page, break out of loop pin and x-lock next page release prior lock and pin (except keep pin on primary bucket page) @@ -383,7 +394,8 @@ The fourth operation is garbage collection (bulk deletion): check if number of buckets changed if so, release content lock and pin and return to for-each-bucket loop else update metapage tuple count - mark meta page dirty and release buffer content lock and pin + mark meta page dirty and write WAL for update of metapage + release buffer content lock and pin Note that this is designed to allow concurrent splits and scans. If a split occurs, tuples relocated into the new bucket will be visited twice by the @@ -425,18 +437,16 @@ Obtaining an overflow page: search for a free page (zero bit in bitmap) if found: set bit in bitmap - mark bitmap page dirty and release content lock + mark bitmap page dirty take metapage buffer content lock in exclusive mode if first-free-bit value did not change, update it and mark meta page dirty - release meta page buffer content lock - return page number else (not found): release bitmap page buffer content lock loop back to try next bitmap page, if any -- here when we have checked all bitmap pages; we hold meta excl. lock extend index to add another overflow page; update meta information - mark meta page dirty and release buffer content lock + mark meta page dirty return page number It is slightly annoying to release and reacquire the metapage lock @@ -456,12 +466,17 @@ like this: -- having determined that no space is free in the target bucket: remember last page of bucket, drop write lock on it - call free-page-acquire routine re-write-lock last page of bucket if it is not last anymore, step to the last page - update (former) last page to point to new page + execute free-page-acquire (obtaining an overflow page) mechanism + described above + update (former) last page to point to the new page and mark buffer dirty write-lock and initialize new page, with back link to former last page - write and release former last page + write WAL for addition of overflow page + release the locks on meta page and bitmap page acquired in + free-page-acquire algorithm + release the lock on former last page + release the lock on new overflow page insert tuple into new page -- etc. @@ -488,12 +503,14 @@ accessors of pages in the bucket. The algorithm is: determine which bitmap page contains the free space bit for page release meta page buffer content lock pin bitmap page and take buffer content lock in exclusive mode - update bitmap bit - mark bitmap page dirty and release buffer content lock and pin - if page number is less than what we saw as first-free-bit in meta: retake meta page buffer content lock in exclusive mode + move (insert) tuples that belong to the overflow page being freed + update bitmap bit + mark bitmap page dirty if page number is still less than first-free-bit, update first-free-bit field and mark meta page dirty + write WAL for delinking overflow page operation + release buffer content lock and pin release meta page buffer content lock and pin We have to do it this way because we must clear the bitmap bit before @@ -504,8 +521,91 @@ page acquirer will scan more bitmap bits than he needs to. What must be avoided is having first-free-bit greater than the actual first free bit, because then that free page would never be found by searchers. -All the freespace operations should be called while holding no buffer -locks. Since they need no lmgr locks, deadlock is not possible. +The reason of moving tuples from overflow page while delinking the later is +to make that as an atomic operation. Not doing so could lead to spurious reads +on standby. Basically, the user might see the same tuple twice. + + +WAL Considerations +------------------ + +The hash index operations like create index, insert, delete, bucket split, +allocate overflow page, and squeeze in themselves don't guarantee hash index +consistency after a crash. To provide robustness, we write WAL for each of +these operations. + +CREATE INDEX writes multiple WAL records. First, we write a record to cover +the initializatoin of the metapage, followed by one for each new bucket +created, followed by one for the initial bitmap page. It's not important for +index creation to appear atomic, because the index isn't yet visible to any +other transaction, and the creating transaction will roll back in the event of +a crash. It would be difficult to cover the whole operation with a single +write-ahead log record anyway, because we can log only a fixed number of +pages, as given by XLR_MAX_BLOCK_ID (32), with current XLog machinery. + +Ordinary item insertions (that don't force a page split or need a new overflow +page) are single WAL entries. They touch a single bucket page and the +metapage. The metapage is updated during replay as it is updated during +original operation. + +If an insertion causes the addition of an overflow page, there will be one +WAL entry for the new overflow page and second entry for insert itself. + +If an insertion causes a bucket split, there will be one WAL entry for insert +itself, followed by a WAL entry for allocating a new bucket, followed by a WAL +entry for each overflow bucket page in the new bucket to which the tuples are +moved from old bucket, followed by a WAL entry to indicate that split is +complete for both old and new buckets. A split operation which requires +overflow pages to complete the operation will need to write a WAL record for +each new allocation of an overflow page. + +As splitting involves multiple atomic actions, it's possible that the system +crashes between moving tuples from bucket pages of the old bucket to new +bucket. In such a case, after recovery, the old and new buckets will be +marked with bucket-being-split and bucket-being-populated flags respectively +which indicates that split is in progress for those buckets. The reader +algorithm works correctly, as it will scan both the old and new buckets when +the split is in progress as explained in the reader algorithm section above. + +We finish the split at next insert or split operation on the old bucket as +explained in insert and split algorithm above. It could be done during +searches, too, but it seems best not to put any extra updates in what would +otherwise be a read-only operation (updating is not possible in hot standby +mode anyway). It would seem natural to complete the split in VACUUM, but since +splitting a bucket might require allocating a new page, it might fail if you +run out of disk space. That would be bad during VACUUM - the reason for +running VACUUM in the first place might be that you run out of disk space, +and now VACUUM won't finish because you're out of disk space. In contrast, +an insertion can require enlarging the physical file anyway. + +Deletion of tuples from a bucket is performed for two reasons: to remove dead +tuples, and to remove tuples that were moved by a bucket split. A WAL entry +is made for each bucket page from which tuples are removed, and then another +WAL entry is made when we clear the needs-split-cleanup flag. If dead tuples +are removed, a separate WAL entry is made to update the metapage. + +As deletion involves multiple atomic operations, it is quite possible that +system crashes after (a) removing tuples from some of the bucket pages, (b) +before clearing the garbage flag, or (c) before updating the metapage. If the +system crashes before completing (b), it will again try to clean the bucket +during next vacuum or insert after recovery which can have some performance +impact, but it will work fine. If the system crashes before completing (c), +after recovery there could be some additional splits until the next vacuum +updates the metapage, but the other operations like insert, delete and scan +will work correctly. We can fix this problem by actually updating the +metapage based on delete operation during replay, but it's not clear whether +it's worth the complication. + +A squeeze operation moves tuples from one of the buckets later in the chain to +one of the bucket earlier in chain and writes WAL record when either the +bucket to which it is writing tuples is filled or bucket from which it +is removing the tuples becomes empty. + +As a squeeze operation involves writing multiple atomic operations, it is +quite possible that the system crashes before completing the operation on +entire bucket. After recovery, the operations will work correctly, but +the index will remain bloated and this can impact performance of read and +insert operations until the next vacuum squeeze the bucket completely. Other Notes diff --git a/src/backend/access/hash/hash.c b/src/backend/access/hash/hash.c index 1f8a7f61c7..641676964b 100644 --- a/src/backend/access/hash/hash.c +++ b/src/backend/access/hash/hash.c @@ -28,6 +28,7 @@ #include "utils/builtins.h" #include "utils/index_selfuncs.h" #include "utils/rel.h" +#include "miscadmin.h" /* Working state for hashbuild and its callback */ @@ -303,6 +304,11 @@ hashgettuple(IndexScanDesc scan, ScanDirection dir) buf = so->hashso_curbuf; Assert(BufferIsValid(buf)); page = BufferGetPage(buf); + + /* + * We don't need test for old snapshot here as the current buffer is + * pinned, so vacuum can't clean the page. + */ maxoffnum = PageGetMaxOffsetNumber(page); for (offnum = ItemPointerGetOffsetNumber(current); offnum <= maxoffnum; @@ -623,6 +629,7 @@ loop_top: } /* Okay, we're really done. Update tuple count in metapage. */ + START_CRIT_SECTION(); if (orig_maxbucket == metap->hashm_maxbucket && orig_ntuples == metap->hashm_ntuples) @@ -649,6 +656,26 @@ loop_top: } MarkBufferDirty(metabuf); + + /* XLOG stuff */ + if (RelationNeedsWAL(rel)) + { + xl_hash_update_meta_page xlrec; + XLogRecPtr recptr; + + xlrec.ntuples = metap->hashm_ntuples; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, sizeof(SizeOfHashUpdateMetaPage)); + + XLogRegisterBuffer(0, metabuf, REGBUF_STANDARD); + + recptr = XLogInsert(RM_HASH_ID, XLOG_HASH_UPDATE_META_PAGE); + PageSetLSN(BufferGetPage(metabuf), recptr); + } + + END_CRIT_SECTION(); + _hash_relbuf(rel, metabuf); /* return statistics */ @@ -816,9 +843,40 @@ hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf, */ if (ndeletable > 0) { + /* No ereport(ERROR) until changes are logged */ + START_CRIT_SECTION(); + PageIndexMultiDelete(page, deletable, ndeletable); bucket_dirty = true; MarkBufferDirty(buf); + + /* XLOG stuff */ + if (RelationNeedsWAL(rel)) + { + xl_hash_delete xlrec; + XLogRecPtr recptr; + + xlrec.is_primary_bucket_page = (buf == bucket_buf) ? true : false; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, SizeOfHashDelete); + + /* + * bucket buffer needs to be registered to ensure that we can + * acquire a cleanup lock on it during replay. + */ + if (!xlrec.is_primary_bucket_page) + XLogRegisterBuffer(0, bucket_buf, REGBUF_STANDARD | REGBUF_NO_IMAGE); + + XLogRegisterBuffer(1, buf, REGBUF_STANDARD); + XLogRegisterBufData(1, (char *) deletable, + ndeletable * sizeof(OffsetNumber)); + + recptr = XLogInsert(RM_HASH_ID, XLOG_HASH_DELETE); + PageSetLSN(BufferGetPage(buf), recptr); + } + + END_CRIT_SECTION(); } /* bail out if there are no more pages to scan. */ @@ -866,8 +924,25 @@ hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf, page = BufferGetPage(bucket_buf); bucket_opaque = (HashPageOpaque) PageGetSpecialPointer(page); + /* No ereport(ERROR) until changes are logged */ + START_CRIT_SECTION(); + bucket_opaque->hasho_flag &= ~LH_BUCKET_NEEDS_SPLIT_CLEANUP; MarkBufferDirty(bucket_buf); + + /* XLOG stuff */ + if (RelationNeedsWAL(rel)) + { + XLogRecPtr recptr; + + XLogBeginInsert(); + XLogRegisterBuffer(0, bucket_buf, REGBUF_STANDARD); + + recptr = XLogInsert(RM_HASH_ID, XLOG_HASH_SPLIT_CLEANUP); + PageSetLSN(page, recptr); + } + + END_CRIT_SECTION(); } /* @@ -881,9 +956,3 @@ hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf, else LockBuffer(bucket_buf, BUFFER_LOCK_UNLOCK); } - -void -hash_redo(XLogReaderState *record) -{ - elog(PANIC, "hash_redo: unimplemented"); -} diff --git a/src/backend/access/hash/hash_xlog.c b/src/backend/access/hash/hash_xlog.c new file mode 100644 index 0000000000..d435215259 --- /dev/null +++ b/src/backend/access/hash/hash_xlog.c @@ -0,0 +1,963 @@ +/*------------------------------------------------------------------------- + * + * hash_xlog.c + * WAL replay logic for hash index. + * + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/hash/hash_xlog.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/hash.h" +#include "access/hash_xlog.h" +#include "access/xlogutils.h" + +/* + * replay a hash index meta page + */ +static void +hash_xlog_init_meta_page(XLogReaderState *record) +{ + XLogRecPtr lsn = record->EndRecPtr; + Page page; + Buffer metabuf; + + xl_hash_init_meta_page *xlrec = (xl_hash_init_meta_page *) XLogRecGetData(record); + + /* create the index' metapage */ + metabuf = XLogInitBufferForRedo(record, 0); + Assert(BufferIsValid(metabuf)); + _hash_init_metabuffer(metabuf, xlrec->num_tuples, xlrec->procid, + xlrec->ffactor, true); + page = (Page) BufferGetPage(metabuf); + PageSetLSN(page, lsn); + MarkBufferDirty(metabuf); + /* all done */ + UnlockReleaseBuffer(metabuf); +} + +/* + * replay a hash index bitmap page + */ +static void +hash_xlog_init_bitmap_page(XLogReaderState *record) +{ + XLogRecPtr lsn = record->EndRecPtr; + Buffer bitmapbuf; + Buffer metabuf; + Page page; + HashMetaPage metap; + uint32 num_buckets; + + xl_hash_init_bitmap_page *xlrec = (xl_hash_init_bitmap_page *) XLogRecGetData(record); + + /* + * Initialize bitmap page + */ + bitmapbuf = XLogInitBufferForRedo(record, 0); + _hash_initbitmapbuffer(bitmapbuf, xlrec->bmsize, true); + PageSetLSN(BufferGetPage(bitmapbuf), lsn); + MarkBufferDirty(bitmapbuf); + UnlockReleaseBuffer(bitmapbuf); + + /* add the new bitmap page to the metapage's list of bitmaps */ + if (XLogReadBufferForRedo(record, 1, &metabuf) == BLK_NEEDS_REDO) + { + /* + * Note: in normal operation, we'd update the metapage while still + * holding lock on the bitmap page. But during replay it's not + * necessary to hold that lock, since nobody can see it yet; the + * creating transaction hasn't yet committed. + */ + page = BufferGetPage(metabuf); + metap = HashPageGetMeta(page); + + num_buckets = metap->hashm_maxbucket + 1; + metap->hashm_mapp[metap->hashm_nmaps] = num_buckets + 1; + metap->hashm_nmaps++; + + PageSetLSN(page, lsn); + MarkBufferDirty(metabuf); + } + if (BufferIsValid(metabuf)) + UnlockReleaseBuffer(metabuf); +} + +/* + * replay a hash index insert without split + */ +static void +hash_xlog_insert(XLogReaderState *record) +{ + HashMetaPage metap; + XLogRecPtr lsn = record->EndRecPtr; + xl_hash_insert *xlrec = (xl_hash_insert *) XLogRecGetData(record); + Buffer buffer; + Page page; + + if (XLogReadBufferForRedo(record, 0, &buffer) == BLK_NEEDS_REDO) + { + Size datalen; + char *datapos = XLogRecGetBlockData(record, 0, &datalen); + + page = BufferGetPage(buffer); + + if (PageAddItem(page, (Item) datapos, datalen, xlrec->offnum, + false, false) == InvalidOffsetNumber) + elog(PANIC, "hash_xlog_insert: failed to add item"); + + PageSetLSN(page, lsn); + MarkBufferDirty(buffer); + } + if (BufferIsValid(buffer)) + UnlockReleaseBuffer(buffer); + + if (XLogReadBufferForRedo(record, 1, &buffer) == BLK_NEEDS_REDO) + { + /* + * Note: in normal operation, we'd update the metapage while still + * holding lock on the page we inserted into. But during replay it's + * not necessary to hold that lock, since no other index updates can + * be happening concurrently. + */ + page = BufferGetPage(buffer); + metap = HashPageGetMeta(page); + metap->hashm_ntuples += 1; + + PageSetLSN(page, lsn); + MarkBufferDirty(buffer); + } + if (BufferIsValid(buffer)) + UnlockReleaseBuffer(buffer); +} + +/* + * replay addition of overflow page for hash index + */ +static void +hash_xlog_add_ovfl_page(XLogReaderState *record) +{ + XLogRecPtr lsn = record->EndRecPtr; + xl_hash_add_ovfl_page *xlrec = (xl_hash_add_ovfl_page *) XLogRecGetData(record); + Buffer leftbuf; + Buffer ovflbuf; + Buffer metabuf; + BlockNumber leftblk; + BlockNumber rightblk; + BlockNumber newmapblk = InvalidBlockNumber; + Page ovflpage; + HashPageOpaque ovflopaque; + uint32 *num_bucket; + char *data; + Size datalen PG_USED_FOR_ASSERTS_ONLY; + bool new_bmpage = false; + + XLogRecGetBlockTag(record, 0, NULL, NULL, &rightblk); + XLogRecGetBlockTag(record, 1, NULL, NULL, &leftblk); + + ovflbuf = XLogInitBufferForRedo(record, 0); + Assert(BufferIsValid(ovflbuf)); + + data = XLogRecGetBlockData(record, 0, &datalen); + num_bucket = (uint32 *) data; + Assert(datalen == sizeof(uint32)); + _hash_initbuf(ovflbuf, InvalidBlockNumber, *num_bucket, LH_OVERFLOW_PAGE, + true); + /* update backlink */ + ovflpage = BufferGetPage(ovflbuf); + ovflopaque = (HashPageOpaque) PageGetSpecialPointer(ovflpage); + ovflopaque->hasho_prevblkno = leftblk; + + PageSetLSN(ovflpage, lsn); + MarkBufferDirty(ovflbuf); + + if (XLogReadBufferForRedo(record, 1, &leftbuf) == BLK_NEEDS_REDO) + { + Page leftpage; + HashPageOpaque leftopaque; + + leftpage = BufferGetPage(leftbuf); + leftopaque = (HashPageOpaque) PageGetSpecialPointer(leftpage); + leftopaque->hasho_nextblkno = rightblk; + + PageSetLSN(leftpage, lsn); + MarkBufferDirty(leftbuf); + } + + if (BufferIsValid(leftbuf)) + UnlockReleaseBuffer(leftbuf); + UnlockReleaseBuffer(ovflbuf); + + /* + * Note: in normal operation, we'd update the bitmap and meta page while + * still holding lock on the overflow pages. But during replay it's not + * necessary to hold those locks, since no other index updates can be + * happening concurrently. + */ + if (XLogRecHasBlockRef(record, 2)) + { + Buffer mapbuffer; + + if (XLogReadBufferForRedo(record, 2, &mapbuffer) == BLK_NEEDS_REDO) + { + Page mappage = (Page) BufferGetPage(mapbuffer); + uint32 *freep = NULL; + char *data; + uint32 *bitmap_page_bit; + + freep = HashPageGetBitmap(mappage); + + data = XLogRecGetBlockData(record, 2, &datalen); + bitmap_page_bit = (uint32 *) data; + + SETBIT(freep, *bitmap_page_bit); + + PageSetLSN(mappage, lsn); + MarkBufferDirty(mapbuffer); + } + if (BufferIsValid(mapbuffer)) + UnlockReleaseBuffer(mapbuffer); + } + + if (XLogRecHasBlockRef(record, 3)) + { + Buffer newmapbuf; + + newmapbuf = XLogInitBufferForRedo(record, 3); + + _hash_initbitmapbuffer(newmapbuf, xlrec->bmsize, true); + + new_bmpage = true; + newmapblk = BufferGetBlockNumber(newmapbuf); + + MarkBufferDirty(newmapbuf); + PageSetLSN(BufferGetPage(newmapbuf), lsn); + + UnlockReleaseBuffer(newmapbuf); + } + + if (XLogReadBufferForRedo(record, 4, &metabuf) == BLK_NEEDS_REDO) + { + HashMetaPage metap; + Page page; + uint32 *firstfree_ovflpage; + + data = XLogRecGetBlockData(record, 4, &datalen); + firstfree_ovflpage = (uint32 *) data; + + page = BufferGetPage(metabuf); + metap = HashPageGetMeta(page); + metap->hashm_firstfree = *firstfree_ovflpage; + + if (!xlrec->bmpage_found) + { + metap->hashm_spares[metap->hashm_ovflpoint]++; + + if (new_bmpage) + { + Assert(BlockNumberIsValid(newmapblk)); + + metap->hashm_mapp[metap->hashm_nmaps] = newmapblk; + metap->hashm_nmaps++; + metap->hashm_spares[metap->hashm_ovflpoint]++; + } + } + + PageSetLSN(page, lsn); + MarkBufferDirty(metabuf); + } + if (BufferIsValid(metabuf)) + UnlockReleaseBuffer(metabuf); +} + +/* + * replay allocation of page for split operation + */ +static void +hash_xlog_split_allocate_page(XLogReaderState *record) +{ + XLogRecPtr lsn = record->EndRecPtr; + xl_hash_split_allocate_page *xlrec = (xl_hash_split_allocate_page *) XLogRecGetData(record); + Buffer oldbuf; + Buffer newbuf; + Buffer metabuf; + Size datalen PG_USED_FOR_ASSERTS_ONLY; + char *data; + XLogRedoAction action; + + /* + * To be consistent with normal operation, here we take cleanup locks on + * both the old and new buckets even though there can't be any concurrent + * inserts. + */ + + /* replay the record for old bucket */ + action = XLogReadBufferForRedoExtended(record, 0, RBM_NORMAL, true, &oldbuf); + + /* + * Note that we still update the page even if it was restored from a full + * page image, because the special space is not included in the image. + */ + if (action == BLK_NEEDS_REDO || action == BLK_RESTORED) + { + Page oldpage; + HashPageOpaque oldopaque; + + oldpage = BufferGetPage(oldbuf); + oldopaque = (HashPageOpaque) PageGetSpecialPointer(oldpage); + + oldopaque->hasho_flag = xlrec->old_bucket_flag; + oldopaque->hasho_prevblkno = xlrec->new_bucket; + + PageSetLSN(oldpage, lsn); + MarkBufferDirty(oldbuf); + } + + /* replay the record for new bucket */ + newbuf = XLogInitBufferForRedo(record, 1); + _hash_initbuf(newbuf, xlrec->new_bucket, xlrec->new_bucket, + xlrec->new_bucket_flag, true); + if (!IsBufferCleanupOK(newbuf)) + elog(PANIC, "hash_xlog_split_allocate_page: failed to acquire cleanup lock"); + MarkBufferDirty(newbuf); + PageSetLSN(BufferGetPage(newbuf), lsn); + + /* + * We can release the lock on old bucket early as well but doing here to + * consistent with normal operation. + */ + if (BufferIsValid(oldbuf)) + UnlockReleaseBuffer(oldbuf); + if (BufferIsValid(newbuf)) + UnlockReleaseBuffer(newbuf); + + /* + * Note: in normal operation, we'd update the meta page while still + * holding lock on the old and new bucket pages. But during replay it's + * not necessary to hold those locks, since no other bucket splits can be + * happening concurrently. + */ + + /* replay the record for metapage changes */ + if (XLogReadBufferForRedo(record, 2, &metabuf) == BLK_NEEDS_REDO) + { + Page page; + HashMetaPage metap; + + page = BufferGetPage(metabuf); + metap = HashPageGetMeta(page); + metap->hashm_maxbucket = xlrec->new_bucket; + + data = XLogRecGetBlockData(record, 2, &datalen); + + if (xlrec->flags & XLH_SPLIT_META_UPDATE_MASKS) + { + uint32 lowmask; + uint32 *highmask; + + /* extract low and high masks. */ + memcpy(&lowmask, data, sizeof(uint32)); + highmask = (uint32 *) ((char *) data + sizeof(uint32)); + + /* update metapage */ + metap->hashm_lowmask = lowmask; + metap->hashm_highmask = *highmask; + + data += sizeof(uint32) * 2; + } + + if (xlrec->flags & XLH_SPLIT_META_UPDATE_SPLITPOINT) + { + uint32 ovflpoint; + uint32 *ovflpages; + + /* extract information of overflow pages. */ + memcpy(&ovflpoint, data, sizeof(uint32)); + ovflpages = (uint32 *) ((char *) data + sizeof(uint32)); + + /* update metapage */ + metap->hashm_spares[ovflpoint] = *ovflpages; + metap->hashm_ovflpoint = ovflpoint; + } + + MarkBufferDirty(metabuf); + PageSetLSN(BufferGetPage(metabuf), lsn); + } + + if (BufferIsValid(metabuf)) + UnlockReleaseBuffer(metabuf); +} + +/* + * replay of split operation + */ +static void +hash_xlog_split_page(XLogReaderState *record) +{ + Buffer buf; + + if (XLogReadBufferForRedo(record, 0, &buf) != BLK_RESTORED) + elog(ERROR, "Hash split record did not contain a full-page image"); + + UnlockReleaseBuffer(buf); +} + +/* + * replay completion of split operation + */ +static void +hash_xlog_split_complete(XLogReaderState *record) +{ + XLogRecPtr lsn = record->EndRecPtr; + xl_hash_split_complete *xlrec = (xl_hash_split_complete *) XLogRecGetData(record); + Buffer oldbuf; + Buffer newbuf; + XLogRedoAction action; + + /* replay the record for old bucket */ + action = XLogReadBufferForRedo(record, 0, &oldbuf); + + /* + * Note that we still update the page even if it was restored from a full + * page image, because the bucket flag is not included in the image. + */ + if (action == BLK_NEEDS_REDO || action == BLK_RESTORED) + { + Page oldpage; + HashPageOpaque oldopaque; + + oldpage = BufferGetPage(oldbuf); + oldopaque = (HashPageOpaque) PageGetSpecialPointer(oldpage); + + oldopaque->hasho_flag = xlrec->old_bucket_flag; + + PageSetLSN(oldpage, lsn); + MarkBufferDirty(oldbuf); + } + if (BufferIsValid(oldbuf)) + UnlockReleaseBuffer(oldbuf); + + /* replay the record for new bucket */ + action = XLogReadBufferForRedo(record, 1, &newbuf); + + /* + * Note that we still update the page even if it was restored from a full + * page image, because the bucket flag is not included in the image. + */ + if (action == BLK_NEEDS_REDO || action == BLK_RESTORED) + { + Page newpage; + HashPageOpaque nopaque; + + newpage = BufferGetPage(newbuf); + nopaque = (HashPageOpaque) PageGetSpecialPointer(newpage); + + nopaque->hasho_flag = xlrec->new_bucket_flag; + + PageSetLSN(newpage, lsn); + MarkBufferDirty(newbuf); + } + if (BufferIsValid(newbuf)) + UnlockReleaseBuffer(newbuf); +} + +/* + * replay move of page contents for squeeze operation of hash index + */ +static void +hash_xlog_move_page_contents(XLogReaderState *record) +{ + XLogRecPtr lsn = record->EndRecPtr; + xl_hash_move_page_contents *xldata = (xl_hash_move_page_contents *) XLogRecGetData(record); + Buffer bucketbuf = InvalidBuffer; + Buffer writebuf = InvalidBuffer; + Buffer deletebuf = InvalidBuffer; + XLogRedoAction action; + + /* + * Ensure we have a cleanup lock on primary bucket page before we start + * with the actual replay operation. This is to ensure that neither a + * scan can start nor a scan can be already-in-progress during the replay + * of this operation. If we allow scans during this operation, then they + * can miss some records or show the same record multiple times. + */ + if (xldata->is_prim_bucket_same_wrt) + action = XLogReadBufferForRedoExtended(record, 1, RBM_NORMAL, true, &writebuf); + else + { + /* + * we don't care for return value as the purpose of reading bucketbuf + * is to ensure a cleanup lock on primary bucket page. + */ + (void) XLogReadBufferForRedoExtended(record, 0, RBM_NORMAL, true, &bucketbuf); + + action = XLogReadBufferForRedo(record, 1, &writebuf); + } + + /* replay the record for adding entries in overflow buffer */ + if (action == BLK_NEEDS_REDO) + { + Page writepage; + char *begin; + char *data; + Size datalen; + uint16 ninserted = 0; + + data = begin = XLogRecGetBlockData(record, 1, &datalen); + + writepage = (Page) BufferGetPage(writebuf); + + if (xldata->ntups > 0) + { + OffsetNumber *towrite = (OffsetNumber *) data; + + data += sizeof(OffsetNumber) * xldata->ntups; + + while (data - begin < datalen) + { + IndexTuple itup = (IndexTuple) data; + Size itemsz; + OffsetNumber l; + + itemsz = IndexTupleDSize(*itup); + itemsz = MAXALIGN(itemsz); + + data += itemsz; + + l = PageAddItem(writepage, (Item) itup, itemsz, towrite[ninserted], false, false); + if (l == InvalidOffsetNumber) + elog(ERROR, "hash_xlog_move_page_contents: failed to add item to hash index page, size %d bytes", + (int) itemsz); + + ninserted++; + } + } + + /* + * number of tuples inserted must be same as requested in REDO record. + */ + Assert(ninserted == xldata->ntups); + + PageSetLSN(writepage, lsn); + MarkBufferDirty(writebuf); + } + + /* replay the record for deleting entries from overflow buffer */ + if (XLogReadBufferForRedo(record, 2, &deletebuf) == BLK_NEEDS_REDO) + { + Page page; + char *ptr; + Size len; + + ptr = XLogRecGetBlockData(record, 2, &len); + + page = (Page) BufferGetPage(deletebuf); + + if (len > 0) + { + OffsetNumber *unused; + OffsetNumber *unend; + + unused = (OffsetNumber *) ptr; + unend = (OffsetNumber *) ((char *) ptr + len); + + if ((unend - unused) > 0) + PageIndexMultiDelete(page, unused, unend - unused); + } + + PageSetLSN(page, lsn); + MarkBufferDirty(deletebuf); + } + + /* + * Replay is complete, now we can release the buffers. We release locks at + * end of replay operation to ensure that we hold lock on primary bucket + * page till end of operation. We can optimize by releasing the lock on + * write buffer as soon as the operation for same is complete, if it is + * not same as primary bucket page, but that doesn't seem to be worth + * complicating the code. + */ + if (BufferIsValid(deletebuf)) + UnlockReleaseBuffer(deletebuf); + + if (BufferIsValid(writebuf)) + UnlockReleaseBuffer(writebuf); + + if (BufferIsValid(bucketbuf)) + UnlockReleaseBuffer(bucketbuf); +} + +/* + * replay squeeze page operation of hash index + */ +static void +hash_xlog_squeeze_page(XLogReaderState *record) +{ + XLogRecPtr lsn = record->EndRecPtr; + xl_hash_squeeze_page *xldata = (xl_hash_squeeze_page *) XLogRecGetData(record); + Buffer bucketbuf = InvalidBuffer; + Buffer writebuf; + Buffer ovflbuf; + Buffer prevbuf = InvalidBuffer; + Buffer mapbuf; + XLogRedoAction action; + + /* + * Ensure we have a cleanup lock on primary bucket page before we start + * with the actual replay operation. This is to ensure that neither a + * scan can start nor a scan can be already-in-progress during the replay + * of this operation. If we allow scans during this operation, then they + * can miss some records or show the same record multiple times. + */ + if (xldata->is_prim_bucket_same_wrt) + action = XLogReadBufferForRedoExtended(record, 1, RBM_NORMAL, true, &writebuf); + else + { + /* + * we don't care for return value as the purpose of reading bucketbuf + * is to ensure a cleanup lock on primary bucket page. + */ + (void) XLogReadBufferForRedoExtended(record, 0, RBM_NORMAL, true, &bucketbuf); + + action = XLogReadBufferForRedo(record, 1, &writebuf); + } + + /* replay the record for adding entries in overflow buffer */ + if (action == BLK_NEEDS_REDO) + { + Page writepage; + char *begin; + char *data; + Size datalen; + uint16 ninserted = 0; + + data = begin = XLogRecGetBlockData(record, 1, &datalen); + + writepage = (Page) BufferGetPage(writebuf); + + if (xldata->ntups > 0) + { + OffsetNumber *towrite = (OffsetNumber *) data; + + data += sizeof(OffsetNumber) * xldata->ntups; + + while (data - begin < datalen) + { + IndexTuple itup = (IndexTuple) data; + Size itemsz; + OffsetNumber l; + + itemsz = IndexTupleDSize(*itup); + itemsz = MAXALIGN(itemsz); + + data += itemsz; + + l = PageAddItem(writepage, (Item) itup, itemsz, towrite[ninserted], false, false); + if (l == InvalidOffsetNumber) + elog(ERROR, "hash_xlog_squeeze_page: failed to add item to hash index page, size %d bytes", + (int) itemsz); + + ninserted++; + } + } + + /* + * number of tuples inserted must be same as requested in REDO record. + */ + Assert(ninserted == xldata->ntups); + + /* + * if the page on which are adding tuples is a page previous to freed + * overflow page, then update its nextblno. + */ + if (xldata->is_prev_bucket_same_wrt) + { + HashPageOpaque writeopaque = (HashPageOpaque) PageGetSpecialPointer(writepage); + + writeopaque->hasho_nextblkno = xldata->nextblkno; + } + + PageSetLSN(writepage, lsn); + MarkBufferDirty(writebuf); + } + + /* replay the record for initializing overflow buffer */ + if (XLogReadBufferForRedo(record, 2, &ovflbuf) == BLK_NEEDS_REDO) + { + Page ovflpage; + + ovflpage = BufferGetPage(ovflbuf); + + _hash_pageinit(ovflpage, BufferGetPageSize(ovflbuf)); + + PageSetLSN(ovflpage, lsn); + MarkBufferDirty(ovflbuf); + } + if (BufferIsValid(ovflbuf)) + UnlockReleaseBuffer(ovflbuf); + + /* replay the record for page previous to the freed overflow page */ + if (!xldata->is_prev_bucket_same_wrt && + XLogReadBufferForRedo(record, 3, &prevbuf) == BLK_NEEDS_REDO) + { + Page prevpage = BufferGetPage(prevbuf); + HashPageOpaque prevopaque = (HashPageOpaque) PageGetSpecialPointer(prevpage); + + prevopaque->hasho_nextblkno = xldata->nextblkno; + + PageSetLSN(prevpage, lsn); + MarkBufferDirty(prevbuf); + } + if (BufferIsValid(prevbuf)) + UnlockReleaseBuffer(prevbuf); + + /* replay the record for page next to the freed overflow page */ + if (XLogRecHasBlockRef(record, 4)) + { + Buffer nextbuf; + + if (XLogReadBufferForRedo(record, 4, &nextbuf) == BLK_NEEDS_REDO) + { + Page nextpage = BufferGetPage(nextbuf); + HashPageOpaque nextopaque = (HashPageOpaque) PageGetSpecialPointer(nextpage); + + nextopaque->hasho_prevblkno = xldata->prevblkno; + + PageSetLSN(nextpage, lsn); + MarkBufferDirty(nextbuf); + } + if (BufferIsValid(nextbuf)) + UnlockReleaseBuffer(nextbuf); + } + + if (BufferIsValid(writebuf)) + UnlockReleaseBuffer(writebuf); + + if (BufferIsValid(bucketbuf)) + UnlockReleaseBuffer(bucketbuf); + + /* + * Note: in normal operation, we'd update the bitmap and meta page while + * still holding lock on the primary bucket page and overflow pages. But + * during replay it's not necessary to hold those locks, since no other + * index updates can be happening concurrently. + */ + /* replay the record for bitmap page */ + if (XLogReadBufferForRedo(record, 5, &mapbuf) == BLK_NEEDS_REDO) + { + Page mappage = (Page) BufferGetPage(mapbuf); + uint32 *freep = NULL; + char *data; + uint32 *bitmap_page_bit; + Size datalen; + + freep = HashPageGetBitmap(mappage); + + data = XLogRecGetBlockData(record, 5, &datalen); + bitmap_page_bit = (uint32 *) data; + + CLRBIT(freep, *bitmap_page_bit); + + PageSetLSN(mappage, lsn); + MarkBufferDirty(mapbuf); + } + if (BufferIsValid(mapbuf)) + UnlockReleaseBuffer(mapbuf); + + /* replay the record for meta page */ + if (XLogRecHasBlockRef(record, 6)) + { + Buffer metabuf; + + if (XLogReadBufferForRedo(record, 6, &metabuf) == BLK_NEEDS_REDO) + { + HashMetaPage metap; + Page page; + char *data; + uint32 *firstfree_ovflpage; + Size datalen; + + data = XLogRecGetBlockData(record, 6, &datalen); + firstfree_ovflpage = (uint32 *) data; + + page = BufferGetPage(metabuf); + metap = HashPageGetMeta(page); + metap->hashm_firstfree = *firstfree_ovflpage; + + PageSetLSN(page, lsn); + MarkBufferDirty(metabuf); + } + if (BufferIsValid(metabuf)) + UnlockReleaseBuffer(metabuf); + } +} + +/* + * replay delete operation of hash index + */ +static void +hash_xlog_delete(XLogReaderState *record) +{ + XLogRecPtr lsn = record->EndRecPtr; + xl_hash_delete *xldata = (xl_hash_delete *) XLogRecGetData(record); + Buffer bucketbuf = InvalidBuffer; + Buffer deletebuf; + Page page; + XLogRedoAction action; + + /* + * Ensure we have a cleanup lock on primary bucket page before we start + * with the actual replay operation. This is to ensure that neither a + * scan can start nor a scan can be already-in-progress during the replay + * of this operation. If we allow scans during this operation, then they + * can miss some records or show the same record multiple times. + */ + if (xldata->is_primary_bucket_page) + action = XLogReadBufferForRedoExtended(record, 1, RBM_NORMAL, true, &deletebuf); + else + { + /* + * we don't care for return value as the purpose of reading bucketbuf + * is to ensure a cleanup lock on primary bucket page. + */ + (void) XLogReadBufferForRedoExtended(record, 0, RBM_NORMAL, true, &bucketbuf); + + action = XLogReadBufferForRedo(record, 1, &deletebuf); + } + + /* replay the record for deleting entries in bucket page */ + if (action == BLK_NEEDS_REDO) + { + char *ptr; + Size len; + + ptr = XLogRecGetBlockData(record, 1, &len); + + page = (Page) BufferGetPage(deletebuf); + + if (len > 0) + { + OffsetNumber *unused; + OffsetNumber *unend; + + unused = (OffsetNumber *) ptr; + unend = (OffsetNumber *) ((char *) ptr + len); + + if ((unend - unused) > 0) + PageIndexMultiDelete(page, unused, unend - unused); + } + + PageSetLSN(page, lsn); + MarkBufferDirty(deletebuf); + } + if (BufferIsValid(deletebuf)) + UnlockReleaseBuffer(deletebuf); + + if (BufferIsValid(bucketbuf)) + UnlockReleaseBuffer(bucketbuf); +} + +/* + * replay split cleanup flag operation for primary bucket page. + */ +static void +hash_xlog_split_cleanup(XLogReaderState *record) +{ + XLogRecPtr lsn = record->EndRecPtr; + Buffer buffer; + Page page; + + if (XLogReadBufferForRedo(record, 0, &buffer) == BLK_NEEDS_REDO) + { + HashPageOpaque bucket_opaque; + + page = (Page) BufferGetPage(buffer); + + bucket_opaque = (HashPageOpaque) PageGetSpecialPointer(page); + bucket_opaque->hasho_flag &= ~LH_BUCKET_NEEDS_SPLIT_CLEANUP; + PageSetLSN(page, lsn); + MarkBufferDirty(buffer); + } + if (BufferIsValid(buffer)) + UnlockReleaseBuffer(buffer); +} + +/* + * replay for update meta page + */ +static void +hash_xlog_update_meta_page(XLogReaderState *record) +{ + HashMetaPage metap; + XLogRecPtr lsn = record->EndRecPtr; + xl_hash_update_meta_page *xldata = (xl_hash_update_meta_page *) XLogRecGetData(record); + Buffer metabuf; + Page page; + + if (XLogReadBufferForRedo(record, 0, &metabuf) == BLK_NEEDS_REDO) + { + page = BufferGetPage(metabuf); + metap = HashPageGetMeta(page); + + metap->hashm_ntuples = xldata->ntuples; + + PageSetLSN(page, lsn); + MarkBufferDirty(metabuf); + } + if (BufferIsValid(metabuf)) + UnlockReleaseBuffer(metabuf); +} + +void +hash_redo(XLogReaderState *record) +{ + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + + switch (info) + { + case XLOG_HASH_INIT_META_PAGE: + hash_xlog_init_meta_page(record); + break; + case XLOG_HASH_INIT_BITMAP_PAGE: + hash_xlog_init_bitmap_page(record); + break; + case XLOG_HASH_INSERT: + hash_xlog_insert(record); + break; + case XLOG_HASH_ADD_OVFL_PAGE: + hash_xlog_add_ovfl_page(record); + break; + case XLOG_HASH_SPLIT_ALLOCATE_PAGE: + hash_xlog_split_allocate_page(record); + break; + case XLOG_HASH_SPLIT_PAGE: + hash_xlog_split_page(record); + break; + case XLOG_HASH_SPLIT_COMPLETE: + hash_xlog_split_complete(record); + break; + case XLOG_HASH_MOVE_PAGE_CONTENTS: + hash_xlog_move_page_contents(record); + break; + case XLOG_HASH_SQUEEZE_PAGE: + hash_xlog_squeeze_page(record); + break; + case XLOG_HASH_DELETE: + hash_xlog_delete(record); + break; + case XLOG_HASH_SPLIT_CLEANUP: + hash_xlog_split_cleanup(record); + break; + case XLOG_HASH_UPDATE_META_PAGE: + hash_xlog_update_meta_page(record); + break; + default: + elog(PANIC, "hash_redo: unknown op code %u", info); + } +} diff --git a/src/backend/access/hash/hashinsert.c b/src/backend/access/hash/hashinsert.c index 354e7339cf..241728fe6b 100644 --- a/src/backend/access/hash/hashinsert.c +++ b/src/backend/access/hash/hashinsert.c @@ -16,6 +16,8 @@ #include "postgres.h" #include "access/hash.h" +#include "access/hash_xlog.h" +#include "miscadmin.h" #include "utils/rel.h" @@ -40,6 +42,7 @@ _hash_doinsert(Relation rel, IndexTuple itup) bool do_expand; uint32 hashkey; Bucket bucket; + OffsetNumber itup_off; /* * Get the hash key for the item (it's stored in the index tuple itself). @@ -158,25 +161,20 @@ restart_insert: Assert(pageopaque->hasho_bucket == bucket); } - /* found page with enough space, so add the item here */ - (void) _hash_pgaddtup(rel, buf, itemsz, itup); - - /* - * dirty and release the modified page. if the page we modified was an - * overflow page, we also need to separately drop the pin we retained on - * the primary bucket page. - */ - MarkBufferDirty(buf); - _hash_relbuf(rel, buf); - if (buf != bucket_buf) - _hash_dropbuf(rel, bucket_buf); - /* * Write-lock the metapage so we can increment the tuple count. After * incrementing it, check to see if it's time for a split. */ LockBuffer(metabuf, BUFFER_LOCK_EXCLUSIVE); + /* Do the update. No ereport(ERROR) until changes are logged */ + START_CRIT_SECTION(); + + /* found page with enough space, so add the item here */ + itup_off = _hash_pgaddtup(rel, buf, itemsz, itup); + MarkBufferDirty(buf); + + /* metapage operations */ metap = HashPageGetMeta(metapage); metap->hashm_ntuples += 1; @@ -184,10 +182,43 @@ restart_insert: do_expand = metap->hashm_ntuples > (double) metap->hashm_ffactor * (metap->hashm_maxbucket + 1); - /* Write out the metapage and drop lock, but keep pin */ MarkBufferDirty(metabuf); + + /* XLOG stuff */ + if (RelationNeedsWAL(rel)) + { + xl_hash_insert xlrec; + XLogRecPtr recptr; + + xlrec.offnum = itup_off; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, SizeOfHashInsert); + + XLogRegisterBuffer(1, metabuf, REGBUF_STANDARD); + + XLogRegisterBuffer(0, buf, REGBUF_STANDARD); + XLogRegisterBufData(0, (char *) itup, IndexTupleDSize(*itup)); + + recptr = XLogInsert(RM_HASH_ID, XLOG_HASH_INSERT); + + PageSetLSN(BufferGetPage(buf), recptr); + PageSetLSN(BufferGetPage(metabuf), recptr); + } + + END_CRIT_SECTION(); + + /* drop lock on metapage, but keep pin */ LockBuffer(metabuf, BUFFER_LOCK_UNLOCK); + /* + * Release the modified page and ensure to release the pin on primary + * page. + */ + _hash_relbuf(rel, buf); + if (buf != bucket_buf) + _hash_dropbuf(rel, bucket_buf); + /* Attempt to split if a split is needed */ if (do_expand) _hash_expandtable(rel, metabuf); diff --git a/src/backend/access/hash/hashovfl.c b/src/backend/access/hash/hashovfl.c index 1087480f7e..a3cae21c60 100644 --- a/src/backend/access/hash/hashovfl.c +++ b/src/backend/access/hash/hashovfl.c @@ -18,6 +18,8 @@ #include "postgres.h" #include "access/hash.h" +#include "access/hash_xlog.h" +#include "miscadmin.h" #include "utils/rel.h" @@ -136,6 +138,13 @@ _hash_addovflpage(Relation rel, Buffer metabuf, Buffer buf, bool retain_pin) * page is released, then finally acquire the lock on new overflow buffer. * We need this locking order to avoid deadlock with backends that are * doing inserts. + * + * Note: We could have avoided locking many buffers here if we made two + * WAL records for acquiring an overflow page (one to allocate an overflow + * page and another to add it to overflow bucket chain). However, doing + * so can leak an overflow page, if the system crashes after allocation. + * Needless to say, it is better to have a single record from a + * performance point of view as well. */ LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); @@ -303,8 +312,12 @@ _hash_addovflpage(Relation rel, Buffer metabuf, Buffer buf, bool retain_pin) found: /* - * Do the update. + * Do the update. No ereport(ERROR) until changes are logged. We want to + * log the changes for bitmap page and overflow page together to avoid + * loss of pages in case the new page is added. */ + START_CRIT_SECTION(); + if (page_found) { Assert(BufferIsValid(mapbuf)); @@ -362,6 +375,51 @@ found: MarkBufferDirty(buf); + /* XLOG stuff */ + if (RelationNeedsWAL(rel)) + { + XLogRecPtr recptr; + xl_hash_add_ovfl_page xlrec; + + xlrec.bmpage_found = page_found; + xlrec.bmsize = metap->hashm_bmsize; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, SizeOfHashAddOvflPage); + + XLogRegisterBuffer(0, ovflbuf, REGBUF_WILL_INIT); + XLogRegisterBufData(0, (char *) &pageopaque->hasho_bucket, sizeof(Bucket)); + + XLogRegisterBuffer(1, buf, REGBUF_STANDARD); + + if (BufferIsValid(mapbuf)) + { + XLogRegisterBuffer(2, mapbuf, REGBUF_STANDARD); + XLogRegisterBufData(2, (char *) &bitmap_page_bit, sizeof(uint32)); + } + + if (BufferIsValid(newmapbuf)) + XLogRegisterBuffer(3, newmapbuf, REGBUF_WILL_INIT); + + XLogRegisterBuffer(4, metabuf, REGBUF_STANDARD); + XLogRegisterBufData(4, (char *) &metap->hashm_firstfree, sizeof(uint32)); + + recptr = XLogInsert(RM_HASH_ID, XLOG_HASH_ADD_OVFL_PAGE); + + PageSetLSN(BufferGetPage(ovflbuf), recptr); + PageSetLSN(BufferGetPage(buf), recptr); + + if (BufferIsValid(mapbuf)) + PageSetLSN(BufferGetPage(mapbuf), recptr); + + if (BufferIsValid(newmapbuf)) + PageSetLSN(BufferGetPage(newmapbuf), recptr); + + PageSetLSN(BufferGetPage(metabuf), recptr); + } + + END_CRIT_SECTION(); + if (retain_pin) LockBuffer(buf, BUFFER_LOCK_UNLOCK); else @@ -408,7 +466,11 @@ _hash_firstfreebit(uint32 map) * Remove this overflow page from its bucket's chain, and mark the page as * free. On entry, ovflbuf is write-locked; it is released before exiting. * - * Add the tuples (itups) to wbuf. + * Add the tuples (itups) to wbuf in this function. We could do that in the + * caller as well, but the advantage of doing it here is we can easily write + * the WAL for XLOG_HASH_SQUEEZE_PAGE operation. Addition of tuples and + * removal of overflow page has to done as an atomic operation, otherwise + * during replay on standby users might find duplicate records. * * Since this function is invoked in VACUUM, we provide an access strategy * parameter that controls fetches of the bucket pages. @@ -430,8 +492,6 @@ _hash_freeovflpage(Relation rel, Buffer bucketbuf, Buffer ovflbuf, HashMetaPage metap; Buffer metabuf; Buffer mapbuf; - Buffer prevbuf = InvalidBuffer; - Buffer nextbuf = InvalidBuffer; BlockNumber ovflblkno; BlockNumber prevblkno; BlockNumber blkno; @@ -445,6 +505,9 @@ _hash_freeovflpage(Relation rel, Buffer bucketbuf, Buffer ovflbuf, int32 bitmappage, bitmapbit; Bucket bucket PG_USED_FOR_ASSERTS_ONLY; + Buffer prevbuf = InvalidBuffer; + Buffer nextbuf = InvalidBuffer; + bool update_metap = false; /* Get information from the doomed page */ _hash_checkpage(rel, ovflbuf, LH_OVERFLOW_PAGE); @@ -508,6 +571,12 @@ _hash_freeovflpage(Relation rel, Buffer bucketbuf, Buffer ovflbuf, /* Get write-lock on metapage to update firstfree */ LockBuffer(metabuf, BUFFER_LOCK_EXCLUSIVE); + /* This operation needs to log multiple tuples, prepare WAL for that */ + if (RelationNeedsWAL(rel)) + XLogEnsureRecordSpace(HASH_XLOG_FREE_OVFL_BUFS, 4 + nitups); + + START_CRIT_SECTION(); + /* * we have to insert tuples on the "write" page, being careful to preserve * hashkey ordering. (If we insert many tuples into the same "write" page @@ -519,7 +588,11 @@ _hash_freeovflpage(Relation rel, Buffer bucketbuf, Buffer ovflbuf, MarkBufferDirty(wbuf); } - /* Initialize the freed overflow page. */ + /* + * Initialize the freed overflow page. Just zeroing the page won't work, + * because WAL replay routines expect pages to be initialized. See + * explanation of RBM_NORMAL mode atop XLogReadBufferExtended. + */ _hash_pageinit(ovflpage, BufferGetPageSize(ovflbuf)); MarkBufferDirty(ovflbuf); @@ -550,9 +623,83 @@ _hash_freeovflpage(Relation rel, Buffer bucketbuf, Buffer ovflbuf, if (ovflbitno < metap->hashm_firstfree) { metap->hashm_firstfree = ovflbitno; + update_metap = true; MarkBufferDirty(metabuf); } + /* XLOG stuff */ + if (RelationNeedsWAL(rel)) + { + xl_hash_squeeze_page xlrec; + XLogRecPtr recptr; + int i; + + xlrec.prevblkno = prevblkno; + xlrec.nextblkno = nextblkno; + xlrec.ntups = nitups; + xlrec.is_prim_bucket_same_wrt = (wbuf == bucketbuf); + xlrec.is_prev_bucket_same_wrt = (wbuf == prevbuf); + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, SizeOfHashSqueezePage); + + /* + * bucket buffer needs to be registered to ensure that we can acquire + * a cleanup lock on it during replay. + */ + if (!xlrec.is_prim_bucket_same_wrt) + XLogRegisterBuffer(0, bucketbuf, REGBUF_STANDARD | REGBUF_NO_IMAGE); + + XLogRegisterBuffer(1, wbuf, REGBUF_STANDARD); + if (xlrec.ntups > 0) + { + XLogRegisterBufData(1, (char *) itup_offsets, + nitups * sizeof(OffsetNumber)); + for (i = 0; i < nitups; i++) + XLogRegisterBufData(1, (char *) itups[i], tups_size[i]); + } + + XLogRegisterBuffer(2, ovflbuf, REGBUF_STANDARD); + + /* + * If prevpage and the writepage (block in which we are moving tuples + * from overflow) are same, then no need to separately register + * prevpage. During replay, we can directly update the nextblock in + * writepage. + */ + if (BufferIsValid(prevbuf) && !xlrec.is_prev_bucket_same_wrt) + XLogRegisterBuffer(3, prevbuf, REGBUF_STANDARD); + + if (BufferIsValid(nextbuf)) + XLogRegisterBuffer(4, nextbuf, REGBUF_STANDARD); + + XLogRegisterBuffer(5, mapbuf, REGBUF_STANDARD); + XLogRegisterBufData(5, (char *) &bitmapbit, sizeof(uint32)); + + if (update_metap) + { + XLogRegisterBuffer(6, metabuf, REGBUF_STANDARD); + XLogRegisterBufData(6, (char *) &metap->hashm_firstfree, sizeof(uint32)); + } + + recptr = XLogInsert(RM_HASH_ID, XLOG_HASH_SQUEEZE_PAGE); + + PageSetLSN(BufferGetPage(wbuf), recptr); + PageSetLSN(BufferGetPage(ovflbuf), recptr); + + if (BufferIsValid(prevbuf) && !xlrec.is_prev_bucket_same_wrt) + PageSetLSN(BufferGetPage(prevbuf), recptr); + if (BufferIsValid(nextbuf)) + PageSetLSN(BufferGetPage(nextbuf), recptr); + + PageSetLSN(BufferGetPage(mapbuf), recptr); + + if (update_metap) + PageSetLSN(BufferGetPage(metabuf), recptr); + } + + END_CRIT_SECTION(); + /* release previous bucket if it is not same as write bucket */ if (BufferIsValid(prevbuf) && prevblkno != writeblkno) _hash_relbuf(rel, prevbuf); @@ -601,7 +748,11 @@ _hash_initbitmapbuffer(Buffer buf, uint16 bmsize, bool initpage) freep = HashPageGetBitmap(pg); MemSet(freep, 0xFF, bmsize); - /* Set pd_lower just past the end of the bitmap page data. */ + /* + * Set pd_lower just past the end of the bitmap page data. We could even + * set pd_lower equal to pd_upper, but this is more precise and makes the + * page look compressible to xlog.c. + */ ((PageHeader) pg)->pd_lower = ((char *) freep + bmsize) - (char *) pg; } @@ -760,6 +911,15 @@ readpage: { Assert(nitups == ndeletable); + /* + * This operation needs to log multiple tuples, prepare + * WAL for that. + */ + if (RelationNeedsWAL(rel)) + XLogEnsureRecordSpace(0, 3 + nitups); + + START_CRIT_SECTION(); + /* * we have to insert tuples on the "write" page, being * careful to preserve hashkey ordering. (If we insert @@ -773,6 +933,43 @@ readpage: PageIndexMultiDelete(rpage, deletable, ndeletable); MarkBufferDirty(rbuf); + /* XLOG stuff */ + if (RelationNeedsWAL(rel)) + { + XLogRecPtr recptr; + xl_hash_move_page_contents xlrec; + + xlrec.ntups = nitups; + xlrec.is_prim_bucket_same_wrt = (wbuf == bucket_buf) ? true : false; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, SizeOfHashMovePageContents); + + /* + * bucket buffer needs to be registered to ensure that + * we can acquire a cleanup lock on it during replay. + */ + if (!xlrec.is_prim_bucket_same_wrt) + XLogRegisterBuffer(0, bucket_buf, REGBUF_STANDARD | REGBUF_NO_IMAGE); + + XLogRegisterBuffer(1, wbuf, REGBUF_STANDARD); + XLogRegisterBufData(1, (char *) itup_offsets, + nitups * sizeof(OffsetNumber)); + for (i = 0; i < nitups; i++) + XLogRegisterBufData(1, (char *) itups[i], tups_size[i]); + + XLogRegisterBuffer(2, rbuf, REGBUF_STANDARD); + XLogRegisterBufData(2, (char *) deletable, + ndeletable * sizeof(OffsetNumber)); + + recptr = XLogInsert(RM_HASH_ID, XLOG_HASH_MOVE_PAGE_CONTENTS); + + PageSetLSN(BufferGetPage(wbuf), recptr); + PageSetLSN(BufferGetPage(rbuf), recptr); + } + + END_CRIT_SECTION(); + tups_moved = true; } diff --git a/src/backend/access/hash/hashpage.c b/src/backend/access/hash/hashpage.c index c73929cebb..dc606f162e 100644 --- a/src/backend/access/hash/hashpage.c +++ b/src/backend/access/hash/hashpage.c @@ -29,6 +29,7 @@ #include "postgres.h" #include "access/hash.h" +#include "access/hash_xlog.h" #include "miscadmin.h" #include "storage/lmgr.h" #include "storage/smgr.h" @@ -43,6 +44,7 @@ static void _hash_splitbucket(Relation rel, Buffer metabuf, HTAB *htab, uint32 maxbucket, uint32 highmask, uint32 lowmask); +static void log_split_page(Relation rel, Buffer buf); /* @@ -381,6 +383,25 @@ _hash_init(Relation rel, double num_tuples, ForkNumber forkNum) pg = BufferGetPage(metabuf); metap = HashPageGetMeta(pg); + /* XLOG stuff */ + if (RelationNeedsWAL(rel)) + { + xl_hash_init_meta_page xlrec; + XLogRecPtr recptr; + + xlrec.num_tuples = num_tuples; + xlrec.procid = metap->hashm_procid; + xlrec.ffactor = metap->hashm_ffactor; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, SizeOfHashInitMetaPage); + XLogRegisterBuffer(0, metabuf, REGBUF_WILL_INIT); + + recptr = XLogInsert(RM_HASH_ID, XLOG_HASH_INIT_META_PAGE); + + PageSetLSN(BufferGetPage(metabuf), recptr); + } + num_buckets = metap->hashm_maxbucket + 1; /* @@ -405,6 +426,12 @@ _hash_init(Relation rel, double num_tuples, ForkNumber forkNum) buf = _hash_getnewbuf(rel, blkno, forkNum); _hash_initbuf(buf, metap->hashm_maxbucket, i, LH_BUCKET_PAGE, false); MarkBufferDirty(buf); + + log_newpage(&rel->rd_node, + forkNum, + blkno, + BufferGetPage(buf), + true); _hash_relbuf(rel, buf); } @@ -431,6 +458,31 @@ _hash_init(Relation rel, double num_tuples, ForkNumber forkNum) metap->hashm_nmaps++; MarkBufferDirty(metabuf); + /* XLOG stuff */ + if (RelationNeedsWAL(rel)) + { + xl_hash_init_bitmap_page xlrec; + XLogRecPtr recptr; + + xlrec.bmsize = metap->hashm_bmsize; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, SizeOfHashInitBitmapPage); + XLogRegisterBuffer(0, bitmapbuf, REGBUF_WILL_INIT); + + /* + * This is safe only because nobody else can be modifying the index at + * this stage; it's only visible to the transaction that is creating + * it. + */ + XLogRegisterBuffer(1, metabuf, REGBUF_STANDARD); + + recptr = XLogInsert(RM_HASH_ID, XLOG_HASH_INIT_BITMAP_PAGE); + + PageSetLSN(BufferGetPage(bitmapbuf), recptr); + PageSetLSN(BufferGetPage(metabuf), recptr); + } + /* all done */ _hash_relbuf(rel, bitmapbuf); _hash_relbuf(rel, metabuf); @@ -525,7 +577,10 @@ _hash_init_metabuffer(Buffer buf, double num_tuples, RegProcedure procid, metap->hashm_ovflpoint = log2_num_buckets; metap->hashm_firstfree = 0; - /* Set pd_lower just past the end of the metadata. */ + /* + * Set pd_lower just past the end of the metadata. This is to log full + * page image of metapage in xloginsert.c. + */ ((PageHeader) page)->pd_lower = ((char *) metap + sizeof(HashMetaPageData)) - (char *) page; } @@ -569,6 +624,8 @@ _hash_expandtable(Relation rel, Buffer metabuf) uint32 maxbucket; uint32 highmask; uint32 lowmask; + bool metap_update_masks = false; + bool metap_update_splitpoint = false; restart_expand: @@ -728,7 +785,11 @@ restart_expand: * The number of buckets in the new splitpoint is equal to the total * number already in existence, i.e. new_bucket. Currently this maps * one-to-one to blocks required, but someday we may need a more - * complicated calculation here. + * complicated calculation here. We treat allocation of buckets as a + * separate WAL-logged action. Even if we fail after this operation, + * won't leak bucket pages; rather, the next split will consume this + * space. In any case, even without failure we don't use all the space + * in one split operation. */ if (!_hash_alloc_buckets(rel, start_nblkno, new_bucket)) { @@ -757,8 +818,7 @@ restart_expand: * Since we are scribbling on the pages in the shared buffers, establish a * critical section. Any failure in this next code leaves us with a big * problem: the metapage is effectively corrupt but could get written back - * to disk. We don't really expect any failure, but just to be sure, - * establish a critical section. + * to disk. */ START_CRIT_SECTION(); @@ -772,6 +832,7 @@ restart_expand: /* Starting a new doubling */ metap->hashm_lowmask = metap->hashm_highmask; metap->hashm_highmask = new_bucket | metap->hashm_lowmask; + metap_update_masks = true; } /* @@ -784,6 +845,7 @@ restart_expand: { metap->hashm_spares[spare_ndx] = metap->hashm_spares[metap->hashm_ovflpoint]; metap->hashm_ovflpoint = spare_ndx; + metap_update_splitpoint = true; } MarkBufferDirty(metabuf); @@ -829,6 +891,49 @@ restart_expand: MarkBufferDirty(buf_nblkno); + /* XLOG stuff */ + if (RelationNeedsWAL(rel)) + { + xl_hash_split_allocate_page xlrec; + XLogRecPtr recptr; + + xlrec.new_bucket = maxbucket; + xlrec.old_bucket_flag = oopaque->hasho_flag; + xlrec.new_bucket_flag = nopaque->hasho_flag; + xlrec.flags = 0; + + XLogBeginInsert(); + + XLogRegisterBuffer(0, buf_oblkno, REGBUF_STANDARD); + XLogRegisterBuffer(1, buf_nblkno, REGBUF_WILL_INIT); + XLogRegisterBuffer(2, metabuf, REGBUF_STANDARD); + + if (metap_update_masks) + { + xlrec.flags |= XLH_SPLIT_META_UPDATE_MASKS; + XLogRegisterBufData(2, (char *) &metap->hashm_lowmask, sizeof(uint32)); + XLogRegisterBufData(2, (char *) &metap->hashm_highmask, sizeof(uint32)); + } + + if (metap_update_splitpoint) + { + xlrec.flags |= XLH_SPLIT_META_UPDATE_SPLITPOINT; + XLogRegisterBufData(2, (char *) &metap->hashm_ovflpoint, + sizeof(uint32)); + XLogRegisterBufData(2, + (char *) &metap->hashm_spares[metap->hashm_ovflpoint], + sizeof(uint32)); + } + + XLogRegisterData((char *) &xlrec, SizeOfHashSplitAllocPage); + + recptr = XLogInsert(RM_HASH_ID, XLOG_HASH_SPLIT_ALLOCATE_PAGE); + + PageSetLSN(BufferGetPage(buf_oblkno), recptr); + PageSetLSN(BufferGetPage(buf_nblkno), recptr); + PageSetLSN(BufferGetPage(metabuf), recptr); + } + END_CRIT_SECTION(); /* drop lock, but keep pin */ @@ -883,6 +988,7 @@ _hash_alloc_buckets(Relation rel, BlockNumber firstblock, uint32 nblocks) { BlockNumber lastblock; char zerobuf[BLCKSZ]; + Page page; lastblock = firstblock + nblocks - 1; @@ -893,7 +999,20 @@ _hash_alloc_buckets(Relation rel, BlockNumber firstblock, uint32 nblocks) if (lastblock < firstblock || lastblock == InvalidBlockNumber) return false; - MemSet(zerobuf, 0, sizeof(zerobuf)); + page = (Page) zerobuf; + + /* + * Initialize the freed overflow page. Just zeroing the page won't work, + * See _hash_freeovflpage for similar usage. + */ + _hash_pageinit(page, BLCKSZ); + + if (RelationNeedsWAL(rel)) + log_newpage(&rel->rd_node, + MAIN_FORKNUM, + lastblock, + zerobuf, + true); RelationOpenSmgr(rel); smgrextend(rel->rd_smgr, MAIN_FORKNUM, lastblock, zerobuf, false); @@ -951,6 +1070,11 @@ _hash_splitbucket(Relation rel, Page npage; HashPageOpaque oopaque; HashPageOpaque nopaque; + OffsetNumber itup_offsets[MaxIndexTuplesPerPage]; + IndexTuple itups[MaxIndexTuplesPerPage]; + Size all_tups_size = 0; + int i; + uint16 nitups = 0; bucket_obuf = obuf; opage = BufferGetPage(obuf); @@ -1029,29 +1153,38 @@ _hash_splitbucket(Relation rel, itemsz = IndexTupleDSize(*new_itup); itemsz = MAXALIGN(itemsz); - if (PageGetFreeSpace(npage) < itemsz) + if (PageGetFreeSpaceForMultipleTuples(npage, nitups + 1) < (all_tups_size + itemsz)) { - /* write out nbuf and drop lock, but keep pin */ + /* + * Change the shared buffer state in critical section, + * otherwise any error could make it unrecoverable. + */ + START_CRIT_SECTION(); + + _hash_pgaddmultitup(rel, nbuf, itups, itup_offsets, nitups); MarkBufferDirty(nbuf); + /* log the split operation before releasing the lock */ + log_split_page(rel, nbuf); + + END_CRIT_SECTION(); + /* drop lock, but keep pin */ LockBuffer(nbuf, BUFFER_LOCK_UNLOCK); + + /* be tidy */ + for (i = 0; i < nitups; i++) + pfree(itups[i]); + nitups = 0; + all_tups_size = 0; + /* chain to a new overflow page */ nbuf = _hash_addovflpage(rel, metabuf, nbuf, (nbuf == bucket_nbuf) ? true : false); npage = BufferGetPage(nbuf); nopaque = (HashPageOpaque) PageGetSpecialPointer(npage); } - /* - * Insert tuple on new page, using _hash_pgaddtup to ensure - * correct ordering by hashkey. This is a tad inefficient - * since we may have to shuffle itempointers repeatedly. - * Possible future improvement: accumulate all the items for - * the new page and qsort them before insertion. - */ - (void) _hash_pgaddtup(rel, nbuf, itemsz, new_itup); - - /* be tidy */ - pfree(new_itup); + itups[nitups++] = new_itup; + all_tups_size += itemsz; } else { @@ -1073,11 +1206,27 @@ _hash_splitbucket(Relation rel, /* Exit loop if no more overflow pages in old bucket */ if (!BlockNumberIsValid(oblkno)) { + /* + * Change the shared buffer state in critical section, otherwise + * any error could make it unrecoverable. + */ + START_CRIT_SECTION(); + + _hash_pgaddmultitup(rel, nbuf, itups, itup_offsets, nitups); MarkBufferDirty(nbuf); + /* log the split operation before releasing the lock */ + log_split_page(rel, nbuf); + + END_CRIT_SECTION(); + if (nbuf == bucket_nbuf) LockBuffer(nbuf, BUFFER_LOCK_UNLOCK); else _hash_relbuf(rel, nbuf); + + /* be tidy */ + for (i = 0; i < nitups; i++) + pfree(itups[i]); break; } @@ -1103,6 +1252,8 @@ _hash_splitbucket(Relation rel, npage = BufferGetPage(bucket_nbuf); nopaque = (HashPageOpaque) PageGetSpecialPointer(npage); + START_CRIT_SECTION(); + oopaque->hasho_flag &= ~LH_BUCKET_BEING_SPLIT; nopaque->hasho_flag &= ~LH_BUCKET_BEING_POPULATED; @@ -1119,6 +1270,29 @@ _hash_splitbucket(Relation rel, */ MarkBufferDirty(bucket_obuf); MarkBufferDirty(bucket_nbuf); + + if (RelationNeedsWAL(rel)) + { + XLogRecPtr recptr; + xl_hash_split_complete xlrec; + + xlrec.old_bucket_flag = oopaque->hasho_flag; + xlrec.new_bucket_flag = nopaque->hasho_flag; + + XLogBeginInsert(); + + XLogRegisterData((char *) &xlrec, SizeOfHashSplitComplete); + + XLogRegisterBuffer(0, bucket_obuf, REGBUF_STANDARD); + XLogRegisterBuffer(1, bucket_nbuf, REGBUF_STANDARD); + + recptr = XLogInsert(RM_HASH_ID, XLOG_HASH_SPLIT_COMPLETE); + + PageSetLSN(BufferGetPage(bucket_obuf), recptr); + PageSetLSN(BufferGetPage(bucket_nbuf), recptr); + } + + END_CRIT_SECTION(); } /* @@ -1244,6 +1418,32 @@ _hash_finish_split(Relation rel, Buffer metabuf, Buffer obuf, Bucket obucket, hash_destroy(tidhtab); } +/* + * log_split_page() -- Log the split operation + * + * We log the split operation when the new page in new bucket gets full, + * so we log the entire page. + * + * 'buf' must be locked by the caller which is also responsible for unlocking + * it. + */ +static void +log_split_page(Relation rel, Buffer buf) +{ + if (RelationNeedsWAL(rel)) + { + XLogRecPtr recptr; + + XLogBeginInsert(); + + XLogRegisterBuffer(0, buf, REGBUF_FORCE_IMAGE | REGBUF_STANDARD); + + recptr = XLogInsert(RM_HASH_ID, XLOG_HASH_SPLIT_PAGE); + + PageSetLSN(BufferGetPage(buf), recptr); + } +} + /* * _hash_getcachedmetap() -- Returns cached metapage data. * diff --git a/src/backend/access/hash/hashsearch.c b/src/backend/access/hash/hashsearch.c index 9e5d7e4bab..d7337703b0 100644 --- a/src/backend/access/hash/hashsearch.c +++ b/src/backend/access/hash/hashsearch.c @@ -123,6 +123,7 @@ _hash_readnext(IndexScanDesc scan, if (block_found) { *pagep = BufferGetPage(*bufp); + TestForOldSnapshot(scan->xs_snapshot, rel, *pagep); *opaquep = (HashPageOpaque) PageGetSpecialPointer(*pagep); } } @@ -168,6 +169,7 @@ _hash_readprev(IndexScanDesc scan, *bufp = _hash_getbuf(rel, blkno, HASH_READ, LH_BUCKET_PAGE | LH_OVERFLOW_PAGE); *pagep = BufferGetPage(*bufp); + TestForOldSnapshot(scan->xs_snapshot, rel, *pagep); *opaquep = (HashPageOpaque) PageGetSpecialPointer(*pagep); /* @@ -283,6 +285,7 @@ _hash_first(IndexScanDesc scan, ScanDirection dir) buf = _hash_getbucketbuf_from_hashkey(rel, hashkey, HASH_READ, NULL); page = BufferGetPage(buf); + TestForOldSnapshot(scan->xs_snapshot, rel, page); opaque = (HashPageOpaque) PageGetSpecialPointer(page); bucket = opaque->hasho_bucket; @@ -318,6 +321,7 @@ _hash_first(IndexScanDesc scan, ScanDirection dir) LockBuffer(buf, BUFFER_LOCK_UNLOCK); old_buf = _hash_getbuf(rel, old_blkno, HASH_READ, LH_BUCKET_PAGE); + TestForOldSnapshot(scan->xs_snapshot, rel, BufferGetPage(old_buf)); /* * remember the split bucket buffer so as to use it later for @@ -520,6 +524,7 @@ _hash_step(IndexScanDesc scan, Buffer *bufP, ScanDirection dir) _hash_readprev(scan, &buf, &page, &opaque); if (BufferIsValid(buf)) { + TestForOldSnapshot(scan->xs_snapshot, rel, page); maxoff = PageGetMaxOffsetNumber(page); offnum = _hash_binsearch_last(page, so->hashso_sk_hash); } diff --git a/src/backend/access/rmgrdesc/hashdesc.c b/src/backend/access/rmgrdesc/hashdesc.c index 7eac8191ca..f1cc9ff951 100644 --- a/src/backend/access/rmgrdesc/hashdesc.c +++ b/src/backend/access/rmgrdesc/hashdesc.c @@ -19,10 +19,142 @@ void hash_desc(StringInfo buf, XLogReaderState *record) { + char *rec = XLogRecGetData(record); + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + + switch (info) + { + case XLOG_HASH_INIT_META_PAGE: + { + xl_hash_init_meta_page *xlrec = (xl_hash_init_meta_page *) rec; + + appendStringInfo(buf, "num_tuples %g, fillfactor %d", + xlrec->num_tuples, xlrec->ffactor); + break; + } + case XLOG_HASH_INIT_BITMAP_PAGE: + { + xl_hash_init_bitmap_page *xlrec = (xl_hash_init_bitmap_page *) rec; + + appendStringInfo(buf, "bmsize %d", xlrec->bmsize); + break; + } + case XLOG_HASH_INSERT: + { + xl_hash_insert *xlrec = (xl_hash_insert *) rec; + + appendStringInfo(buf, "off %u", xlrec->offnum); + break; + } + case XLOG_HASH_ADD_OVFL_PAGE: + { + xl_hash_add_ovfl_page *xlrec = (xl_hash_add_ovfl_page *) rec; + + appendStringInfo(buf, "bmsize %d, bmpage_found %c", + xlrec->bmsize, (xlrec->bmpage_found) ? 'T' : 'F'); + break; + } + case XLOG_HASH_SPLIT_ALLOCATE_PAGE: + { + xl_hash_split_allocate_page *xlrec = (xl_hash_split_allocate_page *) rec; + + appendStringInfo(buf, "new_bucket %u, meta_page_masks_updated %c, issplitpoint_changed %c", + xlrec->new_bucket, + (xlrec->flags & XLH_SPLIT_META_UPDATE_MASKS) ? 'T' : 'F', + (xlrec->flags & XLH_SPLIT_META_UPDATE_SPLITPOINT) ? 'T' : 'F'); + break; + } + case XLOG_HASH_SPLIT_COMPLETE: + { + xl_hash_split_complete *xlrec = (xl_hash_split_complete *) rec; + + appendStringInfo(buf, "old_bucket_flag %u, new_bucket_flag %u", + xlrec->old_bucket_flag, xlrec->new_bucket_flag); + break; + } + case XLOG_HASH_MOVE_PAGE_CONTENTS: + { + xl_hash_move_page_contents *xlrec = (xl_hash_move_page_contents *) rec; + + appendStringInfo(buf, "ntups %d, is_primary %c", + xlrec->ntups, + xlrec->is_prim_bucket_same_wrt ? 'T' : 'F'); + break; + } + case XLOG_HASH_SQUEEZE_PAGE: + { + xl_hash_squeeze_page *xlrec = (xl_hash_squeeze_page *) rec; + + appendStringInfo(buf, "prevblkno %u, nextblkno %u, ntups %d, is_primary %c", + xlrec->prevblkno, + xlrec->nextblkno, + xlrec->ntups, + xlrec->is_prim_bucket_same_wrt ? 'T' : 'F'); + break; + } + case XLOG_HASH_DELETE: + { + xl_hash_delete *xlrec = (xl_hash_delete *) rec; + + appendStringInfo(buf, "is_primary %c", + xlrec->is_primary_bucket_page ? 'T' : 'F'); + break; + } + case XLOG_HASH_UPDATE_META_PAGE: + { + xl_hash_update_meta_page *xlrec = (xl_hash_update_meta_page *) rec; + + appendStringInfo(buf, "ntuples %g", + xlrec->ntuples); + break; + } + } } const char * hash_identify(uint8 info) { - return NULL; + const char *id = NULL; + + switch (info & ~XLR_INFO_MASK) + { + case XLOG_HASH_INIT_META_PAGE: + id = "INIT_META_PAGE"; + break; + case XLOG_HASH_INIT_BITMAP_PAGE: + id = "INIT_BITMAP_PAGE"; + break; + case XLOG_HASH_INSERT: + id = "INSERT"; + break; + case XLOG_HASH_ADD_OVFL_PAGE: + id = "ADD_OVFL_PAGE"; + break; + case XLOG_HASH_SPLIT_ALLOCATE_PAGE: + id = "SPLIT_ALLOCATE_PAGE"; + break; + case XLOG_HASH_SPLIT_PAGE: + id = "SPLIT_PAGE"; + break; + case XLOG_HASH_SPLIT_COMPLETE: + id = "SPLIT_COMPLETE"; + break; + case XLOG_HASH_MOVE_PAGE_CONTENTS: + id = "MOVE_PAGE_CONTENTS"; + break; + case XLOG_HASH_SQUEEZE_PAGE: + id = "SQUEEZE_PAGE"; + break; + case XLOG_HASH_DELETE: + id = "DELETE"; + break; + case XLOG_HASH_SPLIT_CLEANUP: + id = "SPLIT_CLEANUP"; + break; + case XLOG_HASH_UPDATE_META_PAGE: + id = "UPDATE_META_PAGE"; + break; + } + + return id; } diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 72bb06c760..9618032356 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -506,11 +506,6 @@ DefineIndex(Oid relationId, accessMethodForm = (Form_pg_am) GETSTRUCT(tuple); amRoutine = GetIndexAmRoutine(accessMethodForm->amhandler); - if (strcmp(accessMethodName, "hash") == 0 && - RelationNeedsWAL(rel)) - ereport(WARNING, - (errmsg("hash indexes are not WAL-logged and their use is discouraged"))); - if (stmt->unique && !amRoutine->amcanunique) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c index 9001e202b0..ce55fc5277 100644 --- a/src/backend/utils/cache/relcache.c +++ b/src/backend/utils/cache/relcache.c @@ -5880,13 +5880,10 @@ RelationIdIsInInitFile(Oid relationId) /* * Tells whether any index for the relation is unlogged. * - * Any index using the hash AM is implicitly unlogged. - * * Note: There doesn't seem to be any way to have an unlogged index attached - * to a permanent table except to create a hash index, but it seems best to - * keep this general so that it returns sensible results even when they seem - * obvious (like for an unlogged table) and to handle possible future unlogged - * indexes on permanent tables. + * to a permanent table, but it seems best to keep this general so that it + * returns sensible results even when they seem obvious (like for an unlogged + * table) and to handle possible future unlogged indexes on permanent tables. */ bool RelationHasUnloggedIndex(Relation rel) @@ -5908,8 +5905,7 @@ RelationHasUnloggedIndex(Relation rel) elog(ERROR, "cache lookup failed for relation %u", indexoid); reltup = (Form_pg_class) GETSTRUCT(tp); - if (reltup->relpersistence == RELPERSISTENCE_UNLOGGED - || reltup->relam == HASH_AM_OID) + if (reltup->relpersistence == RELPERSISTENCE_UNLOGGED) result = true; ReleaseSysCache(tp); diff --git a/src/include/access/hash_xlog.h b/src/include/access/hash_xlog.h index cc231632e1..2075ab7afa 100644 --- a/src/include/access/hash_xlog.h +++ b/src/include/access/hash_xlog.h @@ -16,7 +16,239 @@ #include "access/xlogreader.h" #include "lib/stringinfo.h" +#include "storage/off.h" +/* Number of buffers required for XLOG_HASH_SQUEEZE_PAGE operation */ +#define HASH_XLOG_FREE_OVFL_BUFS 6 + +/* + * XLOG records for hash operations + */ +#define XLOG_HASH_INIT_META_PAGE 0x00 /* initialize the meta page */ +#define XLOG_HASH_INIT_BITMAP_PAGE 0x10 /* initialize the bitmap page */ +#define XLOG_HASH_INSERT 0x20 /* add index tuple without split */ +#define XLOG_HASH_ADD_OVFL_PAGE 0x30 /* add overflow page */ +#define XLOG_HASH_SPLIT_ALLOCATE_PAGE 0x40 /* allocate new page for split */ +#define XLOG_HASH_SPLIT_PAGE 0x50 /* split page */ +#define XLOG_HASH_SPLIT_COMPLETE 0x60 /* completion of split + * operation */ +#define XLOG_HASH_MOVE_PAGE_CONTENTS 0x70 /* remove tuples from one page + * and add to another page */ +#define XLOG_HASH_SQUEEZE_PAGE 0x80 /* add tuples to one of the previous + * pages in chain and free the ovfl + * page */ +#define XLOG_HASH_DELETE 0x90 /* delete index tuples from a page */ +#define XLOG_HASH_SPLIT_CLEANUP 0xA0 /* clear split-cleanup flag in primary + * bucket page after deleting tuples + * that are moved due to split */ +#define XLOG_HASH_UPDATE_META_PAGE 0xB0 /* update meta page after + * vacuum */ + + +/* + * xl_hash_split_allocate_page flag values, 8 bits are available. + */ +#define XLH_SPLIT_META_UPDATE_MASKS (1<<0) +#define XLH_SPLIT_META_UPDATE_SPLITPOINT (1<<1) + +/* + * This is what we need to know about a HASH index create. + * + * Backup block 0: metapage + */ +typedef struct xl_hash_createidx +{ + double num_tuples; + RegProcedure procid; + uint16 ffactor; +} xl_hash_createidx; +#define SizeOfHashCreateIdx (offsetof(xl_hash_createidx, ffactor) + sizeof(uint16)) + +/* + * This is what we need to know about simple (without split) insert. + * + * This data record is used for XLOG_HASH_INSERT + * + * Backup Blk 0: original page (data contains the inserted tuple) + * Backup Blk 1: metapage (HashMetaPageData) + */ +typedef struct xl_hash_insert +{ + OffsetNumber offnum; +} xl_hash_insert; + +#define SizeOfHashInsert (offsetof(xl_hash_insert, offnum) + sizeof(OffsetNumber)) + +/* + * This is what we need to know about addition of overflow page. + * + * This data record is used for XLOG_HASH_ADD_OVFL_PAGE + * + * Backup Blk 0: newly allocated overflow page + * Backup Blk 1: page before new overflow page in the bucket chain + * Backup Blk 2: bitmap page + * Backup Blk 3: new bitmap page + * Backup Blk 4: metapage + */ +typedef struct xl_hash_add_ovfl_page +{ + uint16 bmsize; + bool bmpage_found; +} xl_hash_add_ovfl_page; + +#define SizeOfHashAddOvflPage \ + (offsetof(xl_hash_add_ovfl_page, bmpage_found) + sizeof(bool)) + +/* + * This is what we need to know about allocating a page for split. + * + * This data record is used for XLOG_HASH_SPLIT_ALLOCATE_PAGE + * + * Backup Blk 0: page for old bucket + * Backup Blk 1: page for new bucket + * Backup Blk 2: metapage + */ +typedef struct xl_hash_split_allocate_page +{ + uint32 new_bucket; + uint16 old_bucket_flag; + uint16 new_bucket_flag; + uint8 flags; +} xl_hash_split_allocate_page; + +#define SizeOfHashSplitAllocPage \ + (offsetof(xl_hash_split_allocate_page, flags) + sizeof(uint8)) + +/* + * This is what we need to know about completing the split operation. + * + * This data record is used for XLOG_HASH_SPLIT_COMPLETE + * + * Backup Blk 0: page for old bucket + * Backup Blk 1: page for new bucket + */ +typedef struct xl_hash_split_complete +{ + uint16 old_bucket_flag; + uint16 new_bucket_flag; +} xl_hash_split_complete; + +#define SizeOfHashSplitComplete \ + (offsetof(xl_hash_split_complete, new_bucket_flag) + sizeof(uint16)) + +/* + * This is what we need to know about move page contents required during + * squeeze operation. + * + * This data record is used for XLOG_HASH_MOVE_PAGE_CONTENTS + * + * Backup Blk 0: bucket page + * Backup Blk 1: page containing moved tuples + * Backup Blk 2: page from which tuples will be removed + */ +typedef struct xl_hash_move_page_contents +{ + uint16 ntups; + bool is_prim_bucket_same_wrt; /* TRUE if the page to which + * tuples are moved is same as + * primary bucket page */ +} xl_hash_move_page_contents; + +#define SizeOfHashMovePageContents \ + (offsetof(xl_hash_move_page_contents, is_prim_bucket_same_wrt) + sizeof(bool)) + +/* + * This is what we need to know about the squeeze page operation. + * + * This data record is used for XLOG_HASH_SQUEEZE_PAGE + * + * Backup Blk 0: page containing tuples moved from freed overflow page + * Backup Blk 1: freed overflow page + * Backup Blk 2: page previous to the freed overflow page + * Backup Blk 3: page next to the freed overflow page + * Backup Blk 4: bitmap page containing info of freed overflow page + * Backup Blk 5: meta page + */ +typedef struct xl_hash_squeeze_page +{ + BlockNumber prevblkno; + BlockNumber nextblkno; + uint16 ntups; + bool is_prim_bucket_same_wrt; /* TRUE if the page to which + * tuples are moved is same as + * primary bucket page */ + bool is_prev_bucket_same_wrt; /* TRUE if the page to which + * tuples are moved is the + * page previous to the freed + * overflow page */ +} xl_hash_squeeze_page; + +#define SizeOfHashSqueezePage \ + (offsetof(xl_hash_squeeze_page, is_prev_bucket_same_wrt) + sizeof(bool)) + +/* + * This is what we need to know about the deletion of index tuples from a page. + * + * This data record is used for XLOG_HASH_DELETE + * + * Backup Blk 0: primary bucket page + * Backup Blk 1: page from which tuples are deleted + */ +typedef struct xl_hash_delete +{ + bool is_primary_bucket_page; /* TRUE if the operation is for + * primary bucket page */ +} xl_hash_delete; + +#define SizeOfHashDelete (offsetof(xl_hash_delete, is_primary_bucket_page) + sizeof(bool)) + +/* + * This is what we need for metapage update operation. + * + * This data record is used for XLOG_HASH_UPDATE_META_PAGE + * + * Backup Blk 0: meta page + */ +typedef struct xl_hash_update_meta_page +{ + double ntuples; +} xl_hash_update_meta_page; + +#define SizeOfHashUpdateMetaPage \ + (offsetof(xl_hash_update_meta_page, ntuples) + sizeof(double)) + +/* + * This is what we need to initialize metapage. + * + * This data record is used for XLOG_HASH_INIT_META_PAGE + * + * Backup Blk 0: meta page + */ +typedef struct xl_hash_init_meta_page +{ + double num_tuples; + RegProcedure procid; + uint16 ffactor; +} xl_hash_init_meta_page; + +#define SizeOfHashInitMetaPage \ + (offsetof(xl_hash_init_meta_page, ffactor) + sizeof(uint16)) + +/* + * This is what we need to initialize bitmap page. + * + * This data record is used for XLOG_HASH_INIT_BITMAP_PAGE + * + * Backup Blk 0: bitmap page + * Backup Blk 1: meta page + */ +typedef struct xl_hash_init_bitmap_page +{ + uint16 bmsize; +} xl_hash_init_bitmap_page; + +#define SizeOfHashInitBitmapPage \ + (offsetof(xl_hash_init_bitmap_page, bmsize) + sizeof(uint16)) extern void hash_redo(XLogReaderState *record); extern void hash_desc(StringInfo buf, XLogReaderState *record); diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index e519fdb0f6..26cd05933c 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2335,13 +2335,9 @@ Options: fastupdate=on, gin_pending_list_limit=128 -- HASH -- CREATE INDEX hash_i4_index ON hash_i4_heap USING hash (random int4_ops); -WARNING: hash indexes are not WAL-logged and their use is discouraged CREATE INDEX hash_name_index ON hash_name_heap USING hash (random name_ops); -WARNING: hash indexes are not WAL-logged and their use is discouraged CREATE INDEX hash_txt_index ON hash_txt_heap USING hash (random text_ops); -WARNING: hash indexes are not WAL-logged and their use is discouraged CREATE INDEX hash_f8_index ON hash_f8_heap USING hash (random float8_ops); -WARNING: hash indexes are not WAL-logged and their use is discouraged CREATE UNLOGGED TABLE unlogged_hash_table (id int4); CREATE INDEX unlogged_hash_index ON unlogged_hash_table USING hash (id int4_ops); DROP TABLE unlogged_hash_table; @@ -2350,7 +2346,6 @@ DROP TABLE unlogged_hash_table; -- maintenance_work_mem setting and fillfactor: SET maintenance_work_mem = '1MB'; CREATE INDEX hash_tuplesort_idx ON tenk1 USING hash (stringu1 name_ops) WITH (fillfactor = 10); -WARNING: hash indexes are not WAL-logged and their use is discouraged EXPLAIN (COSTS OFF) SELECT count(*) FROM tenk1 WHERE stringu1 = 'TVAAAA'; QUERY PLAN diff --git a/src/test/regress/expected/enum.out b/src/test/regress/expected/enum.out index 514d1d01a1..0e6030443f 100644 --- a/src/test/regress/expected/enum.out +++ b/src/test/regress/expected/enum.out @@ -383,7 +383,6 @@ DROP INDEX enumtest_btree; -- Hash index / opclass with the = operator -- CREATE INDEX enumtest_hash ON enumtest USING hash (col); -WARNING: hash indexes are not WAL-logged and their use is discouraged SELECT * FROM enumtest WHERE col = 'orange'; col -------- diff --git a/src/test/regress/expected/hash_index.out b/src/test/regress/expected/hash_index.out index f8b9f029b2..0a18efacfc 100644 --- a/src/test/regress/expected/hash_index.out +++ b/src/test/regress/expected/hash_index.out @@ -201,7 +201,6 @@ SELECT h.seqno AS f20000 -- CREATE TABLE hash_split_heap (keycol INT); CREATE INDEX hash_split_index on hash_split_heap USING HASH (keycol); -WARNING: hash indexes are not WAL-logged and their use is discouraged INSERT INTO hash_split_heap SELECT 1 FROM generate_series(1, 70000) a; VACUUM FULL hash_split_heap; -- Let's do a backward scan. @@ -230,5 +229,4 @@ DROP TABLE hash_temp_heap CASCADE; CREATE TABLE hash_heap_float4 (x float4, y int); INSERT INTO hash_heap_float4 VALUES (1.1,1); CREATE INDEX hash_idx ON hash_heap_float4 USING hash (x); -WARNING: hash indexes are not WAL-logged and their use is discouraged DROP TABLE hash_heap_float4 CASCADE; diff --git a/src/test/regress/expected/macaddr.out b/src/test/regress/expected/macaddr.out index e84ff5f8c0..151f9ce59b 100644 --- a/src/test/regress/expected/macaddr.out +++ b/src/test/regress/expected/macaddr.out @@ -41,7 +41,6 @@ SELECT * FROM macaddr_data; CREATE INDEX macaddr_data_btree ON macaddr_data USING btree (b); CREATE INDEX macaddr_data_hash ON macaddr_data USING hash (b); -WARNING: hash indexes are not WAL-logged and their use is discouraged SELECT a, b, trunc(b) FROM macaddr_data ORDER BY 2, 1; a | b | trunc ----+-------------------+------------------- diff --git a/src/test/regress/expected/replica_identity.out b/src/test/regress/expected/replica_identity.out index fa63235fc9..67c34a92a4 100644 --- a/src/test/regress/expected/replica_identity.out +++ b/src/test/regress/expected/replica_identity.out @@ -12,7 +12,6 @@ CREATE UNIQUE INDEX test_replica_identity_keyab_key ON test_replica_identity (ke CREATE UNIQUE INDEX test_replica_identity_oid_idx ON test_replica_identity (oid); CREATE UNIQUE INDEX test_replica_identity_nonkey ON test_replica_identity (keya, nonkey); CREATE INDEX test_replica_identity_hash ON test_replica_identity USING hash (nonkey); -WARNING: hash indexes are not WAL-logged and their use is discouraged CREATE UNIQUE INDEX test_replica_identity_expr ON test_replica_identity (keya, keyb, (3)); CREATE UNIQUE INDEX test_replica_identity_partial ON test_replica_identity (keya, keyb) WHERE keyb != '3'; -- default is 'd'/DEFAULT for user created tables diff --git a/src/test/regress/expected/uuid.out b/src/test/regress/expected/uuid.out index 423f27787f..db66dc723e 100644 --- a/src/test/regress/expected/uuid.out +++ b/src/test/regress/expected/uuid.out @@ -114,7 +114,6 @@ SELECT COUNT(*) FROM guid1 WHERE guid_field >= '22222222-2222-2222-2222-22222222 -- btree and hash index creation test CREATE INDEX guid1_btree ON guid1 USING BTREE (guid_field); CREATE INDEX guid1_hash ON guid1 USING HASH (guid_field); -WARNING: hash indexes are not WAL-logged and their use is discouraged -- unique index test CREATE UNIQUE INDEX guid1_unique_BTREE ON guid1 USING BTREE (guid_field); -- should fail -- cgit v1.2.3 From bb4a39637ac6155b4ed5a9f83d17921fb57400b6 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Tue, 14 Mar 2017 14:58:56 -0400 Subject: hash: Support WAL consistency checking. Kuntal Ghosh, reviewed by Amit Kapila and Ashutosh Sharma, with a few tweaks by me. Discussion: http://postgr.es/m/CAGz5QCJLERUn_zoO0eDv6_Y_d0o4tNTMPeR7ivTLBg4rUrJdwg@mail.gmail.com --- doc/src/sgml/config.sgml | 6 +++--- src/backend/access/hash/hash_xlog.c | 36 ++++++++++++++++++++++++++++++++++++ src/include/access/hash_xlog.h | 1 + src/include/access/rmgrlist.h | 2 +- 4 files changed, 41 insertions(+), 4 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index eadbfcd22f..b6daf9677c 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -8267,9 +8267,9 @@ LOG: CleanUpLock: deleting: lock(0xb7acd844) id(24688,24696,0,0,0,1) records, or to a comma-separated list of resource managers to check only records originating from those resource managers. Currently, the supported resource managers are heap, - heap2, btree, gin, - gist, sequence, spgist, - brin, and generic. Only + heap2, btree, hash, + gin, gist, sequence, + spgist, brin, and generic. Only superusers can change this setting. diff --git a/src/backend/access/hash/hash_xlog.c b/src/backend/access/hash/hash_xlog.c index d435215259..0c830ab595 100644 --- a/src/backend/access/hash/hash_xlog.c +++ b/src/backend/access/hash/hash_xlog.c @@ -14,6 +14,7 @@ */ #include "postgres.h" +#include "access/bufmask.h" #include "access/hash.h" #include "access/hash_xlog.h" #include "access/xlogutils.h" @@ -961,3 +962,38 @@ hash_redo(XLogReaderState *record) elog(PANIC, "hash_redo: unknown op code %u", info); } } + +/* + * Mask a hash page before performing consistency checks on it. + */ +void +hash_mask(char *pagedata, BlockNumber blkno) +{ + Page page = (Page) pagedata; + HashPageOpaque opaque; + + mask_page_lsn(page); + + mask_page_hint_bits(page); + mask_unused_space(page); + + opaque = (HashPageOpaque) PageGetSpecialPointer(page); + + if (opaque->hasho_flag & LH_UNUSED_PAGE) + { + /* + * Mask everything on a UNUSED page. + */ + mask_page_content(page); + } + else if ((opaque->hasho_flag & LH_BUCKET_PAGE) || + (opaque->hasho_flag & LH_OVERFLOW_PAGE)) + { + /* + * In hash bucket and overflow pages, it is possible to modify the + * LP_FLAGS without emitting any WAL record. Hence, mask the line + * pointer flags. See hashgettuple() for details. + */ + mask_lp_flags(page); + } +} diff --git a/src/include/access/hash_xlog.h b/src/include/access/hash_xlog.h index 2075ab7afa..552d6428ca 100644 --- a/src/include/access/hash_xlog.h +++ b/src/include/access/hash_xlog.h @@ -253,5 +253,6 @@ typedef struct xl_hash_init_bitmap_page extern void hash_redo(XLogReaderState *record); extern void hash_desc(StringInfo buf, XLogReaderState *record); extern const char *hash_identify(uint8 info); +extern void hash_mask(char *pagedata, BlockNumber blkno); #endif /* HASH_XLOG_H */ diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h index b892aea370..2f43c199d3 100644 --- a/src/include/access/rmgrlist.h +++ b/src/include/access/rmgrlist.h @@ -37,7 +37,7 @@ PG_RMGR(RM_STANDBY_ID, "Standby", standby_redo, standby_desc, standby_identify, PG_RMGR(RM_HEAP2_ID, "Heap2", heap2_redo, heap2_desc, heap2_identify, NULL, NULL, heap_mask) PG_RMGR(RM_HEAP_ID, "Heap", heap_redo, heap_desc, heap_identify, NULL, NULL, heap_mask) PG_RMGR(RM_BTREE_ID, "Btree", btree_redo, btree_desc, btree_identify, NULL, NULL, btree_mask) -PG_RMGR(RM_HASH_ID, "Hash", hash_redo, hash_desc, hash_identify, NULL, NULL, NULL) +PG_RMGR(RM_HASH_ID, "Hash", hash_redo, hash_desc, hash_identify, NULL, NULL, hash_mask) PG_RMGR(RM_GIN_ID, "Gin", gin_redo, gin_desc, gin_identify, gin_xlog_startup, gin_xlog_cleanup, gin_mask) PG_RMGR(RM_GIST_ID, "Gist", gist_redo, gist_desc, gist_identify, gist_xlog_startup, gist_xlog_cleanup, gist_mask) PG_RMGR(RM_SEQ_ID, "Sequence", seq_redo, seq_desc, seq_identify, NULL, NULL, seq_mask) -- cgit v1.2.3 From eb4da3e3807d2054bb05c3eb201cb9a363682f09 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 14 Mar 2017 17:13:56 -0400 Subject: Add option to control snapshot export to CREATE_REPLICATION_SLOT We used to export snapshots unconditionally in CREATE_REPLICATION_SLOT in the replication protocol, but several upcoming patches want more control over what happens. Suppress snapshot export in pg_recvlogical, which neither needs nor can use the exported snapshot. Since snapshot exporting can fail this improves reliability. This also paves the way for allowing the creation of replication slots on standbys, which cannot export snapshots because they cannot allocate new XIDs. Author: Petr Jelinek --- doc/src/sgml/logicaldecoding.sgml | 11 +++- doc/src/sgml/protocol.sgml | 19 ++++++- src/backend/commands/subscriptioncmds.c | 6 ++- .../libpqwalreceiver/libpqwalreceiver.c | 15 ++++-- src/backend/replication/repl_gram.y | 43 ++++++++++++---- src/backend/replication/repl_scanner.l | 2 + src/backend/replication/walsender.c | 58 ++++++++++++++++++++-- src/bin/pg_basebackup/streamutil.c | 5 ++ src/include/nodes/replnodes.h | 2 +- src/include/replication/walreceiver.h | 6 +-- 10 files changed, 141 insertions(+), 26 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/logicaldecoding.sgml b/doc/src/sgml/logicaldecoding.sgml index 03c2c691d1..bb22f9ae22 100644 --- a/doc/src/sgml/logicaldecoding.sgml +++ b/doc/src/sgml/logicaldecoding.sgml @@ -271,8 +271,9 @@ $ pg_recvlogical -d postgres --slot test --drop-slot Exported Snapshots - When a new replication slot is created using the streaming replication interface, - a snapshot is exported + When a new replication slot is created using the streaming replication + interface (see ), a + snapshot is exported (see ), which will show exactly the state of the database after which all changes will be included in the change stream. This can be used to create a new replica by @@ -282,6 +283,12 @@ $ pg_recvlogical -d postgres --slot test --drop-slot database's state at that point in time, which afterwards can be updated using the slot's contents without losing any changes. + + Creation of a snapshot is not always possible. In particular, it will + fail when connected to a hot standby. Applications that do not require + snapshot export may suppress it with the NOEXPORT_SNAPSHOT + option. + diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index 67f3f455db..7c82b48845 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -1486,8 +1486,8 @@ The commands accepted in walsender mode are: - - CREATE_REPLICATION_SLOT slot_name [ TEMPORARY ] { PHYSICAL [ RESERVE_WAL ] | LOGICAL output_plugin } + + CREATE_REPLICATION_SLOT slot_name [ TEMPORARY ] { PHYSICAL [ RESERVE_WAL ] | LOGICAL output_plugin [ EXPORT_SNAPSHOT | NOEXPORT_SNAPSHOT ] } CREATE_REPLICATION_SLOT @@ -1538,6 +1538,21 @@ The commands accepted in walsender mode are: + + + EXPORT_SNAPSHOT + NOEXPORT_SNAPSHOT + + + Decides what to do with the snapshot created during logical slot + initialization. EXPORT_SNAPSHOT, which is the default, + will export the snapshot for use in other sessions. This option can't + be used inside a transaction. NOEXPORT_SNAPSHOT will + just use the snapshot for logical decoding as normal but won't do + anything else with it. + + +     diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c index 1868bf5f9e..0198e6d75b 100644 --- a/src/backend/commands/subscriptioncmds.c +++ b/src/backend/commands/subscriptioncmds.c @@ -314,7 +314,11 @@ CreateSubscription(CreateSubscriptionStmt *stmt, bool isTopLevel) PG_TRY(); { - walrcv_create_slot(wrconn, slotname, false, &lsn); + /* + * Create permanent slot for the subscription. We won't use the + * initial snapshot for anything, so no need to export it. + */ + walrcv_create_slot(wrconn, slotname, false, false, &lsn); ereport(NOTICE, (errmsg("created replication slot \"%s\" on publisher", slotname))); diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c index ebadf3680f..cd2e57867c 100644 --- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c +++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c @@ -68,6 +68,7 @@ static void libpqrcv_send(WalReceiverConn *conn, const char *buffer, static char *libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, bool temporary, + bool export_snapshot, XLogRecPtr *lsn); static bool libpqrcv_command(WalReceiverConn *conn, const char *cmd, char **err); @@ -720,7 +721,7 @@ libpqrcv_send(WalReceiverConn *conn, const char *buffer, int nbytes) */ static char * libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, - bool temporary, XLogRecPtr *lsn) + bool temporary, bool export_snapshot, XLogRecPtr *lsn) { PGresult *res; StringInfoData cmd; @@ -728,13 +729,19 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, initStringInfo(&cmd); - appendStringInfo(&cmd, "CREATE_REPLICATION_SLOT \"%s\" ", slotname); + appendStringInfo(&cmd, "CREATE_REPLICATION_SLOT \"%s\"", slotname); if (temporary) - appendStringInfo(&cmd, "TEMPORARY "); + appendStringInfo(&cmd, " TEMPORARY"); if (conn->logical) - appendStringInfo(&cmd, "LOGICAL pgoutput"); + { + appendStringInfo(&cmd, " LOGICAL pgoutput"); + if (export_snapshot) + appendStringInfo(&cmd, " EXPORT_SNAPSHOT"); + else + appendStringInfo(&cmd, " NOEXPORT_SNAPSHOT"); + } res = libpqrcv_PQexec(conn->streamConn, cmd.data); pfree(cmd.data); diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y index b35d0f0cd1..f1e43bc9f3 100644 --- a/src/backend/replication/repl_gram.y +++ b/src/backend/replication/repl_gram.y @@ -79,6 +79,8 @@ Node *replication_parse_result; %token K_SLOT %token K_RESERVE_WAL %token K_TEMPORARY +%token K_EXPORT_SNAPSHOT +%token K_NOEXPORT_SNAPSHOT %type command %type base_backup start_replication start_logical_replication @@ -91,7 +93,9 @@ Node *replication_parse_result; %type plugin_opt_elem %type plugin_opt_arg %type opt_slot var_name -%type opt_reserve_wal opt_temporary +%type opt_temporary +%type create_slot_opt_list +%type create_slot_opt %% @@ -202,18 +206,18 @@ base_backup_opt: create_replication_slot: /* CREATE_REPLICATION_SLOT slot TEMPORARY PHYSICAL RESERVE_WAL */ - K_CREATE_REPLICATION_SLOT IDENT opt_temporary K_PHYSICAL opt_reserve_wal + K_CREATE_REPLICATION_SLOT IDENT opt_temporary K_PHYSICAL create_slot_opt_list { CreateReplicationSlotCmd *cmd; cmd = makeNode(CreateReplicationSlotCmd); cmd->kind = REPLICATION_KIND_PHYSICAL; cmd->slotname = $2; cmd->temporary = $3; - cmd->reserve_wal = $5; + cmd->options = $5; $$ = (Node *) cmd; } /* CREATE_REPLICATION_SLOT slot TEMPORARY LOGICAL plugin */ - | K_CREATE_REPLICATION_SLOT IDENT opt_temporary K_LOGICAL IDENT + | K_CREATE_REPLICATION_SLOT IDENT opt_temporary K_LOGICAL IDENT create_slot_opt_list { CreateReplicationSlotCmd *cmd; cmd = makeNode(CreateReplicationSlotCmd); @@ -221,10 +225,36 @@ create_replication_slot: cmd->slotname = $2; cmd->temporary = $3; cmd->plugin = $5; + cmd->options = $6; $$ = (Node *) cmd; } ; +create_slot_opt_list: + create_slot_opt_list create_slot_opt + { $$ = lappend($1, $2); } + | /* EMPTY */ + { $$ = NIL; } + ; + +create_slot_opt: + K_EXPORT_SNAPSHOT + { + $$ = makeDefElem("export_snapshot", + (Node *)makeInteger(TRUE), -1); + } + | K_NOEXPORT_SNAPSHOT + { + $$ = makeDefElem("export_snapshot", + (Node *)makeInteger(FALSE), -1); + } + | K_RESERVE_WAL + { + $$ = makeDefElem("reserve_wal", + (Node *)makeInteger(TRUE), -1); + } + ; + /* DROP_REPLICATION_SLOT slot */ drop_replication_slot: K_DROP_REPLICATION_SLOT IDENT @@ -291,11 +321,6 @@ opt_physical: | /* EMPTY */ ; -opt_reserve_wal: - K_RESERVE_WAL { $$ = true; } - | /* EMPTY */ { $$ = false; } - ; - opt_temporary: K_TEMPORARY { $$ = true; } | /* EMPTY */ { $$ = false; } diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l index 37f857925e..f56d41d59c 100644 --- a/src/backend/replication/repl_scanner.l +++ b/src/backend/replication/repl_scanner.l @@ -100,6 +100,8 @@ RESERVE_WAL { return K_RESERVE_WAL; } LOGICAL { return K_LOGICAL; } SLOT { return K_SLOT; } TEMPORARY { return K_TEMPORARY; } +EXPORT_SNAPSHOT { return K_EXPORT_SNAPSHOT; } +NOEXPORT_SNAPSHOT { return K_NOEXPORT_SNAPSHOT; } "," { return ','; } ";" { return ';'; } diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index dd3a936fc6..127efecb27 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -51,6 +51,7 @@ #include "catalog/pg_type.h" #include "commands/dbcommands.h" +#include "commands/defrem.h" #include "funcapi.h" #include "libpq/libpq.h" #include "libpq/pqformat.h" @@ -737,6 +738,48 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req return count; } +/* + * Process extra options given to CREATE_REPLICATION_SLOT. + */ +static void +parseCreateReplSlotOptions(CreateReplicationSlotCmd *cmd, + bool *reserve_wal, + bool *export_snapshot) +{ + ListCell *lc; + bool snapshot_action_given = false; + bool reserve_wal_given = false; + + /* Parse options */ + foreach (lc, cmd->options) + { + DefElem *defel = (DefElem *) lfirst(lc); + + if (strcmp(defel->defname, "export_snapshot") == 0) + { + if (snapshot_action_given || cmd->kind != REPLICATION_KIND_LOGICAL) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"))); + + snapshot_action_given = true; + *export_snapshot = defGetBoolean(defel); + } + else if (strcmp(defel->defname, "reserve_wal") == 0) + { + if (reserve_wal_given || cmd->kind != REPLICATION_KIND_PHYSICAL) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"))); + + reserve_wal_given = true; + *reserve_wal = true; + } + else + elog(ERROR, "unrecognized option: %s", defel->defname); + } +} + /* * Create a new replication slot. */ @@ -746,6 +789,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) const char *snapshot_name = NULL; char xpos[MAXFNAMELEN]; char *slot_name; + bool reserve_wal = false; + bool export_snapshot = true; DestReceiver *dest; TupOutputState *tstate; TupleDesc tupdesc; @@ -754,6 +799,8 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) Assert(!MyReplicationSlot); + parseCreateReplSlotOptions(cmd, &reserve_wal, &export_snapshot); + /* setup state for XLogReadPage */ sendTimeLineIsHistoric = false; sendTimeLine = ThisTimeLineID; @@ -799,10 +846,13 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) DecodingContextFindStartpoint(ctx); /* - * Export a plain (not of the snapbuild.c type) snapshot to the user - * that can be imported into another session. + * Export the snapshot if we've been asked to do so. + * + * NB. We will convert the snapbuild.c kind of snapshot to normal + * snapshot when doing this. */ - snapshot_name = SnapBuildExportSnapshot(ctx->snapshot_builder); + if (export_snapshot) + snapshot_name = SnapBuildExportSnapshot(ctx->snapshot_builder); /* don't need the decoding context anymore */ FreeDecodingContext(ctx); @@ -810,7 +860,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) if (!cmd->temporary) ReplicationSlotPersist(); } - else if (cmd->kind == REPLICATION_KIND_PHYSICAL && cmd->reserve_wal) + else if (cmd->kind == REPLICATION_KIND_PHYSICAL && reserve_wal) { ReplicationSlotReserveWal(); diff --git a/src/bin/pg_basebackup/streamutil.c b/src/bin/pg_basebackup/streamutil.c index 1fe42efc21..507da5e76d 100644 --- a/src/bin/pg_basebackup/streamutil.c +++ b/src/bin/pg_basebackup/streamutil.c @@ -338,8 +338,13 @@ CreateReplicationSlot(PGconn *conn, const char *slot_name, const char *plugin, appendPQExpBuffer(query, "CREATE_REPLICATION_SLOT \"%s\" PHYSICAL", slot_name); else + { appendPQExpBuffer(query, "CREATE_REPLICATION_SLOT \"%s\" LOGICAL \"%s\"", slot_name, plugin); + if (PQserverVersion(conn) >= 100000) + /* pg_recvlogical doesn't use an exported snapshot, so suppress */ + appendPQExpBuffer(query, " NOEXPORT_SNAPSHOT"); + } res = PQexec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h index f27354faaf..996da3c02e 100644 --- a/src/include/nodes/replnodes.h +++ b/src/include/nodes/replnodes.h @@ -56,7 +56,7 @@ typedef struct CreateReplicationSlotCmd ReplicationKind kind; char *plugin; bool temporary; - bool reserve_wal; + List *options; } CreateReplicationSlotCmd; diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h index 0857bdc556..78e577c89b 100644 --- a/src/include/replication/walreceiver.h +++ b/src/include/replication/walreceiver.h @@ -183,7 +183,7 @@ typedef void (*walrcv_send_fn) (WalReceiverConn *conn, const char *buffer, int nbytes); typedef char *(*walrcv_create_slot_fn) (WalReceiverConn *conn, const char *slotname, bool temporary, - XLogRecPtr *lsn); + bool export_snapshot, XLogRecPtr *lsn); typedef bool (*walrcv_command_fn) (WalReceiverConn *conn, const char *cmd, char **err); typedef void (*walrcv_disconnect_fn) (WalReceiverConn *conn); @@ -224,8 +224,8 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions; WalReceiverFunctions->walrcv_receive(conn, buffer, wait_fd) #define walrcv_send(conn, buffer, nbytes) \ WalReceiverFunctions->walrcv_send(conn, buffer, nbytes) -#define walrcv_create_slot(conn, slotname, temporary, lsn) \ - WalReceiverFunctions->walrcv_create_slot(conn, slotname, temporary, lsn) +#define walrcv_create_slot(conn, slotname, temporary, export_snapshot, lsn) \ + WalReceiverFunctions->walrcv_create_slot(conn, slotname, temporary, export_snapshot, lsn) #define walrcv_command(conn, cmd, err) \ WalReceiverFunctions->walrcv_command(conn, cmd, err) #define walrcv_disconnect(conn) \ -- cgit v1.2.3 From 0c068205df5299fa41bce8ac33acdc9cfa5e7c6a Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 14 Mar 2017 23:12:18 -0400 Subject: doc: Fix TOC generation for refentries The XSLT stylesheets by default don't show refentries inside sections in table of contents, which for our documentation leads to some regressions compared to the DSSSL output. For example, in the SPI chapter, which is mostly refentries, you don't get any usable table of contents. Tweak things so it's mostly back to what it was before. --- doc/src/sgml/stylesheet.xsl | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'doc/src') diff --git a/doc/src/sgml/stylesheet.xsl b/doc/src/sgml/stylesheet.xsl index 42e8cce368..efcb80ffca 100644 --- a/doc/src/sgml/stylesheet.xsl +++ b/doc/src/sgml/stylesheet.xsl @@ -63,6 +63,18 @@ section toc set toc,title + + + + + + + + + + + -- cgit v1.2.3 From 3f6ea5fc8d9c6d7c85827c87a717be0016d06b89 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 14 Mar 2017 23:30:01 -0400 Subject: doc: Remove useless elements in bibliography Under DSSSL, bibliodiv/para elements didn't show up, but under XSLT they do, but they are kind of useless here, so remove them. --- doc/src/sgml/biblio.sgml | 3 --- 1 file changed, 3 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/biblio.sgml b/doc/src/sgml/biblio.sgml index 0da3a83fae..ab5af16aee 100644 --- a/doc/src/sgml/biblio.sgml +++ b/doc/src/sgml/biblio.sgml @@ -18,7 +18,6 @@ <acronym>SQL</acronym> Reference Books - Reference texts for SQL features. The Practical <acronym>SQL</acronym> Handbook @@ -165,7 +164,6 @@ PostgreSQL-specific Documentation - This section is for related documentation. Enhancement of the ANSI SQL Implementation of PostgreSQL @@ -256,7 +254,6 @@ ssimkovi@ag.or.at Proceedings and Articles - This section is for articles and newsletters. Partial indexing in POSTGRES: research project -- cgit v1.2.3 From aefeb68741fb9456f14b4d690b0c646e532fea6b Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 9 Mar 2017 23:58:48 -0500 Subject: Allow referring to functions without arguments when unique In DDL commands referring to an existing function, allow omitting the argument list if the function name is unique in its schema, per SQL standard. This uses the same logic that the regproc type uses for finding functions by name only. Reviewed-by: Michael Paquier --- doc/src/sgml/ref/alter_extension.sgml | 2 +- doc/src/sgml/ref/alter_function.sgml | 13 +++++---- doc/src/sgml/ref/alter_opfamily.sgml | 7 +++-- doc/src/sgml/ref/comment.sgml | 2 +- doc/src/sgml/ref/create_cast.sgml | 6 ++-- doc/src/sgml/ref/create_transform.sgml | 12 +++++--- doc/src/sgml/ref/drop_function.sgml | 35 ++++++++++++++++++++--- doc/src/sgml/ref/grant.sgml | 2 +- doc/src/sgml/ref/revoke.sgml | 2 +- doc/src/sgml/ref/security_label.sgml | 2 +- src/backend/nodes/copyfuncs.c | 1 + src/backend/nodes/equalfuncs.c | 1 + src/backend/parser/gram.y | 27 ++++++++++++++++++ src/backend/parser/parse_func.c | 37 +++++++++++++++++++++++-- src/include/nodes/parsenodes.h | 3 ++ src/test/regress/expected/create_function_3.out | 11 +++++++- src/test/regress/sql/create_function_3.sql | 8 ++++++ 17 files changed, 143 insertions(+), 28 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/alter_extension.sgml b/doc/src/sgml/ref/alter_extension.sgml index de6d6dca16..a7c0927d1c 100644 --- a/doc/src/sgml/ref/alter_extension.sgml +++ b/doc/src/sgml/ref/alter_extension.sgml @@ -39,7 +39,7 @@ ALTER EXTENSION name DROP object_name | FOREIGN DATA WRAPPER object_name | FOREIGN TABLE object_name | - FUNCTION function_name ( [ [ argmode ] [ argname ] argtype [, ...] ] ) | + FUNCTION function_name [ ( [ [ argmode ] [ argname ] argtype [, ...] ] ) ] | MATERIALIZED VIEW object_name | OPERATOR operator_name (left_type, right_type) | OPERATOR CLASS object_name USING index_method | diff --git a/doc/src/sgml/ref/alter_function.sgml b/doc/src/sgml/ref/alter_function.sgml index 0388d06b95..168eeb7c52 100644 --- a/doc/src/sgml/ref/alter_function.sgml +++ b/doc/src/sgml/ref/alter_function.sgml @@ -21,15 +21,15 @@ PostgreSQL documentation -ALTER FUNCTION name ( [ [ argmode ] [ argname ] argtype [, ...] ] ) +ALTER FUNCTION name [ ( [ [ argmode ] [ argname ] argtype [, ...] ] ) ] action [ ... ] [ RESTRICT ] -ALTER FUNCTION name ( [ [ argmode ] [ argname ] argtype [, ...] ] ) +ALTER FUNCTION name [ ( [ [ argmode ] [ argname ] argtype [, ...] ] ) ] RENAME TO new_name -ALTER FUNCTION name ( [ [ argmode ] [ argname ] argtype [, ...] ] ) +ALTER FUNCTION name [ ( [ [ argmode ] [ argname ] argtype [, ...] ] ) ] OWNER TO { new_owner | CURRENT_USER | SESSION_USER } -ALTER FUNCTION name ( [ [ argmode ] [ argname ] argtype [, ...] ] ) +ALTER FUNCTION name [ ( [ [ argmode ] [ argname ] argtype [, ...] ] ) ] SET SCHEMA new_schema -ALTER FUNCTION name ( [ [ argmode ] [ argname ] argtype [, ...] ] ) +ALTER FUNCTION name [ ( [ [ argmode ] [ argname ] argtype [, ...] ] ) ] DEPENDS ON EXTENSION extension_name where action is one of: @@ -75,7 +75,8 @@ ALTER FUNCTION name ( [ [ name - The name (optionally schema-qualified) of an existing function. + The name (optionally schema-qualified) of an existing function. If no + argument list is specified, the name must be unique in its schema. diff --git a/doc/src/sgml/ref/alter_opfamily.sgml b/doc/src/sgml/ref/alter_opfamily.sgml index 4511c7f7b2..0bafe5b8f8 100644 --- a/doc/src/sgml/ref/alter_opfamily.sgml +++ b/doc/src/sgml/ref/alter_opfamily.sgml @@ -25,7 +25,7 @@ ALTER OPERATOR FAMILY name USING strategy_number operator_name ( op_type, op_type ) [ FOR SEARCH | FOR ORDER BY sort_family_name ] | FUNCTION support_number [ ( op_type [ , op_type ] ) ] - function_name ( argument_type [, ...] ) + function_name [ ( argument_type [, ...] ) ] } [, ... ] ALTER OPERATOR FAMILY name USING index_method DROP @@ -195,8 +195,9 @@ ALTER OPERATOR FAMILY name USING function_name - The name (optionally schema-qualified) of a function that is an - index method support procedure for the operator family. + The name (optionally schema-qualified) of a function that is an index + method support procedure for the operator family. If no argument list + is specified, the name must be unique in its schema. diff --git a/doc/src/sgml/ref/comment.sgml b/doc/src/sgml/ref/comment.sgml index c1cf587cb2..7483c8c03f 100644 --- a/doc/src/sgml/ref/comment.sgml +++ b/doc/src/sgml/ref/comment.sgml @@ -37,7 +37,7 @@ COMMENT ON EVENT TRIGGER object_name | FOREIGN DATA WRAPPER object_name | FOREIGN TABLE object_name | - FUNCTION function_name ( [ [ argmode ] [ argname ] argtype [, ...] ] ) | + FUNCTION function_name [ ( [ [ argmode ] [ argname ] argtype [, ...] ] ) ] | INDEX object_name | LARGE OBJECT large_object_oid | MATERIALIZED VIEW object_name | diff --git a/doc/src/sgml/ref/create_cast.sgml b/doc/src/sgml/ref/create_cast.sgml index 11266755e5..a7d13edc22 100644 --- a/doc/src/sgml/ref/create_cast.sgml +++ b/doc/src/sgml/ref/create_cast.sgml @@ -19,7 +19,7 @@ CREATE CAST (source_type AS target_type) - WITH FUNCTION function_name (argument_type [, ...]) + WITH FUNCTION function_name [ (argument_type [, ...]) ] [ AS ASSIGNMENT | AS IMPLICIT ] CREATE CAST (source_type AS target_type) @@ -192,7 +192,7 @@ SELECT CAST ( 2 AS numeric ) + 4.0; - function_name(argument_type [, ...]) + function_name[(argument_type [, ...])] @@ -200,6 +200,8 @@ SELECT CAST ( 2 AS numeric ) + 4.0; be schema-qualified. If it is not, the function will be looked up in the schema search path. The function's result data type must match the target type of the cast. Its arguments are discussed below. + If no argument list is specified, the function name must be unique in + its schema. diff --git a/doc/src/sgml/ref/create_transform.sgml b/doc/src/sgml/ref/create_transform.sgml index f44ee89d33..647c3b9f05 100644 --- a/doc/src/sgml/ref/create_transform.sgml +++ b/doc/src/sgml/ref/create_transform.sgml @@ -19,8 +19,8 @@ CREATE [ OR REPLACE ] TRANSFORM FOR type_name LANGUAGE lang_name ( - FROM SQL WITH FUNCTION from_sql_function_name (argument_type [, ...]), - TO SQL WITH FUNCTION to_sql_function_name (argument_type [, ...]) + FROM SQL WITH FUNCTION from_sql_function_name [ (argument_type [, ...]) ], + TO SQL WITH FUNCTION to_sql_function_name [ (argument_type [, ...]) ] ); @@ -104,7 +104,7 @@ CREATE [ OR REPLACE ] TRANSFORM FOR type_name LANGUAG - from_sql_function_name(argument_type [, ...]) + from_sql_function_name[(argument_type [, ...])] @@ -116,12 +116,14 @@ CREATE [ OR REPLACE ] TRANSFORM FOR type_name LANGUAG SQL-level function returning internal without at least one argument of type internal.) The actual return value will be something specific to the language implementation. + If no argument list is specified, the function name must be unique in + its schema. - to_sql_function_name(argument_type [, ...]) + to_sql_function_name[(argument_type [, ...])] @@ -130,6 +132,8 @@ CREATE [ OR REPLACE ] TRANSFORM FOR type_name LANGUAG internal and return the type that is the type for the transform. The actual argument value will be something specific to the language implementation. + If no argument list is specified, the function name must be unique in + its schema. diff --git a/doc/src/sgml/ref/drop_function.sgml b/doc/src/sgml/ref/drop_function.sgml index 5969b084b4..0aa984528d 100644 --- a/doc/src/sgml/ref/drop_function.sgml +++ b/doc/src/sgml/ref/drop_function.sgml @@ -21,7 +21,7 @@ PostgreSQL documentation -DROP FUNCTION [ IF EXISTS ] name ( [ [ argmode ] [ argname ] argtype [, ...] ] ) [, ...] +DROP FUNCTION [ IF EXISTS ] name [ ( [ [ argmode ] [ argname ] argtype [, ...] ] ) ] [, ...] [ CASCADE | RESTRICT ] @@ -56,7 +56,8 @@ DROP FUNCTION [ IF EXISTS ] name ( name - The name (optionally schema-qualified) of an existing function. + The name (optionally schema-qualified) of an existing function. If no + argument list is specified, the name must be unique in its schema. @@ -141,14 +142,40 @@ DROP FUNCTION sqrt(integer); DROP FUNCTION sqrt(integer), sqrt(bigint); + + + If the function name is unique in its schema, it can be referred to without + an argument list: + +DROP FUNCTION update_employee_salaries; + + Note that this is different from + +DROP FUNCTION update_employee_salaries(); + + which refers to a function with zero arguments, whereas the first variant + can refer to a function with any number of arguments, including zero, as + long as the name is unique. + Compatibility - A DROP FUNCTION statement is defined in the SQL - standard, but it is not compatible with this command. + This command conforms to the SQL standard, with + these PostgreSQL extensions: + + + The standard only allows one function to be dropped per command. + + + The IF EXISTS option + + + The ability to specify argument modes and names + + diff --git a/doc/src/sgml/ref/grant.sgml b/doc/src/sgml/ref/grant.sgml index d8ca39f869..9fb4c2fd7e 100644 --- a/doc/src/sgml/ref/grant.sgml +++ b/doc/src/sgml/ref/grant.sgml @@ -55,7 +55,7 @@ GRANT { USAGE | ALL [ PRIVILEGES ] } TO role_specification [, ...] [ WITH GRANT OPTION ] GRANT { EXECUTE | ALL [ PRIVILEGES ] } - ON { FUNCTION function_name ( [ [ argmode ] [ arg_name ] arg_type [, ...] ] ) [, ...] + ON { FUNCTION function_name [ ( [ [ argmode ] [ arg_name ] arg_type [, ...] ] ) ] [, ...] | ALL FUNCTIONS IN SCHEMA schema_name [, ...] } TO role_specification [, ...] [ WITH GRANT OPTION ] diff --git a/doc/src/sgml/ref/revoke.sgml b/doc/src/sgml/ref/revoke.sgml index fc00129620..ce532543f0 100644 --- a/doc/src/sgml/ref/revoke.sgml +++ b/doc/src/sgml/ref/revoke.sgml @@ -70,7 +70,7 @@ REVOKE [ GRANT OPTION FOR ] REVOKE [ GRANT OPTION FOR ] { EXECUTE | ALL [ PRIVILEGES ] } - ON { FUNCTION function_name ( [ [ argmode ] [ arg_name ] arg_type [, ...] ] ) [, ...] + ON { FUNCTION function_name [ ( [ [ argmode ] [ arg_name ] arg_type [, ...] ] ) ] [, ...] | ALL FUNCTIONS IN SCHEMA schema_name [, ...] } FROM { [ GROUP ] role_name | PUBLIC } [, ...] [ CASCADE | RESTRICT ] diff --git a/doc/src/sgml/ref/security_label.sgml b/doc/src/sgml/ref/security_label.sgml index 998fe3b7c0..afd86aff3a 100644 --- a/doc/src/sgml/ref/security_label.sgml +++ b/doc/src/sgml/ref/security_label.sgml @@ -30,7 +30,7 @@ SECURITY LABEL [ FOR provider ] ON DOMAIN object_name | EVENT TRIGGER object_name | FOREIGN TABLE object_name - FUNCTION function_name ( [ [ argmode ] [ argname ] argtype [, ...] ] ) | + FUNCTION function_name [ ( [ [ argmode ] [ argname ] argtype [, ...] ] ) ] | LARGE OBJECT large_object_oid | MATERIALIZED VIEW object_name | [ PROCEDURAL ] LANGUAGE object_name | diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index bfc2ac1716..25fd051d6e 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -3067,6 +3067,7 @@ _copyObjectWithArgs(const ObjectWithArgs *from) COPY_NODE_FIELD(objname); COPY_NODE_FIELD(objargs); + COPY_SCALAR_FIELD(args_unspecified); return newnode; } diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index 54e9c983a0..67529e3f86 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -1119,6 +1119,7 @@ _equalObjectWithArgs(const ObjectWithArgs *a, const ObjectWithArgs *b) { COMPARE_NODE_FIELD(objname); COMPARE_NODE_FIELD(objargs); + COMPARE_SCALAR_FIELD(args_unspecified); return true; } diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index e7acc2d9a2..6316688a88 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -7202,6 +7202,33 @@ function_with_argtypes: n->objargs = extractArgTypes($2); $$ = n; } + /* + * Because of reduce/reduce conflicts, we can't use func_name + * below, but we can write it out the long way, which actually + * allows more cases. + */ + | type_func_name_keyword + { + ObjectWithArgs *n = makeNode(ObjectWithArgs); + n->objname = list_make1(makeString(pstrdup($1))); + n->args_unspecified = true; + $$ = n; + } + | ColId + { + ObjectWithArgs *n = makeNode(ObjectWithArgs); + n->objname = list_make1(makeString($1)); + n->args_unspecified = true; + $$ = n; + } + | ColId indirection + { + ObjectWithArgs *n = makeNode(ObjectWithArgs); + n->objname = check_func_name(lcons(makeString($1), $2), + yyscanner); + n->args_unspecified = true; + $$ = n; + } ; /* diff --git a/src/backend/parser/parse_func.c b/src/backend/parser/parse_func.c index dd9749f205..55853c20bb 100644 --- a/src/backend/parser/parse_func.c +++ b/src/backend/parser/parse_func.c @@ -1895,8 +1895,10 @@ func_signature_string(List *funcname, int nargs, /* * LookupFuncName - * Given a possibly-qualified function name and a set of argument types, - * look up the function. + * + * Given a possibly-qualified function name and optionally a set of argument + * types, look up the function. Pass nargs == -1 to indicate that no argument + * types are specified. * * If the function name is not schema-qualified, it is sought in the current * namespace search path. @@ -1914,6 +1916,35 @@ LookupFuncName(List *funcname, int nargs, const Oid *argtypes, bool noError) clist = FuncnameGetCandidates(funcname, nargs, NIL, false, false, noError); + /* + * If no arguments were specified, the name must yield a unique candidate. + */ + if (nargs == -1) + { + if (clist) + { + if (clist->next) + { + if (!noError) + ereport(ERROR, + (errcode(ERRCODE_AMBIGUOUS_FUNCTION), + errmsg("function name \"%s\" is not unique", + NameListToString(funcname)), + errhint("Specify the argument list to select the function unambiguously."))); + } + else + return clist->oid; + } + else + { + if (!noError) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FUNCTION), + errmsg("could not find a function named \"%s\"", + NameListToString(funcname)))); + } + } + while (clist) { if (memcmp(argtypes, clist->args, nargs * sizeof(Oid)) == 0) @@ -1962,7 +1993,7 @@ LookupFuncWithArgs(ObjectWithArgs *func, bool noError) args_item = lnext(args_item); } - return LookupFuncName(func->objname, argcount, argoids, noError); + return LookupFuncName(func->objname, func->args_unspecified ? -1 : argcount, argoids, noError); } /* diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index a44d2178e1..d576523f6a 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -1811,6 +1811,9 @@ typedef struct ObjectWithArgs NodeTag type; List *objname; /* qualified name of function/operator */ List *objargs; /* list of Typename nodes */ + bool args_unspecified; /* argument list was omitted, so name must + * be unique (note that objargs == NIL means + * zero args) */ } ObjectWithArgs; /* diff --git a/src/test/regress/expected/create_function_3.out b/src/test/regress/expected/create_function_3.out index cc4e98a1d4..b5e19485e5 100644 --- a/src/test/regress/expected/create_function_3.out +++ b/src/test/regress/expected/create_function_3.out @@ -218,13 +218,21 @@ SELECT routine_name, ordinal_position, parameter_name, parameter_default (7 rows) DROP FUNCTION functest_IS_1(int, int, text), functest_IS_2(int), functest_IS_3(int); +-- overload +CREATE FUNCTION functest_B_2(bigint) RETURNS bool LANGUAGE 'sql' + IMMUTABLE AS 'SELECT $1 > 0'; +DROP FUNCTION functest_b_1; +DROP FUNCTION functest_b_1; -- error, not found +ERROR: could not find a function named "functest_b_1" +DROP FUNCTION functest_b_2; -- error, ambiguous +ERROR: function name "functest_b_2" is not unique +HINT: Specify the argument list to select the function unambiguously. -- Cleanups DROP SCHEMA temp_func_test CASCADE; NOTICE: drop cascades to 16 other objects DETAIL: drop cascades to function functest_a_1(text,date) drop cascades to function functest_a_2(text[]) drop cascades to function functest_a_3() -drop cascades to function functest_b_1(integer) drop cascades to function functest_b_2(integer) drop cascades to function functest_b_3(integer) drop cascades to function functest_b_4(integer) @@ -237,5 +245,6 @@ drop cascades to function functext_f_1(integer) drop cascades to function functext_f_2(integer) drop cascades to function functext_f_3(integer) drop cascades to function functext_f_4(integer) +drop cascades to function functest_b_2(bigint) DROP USER regress_unpriv_user; RESET search_path; diff --git a/src/test/regress/sql/create_function_3.sql b/src/test/regress/sql/create_function_3.sql index 66a463b089..0a0e407aab 100644 --- a/src/test/regress/sql/create_function_3.sql +++ b/src/test/regress/sql/create_function_3.sql @@ -158,6 +158,14 @@ SELECT routine_name, ordinal_position, parameter_name, parameter_default DROP FUNCTION functest_IS_1(int, int, text), functest_IS_2(int), functest_IS_3(int); +-- overload +CREATE FUNCTION functest_B_2(bigint) RETURNS bool LANGUAGE 'sql' + IMMUTABLE AS 'SELECT $1 > 0'; + +DROP FUNCTION functest_b_1; +DROP FUNCTION functest_b_1; -- error, not found +DROP FUNCTION functest_b_2; -- error, ambiguous + -- Cleanups DROP SCHEMA temp_func_test CASCADE; -- cgit v1.2.3 From c7a9fa399d557c6366222e90b35db31e45d25678 Mon Sep 17 00:00:00 2001 From: Stephen Frost Date: Wed, 15 Mar 2017 11:16:25 -0400 Subject: Add support for EUI-64 MAC addresses as macaddr8 This adds in support for EUI-64 MAC addresses by adding a new data type called 'macaddr8' (using our usual convention of indicating the number of bytes stored). This was largely a copy-and-paste from the macaddr data type, with appropriate adjustments for having 8 bytes instead of 6 and adding support for converting a provided EUI-48 (6 byte format) to the EUI-64 format. Conversion from EUI-48 to EUI-64 inserts FFFE as the 4th and 5th bytes but does not perform the IPv6 modified EUI-64 action of flipping the 7th bit, but we add a function to perform that specific action for the user as it may be commonly done by users who wish to calculate their IPv6 address based on their network prefix and 48-bit MAC address. Author: Haribabu Kommi, with a good bit of rework of macaddr8_in by me. Reviewed by: Vitaly Burovoy, Kuntal Ghosh Discussion: https://postgr.es/m/CAJrrPGcUi8ZH+KkK+=TctNQ+EfkeCEHtMU_yo1mvX8hsk_ghNQ@mail.gmail.com --- contrib/btree_gin/Makefile | 4 +- contrib/btree_gin/btree_gin--1.0--1.1.sql | 35 ++ contrib/btree_gin/btree_gin.c | 10 + contrib/btree_gin/btree_gin.control | 2 +- contrib/btree_gin/expected/macaddr8.out | 51 +++ contrib/btree_gin/sql/macaddr8.sql | 22 ++ contrib/btree_gist/Makefile | 11 +- contrib/btree_gist/btree_gist--1.3--1.4.sql | 64 ++++ contrib/btree_gist/btree_gist.control | 2 +- contrib/btree_gist/btree_gist.h | 1 + contrib/btree_gist/btree_macaddr8.c | 200 ++++++++++ contrib/btree_gist/expected/macaddr8.out | 89 +++++ contrib/btree_gist/sql/macaddr8.sql | 37 ++ doc/src/sgml/brin.sgml | 11 + doc/src/sgml/btree-gin.sgml | 3 +- doc/src/sgml/btree-gist.sgml | 4 +- doc/src/sgml/datatype.sgml | 83 +++++ doc/src/sgml/func.sgml | 56 +++ src/backend/utils/adt/Makefile | 2 +- src/backend/utils/adt/mac.c | 13 +- src/backend/utils/adt/mac8.c | 560 ++++++++++++++++++++++++++++ src/backend/utils/adt/network.c | 10 + src/backend/utils/adt/selfuncs.c | 1 + src/include/catalog/pg_amop.h | 18 + src/include/catalog/pg_amproc.h | 7 + src/include/catalog/pg_cast.h | 6 + src/include/catalog/pg_opclass.h | 3 + src/include/catalog/pg_operator.h | 23 +- src/include/catalog/pg_opfamily.h | 3 + src/include/catalog/pg_proc.h | 37 +- src/include/catalog/pg_type.h | 4 + src/include/utils/inet.h | 22 ++ src/test/regress/expected/macaddr8.out | 354 ++++++++++++++++++ src/test/regress/expected/opr_sanity.out | 6 + src/test/regress/parallel_schedule | 2 +- src/test/regress/serial_schedule | 1 + src/test/regress/sql/macaddr8.sql | 89 +++++ 37 files changed, 1826 insertions(+), 20 deletions(-) create mode 100644 contrib/btree_gin/btree_gin--1.0--1.1.sql create mode 100644 contrib/btree_gin/expected/macaddr8.out create mode 100644 contrib/btree_gin/sql/macaddr8.sql create mode 100644 contrib/btree_gist/btree_gist--1.3--1.4.sql create mode 100644 contrib/btree_gist/btree_macaddr8.c create mode 100644 contrib/btree_gist/expected/macaddr8.out create mode 100644 contrib/btree_gist/sql/macaddr8.sql create mode 100644 src/backend/utils/adt/mac8.c create mode 100644 src/test/regress/expected/macaddr8.out create mode 100644 src/test/regress/sql/macaddr8.sql (limited to 'doc/src') diff --git a/contrib/btree_gin/Makefile b/contrib/btree_gin/Makefile index 0492091599..f22e4af7df 100644 --- a/contrib/btree_gin/Makefile +++ b/contrib/btree_gin/Makefile @@ -4,12 +4,12 @@ MODULE_big = btree_gin OBJS = btree_gin.o $(WIN32RES) EXTENSION = btree_gin -DATA = btree_gin--1.0.sql btree_gin--unpackaged--1.0.sql +DATA = btree_gin--1.0.sql btree_gin--1.0--1.1.sql btree_gin--unpackaged--1.0.sql PGFILEDESC = "btree_gin - B-tree equivalent GIN operator classes" REGRESS = install_btree_gin int2 int4 int8 float4 float8 money oid \ timestamp timestamptz time timetz date interval \ - macaddr inet cidr text varchar char bytea bit varbit \ + macaddr macaddr8 inet cidr text varchar char bytea bit varbit \ numeric ifdef USE_PGXS diff --git a/contrib/btree_gin/btree_gin--1.0--1.1.sql b/contrib/btree_gin/btree_gin--1.0--1.1.sql new file mode 100644 index 0000000000..dd81d27599 --- /dev/null +++ b/contrib/btree_gin/btree_gin--1.0--1.1.sql @@ -0,0 +1,35 @@ +/* contrib/btree_gin/btree_gin--1.0--1.1.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION btree_gin UPDATE TO '1.1'" to load this file. \quit + +-- macaddr8 datatype support new in 10.0. +CREATE FUNCTION gin_extract_value_macaddr8(macaddr8, internal) +RETURNS internal +AS 'MODULE_PATHNAME' +LANGUAGE C STRICT IMMUTABLE; + +CREATE FUNCTION gin_compare_prefix_macaddr8(macaddr8, macaddr8, int2, internal) +RETURNS int4 +AS 'MODULE_PATHNAME' +LANGUAGE C STRICT IMMUTABLE; + +CREATE FUNCTION gin_extract_query_macaddr8(macaddr8, internal, int2, internal, internal) +RETURNS internal +AS 'MODULE_PATHNAME' +LANGUAGE C STRICT IMMUTABLE; + +CREATE OPERATOR CLASS macaddr8_ops +DEFAULT FOR TYPE macaddr8 USING gin +AS + OPERATOR 1 <, + OPERATOR 2 <=, + OPERATOR 3 =, + OPERATOR 4 >=, + OPERATOR 5 >, + FUNCTION 1 macaddr8_cmp(macaddr8, macaddr8), + FUNCTION 2 gin_extract_value_macaddr8(macaddr8, internal), + FUNCTION 3 gin_extract_query_macaddr8(macaddr8, internal, int2, internal, internal), + FUNCTION 4 gin_btree_consistent(internal, int2, anyelement, int4, internal, internal), + FUNCTION 5 gin_compare_prefix_macaddr8(macaddr8, macaddr8, int2, internal), +STORAGE macaddr8; diff --git a/contrib/btree_gin/btree_gin.c b/contrib/btree_gin/btree_gin.c index 030b61097f..725456e940 100644 --- a/contrib/btree_gin/btree_gin.c +++ b/contrib/btree_gin/btree_gin.c @@ -322,6 +322,16 @@ leftmostvalue_macaddr(void) GIN_SUPPORT(macaddr, false, leftmostvalue_macaddr, macaddr_cmp) +static Datum +leftmostvalue_macaddr8(void) +{ + macaddr8 *v = palloc0(sizeof(macaddr8)); + + return Macaddr8PGetDatum(v); +} + +GIN_SUPPORT(macaddr8, false, leftmostvalue_macaddr8, macaddr8_cmp) + static Datum leftmostvalue_inet(void) { diff --git a/contrib/btree_gin/btree_gin.control b/contrib/btree_gin/btree_gin.control index 3b2cb2d709..d96436e8ec 100644 --- a/contrib/btree_gin/btree_gin.control +++ b/contrib/btree_gin/btree_gin.control @@ -1,5 +1,5 @@ # btree_gin extension comment = 'support for indexing common datatypes in GIN' -default_version = '1.0' +default_version = '1.1' module_pathname = '$libdir/btree_gin' relocatable = true diff --git a/contrib/btree_gin/expected/macaddr8.out b/contrib/btree_gin/expected/macaddr8.out new file mode 100644 index 0000000000..025b0c171e --- /dev/null +++ b/contrib/btree_gin/expected/macaddr8.out @@ -0,0 +1,51 @@ +set enable_seqscan=off; +CREATE TABLE test_macaddr8 ( + i macaddr8 +); +INSERT INTO test_macaddr8 VALUES + ( '22:00:5c:03:55:08:01:02' ), + ( '22:00:5c:04:55:08:01:02' ), + ( '22:00:5c:05:55:08:01:02' ), + ( '22:00:5c:08:55:08:01:02' ), + ( '22:00:5c:09:55:08:01:02' ), + ( '22:00:5c:10:55:08:01:02' ) +; +CREATE INDEX idx_macaddr8 ON test_macaddr8 USING gin (i); +SELECT * FROM test_macaddr8 WHERE i<'22:00:5c:08:55:08:01:02'::macaddr8 ORDER BY i; + i +------------------------- + 22:00:5c:03:55:08:01:02 + 22:00:5c:04:55:08:01:02 + 22:00:5c:05:55:08:01:02 +(3 rows) + +SELECT * FROM test_macaddr8 WHERE i<='22:00:5c:08:55:08:01:02'::macaddr8 ORDER BY i; + i +------------------------- + 22:00:5c:03:55:08:01:02 + 22:00:5c:04:55:08:01:02 + 22:00:5c:05:55:08:01:02 + 22:00:5c:08:55:08:01:02 +(4 rows) + +SELECT * FROM test_macaddr8 WHERE i='22:00:5c:08:55:08:01:02'::macaddr8 ORDER BY i; + i +------------------------- + 22:00:5c:08:55:08:01:02 +(1 row) + +SELECT * FROM test_macaddr8 WHERE i>='22:00:5c:08:55:08:01:02'::macaddr8 ORDER BY i; + i +------------------------- + 22:00:5c:08:55:08:01:02 + 22:00:5c:09:55:08:01:02 + 22:00:5c:10:55:08:01:02 +(3 rows) + +SELECT * FROM test_macaddr8 WHERE i>'22:00:5c:08:55:08:01:02'::macaddr8 ORDER BY i; + i +------------------------- + 22:00:5c:09:55:08:01:02 + 22:00:5c:10:55:08:01:02 +(2 rows) + diff --git a/contrib/btree_gin/sql/macaddr8.sql b/contrib/btree_gin/sql/macaddr8.sql new file mode 100644 index 0000000000..86785c3ca9 --- /dev/null +++ b/contrib/btree_gin/sql/macaddr8.sql @@ -0,0 +1,22 @@ +set enable_seqscan=off; + +CREATE TABLE test_macaddr8 ( + i macaddr8 +); + +INSERT INTO test_macaddr8 VALUES + ( '22:00:5c:03:55:08:01:02' ), + ( '22:00:5c:04:55:08:01:02' ), + ( '22:00:5c:05:55:08:01:02' ), + ( '22:00:5c:08:55:08:01:02' ), + ( '22:00:5c:09:55:08:01:02' ), + ( '22:00:5c:10:55:08:01:02' ) +; + +CREATE INDEX idx_macaddr8 ON test_macaddr8 USING gin (i); + +SELECT * FROM test_macaddr8 WHERE i<'22:00:5c:08:55:08:01:02'::macaddr8 ORDER BY i; +SELECT * FROM test_macaddr8 WHERE i<='22:00:5c:08:55:08:01:02'::macaddr8 ORDER BY i; +SELECT * FROM test_macaddr8 WHERE i='22:00:5c:08:55:08:01:02'::macaddr8 ORDER BY i; +SELECT * FROM test_macaddr8 WHERE i>='22:00:5c:08:55:08:01:02'::macaddr8 ORDER BY i; +SELECT * FROM test_macaddr8 WHERE i>'22:00:5c:08:55:08:01:02'::macaddr8 ORDER BY i; diff --git a/contrib/btree_gist/Makefile b/contrib/btree_gist/Makefile index d36f51795d..c70f17869a 100644 --- a/contrib/btree_gist/Makefile +++ b/contrib/btree_gist/Makefile @@ -5,17 +5,18 @@ MODULE_big = btree_gist OBJS = btree_gist.o btree_utils_num.o btree_utils_var.o btree_int2.o \ btree_int4.o btree_int8.o btree_float4.o btree_float8.o btree_cash.o \ btree_oid.o btree_ts.o btree_time.o btree_date.o btree_interval.o \ - btree_macaddr.o btree_inet.o btree_text.o btree_bytea.o btree_bit.o \ - btree_numeric.o btree_uuid.o $(WIN32RES) + btree_macaddr.o btree_macaddr8.o btree_inet.o btree_text.o btree_bytea.o \ + btree_bit.o btree_numeric.o btree_uuid.o $(WIN32RES) EXTENSION = btree_gist DATA = btree_gist--unpackaged--1.0.sql btree_gist--1.0--1.1.sql \ - btree_gist--1.1--1.2.sql btree_gist--1.2.sql btree_gist--1.2--1.3.sql + btree_gist--1.1--1.2.sql btree_gist--1.2.sql btree_gist--1.2--1.3.sql \ + btree_gist--1.3--1.4.sql PGFILEDESC = "btree_gist - B-tree equivalent GiST operator classes" REGRESS = init int2 int4 int8 float4 float8 cash oid timestamp timestamptz \ - time timetz date interval macaddr inet cidr text varchar char bytea \ - bit varbit numeric uuid not_equal + time timetz date interval macaddr macaddr8 inet cidr text varchar char \ + bytea bit varbit numeric uuid not_equal SHLIB_LINK += $(filter -lm, $(LIBS)) diff --git a/contrib/btree_gist/btree_gist--1.3--1.4.sql b/contrib/btree_gist/btree_gist--1.3--1.4.sql new file mode 100644 index 0000000000..f77f6c8380 --- /dev/null +++ b/contrib/btree_gist/btree_gist--1.3--1.4.sql @@ -0,0 +1,64 @@ +/* contrib/btree_gist/btree_gist--1.3--1.4.sql */ + +-- complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION btree_gist UPDATE TO '1.4'" to load this file. \quit + +-- Add support for indexing macaddr8 columns + +-- define the GiST support methods +CREATE FUNCTION gbt_macad8_consistent(internal,macaddr8,int2,oid,internal) +RETURNS bool +AS 'MODULE_PATHNAME' +LANGUAGE C IMMUTABLE STRICT; + +CREATE FUNCTION gbt_macad8_compress(internal) +RETURNS internal +AS 'MODULE_PATHNAME' +LANGUAGE C IMMUTABLE STRICT; + +CREATE FUNCTION gbt_macad8_fetch(internal) +RETURNS internal +AS 'MODULE_PATHNAME' +LANGUAGE C IMMUTABLE STRICT; + +CREATE FUNCTION gbt_macad8_penalty(internal,internal,internal) +RETURNS internal +AS 'MODULE_PATHNAME' +LANGUAGE C IMMUTABLE STRICT; + +CREATE FUNCTION gbt_macad8_picksplit(internal, internal) +RETURNS internal +AS 'MODULE_PATHNAME' +LANGUAGE C IMMUTABLE STRICT; + +CREATE FUNCTION gbt_macad8_union(internal, internal) +RETURNS gbtreekey16 +AS 'MODULE_PATHNAME' +LANGUAGE C IMMUTABLE STRICT; + +CREATE FUNCTION gbt_macad8_same(gbtreekey16, gbtreekey16, internal) +RETURNS internal +AS 'MODULE_PATHNAME' +LANGUAGE C IMMUTABLE STRICT; + +-- Create the operator class +CREATE OPERATOR CLASS gist_macaddr8_ops +DEFAULT FOR TYPE macaddr8 USING gist +AS + OPERATOR 1 < , + OPERATOR 2 <= , + OPERATOR 3 = , + OPERATOR 4 >= , + OPERATOR 5 > , + FUNCTION 1 gbt_macad8_consistent (internal, macaddr8, int2, oid, internal), + FUNCTION 2 gbt_macad8_union (internal, internal), + FUNCTION 3 gbt_macad8_compress (internal), + FUNCTION 4 gbt_decompress (internal), + FUNCTION 5 gbt_macad8_penalty (internal, internal, internal), + FUNCTION 6 gbt_macad8_picksplit (internal, internal), + FUNCTION 7 gbt_macad8_same (gbtreekey16, gbtreekey16, internal), + STORAGE gbtreekey16; + +ALTER OPERATOR FAMILY gist_macaddr8_ops USING gist ADD + OPERATOR 6 <> (macaddr8, macaddr8) , + FUNCTION 9 (macaddr8, macaddr8) gbt_macad8_fetch (internal); diff --git a/contrib/btree_gist/btree_gist.control b/contrib/btree_gist/btree_gist.control index ddbf83dc32..fdf0e6ad9e 100644 --- a/contrib/btree_gist/btree_gist.control +++ b/contrib/btree_gist/btree_gist.control @@ -1,5 +1,5 @@ # btree_gist extension comment = 'support for indexing common datatypes in GiST' -default_version = '1.3' +default_version = '1.4' module_pathname = '$libdir/btree_gist' relocatable = true diff --git a/contrib/btree_gist/btree_gist.h b/contrib/btree_gist/btree_gist.h index 9b3e22c469..f759299bb2 100644 --- a/contrib/btree_gist/btree_gist.h +++ b/contrib/btree_gist/btree_gist.h @@ -27,6 +27,7 @@ enum gbtree_type gbt_t_date, gbt_t_intv, gbt_t_macad, + gbt_t_macad8, gbt_t_text, gbt_t_bpchar, gbt_t_bytea, diff --git a/contrib/btree_gist/btree_macaddr8.c b/contrib/btree_gist/btree_macaddr8.c new file mode 100644 index 0000000000..13238efe32 --- /dev/null +++ b/contrib/btree_gist/btree_macaddr8.c @@ -0,0 +1,200 @@ +/* + * contrib/btree_gist/btree_macaddr8.c + */ +#include "postgres.h" + +#include "btree_gist.h" +#include "btree_utils_num.h" +#include "utils/builtins.h" +#include "utils/inet.h" + +typedef struct +{ + macaddr8 lower; + macaddr8 upper; + /* make struct size = sizeof(gbtreekey16) */ +} mac8KEY; + +/* +** OID ops +*/ +PG_FUNCTION_INFO_V1(gbt_macad8_compress); +PG_FUNCTION_INFO_V1(gbt_macad8_fetch); +PG_FUNCTION_INFO_V1(gbt_macad8_union); +PG_FUNCTION_INFO_V1(gbt_macad8_picksplit); +PG_FUNCTION_INFO_V1(gbt_macad8_consistent); +PG_FUNCTION_INFO_V1(gbt_macad8_penalty); +PG_FUNCTION_INFO_V1(gbt_macad8_same); + + +static bool +gbt_macad8gt(const void *a, const void *b) +{ + return DatumGetBool(DirectFunctionCall2(macaddr8_gt, PointerGetDatum(a), PointerGetDatum(b))); +} +static bool +gbt_macad8ge(const void *a, const void *b) +{ + return DatumGetBool(DirectFunctionCall2(macaddr8_ge, PointerGetDatum(a), PointerGetDatum(b))); +} + +static bool +gbt_macad8eq(const void *a, const void *b) +{ + return DatumGetBool(DirectFunctionCall2(macaddr8_eq, PointerGetDatum(a), PointerGetDatum(b))); +} + +static bool +gbt_macad8le(const void *a, const void *b) +{ + return DatumGetBool(DirectFunctionCall2(macaddr8_le, PointerGetDatum(a), PointerGetDatum(b))); +} + +static bool +gbt_macad8lt(const void *a, const void *b) +{ + return DatumGetBool(DirectFunctionCall2(macaddr8_lt, PointerGetDatum(a), PointerGetDatum(b))); +} + + +static int +gbt_macad8key_cmp(const void *a, const void *b) +{ + mac8KEY *ia = (mac8KEY *) (((const Nsrt *) a)->t); + mac8KEY *ib = (mac8KEY *) (((const Nsrt *) b)->t); + int res; + + res = DatumGetInt32(DirectFunctionCall2(macaddr8_cmp, Macaddr8PGetDatum(&ia->lower), Macaddr8PGetDatum(&ib->lower))); + if (res == 0) + return DatumGetInt32(DirectFunctionCall2(macaddr8_cmp, Macaddr8PGetDatum(&ia->upper), Macaddr8PGetDatum(&ib->upper))); + + return res; +} + + +static const gbtree_ninfo tinfo = +{ + gbt_t_macad8, + sizeof(macaddr8), + 16, /* sizeof(gbtreekey16) */ + gbt_macad8gt, + gbt_macad8ge, + gbt_macad8eq, + gbt_macad8le, + gbt_macad8lt, + gbt_macad8key_cmp, + NULL +}; + + +/************************************************** + * macaddr ops + **************************************************/ + + + +static uint64 +mac8_2_uint64(macaddr8 *m) +{ + unsigned char *mi = (unsigned char *) m; + uint64 res = 0; + int i; + + for (i = 0; i < 8; i++) + res += (((uint64) mi[i]) << ((uint64) ((7 - i) * 8))); + return res; +} + + + +Datum +gbt_macad8_compress(PG_FUNCTION_ARGS) +{ + GISTENTRY *entry = (GISTENTRY *) PG_GETARG_POINTER(0); + + PG_RETURN_POINTER(gbt_num_compress(entry, &tinfo)); +} + +Datum +gbt_macad8_fetch(PG_FUNCTION_ARGS) +{ + GISTENTRY *entry = (GISTENTRY *) PG_GETARG_POINTER(0); + + PG_RETURN_POINTER(gbt_num_fetch(entry, &tinfo)); +} + +Datum +gbt_macad8_consistent(PG_FUNCTION_ARGS) +{ + GISTENTRY *entry = (GISTENTRY *) PG_GETARG_POINTER(0); + macaddr8 *query = (macaddr8 *) PG_GETARG_POINTER(1); + StrategyNumber strategy = (StrategyNumber) PG_GETARG_UINT16(2); + + /* Oid subtype = PG_GETARG_OID(3); */ + bool *recheck = (bool *) PG_GETARG_POINTER(4); + mac8KEY *kkk = (mac8KEY *) DatumGetPointer(entry->key); + GBT_NUMKEY_R key; + + /* All cases served by this function are exact */ + *recheck = false; + + key.lower = (GBT_NUMKEY *) &kkk->lower; + key.upper = (GBT_NUMKEY *) &kkk->upper; + + PG_RETURN_BOOL( + gbt_num_consistent(&key, (void *) query, &strategy, GIST_LEAF(entry), &tinfo) + ); +} + + +Datum +gbt_macad8_union(PG_FUNCTION_ARGS) +{ + GistEntryVector *entryvec = (GistEntryVector *) PG_GETARG_POINTER(0); + void *out = palloc0(sizeof(mac8KEY)); + + *(int *) PG_GETARG_POINTER(1) = sizeof(mac8KEY); + PG_RETURN_POINTER(gbt_num_union((void *) out, entryvec, &tinfo)); +} + + +Datum +gbt_macad8_penalty(PG_FUNCTION_ARGS) +{ + mac8KEY *origentry = (mac8KEY *) DatumGetPointer(((GISTENTRY *) PG_GETARG_POINTER(0))->key); + mac8KEY *newentry = (mac8KEY *) DatumGetPointer(((GISTENTRY *) PG_GETARG_POINTER(1))->key); + float *result = (float *) PG_GETARG_POINTER(2); + uint64 iorg[2], + inew[2]; + + iorg[0] = mac8_2_uint64(&origentry->lower); + iorg[1] = mac8_2_uint64(&origentry->upper); + inew[0] = mac8_2_uint64(&newentry->lower); + inew[1] = mac8_2_uint64(&newentry->upper); + + penalty_num(result, iorg[0], iorg[1], inew[0], inew[1]); + + PG_RETURN_POINTER(result); + +} + +Datum +gbt_macad8_picksplit(PG_FUNCTION_ARGS) +{ + PG_RETURN_POINTER(gbt_num_picksplit( + (GistEntryVector *) PG_GETARG_POINTER(0), + (GIST_SPLITVEC *) PG_GETARG_POINTER(1), + &tinfo + )); +} + +Datum +gbt_macad8_same(PG_FUNCTION_ARGS) +{ + mac8KEY *b1 = (mac8KEY *) PG_GETARG_POINTER(0); + mac8KEY *b2 = (mac8KEY *) PG_GETARG_POINTER(1); + bool *result = (bool *) PG_GETARG_POINTER(2); + + *result = gbt_num_same((void *) b1, (void *) b2, &tinfo); + PG_RETURN_POINTER(result); +} diff --git a/contrib/btree_gist/expected/macaddr8.out b/contrib/btree_gist/expected/macaddr8.out new file mode 100644 index 0000000000..e5ec6a5dea --- /dev/null +++ b/contrib/btree_gist/expected/macaddr8.out @@ -0,0 +1,89 @@ +-- macaddr check +CREATE TABLE macaddr8tmp (a macaddr8); +\copy macaddr8tmp from 'data/macaddr.data' +SET enable_seqscan=on; +SELECT count(*) FROM macaddr8tmp WHERE a < '22:00:5c:e5:9b:0d'; + count +------- + 56 +(1 row) + +SELECT count(*) FROM macaddr8tmp WHERE a <= '22:00:5c:e5:9b:0d'; + count +------- + 60 +(1 row) + +SELECT count(*) FROM macaddr8tmp WHERE a = '22:00:5c:e5:9b:0d'; + count +------- + 4 +(1 row) + +SELECT count(*) FROM macaddr8tmp WHERE a >= '22:00:5c:e5:9b:0d'; + count +------- + 544 +(1 row) + +SELECT count(*) FROM macaddr8tmp WHERE a > '22:00:5c:e5:9b:0d'; + count +------- + 540 +(1 row) + +CREATE INDEX macaddr8idx ON macaddr8tmp USING gist ( a ); +SET enable_seqscan=off; +SELECT count(*) FROM macaddr8tmp WHERE a < '22:00:5c:e5:9b:0d'::macaddr8; + count +------- + 56 +(1 row) + +SELECT count(*) FROM macaddr8tmp WHERE a <= '22:00:5c:e5:9b:0d'::macaddr8; + count +------- + 60 +(1 row) + +SELECT count(*) FROM macaddr8tmp WHERE a = '22:00:5c:e5:9b:0d'::macaddr8; + count +------- + 4 +(1 row) + +SELECT count(*) FROM macaddr8tmp WHERE a >= '22:00:5c:e5:9b:0d'::macaddr8; + count +------- + 544 +(1 row) + +SELECT count(*) FROM macaddr8tmp WHERE a > '22:00:5c:e5:9b:0d'::macaddr8; + count +------- + 540 +(1 row) + +-- Test index-only scans +SET enable_bitmapscan=off; +EXPLAIN (COSTS OFF) +SELECT * FROM macaddr8tmp WHERE a < '02:03:04:05:06:07'::macaddr8; + QUERY PLAN +--------------------------------------------------------- + Index Only Scan using macaddr8idx on macaddr8tmp + Index Cond: (a < '02:03:04:ff:fe:05:06:07'::macaddr8) +(2 rows) + +SELECT * FROM macaddr8tmp WHERE a < '02:03:04:05:06:07'::macaddr8; + a +------------------------- + 01:02:37:ff:fe:05:4f:36 + 01:02:37:ff:fe:05:4f:36 + 01:02:37:ff:fe:05:4f:36 + 01:02:37:ff:fe:05:4f:36 + 01:43:b5:ff:fe:79:eb:0f + 01:43:b5:ff:fe:79:eb:0f + 01:43:b5:ff:fe:79:eb:0f + 01:43:b5:ff:fe:79:eb:0f +(8 rows) + diff --git a/contrib/btree_gist/sql/macaddr8.sql b/contrib/btree_gist/sql/macaddr8.sql new file mode 100644 index 0000000000..61e7d7af40 --- /dev/null +++ b/contrib/btree_gist/sql/macaddr8.sql @@ -0,0 +1,37 @@ +-- macaddr check + +CREATE TABLE macaddr8tmp (a macaddr8); + +\copy macaddr8tmp from 'data/macaddr.data' + +SET enable_seqscan=on; + +SELECT count(*) FROM macaddr8tmp WHERE a < '22:00:5c:e5:9b:0d'; + +SELECT count(*) FROM macaddr8tmp WHERE a <= '22:00:5c:e5:9b:0d'; + +SELECT count(*) FROM macaddr8tmp WHERE a = '22:00:5c:e5:9b:0d'; + +SELECT count(*) FROM macaddr8tmp WHERE a >= '22:00:5c:e5:9b:0d'; + +SELECT count(*) FROM macaddr8tmp WHERE a > '22:00:5c:e5:9b:0d'; + +CREATE INDEX macaddr8idx ON macaddr8tmp USING gist ( a ); + +SET enable_seqscan=off; + +SELECT count(*) FROM macaddr8tmp WHERE a < '22:00:5c:e5:9b:0d'::macaddr8; + +SELECT count(*) FROM macaddr8tmp WHERE a <= '22:00:5c:e5:9b:0d'::macaddr8; + +SELECT count(*) FROM macaddr8tmp WHERE a = '22:00:5c:e5:9b:0d'::macaddr8; + +SELECT count(*) FROM macaddr8tmp WHERE a >= '22:00:5c:e5:9b:0d'::macaddr8; + +SELECT count(*) FROM macaddr8tmp WHERE a > '22:00:5c:e5:9b:0d'::macaddr8; + +-- Test index-only scans +SET enable_bitmapscan=off; +EXPLAIN (COSTS OFF) +SELECT * FROM macaddr8tmp WHERE a < '02:03:04:05:06:07'::macaddr8; +SELECT * FROM macaddr8tmp WHERE a < '02:03:04:05:06:07'::macaddr8; diff --git a/doc/src/sgml/brin.sgml b/doc/src/sgml/brin.sgml index 6448b18e46..5bf11dc2d1 100644 --- a/doc/src/sgml/brin.sgml +++ b/doc/src/sgml/brin.sgml @@ -281,6 +281,17 @@ > + + macaddr8_minmax_ops + macaddr8 + + < + <= + = + >= + > + + name_minmax_ops name diff --git a/doc/src/sgml/btree-gin.sgml b/doc/src/sgml/btree-gin.sgml index 2b081db9d5..0de8eb5c30 100644 --- a/doc/src/sgml/btree-gin.sgml +++ b/doc/src/sgml/btree-gin.sgml @@ -16,7 +16,8 @@ time without time zone, date, interval, oid, money, "char", varchar, text, bytea, bit, - varbit, macaddr, inet, and cidr. + varbit, macaddr, macaddr8, inet, + and cidr. diff --git a/doc/src/sgml/btree-gist.sgml b/doc/src/sgml/btree-gist.sgml index d08647ce05..cfdd5be84a 100644 --- a/doc/src/sgml/btree-gist.sgml +++ b/doc/src/sgml/btree-gist.sgml @@ -16,8 +16,8 @@ time without time zone, date, interval, oid, money, char, varchar, text, bytea, bit, - varbit, macaddr, inet, cidr, - and uuid. + varbit, macaddr, macaddr8, inet, + cidr and uuid. diff --git a/doc/src/sgml/datatype.sgml b/doc/src/sgml/datatype.sgml index 35610307d9..e2f8dee7b6 100644 --- a/doc/src/sgml/datatype.sgml +++ b/doc/src/sgml/datatype.sgml @@ -166,6 +166,12 @@ MAC (Media Access Control) address + + macaddr8 + + MAC (Media Access Control) address (EUI-64 format) + + money @@ -3428,6 +3434,12 @@ SELECT person.name, holidays.num_weeks FROM person, holidays MAC addresses + + macaddr8 + 8 bytes + MAC addresses (EUI-64 format) + +
    @@ -3668,6 +3680,77 @@ SELECT person.name, holidays.num_weeks FROM person, holidays     + + <type>macaddr8</type> + + + macaddr8 (data type) + + + + MAC address (EUI-64 format) + macaddr + + + + The macaddr8 type stores MAC addresses in EUI-64 + format, known for example from Ethernet card hardware addresses + (although MAC addresses are used for other purposes as well). + This type can accept both 6 and 8 byte length MAC addresses + and stores them in 8 byte length format. MAC addresses given + in 6 byte format will be stored in 8 byte length format with the + 4th and 5th bytes set to FF and FE, respectively. + + Note that IPv6 uses a modified EUI-64 format where the 7th bit + should be set to one after the conversion from EUI-48. The + function macaddr8_set7bit is provided to make this + change. + + Generally speaking, any input which is comprised of pairs of hex + digits (on byte boundaries), optionally separated consistently by + one of ':', '-' or '.', is + accepted. The number of hex digits must be either 16 (8 bytes) or + 12 (6 bytes). Leading and trailing whitespace is ignored. + + The following are examples of input formats that are accepted: + + + '08:00:2b:01:02:03:04:05' + '08-00-2b-01-02-03-04-05' + '08002b:0102030405' + '08002b-0102030405' + '0800.2b01.0203.0405' + '0800-2b01-0203-0405' + '08002b01:02030405' + '08002b0102030405' + + + These examples would all specify the same address. Upper and + lower case is accepted for the digits + a through f. Output is always in the + first of the forms shown. + + The last six input formats that are mentioned above are not part + of any standard. + + To convert a traditional 48 bit MAC address in EUI-48 format to + modified EUI-64 format to be included as the host portion of an + IPv6 address, use macaddr8_set7bit as shown: + + +SELECT macaddr8_set7bit('08:00:2b:01:02:03'); + + macaddr8_set7bit +------------------------- + 0a:00:2b:ff:fe:01:02:03 +(1 row) + + + + + + + diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 583b3b241a..a521912317 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -9228,6 +9228,62 @@ CREATE TYPE rainbow AS ENUM ('red', 'orange', 'yellow', 'green', 'blue', 'purple for NOT, AND and OR. + + shows the functions + available for use with the macaddr8 type. The function + trunc(macaddr8) returns a MAC + address with the last 5 bytes set to zero. This can be used to + associate the remaining prefix with a manufacturer. + + + + <type>macaddr8</type> Functions + + + + Function + Return Type + Description + Example + Result + + + + + + + trunc + + trunc(macaddr8) + + macaddr8 + set last 5 bytes to zero + trunc(macaddr8 '12:34:56:78:90:ab:cd:ef') + 12:34:56:00:00:00:00:00 + + + + + macaddr8_set7bit + + macaddr8_set7bit(macaddr8) + + macaddr8 + set 7th bit to one, also known as modified EUI-64, for inclusion in an IPv6 address + macaddr8_set7bit(macaddr8 '00:34:56:ab:cd:ef') + 02:34:56:ff:fe:ab:cd:ef + + + +
    + + + The macaddr8 type also supports the standard relational + operators (>, <=, etc.) for + ordering, and the bitwise arithmetic operators (~, + & and |) for NOT, AND and OR. + +     diff --git a/src/backend/utils/adt/Makefile b/src/backend/utils/adt/Makefile index 0f512753e4..1fb018416e 100644 --- a/src/backend/utils/adt/Makefile +++ b/src/backend/utils/adt/Makefile @@ -16,7 +16,7 @@ OBJS = acl.o amutils.o arrayfuncs.o array_expanded.o array_selfuncs.o \ float.o format_type.o formatting.o genfile.o \ geo_ops.o geo_selfuncs.o geo_spgist.o inet_cidr_ntop.o inet_net_pton.o \ int.o int8.o json.o jsonb.o jsonb_gin.o jsonb_op.o jsonb_util.o \ - jsonfuncs.o like.o lockfuncs.o mac.o misc.o nabstime.o name.o \ + jsonfuncs.o like.o lockfuncs.o mac.o mac8.o misc.o nabstime.o name.o \ network.o network_gist.o network_selfuncs.o network_spgist.o \ numeric.o numutils.o oid.o oracle_compat.o \ orderedsetaggs.o pg_locale.o pg_lsn.o pg_upgrade_support.o \ diff --git a/src/backend/utils/adt/mac.c b/src/backend/utils/adt/mac.c index 2270b223ea..a1e9c53b73 100644 --- a/src/backend/utils/adt/mac.c +++ b/src/backend/utils/adt/mac.c @@ -1,7 +1,14 @@ -/* - * PostgreSQL type definitions for MAC addresses. +/*------------------------------------------------------------------------- + * + * mac.c + * PostgreSQL type definitions for 6 byte, EUI-48, MAC addresses. + * + * Portions Copyright (c) 1998-2017, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/utils/adt/mac.c * - * src/backend/utils/adt/mac.c + *------------------------------------------------------------------------- */ #include "postgres.h" diff --git a/src/backend/utils/adt/mac8.c b/src/backend/utils/adt/mac8.c new file mode 100644 index 0000000000..31f57c3047 --- /dev/null +++ b/src/backend/utils/adt/mac8.c @@ -0,0 +1,560 @@ +/*------------------------------------------------------------------------- + * + * mac8.c + * PostgreSQL type definitions for 8 byte (EUI-64) MAC addresses. + * + * EUI-48 (6 byte) MAC addresses are accepted as input and are stored in + * EUI-64 format, with the 4th and 5th bytes set to FF and FE, respectively. + * + * Output is always in 8 byte (EUI-64) format. + * + * The following code is written with the assumption that the OUI field + * size is 24 bits. + * + * Portions Copyright (c) 1998-2017, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/utils/adt/mac8.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "access/hash.h" +#include "libpq/pqformat.h" +#include "utils/builtins.h" +#include "utils/inet.h" + +/* + * Utility macros used for sorting and comparing: + */ +#define hibits(addr) \ + ((unsigned long)(((addr)->a<<24) | ((addr)->b<<16) | ((addr)->c<<8) | ((addr)->d))) + +#define lobits(addr) \ + ((unsigned long)(((addr)->e<<24) | ((addr)->f<<16) | ((addr)->g<<8) | ((addr)->h))) + +static unsigned char hex2_to_uchar(const char *str, int offset); + +static const int hexlookup[128] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, +}; + +static inline unsigned char +hex2_to_uchar(const char *str, int offset) +{ + unsigned char ret = 0; + int lookup; + const char *ptr = str + offset; + + /* Handle the first character */ + if (*ptr < 0 || *ptr >= 127) + goto invalid_input; + + lookup = hexlookup[(unsigned char) *ptr]; + if (lookup < 0 || lookup > 15) + goto invalid_input; + + ret = lookup << 4; + + /* Move to the second character */ + ptr++; + + if (*ptr < 0 || *ptr > 127) + goto invalid_input; + + lookup = hexlookup[(unsigned char) *ptr]; + if (lookup < 0 || lookup > 15) + goto invalid_input; + + ret += lookup; + + return ret; + +invalid_input: + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("invalid input syntax for type %s: \"%s\"", "macaddr8", + str))); + + /* We do not actually reach here */ + return 0; +} + +/* + * MAC address (EUI-48 and EUI-64) reader. Accepts several common notations. + */ +Datum +macaddr8_in(PG_FUNCTION_ARGS) +{ + const char *str = PG_GETARG_CSTRING(0); + const char *ptr = str; + macaddr8 *result; + unsigned char a = 0, + b = 0, + c = 0, + d = 0, + e = 0, + f = 0, + g = 0, + h = 0; + int count = 0; + char spacer = '\0'; + + /* skip leading spaces */ + while (*ptr && isspace((unsigned char) *ptr)) + ptr++; + + /* digits must always come in pairs */ + while (*ptr && *(ptr + 1)) + { + /* + * Attempt to decode each byte, which must be 2 hex digits in a row. + * If either digit is not hex, hex2_to_uchar will throw ereport() for + * us. Either 6 or 8 byte MAC addresses are supported. + */ + + /* Attempt to collect a byte */ + count++; + + switch (count) + { + case 1: + a = hex2_to_uchar(str, ptr - str); + break; + case 2: + b = hex2_to_uchar(str, ptr - str); + break; + case 3: + c = hex2_to_uchar(str, ptr - str); + break; + case 4: + d = hex2_to_uchar(str, ptr - str); + break; + case 5: + e = hex2_to_uchar(str, ptr - str); + break; + case 6: + f = hex2_to_uchar(str, ptr - str); + break; + case 7: + g = hex2_to_uchar(str, ptr - str); + break; + case 8: + h = hex2_to_uchar(str, ptr - str); + break; + default: + /* must be trailing garbage... */ + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("invalid input syntax for type %s: \"%s\"", "macaddr8", + str))); + } + + /* Move forward to where the next byte should be */ + ptr += 2; + + /* Check for a spacer, these are valid, anything else is not */ + if (*ptr == ':' || *ptr == '-' || *ptr == '.') + { + /* remember the spacer used, if it changes then it isn't valid */ + if (spacer == '\0') + spacer = *ptr; + + /* Have to use the same spacer throughout */ + else if (spacer != *ptr) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("invalid input syntax for type %s: \"%s\"", "macaddr8", + str))); + + /* move past the spacer */ + ptr++; + } + + /* allow trailing whitespace after if we have 6 or 8 bytes */ + if (count == 6 || count == 8) + { + if (isspace((unsigned char) *ptr)) + { + while (*++ptr && isspace((unsigned char) *ptr)); + + /* If we found a space and then non-space, it's invalid */ + if (*ptr) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("invalid input syntax for type %s: \"%s\"", "macaddr8", + str))); + } + } + } + + /* Convert a 6 byte MAC address to macaddr8 */ + if (count == 6) + { + h = f; + g = e; + f = d; + + d = 0xFF; + e = 0xFE; + } + else if (count != 8) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("invalid input syntax for type %s: \"%s\"", "macaddr8", + str))); + + result = (macaddr8 *) palloc0(sizeof(macaddr8)); + + result->a = a; + result->b = b; + result->c = c; + result->d = d; + result->e = e; + result->f = f; + result->g = g; + result->h = h; + + PG_RETURN_MACADDR8_P(result); +} + +/* + * MAC8 address (EUI-64) output function. Fixed format. + */ +Datum +macaddr8_out(PG_FUNCTION_ARGS) +{ + macaddr8 *addr = PG_GETARG_MACADDR8_P(0); + char *result; + + result = (char *) palloc(32); + + snprintf(result, 32, "%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x", + addr->a, addr->b, addr->c, addr->d, + addr->e, addr->f, addr->g, addr->h); + + PG_RETURN_CSTRING(result); +} + +/* + * macaddr8_recv - converts external binary format(EUI-48 and EUI-64) to macaddr8 + * + * The external representation is just the eight bytes, MSB first. + */ +Datum +macaddr8_recv(PG_FUNCTION_ARGS) +{ + StringInfo buf = (StringInfo) PG_GETARG_POINTER(0); + macaddr8 *addr; + + addr = (macaddr8 *) palloc0(sizeof(macaddr8)); + + addr->a = pq_getmsgbyte(buf); + addr->b = pq_getmsgbyte(buf); + addr->c = pq_getmsgbyte(buf); + + if (buf->len == 6) + { + addr->d = 0xFF; + addr->e = 0xFE; + } + else + { + addr->d = pq_getmsgbyte(buf); + addr->e = pq_getmsgbyte(buf); + } + + addr->f = pq_getmsgbyte(buf); + addr->g = pq_getmsgbyte(buf); + addr->h = pq_getmsgbyte(buf); + + PG_RETURN_MACADDR8_P(addr); +} + +/* + * macaddr8_send - converts macaddr8(EUI-64) to binary format + */ +Datum +macaddr8_send(PG_FUNCTION_ARGS) +{ + macaddr8 *addr = PG_GETARG_MACADDR8_P(0); + StringInfoData buf; + + pq_begintypsend(&buf); + pq_sendbyte(&buf, addr->a); + pq_sendbyte(&buf, addr->b); + pq_sendbyte(&buf, addr->c); + pq_sendbyte(&buf, addr->d); + pq_sendbyte(&buf, addr->e); + pq_sendbyte(&buf, addr->f); + pq_sendbyte(&buf, addr->g); + pq_sendbyte(&buf, addr->h); + + PG_RETURN_BYTEA_P(pq_endtypsend(&buf)); +} + + +/* + * macaddr8_cmp_internal - comparison function for sorting: + */ +static int32 +macaddr8_cmp_internal(macaddr8 *a1, macaddr8 *a2) +{ + if (hibits(a1) < hibits(a2)) + return -1; + else if (hibits(a1) > hibits(a2)) + return 1; + else if (lobits(a1) < lobits(a2)) + return -1; + else if (lobits(a1) > lobits(a2)) + return 1; + else + return 0; +} + +Datum +macaddr8_cmp(PG_FUNCTION_ARGS) +{ + macaddr8 *a1 = PG_GETARG_MACADDR8_P(0); + macaddr8 *a2 = PG_GETARG_MACADDR8_P(1); + + PG_RETURN_INT32(macaddr8_cmp_internal(a1, a2)); +} + +/* + * Boolean comparison functions. + */ + +Datum +macaddr8_lt(PG_FUNCTION_ARGS) +{ + macaddr8 *a1 = PG_GETARG_MACADDR8_P(0); + macaddr8 *a2 = PG_GETARG_MACADDR8_P(1); + + PG_RETURN_BOOL(macaddr8_cmp_internal(a1, a2) < 0); +} + +Datum +macaddr8_le(PG_FUNCTION_ARGS) +{ + macaddr8 *a1 = PG_GETARG_MACADDR8_P(0); + macaddr8 *a2 = PG_GETARG_MACADDR8_P(1); + + PG_RETURN_BOOL(macaddr8_cmp_internal(a1, a2) <= 0); +} + +Datum +macaddr8_eq(PG_FUNCTION_ARGS) +{ + macaddr8 *a1 = PG_GETARG_MACADDR8_P(0); + macaddr8 *a2 = PG_GETARG_MACADDR8_P(1); + + PG_RETURN_BOOL(macaddr8_cmp_internal(a1, a2) == 0); +} + +Datum +macaddr8_ge(PG_FUNCTION_ARGS) +{ + macaddr8 *a1 = PG_GETARG_MACADDR8_P(0); + macaddr8 *a2 = PG_GETARG_MACADDR8_P(1); + + PG_RETURN_BOOL(macaddr8_cmp_internal(a1, a2) >= 0); +} + +Datum +macaddr8_gt(PG_FUNCTION_ARGS) +{ + macaddr8 *a1 = PG_GETARG_MACADDR8_P(0); + macaddr8 *a2 = PG_GETARG_MACADDR8_P(1); + + PG_RETURN_BOOL(macaddr8_cmp_internal(a1, a2) > 0); +} + +Datum +macaddr8_ne(PG_FUNCTION_ARGS) +{ + macaddr8 *a1 = PG_GETARG_MACADDR8_P(0); + macaddr8 *a2 = PG_GETARG_MACADDR8_P(1); + + PG_RETURN_BOOL(macaddr8_cmp_internal(a1, a2) != 0); +} + +/* + * Support function for hash indexes on macaddr8. + */ +Datum +hashmacaddr8(PG_FUNCTION_ARGS) +{ + macaddr8 *key = PG_GETARG_MACADDR8_P(0); + + return hash_any((unsigned char *) key, sizeof(macaddr8)); +} + +/* + * Arithmetic functions: bitwise NOT, AND, OR. + */ +Datum +macaddr8_not(PG_FUNCTION_ARGS) +{ + macaddr8 *addr = PG_GETARG_MACADDR8_P(0); + macaddr8 *result; + + result = (macaddr8 *) palloc0(sizeof(macaddr8)); + result->a = ~addr->a; + result->b = ~addr->b; + result->c = ~addr->c; + result->d = ~addr->d; + result->e = ~addr->e; + result->f = ~addr->f; + result->g = ~addr->g; + result->h = ~addr->h; + + PG_RETURN_MACADDR8_P(result); +} + +Datum +macaddr8_and(PG_FUNCTION_ARGS) +{ + macaddr8 *addr1 = PG_GETARG_MACADDR8_P(0); + macaddr8 *addr2 = PG_GETARG_MACADDR8_P(1); + macaddr8 *result; + + result = (macaddr8 *) palloc0(sizeof(macaddr8)); + result->a = addr1->a & addr2->a; + result->b = addr1->b & addr2->b; + result->c = addr1->c & addr2->c; + result->d = addr1->d & addr2->d; + result->e = addr1->e & addr2->e; + result->f = addr1->f & addr2->f; + result->g = addr1->g & addr2->g; + result->h = addr1->h & addr2->h; + + PG_RETURN_MACADDR8_P(result); +} + +Datum +macaddr8_or(PG_FUNCTION_ARGS) +{ + macaddr8 *addr1 = PG_GETARG_MACADDR8_P(0); + macaddr8 *addr2 = PG_GETARG_MACADDR8_P(1); + macaddr8 *result; + + result = (macaddr8 *) palloc0(sizeof(macaddr8)); + result->a = addr1->a | addr2->a; + result->b = addr1->b | addr2->b; + result->c = addr1->c | addr2->c; + result->d = addr1->d | addr2->d; + result->e = addr1->e | addr2->e; + result->f = addr1->f | addr2->f; + result->g = addr1->g | addr2->g; + result->h = addr1->h | addr2->h; + + PG_RETURN_MACADDR8_P(result); +} + +/* + * Truncation function to allow comparing macaddr8 manufacturers. + */ +Datum +macaddr8_trunc(PG_FUNCTION_ARGS) +{ + macaddr8 *addr = PG_GETARG_MACADDR8_P(0); + macaddr8 *result; + + result = (macaddr8 *) palloc0(sizeof(macaddr8)); + + result->a = addr->a; + result->b = addr->b; + result->c = addr->c; + result->d = 0; + result->e = 0; + result->f = 0; + result->g = 0; + result->h = 0; + + PG_RETURN_MACADDR8_P(result); +} + +/* + * Set 7th bit for modified EUI-64 as used in IPv6. + */ +Datum +macaddr8_set7bit(PG_FUNCTION_ARGS) +{ + macaddr8 *addr = PG_GETARG_MACADDR8_P(0); + macaddr8 *result; + + result = (macaddr8 *) palloc0(sizeof(macaddr8)); + + result->a = addr->a | 0x02; + result->b = addr->b; + result->c = addr->c; + result->d = addr->d; + result->e = addr->e; + result->f = addr->f; + result->g = addr->g; + result->h = addr->h; + + PG_RETURN_MACADDR8_P(result); +} + +/*---------------------------------------------------------- + * Conversion operators. + *---------------------------------------------------------*/ + +Datum +macaddrtomacaddr8(PG_FUNCTION_ARGS) +{ + macaddr *addr6 = PG_GETARG_MACADDR_P(0); + macaddr8 *result; + + result = (macaddr8 *) palloc0(sizeof(macaddr8)); + + result->a = addr6->a; + result->b = addr6->b; + result->c = addr6->c; + result->d = 0xFF; + result->e = 0xFE; + result->f = addr6->d; + result->g = addr6->e; + result->h = addr6->f; + + + PG_RETURN_MACADDR8_P(result); +} + +Datum +macaddr8tomacaddr(PG_FUNCTION_ARGS) +{ + macaddr8 *addr = PG_GETARG_MACADDR8_P(0); + macaddr *result; + + result = (macaddr *) palloc0(sizeof(macaddr)); + + if ((addr->d != 0xFF) || (addr->e != 0xFE)) + ereport(ERROR, + (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), + errmsg("macaddr8 data out of range to convert to macaddr"), + errhint("Only addresses that have FF and FE as values in the " + "4th and 5th bytes, from the left, for example: " + "XX-XX-XX-FF-FE-XX-XX-XX, are eligible to be converted " + "from macaddr8 to macaddr."))); + + result->a = addr->a; + result->b = addr->b; + result->c = addr->c; + result->d = addr->f; + result->e = addr->g; + result->f = addr->h; + + PG_RETURN_MACADDR_P(result); +} diff --git a/src/backend/utils/adt/network.c b/src/backend/utils/adt/network.c index dbc557e583..2459adcf9f 100644 --- a/src/backend/utils/adt/network.c +++ b/src/backend/utils/adt/network.c @@ -934,6 +934,16 @@ convert_network_to_scalar(Datum value, Oid typid) res += (mac->d << 16) | (mac->e << 8) | (mac->f); return res; } + case MACADDR8OID: + { + macaddr8 *mac = DatumGetMacaddr8P(value); + double res; + + res = (mac->a << 24) | (mac->b << 16) | (mac->c << 8) | (mac->d); + res *= ((double) 256) * 256 * 256 * 256; + res += (mac->e << 24) | (mac->f << 16) | (mac->g << 8) | (mac->h); + return res; + } } /* diff --git a/src/backend/utils/adt/selfuncs.c b/src/backend/utils/adt/selfuncs.c index 04bd9b95b2..bb9a544686 100644 --- a/src/backend/utils/adt/selfuncs.c +++ b/src/backend/utils/adt/selfuncs.c @@ -3800,6 +3800,7 @@ convert_to_scalar(Datum value, Oid valuetypid, double *scaledvalue, case INETOID: case CIDROID: case MACADDROID: + case MACADDR8OID: *scaledvalue = convert_network_to_scalar(value, valuetypid); *scaledlobound = convert_network_to_scalar(lobound, boundstypid); *scaledhibound = convert_network_to_scalar(hibound, boundstypid); diff --git a/src/include/catalog/pg_amop.h b/src/include/catalog/pg_amop.h index 0251664e4a..da0228de6b 100644 --- a/src/include/catalog/pg_amop.h +++ b/src/include/catalog/pg_amop.h @@ -372,6 +372,16 @@ DATA(insert ( 1984 829 829 3 s 1220 403 0 )); DATA(insert ( 1984 829 829 4 s 1225 403 0 )); DATA(insert ( 1984 829 829 5 s 1224 403 0 )); +/* + * btree macaddr8 + */ + +DATA(insert ( 3371 774 774 1 s 3364 403 0 )); +DATA(insert ( 3371 774 774 2 s 3365 403 0 )); +DATA(insert ( 3371 774 774 3 s 3362 403 0 )); +DATA(insert ( 3371 774 774 4 s 3367 403 0 )); +DATA(insert ( 3371 774 774 5 s 3366 403 0 )); + /* * btree network */ @@ -553,6 +563,8 @@ DATA(insert ( 1977 20 23 1 s 416 405 0 )); DATA(insert ( 1983 1186 1186 1 s 1330 405 0 )); /* macaddr_ops */ DATA(insert ( 1985 829 829 1 s 1220 405 0 )); +/* macaddr8_ops */ +DATA(insert ( 3372 774 774 1 s 3362 405 0 )); /* name_ops */ DATA(insert ( 1987 19 19 1 s 93 405 0 )); /* oid_ops */ @@ -999,6 +1011,12 @@ DATA(insert ( 4074 829 829 2 s 1223 3580 0 )); DATA(insert ( 4074 829 829 3 s 1220 3580 0 )); DATA(insert ( 4074 829 829 4 s 1225 3580 0 )); DATA(insert ( 4074 829 829 5 s 1224 3580 0 )); +/* minmax macaddr8 */ +DATA(insert ( 4109 774 774 1 s 3364 3580 0 )); +DATA(insert ( 4109 774 774 2 s 3365 3580 0 )); +DATA(insert ( 4109 774 774 3 s 3362 3580 0 )); +DATA(insert ( 4109 774 774 4 s 3367 3580 0 )); +DATA(insert ( 4109 774 774 5 s 3366 3580 0 )); /* minmax inet */ DATA(insert ( 4075 869 869 1 s 1203 3580 0 )); DATA(insert ( 4075 869 869 2 s 1204 3580 0 )); diff --git a/src/include/catalog/pg_amproc.h b/src/include/catalog/pg_amproc.h index f1a52ce3e0..a87ec423e1 100644 --- a/src/include/catalog/pg_amproc.h +++ b/src/include/catalog/pg_amproc.h @@ -142,6 +142,7 @@ DATA(insert ( 2968 2950 2950 2 3300 )); DATA(insert ( 2994 2249 2249 1 2987 )); DATA(insert ( 3194 2249 2249 1 3187 )); DATA(insert ( 3253 3220 3220 1 3251 )); +DATA(insert ( 3371 774 774 1 4119 )); DATA(insert ( 3522 3500 3500 1 3514 )); DATA(insert ( 3626 3614 3614 1 3622 )); DATA(insert ( 3683 3615 3615 1 3668 )); @@ -182,6 +183,7 @@ DATA(insert ( 2231 1042 1042 1 1080 )); DATA(insert ( 2235 1033 1033 1 329 )); DATA(insert ( 2969 2950 2950 1 2963 )); DATA(insert ( 3254 3220 3220 1 3252 )); +DATA(insert ( 3372 774 774 1 328 )); DATA(insert ( 3523 3500 3500 1 3515 )); DATA(insert ( 3903 3831 3831 1 3902 )); DATA(insert ( 4034 3802 3802 1 4045 )); @@ -414,6 +416,11 @@ DATA(insert ( 4074 829 829 1 3383 )); DATA(insert ( 4074 829 829 2 3384 )); DATA(insert ( 4074 829 829 3 3385 )); DATA(insert ( 4074 829 829 4 3386 )); +/* minmax macaddr8 */ +DATA(insert ( 4109 774 774 1 3383 )); +DATA(insert ( 4109 774 774 2 3384 )); +DATA(insert ( 4109 774 774 3 3385 )); +DATA(insert ( 4109 774 774 4 3386 )); /* minmax inet */ DATA(insert ( 4075 869 869 1 3383 )); DATA(insert ( 4075 869 869 2 3384 )); diff --git a/src/include/catalog/pg_cast.h b/src/include/catalog/pg_cast.h index 80a40ab128..ce8dc59e5a 100644 --- a/src/include/catalog/pg_cast.h +++ b/src/include/catalog/pg_cast.h @@ -303,6 +303,12 @@ DATA(insert ( 718 600 1416 e f )); DATA(insert ( 718 603 1480 e f )); DATA(insert ( 718 604 1544 e f )); +/* + * MAC address category + */ +DATA(insert ( 829 774 4123 i f )); +DATA(insert ( 774 829 4124 i f )); + /* * INET category */ diff --git a/src/include/catalog/pg_opclass.h b/src/include/catalog/pg_opclass.h index 0cde14c25d..5819d5309f 100644 --- a/src/include/catalog/pg_opclass.h +++ b/src/include/catalog/pg_opclass.h @@ -127,6 +127,8 @@ DATA(insert ( 403 interval_ops PGNSP PGUID 1982 1186 t 0 )); DATA(insert ( 405 interval_ops PGNSP PGUID 1983 1186 t 0 )); DATA(insert ( 403 macaddr_ops PGNSP PGUID 1984 829 t 0 )); DATA(insert ( 405 macaddr_ops PGNSP PGUID 1985 829 t 0 )); +DATA(insert ( 403 macaddr8_ops PGNSP PGUID 3371 774 t 0 )); +DATA(insert ( 405 macaddr8_ops PGNSP PGUID 3372 774 t 0 )); /* * Here's an ugly little hack to save space in the system catalog indexes. * btree doesn't ordinarily allow a storage type different from input type; @@ -224,6 +226,7 @@ DATA(insert ( 3580 float8_minmax_ops PGNSP PGUID 4070 701 t 701 )); DATA(insert ( 3580 abstime_minmax_ops PGNSP PGUID 4072 702 t 702 )); DATA(insert ( 3580 reltime_minmax_ops PGNSP PGUID 4073 703 t 703 )); DATA(insert ( 3580 macaddr_minmax_ops PGNSP PGUID 4074 829 t 829 )); +DATA(insert ( 3580 macaddr8_minmax_ops PGNSP PGUID 4109 774 t 774 )); DATA(insert ( 3580 inet_minmax_ops PGNSP PGUID 4075 869 f 869 )); DATA(insert ( 3580 inet_inclusion_ops PGNSP PGUID 4102 869 t 869 )); DATA(insert ( 3580 bpchar_minmax_ops PGNSP PGUID 4076 1042 t 1042 )); diff --git a/src/include/catalog/pg_operator.h b/src/include/catalog/pg_operator.h index 45feb69b93..fe8795ac8b 100644 --- a/src/include/catalog/pg_operator.h +++ b/src/include/catalog/pg_operator.h @@ -1119,7 +1119,7 @@ DESCR("equal"); DATA(insert OID = 1617 ( "#" PGNSP PGUID b f f 628 628 600 1617 0 line_interpt - - )); DESCR("intersection point"); -/* MAC type */ +/* MACADDR type */ DATA(insert OID = 1220 ( "=" PGNSP PGUID b t t 829 829 16 1220 1221 macaddr_eq eqsel eqjoinsel )); DESCR("equal"); DATA(insert OID = 1221 ( "<>" PGNSP PGUID b f f 829 829 16 1221 1220 macaddr_ne neqsel neqjoinsel )); @@ -1140,6 +1140,27 @@ DESCR("bitwise and"); DATA(insert OID = 3149 ( "|" PGNSP PGUID b f f 829 829 829 0 0 macaddr_or - - )); DESCR("bitwise or"); +/* MACADDR8 type */ +DATA(insert OID = 3362 ( "=" PGNSP PGUID b t t 774 774 16 3362 3363 macaddr8_eq eqsel eqjoinsel )); +DESCR("equal"); +DATA(insert OID = 3363 ( "<>" PGNSP PGUID b f f 774 774 16 3363 3362 macaddr8_ne neqsel neqjoinsel )); +DESCR("not equal"); +DATA(insert OID = 3364 ( "<" PGNSP PGUID b f f 774 774 16 3366 3367 macaddr8_lt scalarltsel scalarltjoinsel )); +DESCR("less than"); +DATA(insert OID = 3365 ( "<=" PGNSP PGUID b f f 774 774 16 3367 3366 macaddr8_le scalarltsel scalarltjoinsel )); +DESCR("less than or equal"); +DATA(insert OID = 3366 ( ">" PGNSP PGUID b f f 774 774 16 3364 3365 macaddr8_gt scalargtsel scalargtjoinsel )); +DESCR("greater than"); +DATA(insert OID = 3367 ( ">=" PGNSP PGUID b f f 774 774 16 3365 3364 macaddr8_ge scalargtsel scalargtjoinsel )); +DESCR("greater than or equal"); + +DATA(insert OID = 3368 ( "~" PGNSP PGUID l f f 0 774 774 0 0 macaddr8_not - - )); +DESCR("bitwise not"); +DATA(insert OID = 3369 ( "&" PGNSP PGUID b f f 774 774 774 0 0 macaddr8_and - - )); +DESCR("bitwise and"); +DATA(insert OID = 3370 ( "|" PGNSP PGUID b f f 774 774 774 0 0 macaddr8_or - - )); +DESCR("bitwise or"); + /* INET type (these also support CIDR via implicit cast) */ DATA(insert OID = 1201 ( "=" PGNSP PGUID b t t 869 869 16 1201 1202 network_eq eqsel eqjoinsel )); DESCR("equal"); diff --git a/src/include/catalog/pg_opfamily.h b/src/include/catalog/pg_opfamily.h index bd673fe59b..546527aa8e 100644 --- a/src/include/catalog/pg_opfamily.h +++ b/src/include/catalog/pg_opfamily.h @@ -87,6 +87,8 @@ DATA(insert OID = 1982 ( 403 interval_ops PGNSP PGUID )); DATA(insert OID = 1983 ( 405 interval_ops PGNSP PGUID )); DATA(insert OID = 1984 ( 403 macaddr_ops PGNSP PGUID )); DATA(insert OID = 1985 ( 405 macaddr_ops PGNSP PGUID )); +DATA(insert OID = 3371 ( 403 macaddr8_ops PGNSP PGUID )); +DATA(insert OID = 3372 ( 405 macaddr8_ops PGNSP PGUID )); DATA(insert OID = 1986 ( 403 name_ops PGNSP PGUID )); #define NAME_BTREE_FAM_OID 1986 DATA(insert OID = 1987 ( 405 name_ops PGNSP PGUID )); @@ -171,6 +173,7 @@ DATA(insert OID = 4070 ( 3580 float_minmax_ops PGNSP PGUID )); DATA(insert OID = 4072 ( 3580 abstime_minmax_ops PGNSP PGUID )); DATA(insert OID = 4073 ( 3580 reltime_minmax_ops PGNSP PGUID )); DATA(insert OID = 4074 ( 3580 macaddr_minmax_ops PGNSP PGUID )); +DATA(insert OID = 4109 ( 3580 macaddr8_minmax_ops PGNSP PGUID )); DATA(insert OID = 4075 ( 3580 network_minmax_ops PGNSP PGUID )); DATA(insert OID = 4102 ( 3580 network_inclusion_ops PGNSP PGUID )); DATA(insert OID = 4076 ( 3580 bpchar_minmax_ops PGNSP PGUID )); diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index ec4aedb851..3d5d866071 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -692,6 +692,8 @@ DATA(insert OID = 422 ( hashinet PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 DESCR("hash"); DATA(insert OID = 432 ( hash_numeric PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 23 "1700" _null_ _null_ _null_ _null_ _null_ hash_numeric _null_ _null_ _null_ )); DESCR("hash"); +DATA(insert OID = 328 ( hashmacaddr8 PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 23 "774" _null_ _null_ _null_ _null_ _null_ hashmacaddr8 _null_ _null_ _null_ )); +DESCR("hash"); DATA(insert OID = 438 ( num_nulls PGNSP PGUID 12 1 0 2276 0 f f f f f f i s 1 0 23 "2276" "{2276}" "{v}" _null_ _null_ _null_ pg_num_nulls _null_ _null_ _null_ )); DESCR("count the number of NULL arguments"); @@ -2098,14 +2100,14 @@ DESCR("get bit"); DATA(insert OID = 3033 ( set_bit PGNSP PGUID 12 1 0 0 0 f f f f t f i s 3 0 1560 "1560 23 23" _null_ _null_ _null_ _null_ _null_ bitsetbit _null_ _null_ _null_ )); DESCR("set bit"); -/* for mac type support */ +/* for macaddr type support */ DATA(insert OID = 436 ( macaddr_in PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 829 "2275" _null_ _null_ _null_ _null_ _null_ macaddr_in _null_ _null_ _null_ )); DESCR("I/O"); DATA(insert OID = 437 ( macaddr_out PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 2275 "829" _null_ _null_ _null_ _null_ _null_ macaddr_out _null_ _null_ _null_ )); DESCR("I/O"); DATA(insert OID = 753 ( trunc PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 829 "829" _null_ _null_ _null_ _null_ _null_ macaddr_trunc _null_ _null_ _null_ )); -DESCR("MAC manufacturer fields"); +DESCR("MACADDR manufacturer fields"); DATA(insert OID = 830 ( macaddr_eq PGNSP PGUID 12 1 0 0 0 f f f t t f i s 2 0 16 "829 829" _null_ _null_ _null_ _null_ _null_ macaddr_eq _null_ _null_ _null_ )); DATA(insert OID = 831 ( macaddr_lt PGNSP PGUID 12 1 0 0 0 f f f t t f i s 2 0 16 "829 829" _null_ _null_ _null_ _null_ _null_ macaddr_lt _null_ _null_ _null_ )); @@ -2119,6 +2121,33 @@ DATA(insert OID = 3144 ( macaddr_not PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 DATA(insert OID = 3145 ( macaddr_and PGNSP PGUID 12 1 0 0 0 f f f f t f i s 2 0 829 "829 829" _null_ _null_ _null_ _null_ _null_ macaddr_and _null_ _null_ _null_ )); DATA(insert OID = 3146 ( macaddr_or PGNSP PGUID 12 1 0 0 0 f f f f t f i s 2 0 829 "829 829" _null_ _null_ _null_ _null_ _null_ macaddr_or _null_ _null_ _null_ )); +/* for macaddr8 type support */ +DATA(insert OID = 4110 ( macaddr8_in PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 774 "2275" _null_ _null_ _null_ _null_ _null_ macaddr8_in _null_ _null_ _null_ )); +DESCR("I/O"); +DATA(insert OID = 4111 ( macaddr8_out PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 2275 "774" _null_ _null_ _null_ _null_ _null_ macaddr8_out _null_ _null_ _null_ )); +DESCR("I/O"); + +DATA(insert OID = 4112 ( trunc PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 774 "774" _null_ _null_ _null_ _null_ _null_ macaddr8_trunc _null_ _null_ _null_ )); +DESCR("MACADDR8 manufacturer fields"); + +DATA(insert OID = 4113 ( macaddr8_eq PGNSP PGUID 12 1 0 0 0 f f f t t f i s 2 0 16 "774 774" _null_ _null_ _null_ _null_ _null_ macaddr8_eq _null_ _null_ _null_ )); +DATA(insert OID = 4114 ( macaddr8_lt PGNSP PGUID 12 1 0 0 0 f f f t t f i s 2 0 16 "774 774" _null_ _null_ _null_ _null_ _null_ macaddr8_lt _null_ _null_ _null_ )); +DATA(insert OID = 4115 ( macaddr8_le PGNSP PGUID 12 1 0 0 0 f f f t t f i s 2 0 16 "774 774" _null_ _null_ _null_ _null_ _null_ macaddr8_le _null_ _null_ _null_ )); +DATA(insert OID = 4116 ( macaddr8_gt PGNSP PGUID 12 1 0 0 0 f f f t t f i s 2 0 16 "774 774" _null_ _null_ _null_ _null_ _null_ macaddr8_gt _null_ _null_ _null_ )); +DATA(insert OID = 4117 ( macaddr8_ge PGNSP PGUID 12 1 0 0 0 f f f t t f i s 2 0 16 "774 774" _null_ _null_ _null_ _null_ _null_ macaddr8_ge _null_ _null_ _null_ )); +DATA(insert OID = 4118 ( macaddr8_ne PGNSP PGUID 12 1 0 0 0 f f f t t f i s 2 0 16 "774 774" _null_ _null_ _null_ _null_ _null_ macaddr8_ne _null_ _null_ _null_ )); +DATA(insert OID = 4119 ( macaddr8_cmp PGNSP PGUID 12 1 0 0 0 f f f f t f i s 2 0 23 "774 774" _null_ _null_ _null_ _null_ _null_ macaddr8_cmp _null_ _null_ _null_ )); +DESCR("less-equal-greater"); +DATA(insert OID = 4120 ( macaddr8_not PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 774 "774" _null_ _null_ _null_ _null_ _null_ macaddr8_not _null_ _null_ _null_ )); +DATA(insert OID = 4121 ( macaddr8_and PGNSP PGUID 12 1 0 0 0 f f f f t f i s 2 0 774 "774 774" _null_ _null_ _null_ _null_ _null_ macaddr8_and _null_ _null_ _null_ )); +DATA(insert OID = 4122 ( macaddr8_or PGNSP PGUID 12 1 0 0 0 f f f f t f i s 2 0 774 "774 774" _null_ _null_ _null_ _null_ _null_ macaddr8_or _null_ _null_ _null_ )); +DATA(insert OID = 4123 ( macaddr8 PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 774 "829" _null_ _null_ _null_ _null_ _null_ macaddrtomacaddr8 _null_ _null_ _null_ )); +DESCR("convert macaddr to macaddr8"); +DATA(insert OID = 4124 ( macaddr PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 829 "774" _null_ _null_ _null_ _null_ _null_ macaddr8tomacaddr _null_ _null_ _null_ )); +DESCR("convert macaddr8 to macaddr"); +DATA(insert OID = 4125 ( macaddr8_set7bit PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 774 "774" _null_ _null_ _null_ _null_ _null_ macaddr8_set7bit _null_ _null_ _null_ )); +DESCR("set 7th bit in macaddr8"); + /* for inet type support */ DATA(insert OID = 910 ( inet_in PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 869 "2275" _null_ _null_ _null_ _null_ _null_ inet_in _null_ _null_ _null_ )); DESCR("I/O"); @@ -4056,6 +4085,10 @@ DATA(insert OID = 3120 ( void_recv PGNSP PGUID 12 1 0 0 0 f f f f t f i s DESCR("I/O"); DATA(insert OID = 3121 ( void_send PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 17 "2278" _null_ _null_ _null_ _null_ _null_ void_send _null_ _null_ _null_ )); DESCR("I/O"); +DATA(insert OID = 3446 ( macaddr8_recv PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 774 "2281" _null_ _null_ _null_ _null_ _null_ macaddr8_recv _null_ _null_ _null_ )); +DESCR("I/O"); +DATA(insert OID = 3447 ( macaddr8_send PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 17 "774" _null_ _null_ _null_ _null_ _null_ macaddr8_send _null_ _null_ _null_ )); +DESCR("I/O"); /* System-view support functions with pretty-print option */ DATA(insert OID = 2504 ( pg_get_ruledef PGNSP PGUID 12 1 0 0 0 f f f f t f s s 2 0 25 "26 16" _null_ _null_ _null_ _null_ _null_ pg_get_ruledef_ext _null_ _null_ _null_ )); diff --git a/src/include/catalog/pg_type.h b/src/include/catalog/pg_type.h index 6e4c65e6ad..9f61238179 100644 --- a/src/include/catalog/pg_type.h +++ b/src/include/catalog/pg_type.h @@ -441,6 +441,9 @@ DESCR("IP address/netmask, host address, netmask optional"); DATA(insert OID = 650 ( cidr PGNSP PGUID -1 f b I f t \054 0 0 651 cidr_in cidr_out cidr_recv cidr_send - - - i m f 0 -1 0 0 _null_ _null_ _null_ )); DESCR("network IP address/netmask, network address"); #define CIDROID 650 +DATA(insert OID = 774 ( macaddr8 PGNSP PGUID 8 f b U f t \054 0 0 775 macaddr8_in macaddr8_out macaddr8_recv macaddr8_send - - - i p f 0 -1 0 0 _null_ _null_ _null_ )); +DESCR("XX:XX:XX:XX:XX:XX:XX:XX, MAC address"); +#define MACADDR8OID 774 /* OIDS 900 - 999 */ @@ -482,6 +485,7 @@ DESCR("access control list"); #define ACLITEMOID 1033 DATA(insert OID = 1034 ( _aclitem PGNSP PGUID -1 f b A f t \054 0 1033 0 array_in array_out array_recv array_send - - array_typanalyze i x f 0 -1 0 0 _null_ _null_ _null_ )); DATA(insert OID = 1040 ( _macaddr PGNSP PGUID -1 f b A f t \054 0 829 0 array_in array_out array_recv array_send - - array_typanalyze i x f 0 -1 0 0 _null_ _null_ _null_ )); +DATA(insert OID = 775 ( _macaddr8 PGNSP PGUID -1 f b A f t \054 0 774 0 array_in array_out array_recv array_send - - array_typanalyze i x f 0 -1 0 0 _null_ _null_ _null_ )); DATA(insert OID = 1041 ( _inet PGNSP PGUID -1 f b A f t \054 0 869 0 array_in array_out array_recv array_send - - array_typanalyze i x f 0 -1 0 0 _null_ _null_ _null_ )); DATA(insert OID = 651 ( _cidr PGNSP PGUID -1 f b A f t \054 0 650 0 array_in array_out array_recv array_send - - array_typanalyze i x f 0 -1 0 0 _null_ _null_ _null_ )); DATA(insert OID = 1263 ( _cstring PGNSP PGUID -1 f b A f t \054 0 2275 0 array_in array_out array_recv array_send - - array_typanalyze i x f 0 -1 0 0 _null_ _null_ _null_ )); diff --git a/src/include/utils/inet.h b/src/include/utils/inet.h index b4d7359f19..7dc179e255 100644 --- a/src/include/utils/inet.h +++ b/src/include/utils/inet.h @@ -101,6 +101,21 @@ typedef struct macaddr unsigned char f; } macaddr; +/* + * This is the internal storage format for MAC8 addresses: + */ +typedef struct macaddr8 +{ + unsigned char a; + unsigned char b; + unsigned char c; + unsigned char d; + unsigned char e; + unsigned char f; + unsigned char g; + unsigned char h; +} macaddr8; + /* * fmgr interface macros */ @@ -111,12 +126,19 @@ typedef struct macaddr /* obsolescent variants */ #define DatumGetInetP(X) ((inet *) PG_DETOAST_DATUM(X)) #define PG_GETARG_INET_P(n) DatumGetInetP(PG_GETARG_DATUM(n)) + /* macaddr is a fixed-length pass-by-reference datatype */ #define DatumGetMacaddrP(X) ((macaddr *) DatumGetPointer(X)) #define MacaddrPGetDatum(X) PointerGetDatum(X) #define PG_GETARG_MACADDR_P(n) DatumGetMacaddrP(PG_GETARG_DATUM(n)) #define PG_RETURN_MACADDR_P(x) return MacaddrPGetDatum(x) +/* macaddr8 is a fixed-length pass-by-reference datatype */ +#define DatumGetMacaddr8P(X) ((macaddr8 *) DatumGetPointer(X)) +#define Macaddr8PGetDatum(X) PointerGetDatum(X) +#define PG_GETARG_MACADDR8_P(n) DatumGetMacaddr8P(PG_GETARG_DATUM(n)) +#define PG_RETURN_MACADDR8_P(x) return Macaddr8PGetDatum(x) + /* * Support functions in network.c */ diff --git a/src/test/regress/expected/macaddr8.out b/src/test/regress/expected/macaddr8.out new file mode 100644 index 0000000000..74f53a121f --- /dev/null +++ b/src/test/regress/expected/macaddr8.out @@ -0,0 +1,354 @@ +-- +-- macaddr8 +-- +-- test various cases of valid and invalid input +-- valid +SELECT '08:00:2b:01:02:03 '::macaddr8; + macaddr8 +------------------------- + 08:00:2b:ff:fe:01:02:03 +(1 row) + +SELECT ' 08:00:2b:01:02:03 '::macaddr8; + macaddr8 +------------------------- + 08:00:2b:ff:fe:01:02:03 +(1 row) + +SELECT ' 08:00:2b:01:02:03'::macaddr8; + macaddr8 +------------------------- + 08:00:2b:ff:fe:01:02:03 +(1 row) + +SELECT '08:00:2b:01:02:03:04:05 '::macaddr8; + macaddr8 +------------------------- + 08:00:2b:01:02:03:04:05 +(1 row) + +SELECT ' 08:00:2b:01:02:03:04:05 '::macaddr8; + macaddr8 +------------------------- + 08:00:2b:01:02:03:04:05 +(1 row) + +SELECT ' 08:00:2b:01:02:03:04:05'::macaddr8; + macaddr8 +------------------------- + 08:00:2b:01:02:03:04:05 +(1 row) + +SELECT '123 08:00:2b:01:02:03'::macaddr8; -- invalid +ERROR: invalid input syntax for type macaddr8: "123 08:00:2b:01:02:03" +LINE 1: SELECT '123 08:00:2b:01:02:03'::macaddr8; + ^ +SELECT '08:00:2b:01:02:03 123'::macaddr8; -- invalid +ERROR: invalid input syntax for type macaddr8: "08:00:2b:01:02:03 123" +LINE 1: SELECT '08:00:2b:01:02:03 123'::macaddr8; + ^ +SELECT '123 08:00:2b:01:02:03:04:05'::macaddr8; -- invalid +ERROR: invalid input syntax for type macaddr8: "123 08:00:2b:01:02:03:04:05" +LINE 1: SELECT '123 08:00:2b:01:02:03:04:05'::macaddr8; + ^ +SELECT '08:00:2b:01:02:03:04:05 123'::macaddr8; -- invalid +ERROR: invalid input syntax for type macaddr8: "08:00:2b:01:02:03:04:05 123" +LINE 1: SELECT '08:00:2b:01:02:03:04:05 123'::macaddr8; + ^ +SELECT '08:00:2b:01:02:03:04:05:06:07'::macaddr8; -- invalid +ERROR: invalid input syntax for type macaddr8: "08:00:2b:01:02:03:04:05:06:07" +LINE 1: SELECT '08:00:2b:01:02:03:04:05:06:07'::macaddr8; + ^ +SELECT '08-00-2b-01-02-03-04-05-06-07'::macaddr8; -- invalid +ERROR: invalid input syntax for type macaddr8: "08-00-2b-01-02-03-04-05-06-07" +LINE 1: SELECT '08-00-2b-01-02-03-04-05-06-07'::macaddr8; + ^ +SELECT '08002b:01020304050607'::macaddr8; -- invalid +ERROR: invalid input syntax for type macaddr8: "08002b:01020304050607" +LINE 1: SELECT '08002b:01020304050607'::macaddr8; + ^ +SELECT '08002b01020304050607'::macaddr8; -- invalid +ERROR: invalid input syntax for type macaddr8: "08002b01020304050607" +LINE 1: SELECT '08002b01020304050607'::macaddr8; + ^ +SELECT '0z002b0102030405'::macaddr8; -- invalid +ERROR: invalid input syntax for type macaddr8: "0z002b0102030405" +LINE 1: SELECT '0z002b0102030405'::macaddr8; + ^ +SELECT '08002b010203xyza'::macaddr8; -- invalid +ERROR: invalid input syntax for type macaddr8: "08002b010203xyza" +LINE 1: SELECT '08002b010203xyza'::macaddr8; + ^ +SELECT '08:00-2b:01:02:03:04:05'::macaddr8; -- invalid +ERROR: invalid input syntax for type macaddr8: "08:00-2b:01:02:03:04:05" +LINE 1: SELECT '08:00-2b:01:02:03:04:05'::macaddr8; + ^ +SELECT '08:00-2b:01:02:03:04:05'::macaddr8; -- invalid +ERROR: invalid input syntax for type macaddr8: "08:00-2b:01:02:03:04:05" +LINE 1: SELECT '08:00-2b:01:02:03:04:05'::macaddr8; + ^ +SELECT '08:00:2b:01.02:03:04:05'::macaddr8; -- invalid +ERROR: invalid input syntax for type macaddr8: "08:00:2b:01.02:03:04:05" +LINE 1: SELECT '08:00:2b:01.02:03:04:05'::macaddr8; + ^ +SELECT '08:00:2b:01.02:03:04:05'::macaddr8; -- invalid +ERROR: invalid input syntax for type macaddr8: "08:00:2b:01.02:03:04:05" +LINE 1: SELECT '08:00:2b:01.02:03:04:05'::macaddr8; + ^ +-- test converting a MAC address to modified EUI-64 for inclusion +-- in an ipv6 address +SELECT macaddr8_set7bit('00:08:2b:01:02:03'::macaddr8); + macaddr8_set7bit +------------------------- + 02:08:2b:ff:fe:01:02:03 +(1 row) + +CREATE TABLE macaddr8_data (a int, b macaddr8); +INSERT INTO macaddr8_data VALUES (1, '08:00:2b:01:02:03'); +INSERT INTO macaddr8_data VALUES (2, '08-00-2b-01-02-03'); +INSERT INTO macaddr8_data VALUES (3, '08002b:010203'); +INSERT INTO macaddr8_data VALUES (4, '08002b-010203'); +INSERT INTO macaddr8_data VALUES (5, '0800.2b01.0203'); +INSERT INTO macaddr8_data VALUES (6, '0800-2b01-0203'); +INSERT INTO macaddr8_data VALUES (7, '08002b010203'); +INSERT INTO macaddr8_data VALUES (8, '0800:2b01:0203'); +INSERT INTO macaddr8_data VALUES (9, 'not even close'); -- invalid +ERROR: invalid input syntax for type macaddr8: "not even close" +LINE 1: INSERT INTO macaddr8_data VALUES (9, 'not even close'); + ^ +INSERT INTO macaddr8_data VALUES (10, '08:00:2b:01:02:04'); +INSERT INTO macaddr8_data VALUES (11, '08:00:2b:01:02:02'); +INSERT INTO macaddr8_data VALUES (12, '08:00:2a:01:02:03'); +INSERT INTO macaddr8_data VALUES (13, '08:00:2c:01:02:03'); +INSERT INTO macaddr8_data VALUES (14, '08:00:2a:01:02:04'); +INSERT INTO macaddr8_data VALUES (15, '08:00:2b:01:02:03:04:05'); +INSERT INTO macaddr8_data VALUES (16, '08-00-2b-01-02-03-04-05'); +INSERT INTO macaddr8_data VALUES (17, '08002b:0102030405'); +INSERT INTO macaddr8_data VALUES (18, '08002b-0102030405'); +INSERT INTO macaddr8_data VALUES (19, '0800.2b01.0203.0405'); +INSERT INTO macaddr8_data VALUES (20, '08002b01:02030405'); +INSERT INTO macaddr8_data VALUES (21, '08002b0102030405'); +SELECT * FROM macaddr8_data ORDER BY 1; + a | b +----+------------------------- + 1 | 08:00:2b:ff:fe:01:02:03 + 2 | 08:00:2b:ff:fe:01:02:03 + 3 | 08:00:2b:ff:fe:01:02:03 + 4 | 08:00:2b:ff:fe:01:02:03 + 5 | 08:00:2b:ff:fe:01:02:03 + 6 | 08:00:2b:ff:fe:01:02:03 + 7 | 08:00:2b:ff:fe:01:02:03 + 8 | 08:00:2b:ff:fe:01:02:03 + 10 | 08:00:2b:ff:fe:01:02:04 + 11 | 08:00:2b:ff:fe:01:02:02 + 12 | 08:00:2a:ff:fe:01:02:03 + 13 | 08:00:2c:ff:fe:01:02:03 + 14 | 08:00:2a:ff:fe:01:02:04 + 15 | 08:00:2b:01:02:03:04:05 + 16 | 08:00:2b:01:02:03:04:05 + 17 | 08:00:2b:01:02:03:04:05 + 18 | 08:00:2b:01:02:03:04:05 + 19 | 08:00:2b:01:02:03:04:05 + 20 | 08:00:2b:01:02:03:04:05 + 21 | 08:00:2b:01:02:03:04:05 +(20 rows) + +CREATE INDEX macaddr8_data_btree ON macaddr8_data USING btree (b); +CREATE INDEX macaddr8_data_hash ON macaddr8_data USING hash (b); +SELECT a, b, trunc(b) FROM macaddr8_data ORDER BY 2, 1; + a | b | trunc +----+-------------------------+------------------------- + 12 | 08:00:2a:ff:fe:01:02:03 | 08:00:2a:00:00:00:00:00 + 14 | 08:00:2a:ff:fe:01:02:04 | 08:00:2a:00:00:00:00:00 + 15 | 08:00:2b:01:02:03:04:05 | 08:00:2b:00:00:00:00:00 + 16 | 08:00:2b:01:02:03:04:05 | 08:00:2b:00:00:00:00:00 + 17 | 08:00:2b:01:02:03:04:05 | 08:00:2b:00:00:00:00:00 + 18 | 08:00:2b:01:02:03:04:05 | 08:00:2b:00:00:00:00:00 + 19 | 08:00:2b:01:02:03:04:05 | 08:00:2b:00:00:00:00:00 + 20 | 08:00:2b:01:02:03:04:05 | 08:00:2b:00:00:00:00:00 + 21 | 08:00:2b:01:02:03:04:05 | 08:00:2b:00:00:00:00:00 + 11 | 08:00:2b:ff:fe:01:02:02 | 08:00:2b:00:00:00:00:00 + 1 | 08:00:2b:ff:fe:01:02:03 | 08:00:2b:00:00:00:00:00 + 2 | 08:00:2b:ff:fe:01:02:03 | 08:00:2b:00:00:00:00:00 + 3 | 08:00:2b:ff:fe:01:02:03 | 08:00:2b:00:00:00:00:00 + 4 | 08:00:2b:ff:fe:01:02:03 | 08:00:2b:00:00:00:00:00 + 5 | 08:00:2b:ff:fe:01:02:03 | 08:00:2b:00:00:00:00:00 + 6 | 08:00:2b:ff:fe:01:02:03 | 08:00:2b:00:00:00:00:00 + 7 | 08:00:2b:ff:fe:01:02:03 | 08:00:2b:00:00:00:00:00 + 8 | 08:00:2b:ff:fe:01:02:03 | 08:00:2b:00:00:00:00:00 + 10 | 08:00:2b:ff:fe:01:02:04 | 08:00:2b:00:00:00:00:00 + 13 | 08:00:2c:ff:fe:01:02:03 | 08:00:2c:00:00:00:00:00 +(20 rows) + +SELECT b < '08:00:2b:01:02:04' FROM macaddr8_data WHERE a = 1; -- true + ?column? +---------- + t +(1 row) + +SELECT b > '08:00:2b:ff:fe:01:02:04' FROM macaddr8_data WHERE a = 1; -- false + ?column? +---------- + f +(1 row) + +SELECT b > '08:00:2b:ff:fe:01:02:03' FROM macaddr8_data WHERE a = 1; -- false + ?column? +---------- + f +(1 row) + +SELECT b::macaddr <= '08:00:2b:01:02:04' FROM macaddr8_data WHERE a = 1; -- true + ?column? +---------- + t +(1 row) + +SELECT b::macaddr >= '08:00:2b:01:02:04' FROM macaddr8_data WHERE a = 1; -- false + ?column? +---------- + f +(1 row) + +SELECT b = '08:00:2b:ff:fe:01:02:03' FROM macaddr8_data WHERE a = 1; -- true + ?column? +---------- + t +(1 row) + +SELECT b::macaddr <> '08:00:2b:01:02:04'::macaddr FROM macaddr8_data WHERE a = 1; -- true + ?column? +---------- + t +(1 row) + +SELECT b::macaddr <> '08:00:2b:01:02:03'::macaddr FROM macaddr8_data WHERE a = 1; -- false + ?column? +---------- + f +(1 row) + +SELECT b < '08:00:2b:01:02:03:04:06' FROM macaddr8_data WHERE a = 15; -- true + ?column? +---------- + t +(1 row) + +SELECT b > '08:00:2b:01:02:03:04:06' FROM macaddr8_data WHERE a = 15; -- false + ?column? +---------- + f +(1 row) + +SELECT b > '08:00:2b:01:02:03:04:05' FROM macaddr8_data WHERE a = 15; -- false + ?column? +---------- + f +(1 row) + +SELECT b <= '08:00:2b:01:02:03:04:06' FROM macaddr8_data WHERE a = 15; -- true + ?column? +---------- + t +(1 row) + +SELECT b >= '08:00:2b:01:02:03:04:06' FROM macaddr8_data WHERE a = 15; -- false + ?column? +---------- + f +(1 row) + +SELECT b = '08:00:2b:01:02:03:04:05' FROM macaddr8_data WHERE a = 15; -- true + ?column? +---------- + t +(1 row) + +SELECT b <> '08:00:2b:01:02:03:04:06' FROM macaddr8_data WHERE a = 15; -- true + ?column? +---------- + t +(1 row) + +SELECT b <> '08:00:2b:01:02:03:04:05' FROM macaddr8_data WHERE a = 15; -- false + ?column? +---------- + f +(1 row) + +SELECT ~b FROM macaddr8_data; + ?column? +------------------------- + f7:ff:d4:00:01:fe:fd:fc + f7:ff:d4:00:01:fe:fd:fc + f7:ff:d4:00:01:fe:fd:fc + f7:ff:d4:00:01:fe:fd:fc + f7:ff:d4:00:01:fe:fd:fc + f7:ff:d4:00:01:fe:fd:fc + f7:ff:d4:00:01:fe:fd:fc + f7:ff:d4:00:01:fe:fd:fc + f7:ff:d4:00:01:fe:fd:fb + f7:ff:d4:00:01:fe:fd:fd + f7:ff:d5:00:01:fe:fd:fc + f7:ff:d3:00:01:fe:fd:fc + f7:ff:d5:00:01:fe:fd:fb + f7:ff:d4:fe:fd:fc:fb:fa + f7:ff:d4:fe:fd:fc:fb:fa + f7:ff:d4:fe:fd:fc:fb:fa + f7:ff:d4:fe:fd:fc:fb:fa + f7:ff:d4:fe:fd:fc:fb:fa + f7:ff:d4:fe:fd:fc:fb:fa + f7:ff:d4:fe:fd:fc:fb:fa +(20 rows) + +SELECT b & '00:00:00:ff:ff:ff' FROM macaddr8_data; + ?column? +------------------------- + 00:00:00:ff:fe:01:02:03 + 00:00:00:ff:fe:01:02:03 + 00:00:00:ff:fe:01:02:03 + 00:00:00:ff:fe:01:02:03 + 00:00:00:ff:fe:01:02:03 + 00:00:00:ff:fe:01:02:03 + 00:00:00:ff:fe:01:02:03 + 00:00:00:ff:fe:01:02:03 + 00:00:00:ff:fe:01:02:04 + 00:00:00:ff:fe:01:02:02 + 00:00:00:ff:fe:01:02:03 + 00:00:00:ff:fe:01:02:03 + 00:00:00:ff:fe:01:02:04 + 00:00:00:01:02:03:04:05 + 00:00:00:01:02:03:04:05 + 00:00:00:01:02:03:04:05 + 00:00:00:01:02:03:04:05 + 00:00:00:01:02:03:04:05 + 00:00:00:01:02:03:04:05 + 00:00:00:01:02:03:04:05 +(20 rows) + +SELECT b | '01:02:03:04:05:06' FROM macaddr8_data; + ?column? +------------------------- + 09:02:2b:ff:fe:05:07:07 + 09:02:2b:ff:fe:05:07:07 + 09:02:2b:ff:fe:05:07:07 + 09:02:2b:ff:fe:05:07:07 + 09:02:2b:ff:fe:05:07:07 + 09:02:2b:ff:fe:05:07:07 + 09:02:2b:ff:fe:05:07:07 + 09:02:2b:ff:fe:05:07:07 + 09:02:2b:ff:fe:05:07:06 + 09:02:2b:ff:fe:05:07:06 + 09:02:2b:ff:fe:05:07:07 + 09:02:2f:ff:fe:05:07:07 + 09:02:2b:ff:fe:05:07:06 + 09:02:2b:ff:fe:07:05:07 + 09:02:2b:ff:fe:07:05:07 + 09:02:2b:ff:fe:07:05:07 + 09:02:2b:ff:fe:07:05:07 + 09:02:2b:ff:fe:07:05:07 + 09:02:2b:ff:fe:07:05:07 + 09:02:2b:ff:fe:07:05:07 +(20 rows) + +DROP TABLE macaddr8_data; diff --git a/src/test/regress/expected/opr_sanity.out b/src/test/regress/expected/opr_sanity.out index 0bcec136c5..64d9dd605f 100644 --- a/src/test/regress/expected/opr_sanity.out +++ b/src/test/regress/expected/opr_sanity.out @@ -685,6 +685,12 @@ uuid_gt(uuid,uuid) uuid_ne(uuid,uuid) xidneq(xid,xid) xidneqint4(xid,integer) +macaddr8_eq(macaddr8,macaddr8) +macaddr8_lt(macaddr8,macaddr8) +macaddr8_le(macaddr8,macaddr8) +macaddr8_gt(macaddr8,macaddr8) +macaddr8_ge(macaddr8,macaddr8) +macaddr8_ne(macaddr8,macaddr8) -- restore normal output mode \a\t -- List of functions used by libpq's fe-lobj.c diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 9f38349e90..ea7b5b4aa2 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -23,7 +23,7 @@ test: numerology # ---------- # The second group of parallel tests # ---------- -test: point lseg line box path polygon circle date time timetz timestamp timestamptz interval abstime reltime tinterval inet macaddr tstypes comments +test: point lseg line box path polygon circle date time timetz timestamp timestamptz interval abstime reltime tinterval inet macaddr macaddr8 tstypes comments # ---------- # Another group of parallel tests diff --git a/src/test/regress/serial_schedule b/src/test/regress/serial_schedule index 2987b24ebb..cf48ea7cc8 100644 --- a/src/test/regress/serial_schedule +++ b/src/test/regress/serial_schedule @@ -41,6 +41,7 @@ test: reltime test: tinterval test: inet test: macaddr +test: macaddr8 test: tstypes test: comments test: geometry diff --git a/src/test/regress/sql/macaddr8.sql b/src/test/regress/sql/macaddr8.sql new file mode 100644 index 0000000000..57a227c5ab --- /dev/null +++ b/src/test/regress/sql/macaddr8.sql @@ -0,0 +1,89 @@ +-- +-- macaddr8 +-- + +-- test various cases of valid and invalid input +-- valid +SELECT '08:00:2b:01:02:03 '::macaddr8; +SELECT ' 08:00:2b:01:02:03 '::macaddr8; +SELECT ' 08:00:2b:01:02:03'::macaddr8; +SELECT '08:00:2b:01:02:03:04:05 '::macaddr8; +SELECT ' 08:00:2b:01:02:03:04:05 '::macaddr8; +SELECT ' 08:00:2b:01:02:03:04:05'::macaddr8; + +SELECT '123 08:00:2b:01:02:03'::macaddr8; -- invalid +SELECT '08:00:2b:01:02:03 123'::macaddr8; -- invalid +SELECT '123 08:00:2b:01:02:03:04:05'::macaddr8; -- invalid +SELECT '08:00:2b:01:02:03:04:05 123'::macaddr8; -- invalid +SELECT '08:00:2b:01:02:03:04:05:06:07'::macaddr8; -- invalid +SELECT '08-00-2b-01-02-03-04-05-06-07'::macaddr8; -- invalid +SELECT '08002b:01020304050607'::macaddr8; -- invalid +SELECT '08002b01020304050607'::macaddr8; -- invalid +SELECT '0z002b0102030405'::macaddr8; -- invalid +SELECT '08002b010203xyza'::macaddr8; -- invalid + +SELECT '08:00-2b:01:02:03:04:05'::macaddr8; -- invalid +SELECT '08:00-2b:01:02:03:04:05'::macaddr8; -- invalid +SELECT '08:00:2b:01.02:03:04:05'::macaddr8; -- invalid +SELECT '08:00:2b:01.02:03:04:05'::macaddr8; -- invalid + +-- test converting a MAC address to modified EUI-64 for inclusion +-- in an ipv6 address +SELECT macaddr8_set7bit('00:08:2b:01:02:03'::macaddr8); + +CREATE TABLE macaddr8_data (a int, b macaddr8); + +INSERT INTO macaddr8_data VALUES (1, '08:00:2b:01:02:03'); +INSERT INTO macaddr8_data VALUES (2, '08-00-2b-01-02-03'); +INSERT INTO macaddr8_data VALUES (3, '08002b:010203'); +INSERT INTO macaddr8_data VALUES (4, '08002b-010203'); +INSERT INTO macaddr8_data VALUES (5, '0800.2b01.0203'); +INSERT INTO macaddr8_data VALUES (6, '0800-2b01-0203'); +INSERT INTO macaddr8_data VALUES (7, '08002b010203'); +INSERT INTO macaddr8_data VALUES (8, '0800:2b01:0203'); +INSERT INTO macaddr8_data VALUES (9, 'not even close'); -- invalid + +INSERT INTO macaddr8_data VALUES (10, '08:00:2b:01:02:04'); +INSERT INTO macaddr8_data VALUES (11, '08:00:2b:01:02:02'); +INSERT INTO macaddr8_data VALUES (12, '08:00:2a:01:02:03'); +INSERT INTO macaddr8_data VALUES (13, '08:00:2c:01:02:03'); +INSERT INTO macaddr8_data VALUES (14, '08:00:2a:01:02:04'); + +INSERT INTO macaddr8_data VALUES (15, '08:00:2b:01:02:03:04:05'); +INSERT INTO macaddr8_data VALUES (16, '08-00-2b-01-02-03-04-05'); +INSERT INTO macaddr8_data VALUES (17, '08002b:0102030405'); +INSERT INTO macaddr8_data VALUES (18, '08002b-0102030405'); +INSERT INTO macaddr8_data VALUES (19, '0800.2b01.0203.0405'); +INSERT INTO macaddr8_data VALUES (20, '08002b01:02030405'); +INSERT INTO macaddr8_data VALUES (21, '08002b0102030405'); + +SELECT * FROM macaddr8_data ORDER BY 1; + +CREATE INDEX macaddr8_data_btree ON macaddr8_data USING btree (b); +CREATE INDEX macaddr8_data_hash ON macaddr8_data USING hash (b); + +SELECT a, b, trunc(b) FROM macaddr8_data ORDER BY 2, 1; + +SELECT b < '08:00:2b:01:02:04' FROM macaddr8_data WHERE a = 1; -- true +SELECT b > '08:00:2b:ff:fe:01:02:04' FROM macaddr8_data WHERE a = 1; -- false +SELECT b > '08:00:2b:ff:fe:01:02:03' FROM macaddr8_data WHERE a = 1; -- false +SELECT b::macaddr <= '08:00:2b:01:02:04' FROM macaddr8_data WHERE a = 1; -- true +SELECT b::macaddr >= '08:00:2b:01:02:04' FROM macaddr8_data WHERE a = 1; -- false +SELECT b = '08:00:2b:ff:fe:01:02:03' FROM macaddr8_data WHERE a = 1; -- true +SELECT b::macaddr <> '08:00:2b:01:02:04'::macaddr FROM macaddr8_data WHERE a = 1; -- true +SELECT b::macaddr <> '08:00:2b:01:02:03'::macaddr FROM macaddr8_data WHERE a = 1; -- false + +SELECT b < '08:00:2b:01:02:03:04:06' FROM macaddr8_data WHERE a = 15; -- true +SELECT b > '08:00:2b:01:02:03:04:06' FROM macaddr8_data WHERE a = 15; -- false +SELECT b > '08:00:2b:01:02:03:04:05' FROM macaddr8_data WHERE a = 15; -- false +SELECT b <= '08:00:2b:01:02:03:04:06' FROM macaddr8_data WHERE a = 15; -- true +SELECT b >= '08:00:2b:01:02:03:04:06' FROM macaddr8_data WHERE a = 15; -- false +SELECT b = '08:00:2b:01:02:03:04:05' FROM macaddr8_data WHERE a = 15; -- true +SELECT b <> '08:00:2b:01:02:03:04:06' FROM macaddr8_data WHERE a = 15; -- true +SELECT b <> '08:00:2b:01:02:03:04:05' FROM macaddr8_data WHERE a = 15; -- false + +SELECT ~b FROM macaddr8_data; +SELECT b & '00:00:00:ff:ff:ff' FROM macaddr8_data; +SELECT b | '01:02:03:04:05:06' FROM macaddr8_data; + +DROP TABLE macaddr8_data; -- cgit v1.2.3 From e76db009f079ece9408e37336887bc6457cc1fc6 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 15 Mar 2017 13:52:07 -0400 Subject: Add more documentation and tests for publications Add/correct documentation and add some tests related to how access control around adding tables to publications works. --- doc/src/sgml/logical-replication.sgml | 6 ++++++ doc/src/sgml/ref/create_publication.sgml | 6 +++--- src/test/regress/expected/publication.out | 20 +++++++++++++++++++- src/test/regress/sql/publication.sql | 24 +++++++++++++++++++++++- 4 files changed, 51 insertions(+), 5 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml index 7b351f2727..a6c04e923d 100644 --- a/doc/src/sgml/logical-replication.sgml +++ b/doc/src/sgml/logical-replication.sgml @@ -307,6 +307,12 @@ privilege in the database. + + To add tables to a publication, the user must have ownership rights on the + table. To create a publication that publishes all tables automatically, + the user must be a superuser. + + To create a subscription, the user must be a superuser. diff --git a/doc/src/sgml/ref/create_publication.sgml b/doc/src/sgml/ref/create_publication.sgml index 995f2bcf3c..b00a91ef8a 100644 --- a/doc/src/sgml/ref/create_publication.sgml +++ b/doc/src/sgml/ref/create_publication.sgml @@ -143,9 +143,9 @@ CREATE PUBLICATION name - To add a table to a publication, the invoking user must have - SELECT privilege on given table. The - FOR ALL TABLES clause requires superuser. + To add a table to a publication, the invoking user must have ownership + rights on the table. The FOR ALL TABLES clause requires + the invoking user to be a superuser. diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out index 34320267c3..f3d60db86b 100644 --- a/src/test/regress/expected/publication.out +++ b/src/test/regress/expected/publication.out @@ -2,6 +2,7 @@ -- PUBLICATION -- CREATE ROLE regress_publication_user LOGIN SUPERUSER; +CREATE ROLE regress_publication_user2; CREATE ROLE regress_publication_user_dummy LOGIN NOSUPERUSER; SET SESSION AUTHORIZATION 'regress_publication_user'; CREATE PUBLICATION testpub_default; @@ -140,6 +141,23 @@ Publications: "testpib_ins_trunct" "testpub_fortbl" +-- permissions +SET ROLE regress_publication_user2; +CREATE PUBLICATION testpub2; -- fail +ERROR: permission denied for database regression +SET ROLE regress_publication_user; +GRANT CREATE ON DATABASE regression TO regress_publication_user2; +SET ROLE regress_publication_user2; +CREATE PUBLICATION testpub2; -- ok +ALTER PUBLICATION testpub2 ADD TABLE testpub_tbl1; -- fail +ERROR: must be owner of relation testpub_tbl1 +SET ROLE regress_publication_user; +GRANT regress_publication_user TO regress_publication_user2; +SET ROLE regress_publication_user2; +ALTER PUBLICATION testpub2 ADD TABLE testpub_tbl1; -- ok +DROP PUBLICATION testpub2; +SET ROLE regress_publication_user; +REVOKE CREATE ON DATABASE regression FROM regress_publication_user2; DROP VIEW testpub_view; DROP TABLE testpub_tbl1; \dRp+ testpub_default @@ -168,5 +186,5 @@ DROP PUBLICATION testpub_fortbl; DROP SCHEMA pub_test CASCADE; NOTICE: drop cascades to table pub_test.testpub_nopk RESET SESSION AUTHORIZATION; -DROP ROLE regress_publication_user; +DROP ROLE regress_publication_user, regress_publication_user2; DROP ROLE regress_publication_user_dummy; diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql index de68e61cb3..522c39029e 100644 --- a/src/test/regress/sql/publication.sql +++ b/src/test/regress/sql/publication.sql @@ -2,6 +2,7 @@ -- PUBLICATION -- CREATE ROLE regress_publication_user LOGIN SUPERUSER; +CREATE ROLE regress_publication_user2; CREATE ROLE regress_publication_user_dummy LOGIN NOSUPERUSER; SET SESSION AUTHORIZATION 'regress_publication_user'; @@ -69,6 +70,27 @@ ALTER PUBLICATION testpub_default DROP TABLE pub_test.testpub_nopk; \d+ testpub_tbl1 +-- permissions +SET ROLE regress_publication_user2; +CREATE PUBLICATION testpub2; -- fail + +SET ROLE regress_publication_user; +GRANT CREATE ON DATABASE regression TO regress_publication_user2; +SET ROLE regress_publication_user2; +CREATE PUBLICATION testpub2; -- ok + +ALTER PUBLICATION testpub2 ADD TABLE testpub_tbl1; -- fail + +SET ROLE regress_publication_user; +GRANT regress_publication_user TO regress_publication_user2; +SET ROLE regress_publication_user2; +ALTER PUBLICATION testpub2 ADD TABLE testpub_tbl1; -- ok + +DROP PUBLICATION testpub2; + +SET ROLE regress_publication_user; +REVOKE CREATE ON DATABASE regression FROM regress_publication_user2; + DROP VIEW testpub_view; DROP TABLE testpub_tbl1; @@ -90,5 +112,5 @@ DROP PUBLICATION testpub_fortbl; DROP SCHEMA pub_test CASCADE; RESET SESSION AUTHORIZATION; -DROP ROLE regress_publication_user; +DROP ROLE regress_publication_user, regress_publication_user2; DROP ROLE regress_publication_user_dummy; -- cgit v1.2.3 From befd73c50f11a6c6a6719dae20f0de7b7585bef4 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Thu, 16 Mar 2017 15:05:02 -0400 Subject: Add pg_ls_logdir() and pg_ls_waldir() functions. These functions are intended to be used by monitoring tools, and, unlike pg_ls_dir(), access to them can be granted to non-superusers, so that those monitoring tools can observe the principle of least privilege. Dave Page, revised by me, and also reviewed a bit by Thomas Munro. Discussion: http://postgr.es/m/CA+OCxow-X=D2fWdKy+HP+vQ1LtrgbsYQ=CshzZBqyFT5jOYrFw@mail.gmail.com --- doc/src/sgml/func.sgml | 45 ++++++++++++++++- src/backend/catalog/system_views.sql | 3 ++ src/backend/utils/adt/genfile.c | 94 ++++++++++++++++++++++++++++++++++++ src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_proc.h | 6 +++ 5 files changed, 147 insertions(+), 3 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index a521912317..9518fa2038 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -19646,7 +19646,8 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); database cluster directory and the log_directory can be accessed. Use a relative path for files in the cluster directory, and a path matching the log_directory configuration setting - for log files. Use of these functions is restricted to superusers. + for log files. Use of these functions is restricted to superusers + except where stated otherwise. @@ -19667,6 +19668,26 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); List the contents of a directory. + + + pg_ls_logdir() + + setof record + + List the name, size, and last modification time of files in the log + directory. Access may be granted to non-superuser roles. + + + + + pg_ls_waldir() + + setof record + + List the name, size, and last modification time of files in the WAL + directory. Access may be granted to non-superuser roles. + + pg_read_file(filename text [, offset bigint, length bigint [, missing_ok boolean] ]) @@ -19699,7 +19720,7 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup());
    - All of these functions take an optional missing_ok parameter, + Some of these functions take an optional missing_ok parameter, which specifies the behavior when the file or directory does not exist. If true, the function returns NULL (except pg_ls_dir, which returns an empty result set). If @@ -19719,6 +19740,26 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); empty directory from an non-existent directory. + + pg_ls_logdir + + + pg_ls_logdir returns the name, size, and last modified time + (mtime) of each file in the log directory. By default, only superusers + can use this function, but access may be granted to others using + GRANT. + + + + pg_ls_waldir + + + pg_ls_waldir returns the name, size, and last modified time + (mtime) of each file in the write ahead log (WAL) directory. By + default only superusers can use this function, but access may be granted + to others using GRANT. + + pg_read_file diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 0bce20914e..b6552da4b0 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1102,3 +1102,6 @@ REVOKE EXECUTE ON FUNCTION pg_stat_reset() FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_reset_shared(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_table_counters(oid) FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_function_counters(oid) FROM public; + +REVOKE EXECUTE ON FUNCTION pg_ls_logdir() FROM public; +REVOKE EXECUTE ON FUNCTION pg_ls_waldir() FROM public; diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c index 2147936dd8..8d0a236e6d 100644 --- a/src/backend/utils/adt/genfile.c +++ b/src/backend/utils/adt/genfile.c @@ -21,6 +21,7 @@ #include #include "access/htup_details.h" +#include "access/xlog_internal.h" #include "catalog/pg_type.h" #include "funcapi.h" #include "mb/pg_wchar.h" @@ -473,3 +474,96 @@ pg_ls_dir_1arg(PG_FUNCTION_ARGS) { return pg_ls_dir(fcinfo); } + +/* Generic function to return a directory listing of files */ +static Datum +pg_ls_dir_files(FunctionCallInfo fcinfo, char *dir) +{ + FuncCallContext *funcctx; + struct dirent *de; + directory_fctx *fctx; + + if (SRF_IS_FIRSTCALL()) + { + MemoryContext oldcontext; + TupleDesc tupdesc; + + funcctx = SRF_FIRSTCALL_INIT(); + oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + + fctx = palloc(sizeof(directory_fctx)); + + tupdesc = CreateTemplateTupleDesc(3, false); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "name", + TEXTOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 2, "size", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "modification", + TIMESTAMPTZOID, -1, 0); + funcctx->tuple_desc = BlessTupleDesc(tupdesc); + + fctx->location = pstrdup(dir); + fctx->dirdesc = AllocateDir(fctx->location); + + if (!fctx->dirdesc) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read directory \"%s\": %m", + fctx->location))); + + funcctx->user_fctx = fctx; + MemoryContextSwitchTo(oldcontext); + } + + funcctx = SRF_PERCALL_SETUP(); + fctx = (directory_fctx *) funcctx->user_fctx; + + while ((de = ReadDir(fctx->dirdesc, fctx->location)) != NULL) + { + Datum values[3]; + bool nulls[3]; + char path[MAXPGPATH]; + struct stat attrib; + HeapTuple tuple; + + /* Skip hidden files */ + if (de->d_name[0] == '.') + continue; + + /* Get the file info */ + snprintf(path, MAXPGPATH, "%s/%s", fctx->location, de->d_name); + if (stat(path, &attrib) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat directory \"%s\": %m", dir))); + + /* Ignore anything but regular files */ + if (!S_ISREG(attrib.st_mode)) + continue; + + values[0] = CStringGetTextDatum(de->d_name); + values[1] = Int64GetDatum((int64) attrib.st_size); + values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime)); + memset(nulls, 0, sizeof(nulls)); + + tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls); + SRF_RETURN_NEXT(funcctx, HeapTupleGetDatum(tuple)); + } + + FreeDir(fctx->dirdesc); + SRF_RETURN_DONE(funcctx); +} + +/* Function to return the list of files in the log directory */ +Datum +pg_ls_logdir(PG_FUNCTION_ARGS) +{ + return pg_ls_dir_files(fcinfo, Log_directory); +} + +/* Function to return the list of files in the WAL directory */ +Datum +pg_ls_waldir(PG_FUNCTION_ARGS) +{ + return pg_ls_dir_files(fcinfo, XLOGDIR); +} diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index b24e3953a1..b4f1b9a6c2 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201703151 +#define CATALOG_VERSION_NO 201703161 #endif diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index 3d5d866071..836d6ff0b2 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -5398,6 +5398,12 @@ DESCR("pg_controldata init state information as a function"); DATA(insert OID = 3445 ( pg_import_system_collations PGNSP PGUID 12 100 0 0 0 f f f f t f v r 2 0 2278 "16 4089" _null_ _null_ "{if_not_exists,schema}" _null_ _null_ pg_import_system_collations _null_ _null_ _null_ )); DESCR("import collations from operating system"); +/* system management/monitoring related functions */ +DATA(insert OID = 3353 ( pg_ls_logdir PGNSP PGUID 12 10 20 0 0 f f f f t t v s 0 0 2249 "" "{25,20,1184}" "{o,o,o}" "{name,size,modification}" _null_ _null_ pg_ls_logdir _null_ _null_ _null_ )); +DESCR("list files in the log directory"); +DATA(insert OID = 3354 ( pg_ls_waldir PGNSP PGUID 12 10 20 0 0 f f f f t t v s 0 0 2249 "" "{25,20,1184}" "{o,o,o}" "{name,size,modification}" _null_ _null_ pg_ls_waldir _null_ _null_ _null_ )); +DESCR("list of files in the WAL directory"); + /* * Symbolic values for provolatile column: these indicate whether the result * of a function is dependent *only* on the values of its explicit arguments, -- cgit v1.2.3 From b4ff8609dbad541d287b332846442b076a25a6df Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Thu, 16 Mar 2017 20:29:11 -0400 Subject: Fix grammar. This would have ben grammatical if the closing tag name were actually part of the output, but of course it's not. --- doc/src/sgml/func.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 9518fa2038..502f99b22b 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -18163,7 +18163,7 @@ SELECT set_config('log_statement_stats', 'off', false); The functions shown in send control signals to other server processes. Use of these functions is restricted to - superusers by default but access may be granted to others with the + superusers by default but access may be granted to others using GRANT, with noted exceptions. -- cgit v1.2.3 From bd9028b824c9c5f97795bb6b7b6f5ce38eb69f98 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Fri, 17 Mar 2017 09:32:34 -0400 Subject: Remove dead link. David Christensen Discussion: http://postgr.es/m/82299377-1480-4439-9ABA-5828D71AA22E@endpoint.com --- doc/src/sgml/high-availability.sgml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml index 0e619912d8..cc84b911b0 100644 --- a/doc/src/sgml/high-availability.sgml +++ b/doc/src/sgml/high-availability.sgml @@ -76,9 +76,7 @@ The remainder of this section outlines various failover, replication, - and load balancing solutions. A glossary is - also available. + and load balancing solutions. -- cgit v1.2.3 From 88e66d193fbaf756b3cc9bf94cad116aacbb355b Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Fri, 17 Mar 2017 09:46:58 -0400 Subject: Rename "pg_clog" directory to "pg_xact". Names containing the letters "log" sometimes confuse users into believing that only non-critical data is present. It is hoped this renaming will discourage ill-considered removals of transaction status data. Michael Paquier Discussion: http://postgr.es/m/CA+Tgmoa9xFQyjRZupbdEFuwUerFTvC6HjZq1ud6GYragGDFFgA@mail.gmail.com --- doc/src/sgml/backup.sgml | 4 ++-- doc/src/sgml/catalogs.sgml | 4 ++-- doc/src/sgml/config.sgml | 2 +- doc/src/sgml/func.sgml | 2 +- doc/src/sgml/maintenance.sgml | 8 ++++---- doc/src/sgml/ref/pg_resetwal.sgml | 4 ++-- doc/src/sgml/ref/pg_rewind.sgml | 2 +- doc/src/sgml/storage.sgml | 10 +++++----- doc/src/sgml/wal.sgml | 2 +- src/backend/access/heap/heapam.c | 4 ++-- src/backend/access/transam/README | 10 +++++----- src/backend/access/transam/clog.c | 2 +- src/backend/access/transam/commit_ts.c | 2 +- src/backend/access/transam/multixact.c | 2 +- src/backend/access/transam/subtrans.c | 5 ++--- src/backend/access/transam/transam.c | 2 +- src/backend/access/transam/twophase.c | 4 ++-- src/backend/access/transam/xact.c | 18 +++++++++--------- src/backend/access/transam/xlog.c | 2 +- src/backend/commands/vacuum.c | 10 +++++----- src/backend/postmaster/autovacuum.c | 2 +- src/backend/storage/buffer/README | 2 +- src/backend/storage/ipc/procarray.c | 4 ++-- src/backend/utils/time/tqual.c | 6 +++--- src/bin/initdb/initdb.c | 2 +- src/bin/pg_upgrade/exec.c | 8 +++++++- src/bin/pg_upgrade/pg_upgrade.c | 30 ++++++++++++++++++------------ src/include/access/slru.h | 4 ++-- 28 files changed, 84 insertions(+), 73 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml index 69c599e180..12f2efe4e2 100644 --- a/doc/src/sgml/backup.sgml +++ b/doc/src/sgml/backup.sgml @@ -382,10 +382,10 @@ tar -cf backup.tar /usr/local/pgsql/data directories. This will not work because the information contained in these files is not usable without the commit log files, - pg_clog/*, which contain the commit status of + pg_xact/*, which contain the commit status of all transactions. A table file is only usable with this information. Of course it is also impossible to restore only a - table and the associated pg_clog data + table and the associated pg_xact data because that would render all other tables in the database cluster useless. So file system backups only work for complete backup and restoration of an entire database cluster. diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 2c2da2ad8a..df0435c3f0 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -1893,7 +1893,7 @@ All transaction IDs before this one have been replaced with a permanent (frozen) transaction ID in this table. This is used to track whether the table needs to be vacuumed in order to prevent transaction - ID wraparound or to allow pg_clog to be shrunk. Zero + ID wraparound or to allow pg_xact to be shrunk. Zero (InvalidTransactionId) if the relation is not a table. @@ -2570,7 +2570,7 @@ All transaction IDs before this one have been replaced with a permanent (frozen) transaction ID in this database. This is used to track whether the database needs to be vacuumed in order to prevent - transaction ID wraparound or to allow pg_clog to be shrunk. + transaction ID wraparound or to allow pg_xact to be shrunk. It is the minimum of the per-table pg_class.relfrozenxid values. diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index b6daf9677c..9963cd61a1 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -6004,7 +6004,7 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; Vacuum also allows removal of old files from the - pg_clog subdirectory, which is why the default + pg_xact subdirectory, which is why the default is a relatively low 200 million transactions. This parameter can only be set at server start, but the setting can be reduced for individual tables by diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 502f99b22b..9408a255dc 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -15616,7 +15616,7 @@ SELECT * FROM pg_ls_dir('.') WITH ORDINALITY AS t(ls,n); postmaster.pid | 9 pg_ident.conf | 10 global | 11 - pg_clog | 12 + pg_xact | 12 pg_snapshots | 13 pg_multixact | 14 PG_VERSION | 15 diff --git a/doc/src/sgml/maintenance.sgml b/doc/src/sgml/maintenance.sgml index f87f3e00de..65a64c85ec 100644 --- a/doc/src/sgml/maintenance.sgml +++ b/doc/src/sgml/maintenance.sgml @@ -527,18 +527,18 @@ The sole disadvantage of increasing autovacuum_freeze_max_age (and vacuum_freeze_table_age along with it) - is that the pg_clog subdirectory of the database cluster + is that the pg_xact subdirectory of the database cluster will take more space, because it must store the commit status of all transactions back to the autovacuum_freeze_max_age horizon. The commit status uses two bits per transaction, so if autovacuum_freeze_max_age is set to its maximum allowed - value of two billion, pg_clog can be expected to + value of two billion, pg_xact can be expected to grow to about half a gigabyte. If this is trivial compared to your total database size, setting autovacuum_freeze_max_age to its maximum allowed value is recommended. Otherwise, set it depending - on what you are willing to allow for pg_clog storage. + on what you are willing to allow for pg_xact storage. (The default, 200 million transactions, translates to about 50MB of - pg_clog storage.) + pg_xact storage.) diff --git a/doc/src/sgml/ref/pg_resetwal.sgml b/doc/src/sgml/ref/pg_resetwal.sgml index 0cc6fb4c4d..0d93b56ddd 100644 --- a/doc/src/sgml/ref/pg_resetwal.sgml +++ b/doc/src/sgml/ref/pg_resetwal.sgml @@ -256,12 +256,12 @@ PostgreSQL documentation A safe value can be determined by looking for the numerically largest - file name in the directory pg_clog under the data directory, + file name in the directory pg_xact under the data directory, adding one, and then multiplying by 1048576 (0x100000). Note that the file names are in hexadecimal. It is usually easiest to specify the option value in hexadecimal too. For example, if 0011 is the largest entry - in pg_clog, -x 0x1200000 will work (five + in pg_xact, -x 0x1200000 will work (five trailing zeroes provide the proper multiplier). diff --git a/doc/src/sgml/ref/pg_rewind.sgml b/doc/src/sgml/ref/pg_rewind.sgml index 371c4a475f..d5430d4324 100644 --- a/doc/src/sgml/ref/pg_rewind.sgml +++ b/doc/src/sgml/ref/pg_rewind.sgml @@ -229,7 +229,7 @@ PostgreSQL documentation - Copy all other files such as pg_clog and + Copy all other files such as pg_xact and configuration files from the source cluster to the target cluster (everything except the relation files). diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml index e0a89861f8..a156693ec8 100644 --- a/doc/src/sgml/storage.sgml +++ b/doc/src/sgml/storage.sgml @@ -77,11 +77,6 @@ Item Subdirectory containing transaction commit timestamp data - - pg_clog - Subdirectory containing transaction commit status data - - pg_dynshmem Subdirectory containing files used by the dynamic shared memory @@ -151,6 +146,11 @@ Item Subdirectory containing WAL (Write Ahead Log) files + + pg_xact + Subdirectory containing transaction commit status data + + postgresql.auto.conf A file used for storing configuration parameters that are set by diff --git a/doc/src/sgml/wal.sgml b/doc/src/sgml/wal.sgml index 1468ba52a4..a749b83dc0 100644 --- a/doc/src/sgml/wal.sgml +++ b/doc/src/sgml/wal.sgml @@ -201,7 +201,7 @@ - Internal data structures such as pg_clog, pg_subtrans, pg_multixact, + Internal data structures such as pg_xact, pg_subtrans, pg_multixact, pg_serial, pg_notify, pg_stat, pg_snapshots are not directly checksummed, nor are pages protected by full page writes. However, where such data structures are persistent, WAL records are written that allow diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index bffc971d68..85261379b1 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -6790,8 +6790,8 @@ heap_prepare_freeze_tuple(HeapTupleHeader tuple, TransactionId cutoff_xid, * Note: it might seem we could make the changes without exclusive lock, since * TransactionId read/write is assumed atomic anyway. However there is a race * condition: someone who just fetched an old XID that we overwrite here could - * conceivably not finish checking the XID against pg_clog before we finish - * the VACUUM and perhaps truncate off the part of pg_clog he needs. Getting + * conceivably not finish checking the XID against pg_xact before we finish + * the VACUUM and perhaps truncate off the part of pg_xact he needs. Getting * exclusive lock ensures no other backend is in process of checking the * tuple status. Also, getting exclusive lock makes it safe to adjust the * infomask bits. diff --git a/src/backend/access/transam/README b/src/backend/access/transam/README index 4ae4715339..e7dd19fd7b 100644 --- a/src/backend/access/transam/README +++ b/src/backend/access/transam/README @@ -331,17 +331,17 @@ of the xid fields is atomic, so assuming it for xmin as well is no extra risk. -pg_clog and pg_subtrans +pg_xact and pg_subtrans ----------------------- -pg_clog and pg_subtrans are permanent (on-disk) storage of transaction related +pg_xact and pg_subtrans are permanent (on-disk) storage of transaction related information. There is a limited number of pages of each kept in memory, so in many cases there is no need to actually read from disk. However, if there's a long running transaction or a backend sitting idle with an open transaction, it may be necessary to be able to read and write this information from disk. They also allow information to be permanent across server restarts. -pg_clog records the commit status for each transaction that has been assigned +pg_xact records the commit status for each transaction that has been assigned an XID. A transaction can be in progress, committed, aborted, or "sub-committed". This last state means that it's a subtransaction that's no longer running, but its parent has not updated its state yet. It is not @@ -381,9 +381,9 @@ each transaction we keep a "cache" of Xids that are known to be part of the transaction tree, so we can skip looking at pg_subtrans unless we know the cache has been overflowed. See storage/ipc/procarray.c for the gory details. -slru.c is the supporting mechanism for both pg_clog and pg_subtrans. It +slru.c is the supporting mechanism for both pg_xact and pg_subtrans. It implements the LRU policy for in-memory buffer pages. The high-level routines -for pg_clog are implemented in transam.c, while the low-level functions are in +for pg_xact are implemented in transam.c, while the low-level functions are in clog.c. pg_subtrans is contained completely in subtrans.c. diff --git a/src/backend/access/transam/clog.c b/src/backend/access/transam/clog.c index 1a43819736..5b1d13dac1 100644 --- a/src/backend/access/transam/clog.c +++ b/src/backend/access/transam/clog.c @@ -450,7 +450,7 @@ CLOGShmemInit(void) { ClogCtl->PagePrecedes = CLOGPagePrecedes; SimpleLruInit(ClogCtl, "clog", CLOGShmemBuffers(), CLOG_LSNS_PER_PAGE, - CLogControlLock, "pg_clog", LWTRANCHE_CLOG_BUFFERS); + CLogControlLock, "pg_xact", LWTRANCHE_CLOG_BUFFERS); } /* diff --git a/src/backend/access/transam/commit_ts.c b/src/backend/access/transam/commit_ts.c index 20f60bc023..8e1df6e0ea 100644 --- a/src/backend/access/transam/commit_ts.c +++ b/src/backend/access/transam/commit_ts.c @@ -3,7 +3,7 @@ * commit_ts.c * PostgreSQL commit timestamp manager * - * This module is a pg_clog-like system that stores the commit timestamp + * This module is a pg_xact-like system that stores the commit timestamp * for each transaction. * * XLOG interactions: this module generates an XLOG record whenever a new diff --git a/src/backend/access/transam/multixact.c b/src/backend/access/transam/multixact.c index c350dfa17f..1a7824b5d4 100644 --- a/src/backend/access/transam/multixact.c +++ b/src/backend/access/transam/multixact.c @@ -3,7 +3,7 @@ * multixact.c * PostgreSQL multi-transaction-log manager * - * The pg_multixact manager is a pg_clog-like manager that stores an array of + * The pg_multixact manager is a pg_xact-like manager that stores an array of * MultiXactMember for each MultiXactId. It is a fundamental part of the * shared-row-lock implementation. Each MultiXactMember is comprised of a * TransactionId and a set of flag bits. The name is a bit historical: diff --git a/src/backend/access/transam/subtrans.c b/src/backend/access/transam/subtrans.c index a66fc0f2ad..70828e5170 100644 --- a/src/backend/access/transam/subtrans.c +++ b/src/backend/access/transam/subtrans.c @@ -3,15 +3,14 @@ * subtrans.c * PostgreSQL subtransaction-log manager * - * The pg_subtrans manager is a pg_clog-like manager that stores the parent + * The pg_subtrans manager is a pg_xact-like manager that stores the parent * transaction Id for each transaction. It is a fundamental part of the * nested transactions implementation. A main transaction has a parent * of InvalidTransactionId, and each subtransaction has its immediate parent. * The tree can easily be walked from child to parent, but not in the * opposite direction. * - * This code is based on clog.c, but the robustness requirements - * are completely different from pg_clog, because we only need to remember + * are completely different from pg_xact, because we only need to remember * pg_subtrans information for currently-open transactions. Thus, there is * no need to preserve data over a crash and restart. * diff --git a/src/backend/access/transam/transam.c b/src/backend/access/transam/transam.c index c2b4318bd3..b91a259e80 100644 --- a/src/backend/access/transam/transam.c +++ b/src/backend/access/transam/transam.c @@ -224,7 +224,7 @@ TransactionIdDidAbort(TransactionId transactionId) * True iff transaction associated with the identifier is currently * known to have either committed or aborted. * - * This does NOT look into pg_clog but merely probes our local cache + * This does NOT look into pg_xact but merely probes our local cache * (and so it's not named TransactionIdDidComplete, which would be the * appropriate name for a function that worked that way). The intended * use is just to short-circuit TransactionIdIsInProgress calls when doing diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index 83ca6e0408..da77e3efde 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1379,7 +1379,7 @@ FinishPreparedTransaction(const char *gid, bool isCommit) /* * The order of operations here is critical: make the XLOG entry for * commit or abort, then mark the transaction committed or aborted in - * pg_clog, then remove its PGPROC from the global ProcArray (which means + * pg_xact, then remove its PGPROC from the global ProcArray (which means * TransactionIdIsInProgress will stop saying the prepared xact is in * progress), then run the post-commit or post-abort callbacks. The * callbacks will release the locks the transaction held. @@ -2093,7 +2093,7 @@ RecordTransactionCommitPrepared(TransactionId xid, /* Flush XLOG to disk */ XLogFlush(recptr); - /* Mark the transaction committed in pg_clog */ + /* Mark the transaction committed in pg_xact */ TransactionIdCommitTree(xid, nchildren, children); /* Checkpoint can proceed now */ diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c index 82f9a3c5c6..02e0779f32 100644 --- a/src/backend/access/transam/xact.c +++ b/src/backend/access/transam/xact.c @@ -1208,8 +1208,8 @@ RecordTransactionCommit(void) /* * Mark ourselves as within our "commit critical section". This * forces any concurrent checkpoint to wait until we've updated - * pg_clog. Without this, it is possible for the checkpoint to set - * REDO after the XLOG record but fail to flush the pg_clog update to + * pg_xact. Without this, it is possible for the checkpoint to set + * REDO after the XLOG record but fail to flush the pg_xact update to * disk, leading to loss of the transaction commit if the system * crashes a little later. * @@ -2035,7 +2035,7 @@ CommitTransaction(void) if (!is_parallel_worker) { /* - * We need to mark our XIDs as committed in pg_clog. This is where we + * We need to mark our XIDs as committed in pg_xact. This is where we * durably commit. */ latestXid = RecordTransactionCommit(); @@ -2545,7 +2545,7 @@ AbortTransaction(void) AtAbort_Twophase(); /* - * Advertise the fact that we aborted in pg_clog (assuming that we got as + * Advertise the fact that we aborted in pg_xact (assuming that we got as * far as assigning an XID to advertise). But if we're inside a parallel * worker, skip this; the user backend must be the one to write the abort * record. @@ -4631,7 +4631,7 @@ AbortSubTransaction(void) s->parent->subTransactionId); AtSubAbort_Notify(); - /* Advertise the fact that we aborted in pg_clog. */ + /* Advertise the fact that we aborted in pg_xact. */ (void) RecordTransactionAbort(true); /* Post-abort cleanup */ @@ -5375,7 +5375,7 @@ xact_redo_commit(xl_xact_parsed_commit *parsed, if (standbyState == STANDBY_DISABLED) { /* - * Mark the transaction committed in pg_clog. + * Mark the transaction committed in pg_xact. */ TransactionIdCommitTree(xid, parsed->nsubxacts, parsed->subxacts); } @@ -5393,7 +5393,7 @@ xact_redo_commit(xl_xact_parsed_commit *parsed, RecordKnownAssignedTransactionIds(max_xid); /* - * Mark the transaction committed in pg_clog. We use async commit + * Mark the transaction committed in pg_xact. We use async commit * protocol during recovery to provide information on database * consistency for when users try to set hint bits. It is important * that we do not set hint bits until the minRecoveryPoint is past @@ -5530,7 +5530,7 @@ xact_redo_abort(xl_xact_parsed_abort *parsed, TransactionId xid) if (standbyState == STANDBY_DISABLED) { - /* Mark the transaction aborted in pg_clog, no need for async stuff */ + /* Mark the transaction aborted in pg_xact, no need for async stuff */ TransactionIdAbortTree(xid, parsed->nsubxacts, parsed->subxacts); } else @@ -5546,7 +5546,7 @@ xact_redo_abort(xl_xact_parsed_abort *parsed, TransactionId xid) */ RecordKnownAssignedTransactionIds(max_xid); - /* Mark the transaction aborted in pg_clog, no need for async stuff */ + /* Mark the transaction aborted in pg_xact, no need for async stuff */ TransactionIdAbortTree(xid, parsed->nsubxacts, parsed->subxacts); /* diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 64335f909e..cdb3a8ac1d 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -8710,7 +8710,7 @@ CreateCheckPoint(int flags) * that are currently in commit critical sections. If an xact inserted * its commit record into XLOG just before the REDO point, then a crash * restart from the REDO point would not replay that record, which means - * that our flushing had better include the xact's update of pg_clog. So + * that our flushing had better include the xact's update of pg_xact. So * we wait till he's out of his commit critical section before proceeding. * See notes in RecordTransactionCommit(). * diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index 3b3dfeead4..ff633faca6 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -357,7 +357,7 @@ vacuum(int options, RangeVar *relation, Oid relid, VacuumParams *params, if ((options & VACOPT_VACUUM) && !IsAutoVacuumWorkerProcess()) { /* - * Update pg_database.datfrozenxid, and truncate pg_clog if possible. + * Update pg_database.datfrozenxid, and truncate pg_xact if possible. * (autovacuum.c does this for itself.) */ vac_update_datfrozenxid(); @@ -910,7 +910,7 @@ vac_update_relstats(Relation relation, * pg_class.relminmxid values. * * If we are able to advance either pg_database value, also try to - * truncate pg_clog and pg_multixact. + * truncate pg_xact and pg_multixact. * * We violate transaction semantics here by overwriting the database's * existing pg_database tuple with the new values. This is reasonably @@ -1056,7 +1056,7 @@ vac_update_datfrozenxid(void) /* * If we were able to advance datfrozenxid or datminmxid, see if we can - * truncate pg_clog and/or pg_multixact. Also do it if the shared + * truncate pg_xact and/or pg_multixact. Also do it if the shared * XID-wrap-limit info is stale, since this action will update that too. */ if (dirty || ForceTransactionIdLimitUpdate()) @@ -1069,7 +1069,7 @@ vac_update_datfrozenxid(void) * vac_truncate_clog() -- attempt to truncate the commit log * * Scan pg_database to determine the system-wide oldest datfrozenxid, - * and use it to truncate the transaction commit log (pg_clog). + * and use it to truncate the transaction commit log (pg_xact). * Also update the XID wrap limit info maintained by varsup.c. * Likewise for datminmxid. * @@ -1116,7 +1116,7 @@ vac_truncate_clog(TransactionId frozenXID, * of the interlock against copying a DB containing an active backend. * Hence the new entry will not reduce the minimum. Also, if two VACUUMs * concurrently modify the datfrozenxid's of different databases, the - * worst possible outcome is that pg_clog is not truncated as aggressively + * worst possible outcome is that pg_xact is not truncated as aggressively * as it could be. */ relation = heap_open(DatabaseRelationId, AccessShareLock); diff --git a/src/backend/postmaster/autovacuum.c b/src/backend/postmaster/autovacuum.c index fa8de1390e..33ca749b52 100644 --- a/src/backend/postmaster/autovacuum.c +++ b/src/backend/postmaster/autovacuum.c @@ -2472,7 +2472,7 @@ deleted: */ /* - * Update pg_database.datfrozenxid, and truncate pg_clog if possible. We + * Update pg_database.datfrozenxid, and truncate pg_xact if possible. We * only need to do this once, not after each table. * * Even if we didn't vacuum anything, it may still be important to do diff --git a/src/backend/storage/buffer/README b/src/backend/storage/buffer/README index 248883f0da..bc80777bb1 100644 --- a/src/backend/storage/buffer/README +++ b/src/backend/storage/buffer/README @@ -63,7 +63,7 @@ at about the same time would OR the same bits into the field, so there is little or no risk of conflicting update; what's more, if there did manage to be a conflict it would merely mean that one bit-update would be lost and need to be done again later. These four bits are only hints -(they cache the results of transaction status lookups in pg_clog), so no +(they cache the results of transaction status lookups in pg_xact), so no great harm is done if they get reset to zero by conflicting updates. Note, however, that a tuple is frozen by setting both HEAP_XMIN_INVALID and HEAP_XMIN_COMMITTED; this is a critical update and accordingly requires diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c index c724a0e952..0f8f435faf 100644 --- a/src/backend/storage/ipc/procarray.c +++ b/src/backend/storage/ipc/procarray.c @@ -381,7 +381,7 @@ ProcArrayRemove(PGPROC *proc, TransactionId latestXid) * ProcArrayEndTransaction -- mark a transaction as no longer running * * This is used interchangeably for commit and abort cases. The transaction - * commit/abort must already be reported to WAL and pg_clog. + * commit/abort must already be reported to WAL and pg_xact. * * proc is currently always MyProc, but we pass it explicitly for flexibility. * latestXid is the latest Xid among the transaction's main XID and @@ -1174,7 +1174,7 @@ TransactionIdIsInProgress(TransactionId xid) * At this point, we know it's either a subtransaction of one of the Xids * in xids[], or it's not running. If it's an already-failed * subtransaction, we want to say "not running" even though its parent may - * still be running. So first, check pg_clog to see if it's been aborted. + * still be running. So first, check pg_xact to see if it's been aborted. */ xc_slow_answer_inc(); diff --git a/src/backend/utils/time/tqual.c b/src/backend/utils/time/tqual.c index 703bdcedaf..519f3b6803 100644 --- a/src/backend/utils/time/tqual.c +++ b/src/backend/utils/time/tqual.c @@ -13,9 +13,9 @@ * NOTE: When using a non-MVCC snapshot, we must check * TransactionIdIsInProgress (which looks in the PGXACT array) * before TransactionIdDidCommit/TransactionIdDidAbort (which look in - * pg_clog). Otherwise we have a race condition: we might decide that a + * pg_xact). Otherwise we have a race condition: we might decide that a * just-committed transaction crashed, because none of the tests succeed. - * xact.c is careful to record commit/abort in pg_clog before it unsets + * xact.c is careful to record commit/abort in pg_xact before it unsets * MyPgXact->xid in the PGXACT array. That fixes that problem, but it * also means there is a window where TransactionIdIsInProgress and * TransactionIdDidCommit will both return true. If we check only @@ -29,7 +29,7 @@ * * When using an MVCC snapshot, we rely on XidInMVCCSnapshot rather than * TransactionIdIsInProgress, but the logic is otherwise the same: do not - * check pg_clog until after deciding that the xact is no longer in progress. + * check pg_xact until after deciding that the xact is no longer in progress. * * * Summary of visibility functions: diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index 9b71e0042d..e0c72fbb80 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -197,7 +197,6 @@ static const char *backend_options = "--single -F -O -j -c search_path=pg_catalo static const char *const subdirs[] = { "global", "pg_wal/archive_status", - "pg_clog", "pg_commit_ts", "pg_dynshmem", "pg_notify", @@ -214,6 +213,7 @@ static const char *const subdirs[] = { "pg_tblspc", "pg_stat", "pg_stat_tmp", + "pg_xact", "pg_logical", "pg_logical/snapshots", "pg_logical/mappings" diff --git a/src/bin/pg_upgrade/exec.c b/src/bin/pg_upgrade/exec.c index 91fa71728e..cdce4be211 100644 --- a/src/bin/pg_upgrade/exec.c +++ b/src/bin/pg_upgrade/exec.c @@ -338,11 +338,17 @@ check_data_dir(ClusterInfo *cluster) check_single_dir(pg_data, "pg_tblspc"); check_single_dir(pg_data, "pg_twophase"); - /* pg_xlog has been renamed to pg_wal in post-10 cluster */ + /* pg_xlog has been renamed to pg_wal in v10 */ if (GET_MAJOR_VERSION(cluster->major_version) < 1000) check_single_dir(pg_data, "pg_xlog"); else check_single_dir(pg_data, "pg_wal"); + + /* pg_clog has been renamed to pg_xact in v10 */ + if (GET_MAJOR_VERSION(cluster->major_version) < 1000) + check_single_dir(pg_data, "pg_clog"); + else + check_single_dir(pg_data, "pg_xact"); } diff --git a/src/bin/pg_upgrade/pg_upgrade.c b/src/bin/pg_upgrade/pg_upgrade.c index fc66d93dd6..ca1aa5cbb8 100644 --- a/src/bin/pg_upgrade/pg_upgrade.c +++ b/src/bin/pg_upgrade/pg_upgrade.c @@ -48,7 +48,7 @@ static void prepare_new_cluster(void); static void prepare_new_databases(void); static void create_new_objects(void); -static void copy_clog_xlog_xid(void); +static void copy_xact_xlog_xid(void); static void set_frozenxids(bool minmxid_only); static void setup(char *argv0, bool *live_check); static void cleanup(void); @@ -115,7 +115,7 @@ main(int argc, char **argv) * Destructive Changes to New Cluster */ - copy_clog_xlog_xid(); + copy_xact_xlog_xid(); /* New now using xids of the old system */ @@ -376,17 +376,17 @@ remove_new_subdir(char *subdir, bool rmtopdir) * Copy the files from the old cluster into it */ static void -copy_subdir_files(char *subdir) +copy_subdir_files(char *old_subdir, char *new_subdir) { char old_path[MAXPGPATH]; char new_path[MAXPGPATH]; - remove_new_subdir(subdir, true); + remove_new_subdir(new_subdir, true); - snprintf(old_path, sizeof(old_path), "%s/%s", old_cluster.pgdata, subdir); - snprintf(new_path, sizeof(new_path), "%s/%s", new_cluster.pgdata, subdir); + snprintf(old_path, sizeof(old_path), "%s/%s", old_cluster.pgdata, old_subdir); + snprintf(new_path, sizeof(new_path), "%s/%s", new_cluster.pgdata, new_subdir); - prep_status("Copying old %s to new server", subdir); + prep_status("Copying old %s to new server", old_subdir); exec_prog(UTILITY_LOG_FILE, NULL, true, #ifndef WIN32 @@ -401,10 +401,16 @@ copy_subdir_files(char *subdir) } static void -copy_clog_xlog_xid(void) +copy_xact_xlog_xid(void) { - /* copy old commit logs to new data dir */ - copy_subdir_files("pg_clog"); + /* + * Copy old commit logs to new data dir. pg_clog has been renamed to + * pg_xact in post-10 clusters. + */ + copy_subdir_files(GET_MAJOR_VERSION(old_cluster.major_version) < 1000 ? + "pg_clog" : "pg_xact", + GET_MAJOR_VERSION(new_cluster.major_version) < 1000 ? + "pg_clog" : "pg_xact"); /* set the next transaction id and epoch of the new cluster */ prep_status("Setting next transaction ID and epoch for new cluster"); @@ -434,8 +440,8 @@ copy_clog_xlog_xid(void) if (old_cluster.controldata.cat_ver >= MULTIXACT_FORMATCHANGE_CAT_VER && new_cluster.controldata.cat_ver >= MULTIXACT_FORMATCHANGE_CAT_VER) { - copy_subdir_files("pg_multixact/offsets"); - copy_subdir_files("pg_multixact/members"); + copy_subdir_files("pg_multixact/offsets", "pg_multixact/offsets"); + copy_subdir_files("pg_multixact/members", "pg_multixact/members"); prep_status("Setting next multixact ID and offset for new cluster"); diff --git a/src/include/access/slru.h b/src/include/access/slru.h index cf6edc3bdb..722867d5d2 100644 --- a/src/include/access/slru.h +++ b/src/include/access/slru.h @@ -76,7 +76,7 @@ typedef struct SlruSharedData /* * Optional array of WAL flush LSNs associated with entries in the SLRU * pages. If not zero/NULL, we must flush WAL before writing pages (true - * for pg_clog, false for multixact, pg_subtrans, pg_notify). group_lsn[] + * for pg_xact, false for multixact, pg_subtrans, pg_notify). group_lsn[] * has lsn_groups_per_page entries per buffer slot, each containing the * highest LSN known for a contiguous group of SLRU entries on that slot's * page. @@ -119,7 +119,7 @@ typedef struct SlruCtlData SlruShared shared; /* - * This flag tells whether to fsync writes (true for pg_clog and multixact + * This flag tells whether to fsync writes (true for pg_xact and multixact * stuff, false for pg_subtrans and pg_notify). */ bool do_fsync; -- cgit v1.2.3 From fef2bcdcba0888c95ddf2a7535179c3b9a6a2f0e Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 17 Mar 2017 09:49:10 -0400 Subject: pageinspect: Add page_checksum function Author: Tomas Vondra Reviewed-by: Ashutosh Sharma --- contrib/pageinspect/expected/page.out | 6 ++++ contrib/pageinspect/pageinspect--1.5--1.6.sql | 8 +++++ contrib/pageinspect/rawpage.c | 37 +++++++++++++++++++++ contrib/pageinspect/sql/page.sql | 2 ++ doc/src/sgml/pageinspect.sgml | 47 +++++++++++++++++++++++++-- 5 files changed, 98 insertions(+), 2 deletions(-) (limited to 'doc/src') diff --git a/contrib/pageinspect/expected/page.out b/contrib/pageinspect/expected/page.out index 08599af774..8e15947a81 100644 --- a/contrib/pageinspect/expected/page.out +++ b/contrib/pageinspect/expected/page.out @@ -49,6 +49,12 @@ SELECT pagesize, version FROM page_header(get_raw_page('test1', 0)); 8192 | 4 (1 row) +SELECT page_checksum(get_raw_page('test1', 0), 0) IS NOT NULL AS silly_checksum_test; + silly_checksum_test +--------------------- + t +(1 row) + SELECT tuple_data_split('test1'::regclass, t_data, t_infomask, t_infomask2, t_bits) FROM heap_page_items(get_raw_page('test1', 0)); tuple_data_split diff --git a/contrib/pageinspect/pageinspect--1.5--1.6.sql b/contrib/pageinspect/pageinspect--1.5--1.6.sql index ac3956882c..6ac2a8011d 100644 --- a/contrib/pageinspect/pageinspect--1.5--1.6.sql +++ b/contrib/pageinspect/pageinspect--1.5--1.6.sql @@ -75,3 +75,11 @@ CREATE FUNCTION hash_metapage_info(IN page bytea, OUT mapp int8[]) AS 'MODULE_PATHNAME', 'hash_metapage_info' LANGUAGE C STRICT PARALLEL SAFE; + +-- +-- page_checksum() +-- +CREATE FUNCTION page_checksum(IN page bytea, IN blkno int4) +RETURNS smallint +AS 'MODULE_PATHNAME', 'page_checksum' +LANGUAGE C STRICT PARALLEL SAFE; diff --git a/contrib/pageinspect/rawpage.c b/contrib/pageinspect/rawpage.c index a5def91751..631e435a93 100644 --- a/contrib/pageinspect/rawpage.c +++ b/contrib/pageinspect/rawpage.c @@ -24,6 +24,7 @@ #include "funcapi.h" #include "miscadmin.h" #include "storage/bufmgr.h" +#include "storage/checksum.h" #include "utils/builtins.h" #include "utils/pg_lsn.h" #include "utils/rel.h" @@ -280,3 +281,39 @@ page_header(PG_FUNCTION_ARGS) PG_RETURN_DATUM(result); } + +/* + * page_checksum + * + * Compute checksum of a raw page + */ + +PG_FUNCTION_INFO_V1(page_checksum); + +Datum +page_checksum(PG_FUNCTION_ARGS) +{ + bytea *raw_page = PG_GETARG_BYTEA_P(0); + uint32 blkno = PG_GETARG_INT32(1); + int raw_page_size; + PageHeader page; + + if (!superuser()) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + (errmsg("must be superuser to use raw page functions")))); + + raw_page_size = VARSIZE(raw_page) - VARHDRSZ; + + /* + * Check that the supplied page is of the right size. + */ + if (raw_page_size != BLCKSZ) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("incorrect size of input page (%d bytes)", raw_page_size))); + + page = (PageHeader) VARDATA(raw_page); + + PG_RETURN_INT16(pg_checksum_page((char *)page, blkno)); +} diff --git a/contrib/pageinspect/sql/page.sql b/contrib/pageinspect/sql/page.sql index 53e9f8022d..493ca9b211 100644 --- a/contrib/pageinspect/sql/page.sql +++ b/contrib/pageinspect/sql/page.sql @@ -24,6 +24,8 @@ SELECT get_raw_page('test1', 0) = get_raw_page('test1', 'main', 0); SELECT pagesize, version FROM page_header(get_raw_page('test1', 0)); +SELECT page_checksum(get_raw_page('test1', 0), 0) IS NOT NULL AS silly_checksum_test; + SELECT tuple_data_split('test1'::regclass, t_data, t_infomask, t_infomask2, t_bits) FROM heap_page_items(get_raw_page('test1', 0)); diff --git a/doc/src/sgml/pageinspect.sgml b/doc/src/sgml/pageinspect.sgml index 5e6712f9cd..9f41bb21eb 100644 --- a/doc/src/sgml/pageinspect.sgml +++ b/doc/src/sgml/pageinspect.sgml @@ -73,12 +73,55 @@ test=# SELECT * FROM page_header(get_raw_page('pg_class', 0)); lsn | checksum | flags | lower | upper | special | pagesize | version | prune_xid -----------+----------+--------+-------+-------+---------+----------+---------+----------- - 0/24A1B50 | 1 | 1 | 232 | 368 | 8192 | 8192 | 4 | 0 + 0/24A1B50 | 0 | 1 | 232 | 368 | 8192 | 8192 | 4 | 0 The returned columns correspond to the fields in the PageHeaderData struct. See src/include/storage/bufpage.h for details. - + + + + The checksum field is the checksum stored in + the page, which might be incorrect if the page is somehow corrupted. If + data checksums are not enabled for this instance, then the value stored + is meaningless. + + +     + + + + page_checksum(page bytea, blkno int4) returns smallint + + page_checksum + + + + + + page_checksum computes the checksum for the page, as if + it was located at the given block. + + + + A page image obtained with get_raw_page should be + passed as argument. For example: + +test=# SELECT page_checksum(get_raw_page('pg_class', 0), 0); + page_checksum +--------------- + 13443 + + Note that the checksum depends on the block number, so matching block + numbers should be passed (except when doing esoteric debugging). + + + + The checksum computed with this function can be compared with + the checksum result field of the + function page_header. If data checksums are + enabled for this instance, then the two values should be equal. + -- cgit v1.2.3 From 249cf070e36721a65be74838c53acf8249faf935 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Sat, 18 Mar 2017 07:43:01 -0400 Subject: Create and use wait events for read, write, and fsync operations. Previous commits, notably 53be0b1add7064ca5db3cd884302dfc3268d884e and 6f3bd98ebfc008cbd676da777bb0b2376c4c4bfa, made it possible to see from pg_stat_activity when a backend was stuck waiting for another backend, but it's also fairly common for a backend to be stuck waiting for an I/O. Add wait events for those operations, too. Rushabh Lathia, with further hacking by me. Reviewed and tested by Michael Paquier, Amit Kapila, Rajkumar Raghuwanshi, and Rahila Syed. Discussion: http://postgr.es/m/CAGPqQf0LsYHXREPAZqYGVkDqHSyjf=KsD=k0GTVPAuzyThh-VQ@mail.gmail.com --- doc/src/sgml/monitoring.sgml | 271 ++++++++++++++++++++++++ src/backend/access/heap/rewriteheap.c | 17 +- src/backend/access/transam/slru.c | 12 ++ src/backend/access/transam/timeline.c | 11 + src/backend/access/transam/twophase.c | 9 + src/backend/access/transam/xlog.c | 31 +++ src/backend/access/transam/xlogutils.c | 3 + src/backend/postmaster/pgstat.c | 233 ++++++++++++++++++++ src/backend/replication/logical/reorderbuffer.c | 9 + src/backend/replication/logical/snapbuild.c | 14 ++ src/backend/replication/slot.c | 13 ++ src/backend/replication/walsender.c | 4 + src/backend/storage/file/buffile.c | 11 +- src/backend/storage/file/copydir.c | 7 +- src/backend/storage/file/fd.c | 28 ++- src/backend/storage/ipc/dsm_impl.c | 3 + src/backend/storage/smgr/md.c | 21 +- src/backend/utils/cache/relmapper.c | 7 + src/backend/utils/init/miscinit.c | 17 ++ src/include/pgstat.h | 78 +++++++ src/include/storage/fd.h | 12 +- 21 files changed, 782 insertions(+), 29 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 9eaf43adb1..9b50fb9c13 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -716,6 +716,12 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser point. + + + IO: The server process is waiting for a IO to complete. + wait_event will identify the specific wait point. + + @@ -1272,6 +1278,271 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser RecoveryApplyDelay Waiting to apply WAL at recovery because it is delayed. + + IO + BufFileRead + Waiting for a read from a buffered file. + + + BufFileWrite + Waiting for a write to a buffered file. + + + ControlFileRead + Waiting for a read from the control file. + + + ControlFileSync + Waiting for the control file to reach stable storage. + + + ControlFileSyncUpdate + Waiting for an update to the control file to reach stable storage. + + + ControlFileWrite + Waiting for a write to the control file. + + + ControlFileWriteUpdate + Waiting for a write to update the control file. + + + CopyFileRead + Waiting for a read during a file copy operation. + + + CopyFileWrite + Waiting for a write during a file copy operation. + + + DataFileExtend + Waiting for a relation data file to be extended. + + + DataFileFlush + Waiting for a relation data file to reach stable storage. + + + DataFileImmediateSync + Waiting for an immediate synchronization of a relation data file to stable storage. + + + DataFilePrefetch + Waiting for an asynchronous prefetch from a relation data file. + + + DataFileRead + Waiting for a read from a relation data file. + + + DataFileSync + Waiting for changes to a relation data file to reach stable storage. + + + DataFileTruncate + Waiting for a relation data file to be truncated. + + + DataFileWrite + Waiting for a write to a relation data file. + + + DSMFillZeroWrite + Waiting to write zero bytes to a dynamic shared memory backing file. + + + LockFileAddToDataDirRead + Waiting for a read while adding a line to the data directory lock file. + + + LockFileAddToDataDirSync + Waiting for data to reach stable storage while adding a line to the data directory lock file. + + + LockFileAddToDataDirWrite + Waiting for a write while adding a line to the data directory lock file. + + + LockFileCreateRead + Waiting to read while creating the data directory lock file. + + + LockFileCreateSync + Waiting for data to reach stable storage while creating the data directory lock file. + + + LockFileCreateWrite + Waiting for a write while creating the data directory lock file. + + + LockFileReCheckDataDirRead + Waiting for a read during recheck of the data directory lock file. + + + LogicalRewriteCheckpointSync + Waiting for logical rewrite mappings to reach stable storage during a checkpoint. + + + LogicalRewriteMappingSync + Waiting for mapping data to reach stable storage during a logical rewrite. + + + LogicalRewriteMappingWrite + Waiting for a write of mapping data during a logical rewrite. + + + LogicalRewriteSync + Waiting for logical rewrite mappings to reach stable storage. + + + LogicalRewriteWrite + Waiting for a write of logical rewrite mappings. + + + RelationMapRead + Waiting for a read of the relation map file. + + + RelationMapSync + Waiting for the relation map file to reach stable storage. + + + RelationMapWrite + Waiting for a write to the relation map file. + + + ReorderBufferRead + Waiting for a read during reorder buffer management. + + + ReorderBufferWrite + Waiting for a write during reorder buffer management. + + + ReorderLogicalMappingRead + Waiting for a read of a logical mapping during reorder buffer management. + + + ReplicationSlotRead + Waiting for a read from a replication slot control file. + + + ReplicationSlotRestoreSync + Waiting for a replication slot control file to reach stable storage while restoring it to memory. + + + ReplicationSlotSync + Waiting for a replication slot control file to reach stable storage. + + + ReplicationSlotWrite + Waiting for a write to a replication slot control file. + + + SLRUFlushSync + Waiting for SLRU data to reach stable storage during a checkpoint or database shutdown. + + + SLRURead + Waiting for a read of an SLRU page. + + + SLRUSync + Waiting for SLRU data to reach stable storage following a page write. + + + SLRUWrite + Waiting for a write of an SLRU page. + + + SnapbuildRead + Waiting for a read of a serialized historical catalog snapshot. + + + SnapbuildSync + Waiting for a serialized historical catalog snapshot to reach stable storage. + + + SnapbuildWrite + Waiting for a write of a serialized historical catalog snapshot. + + + TimelineHistoryFileSync + Waiting for a timeline history file received via streaming replication to reach stable storage. + + + TimelineHistoryFileWrite + Waiting for a write of a timeline history file received via streaming replication. + + + TimelineHistoryRead + Waiting for a read of a timeline history file. + + + TimelineHistorySync + Waiting for a newly created timeline history file to reach stable storage. + + + TimelineHistoryWrite + Waiting for a write of a newly created timeline history file. + + + TwophaseFileRead + Waiting for a read of a two phase state file. + + + TwophaseFileSync + Waiting for a two phase state file to reach stable storage. + + + TwophaseFileWrite + Waiting for a write of a two phase state file. + + + WALBootstrapSync + Waiting for WAL to reach stable storage during bootstrapping. + + + WALBootstrapWrite + Waiting for a write of a WAL page during bootstrapping. + + + WALCopyRead + Waiting for a read when creating a new WAL segment by copying an existing one. + + + WALCopySync + Waiting a new WAL segment created by copying an existing one to reach stable storage. + + + WALCopyWrite + Waiting for a write when creating a new WAL segment by copying an existing one. + + + WALInitSync + Waiting for a newly initialized WAL file to reach stable storage. + + + WALInitWrite + Waiting for a write while initializing a new WAL file. + + + WALRead + Waiting for a read from a WAL file. + + + WALSenderTimelineHistoryRead + Waiting for a read from a timeline history file during walsender timeline command. + + + WALSyncMethodAssign + Waiting for data to reach stable storage while assigning WAL sync method. + + + WALWrite + Waiting for a write to a WAL file. + diff --git a/src/backend/access/heap/rewriteheap.c b/src/backend/access/heap/rewriteheap.c index c7b283c198..d7f65a5e99 100644 --- a/src/backend/access/heap/rewriteheap.c +++ b/src/backend/access/heap/rewriteheap.c @@ -119,6 +119,8 @@ #include "lib/ilist.h" +#include "pgstat.h" + #include "replication/logical.h" #include "replication/slot.h" @@ -916,7 +918,8 @@ logical_heap_rewrite_flush_mappings(RewriteState state) * Note that we deviate from the usual WAL coding practices here, * check the above "Logical rewrite support" comment for reasoning. */ - written = FileWrite(src->vfd, waldata_start, len); + written = FileWrite(src->vfd, waldata_start, len, + WAIT_EVENT_LOGICAL_REWRITE_WRITE); if (written != len) ereport(ERROR, (errcode_for_file_access(), @@ -957,7 +960,7 @@ logical_end_heap_rewrite(RewriteState state) hash_seq_init(&seq_status, state->rs_logical_mappings); while ((src = (RewriteMappingFile *) hash_seq_search(&seq_status)) != NULL) { - if (FileSync(src->vfd) != 0) + if (FileSync(src->vfd, WAIT_EVENT_LOGICAL_REWRITE_SYNC) != 0) ereport(ERROR, (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", src->path))); @@ -1141,11 +1144,13 @@ heap_xlog_logical_rewrite(XLogReaderState *r) * Truncate all data that's not guaranteed to have been safely fsynced (by * previous record or by the last checkpoint). */ + pgstat_report_wait_start(WAIT_EVENT_LOGICAL_REWRITE_TRUNCATE); if (ftruncate(fd, xlrec->offset) != 0) ereport(ERROR, (errcode_for_file_access(), errmsg("could not truncate file \"%s\" to %u: %m", path, (uint32) xlrec->offset))); + pgstat_report_wait_end(); /* now seek to the position we want to write our data to */ if (lseek(fd, xlrec->offset, SEEK_SET) != xlrec->offset) @@ -1159,20 +1164,24 @@ heap_xlog_logical_rewrite(XLogReaderState *r) len = xlrec->num_mappings * sizeof(LogicalRewriteMappingData); /* write out tail end of mapping file (again) */ + pgstat_report_wait_start(WAIT_EVENT_LOGICAL_REWRITE_MAPPING_WRITE); if (write(fd, data, len) != len) ereport(ERROR, (errcode_for_file_access(), errmsg("could not write to file \"%s\": %m", path))); + pgstat_report_wait_end(); /* * Now fsync all previously written data. We could improve things and only * do this for the last write to a file, but the required bookkeeping * doesn't seem worth the trouble. */ + pgstat_report_wait_start(WAIT_EVENT_LOGICAL_REWRITE_MAPPING_SYNC); if (pg_fsync(fd) != 0) ereport(ERROR, (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", path))); + pgstat_report_wait_end(); CloseTransientFile(fd); } @@ -1266,10 +1275,12 @@ CheckPointLogicalRewriteHeap(void) * changed or have only been created since the checkpoint's start, * but it's currently not deemed worth the effort. */ - else if (pg_fsync(fd) != 0) + pgstat_report_wait_start(WAIT_EVENT_LOGICAL_REWRITE_CHECKPOINT_SYNC); + if (pg_fsync(fd) != 0) ereport(ERROR, (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", path))); + pgstat_report_wait_end(); CloseTransientFile(fd); } } diff --git a/src/backend/access/transam/slru.c b/src/backend/access/transam/slru.c index a66ef5c639..7ae783102a 100644 --- a/src/backend/access/transam/slru.c +++ b/src/backend/access/transam/slru.c @@ -54,6 +54,7 @@ #include "access/slru.h" #include "access/transam.h" #include "access/xlog.h" +#include "pgstat.h" #include "storage/fd.h" #include "storage/shmem.h" #include "miscadmin.h" @@ -675,13 +676,16 @@ SlruPhysicalReadPage(SlruCtl ctl, int pageno, int slotno) } errno = 0; + pgstat_report_wait_start(WAIT_EVENT_SLRU_READ); if (read(fd, shared->page_buffer[slotno], BLCKSZ) != BLCKSZ) { + pgstat_report_wait_end(); slru_errcause = SLRU_READ_FAILED; slru_errno = errno; CloseTransientFile(fd); return false; } + pgstat_report_wait_end(); if (CloseTransientFile(fd)) { @@ -834,8 +838,10 @@ SlruPhysicalWritePage(SlruCtl ctl, int pageno, int slotno, SlruFlush fdata) } errno = 0; + pgstat_report_wait_start(WAIT_EVENT_SLRU_WRITE); if (write(fd, shared->page_buffer[slotno], BLCKSZ) != BLCKSZ) { + pgstat_report_wait_end(); /* if write didn't set errno, assume problem is no disk space */ if (errno == 0) errno = ENOSPC; @@ -845,6 +851,7 @@ SlruPhysicalWritePage(SlruCtl ctl, int pageno, int slotno, SlruFlush fdata) CloseTransientFile(fd); return false; } + pgstat_report_wait_end(); /* * If not part of Flush, need to fsync now. We assume this happens @@ -852,13 +859,16 @@ SlruPhysicalWritePage(SlruCtl ctl, int pageno, int slotno, SlruFlush fdata) */ if (!fdata) { + pgstat_report_wait_start(WAIT_EVENT_SLRU_SYNC); if (ctl->do_fsync && pg_fsync(fd)) { + pgstat_report_wait_end(); slru_errcause = SLRU_FSYNC_FAILED; slru_errno = errno; CloseTransientFile(fd); return false; } + pgstat_report_wait_end(); if (CloseTransientFile(fd)) { @@ -1126,6 +1136,7 @@ SimpleLruFlush(SlruCtl ctl, bool allow_redirtied) ok = true; for (i = 0; i < fdata.num_files; i++) { + pgstat_report_wait_start(WAIT_EVENT_SLRU_FLUSH_SYNC); if (ctl->do_fsync && pg_fsync(fdata.fd[i])) { slru_errcause = SLRU_FSYNC_FAILED; @@ -1133,6 +1144,7 @@ SimpleLruFlush(SlruCtl ctl, bool allow_redirtied) pageno = fdata.segno[i] * SLRU_PAGES_PER_SEGMENT; ok = false; } + pgstat_report_wait_end(); if (CloseTransientFile(fdata.fd[i])) { diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index 1fdc59190c..a11f0f8526 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -38,6 +38,7 @@ #include "access/xlog.h" #include "access/xlog_internal.h" #include "access/xlogdefs.h" +#include "pgstat.h" #include "storage/fd.h" /* @@ -338,7 +339,9 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, for (;;) { errno = 0; + pgstat_report_wait_start(WAIT_EVENT_TIMELINE_HISTORY_READ); nbytes = (int) read(srcfd, buffer, sizeof(buffer)); + pgstat_report_wait_end(); if (nbytes < 0 || errno != 0) ereport(ERROR, (errcode_for_file_access(), @@ -346,6 +349,7 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, if (nbytes == 0) break; errno = 0; + pgstat_report_wait_start(WAIT_EVENT_TIMELINE_HISTORY_WRITE); if ((int) write(fd, buffer, nbytes) != nbytes) { int save_errno = errno; @@ -365,6 +369,7 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, (errcode_for_file_access(), errmsg("could not write to file \"%s\": %m", tmppath))); } + pgstat_report_wait_end(); } CloseTransientFile(srcfd); } @@ -400,10 +405,12 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, errmsg("could not write to file \"%s\": %m", tmppath))); } + pgstat_report_wait_start(WAIT_EVENT_TIMELINE_HISTORY_SYNC); if (pg_fsync(fd) != 0) ereport(ERROR, (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", tmppath))); + pgstat_report_wait_end(); if (CloseTransientFile(fd)) ereport(ERROR, @@ -460,6 +467,7 @@ writeTimeLineHistoryFile(TimeLineID tli, char *content, int size) errmsg("could not create file \"%s\": %m", tmppath))); errno = 0; + pgstat_report_wait_start(WAIT_EVENT_TIMELINE_HISTORY_FILE_WRITE); if ((int) write(fd, content, size) != size) { int save_errno = errno; @@ -475,11 +483,14 @@ writeTimeLineHistoryFile(TimeLineID tli, char *content, int size) (errcode_for_file_access(), errmsg("could not write to file \"%s\": %m", tmppath))); } + pgstat_report_wait_end(); + pgstat_report_wait_start(WAIT_EVENT_TIMELINE_HISTORY_FILE_SYNC); if (pg_fsync(fd) != 0) ereport(ERROR, (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", tmppath))); + pgstat_report_wait_end(); if (CloseTransientFile(fd)) ereport(ERROR, diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index da77e3efde..f09941d0ec 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -1200,8 +1200,10 @@ ReadTwoPhaseFile(TransactionId xid, bool give_warnings) */ buf = (char *) palloc(stat.st_size); + pgstat_report_wait_start(WAIT_EVENT_TWOPHASE_FILE_READ); if (read(fd, buf, stat.st_size) != stat.st_size) { + pgstat_report_wait_end(); CloseTransientFile(fd); if (give_warnings) ereport(WARNING, @@ -1212,6 +1214,7 @@ ReadTwoPhaseFile(TransactionId xid, bool give_warnings) return NULL; } + pgstat_report_wait_end(); CloseTransientFile(fd); hdr = (TwoPhaseFileHeader *) buf; @@ -1542,8 +1545,10 @@ RecreateTwoPhaseFile(TransactionId xid, void *content, int len) path))); /* Write content and CRC */ + pgstat_report_wait_start(WAIT_EVENT_TWOPHASE_FILE_WRITE); if (write(fd, content, len) != len) { + pgstat_report_wait_end(); CloseTransientFile(fd); ereport(ERROR, (errcode_for_file_access(), @@ -1551,16 +1556,19 @@ RecreateTwoPhaseFile(TransactionId xid, void *content, int len) } if (write(fd, &statefile_crc, sizeof(pg_crc32c)) != sizeof(pg_crc32c)) { + pgstat_report_wait_end(); CloseTransientFile(fd); ereport(ERROR, (errcode_for_file_access(), errmsg("could not write two-phase state file: %m"))); } + pgstat_report_wait_end(); /* * We must fsync the file because the end-of-replay checkpoint will not do * so, there being no GXACT in shared memory yet to tell it to. */ + pgstat_report_wait_start(WAIT_EVENT_TWOPHASE_FILE_SYNC); if (pg_fsync(fd) != 0) { CloseTransientFile(fd); @@ -1568,6 +1576,7 @@ RecreateTwoPhaseFile(TransactionId xid, void *content, int len) (errcode_for_file_access(), errmsg("could not fsync two-phase state file: %m"))); } + pgstat_report_wait_end(); if (CloseTransientFile(fd) != 0) ereport(ERROR, diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index cdb3a8ac1d..9480377611 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -2456,7 +2456,9 @@ XLogWrite(XLogwrtRqst WriteRqst, bool flexible) do { errno = 0; + pgstat_report_wait_start(WAIT_EVENT_WAL_WRITE); written = write(openLogFile, from, nleft); + pgstat_report_wait_end(); if (written <= 0) { if (errno == EINTR) @@ -3207,6 +3209,7 @@ XLogFileInit(XLogSegNo logsegno, bool *use_existent, bool use_lock) for (nbytes = 0; nbytes < XLogSegSize; nbytes += XLOG_BLCKSZ) { errno = 0; + pgstat_report_wait_start(WAIT_EVENT_WAL_INIT_WRITE); if ((int) write(fd, zbuffer, XLOG_BLCKSZ) != (int) XLOG_BLCKSZ) { int save_errno = errno; @@ -3225,8 +3228,10 @@ XLogFileInit(XLogSegNo logsegno, bool *use_existent, bool use_lock) (errcode_for_file_access(), errmsg("could not write to file \"%s\": %m", tmppath))); } + pgstat_report_wait_end(); } + pgstat_report_wait_start(WAIT_EVENT_WAL_INIT_SYNC); if (pg_fsync(fd) != 0) { close(fd); @@ -3234,6 +3239,7 @@ XLogFileInit(XLogSegNo logsegno, bool *use_existent, bool use_lock) (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", tmppath))); } + pgstat_report_wait_end(); if (close(fd)) ereport(ERROR, @@ -3360,6 +3366,7 @@ XLogFileCopy(XLogSegNo destsegno, TimeLineID srcTLI, XLogSegNo srcsegno, if (nread > sizeof(buffer)) nread = sizeof(buffer); errno = 0; + pgstat_report_wait_start(WAIT_EVENT_WAL_COPY_READ); if (read(srcfd, buffer, nread) != nread) { if (errno != 0) @@ -3372,8 +3379,10 @@ XLogFileCopy(XLogSegNo destsegno, TimeLineID srcTLI, XLogSegNo srcsegno, (errmsg("not enough data in file \"%s\"", path))); } + pgstat_report_wait_end(); } errno = 0; + pgstat_report_wait_start(WAIT_EVENT_WAL_COPY_WRITE); if ((int) write(fd, buffer, sizeof(buffer)) != (int) sizeof(buffer)) { int save_errno = errno; @@ -3389,12 +3398,15 @@ XLogFileCopy(XLogSegNo destsegno, TimeLineID srcTLI, XLogSegNo srcsegno, (errcode_for_file_access(), errmsg("could not write to file \"%s\": %m", tmppath))); } + pgstat_report_wait_end(); } + pgstat_report_wait_start(WAIT_EVENT_WAL_COPY_SYNC); if (pg_fsync(fd) != 0) ereport(ERROR, (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", tmppath))); + pgstat_report_wait_end(); if (CloseTransientFile(fd)) ereport(ERROR, @@ -4414,6 +4426,7 @@ WriteControlFile(void) XLOG_CONTROL_FILE))); errno = 0; + pgstat_report_wait_start(WAIT_EVENT_CONTROL_FILE_WRITE); if (write(fd, buffer, PG_CONTROL_SIZE) != PG_CONTROL_SIZE) { /* if write didn't set errno, assume problem is no disk space */ @@ -4423,11 +4436,14 @@ WriteControlFile(void) (errcode_for_file_access(), errmsg("could not write to control file: %m"))); } + pgstat_report_wait_end(); + pgstat_report_wait_start(WAIT_EVENT_CONTROL_FILE_SYNC); if (pg_fsync(fd) != 0) ereport(PANIC, (errcode_for_file_access(), errmsg("could not fsync control file: %m"))); + pgstat_report_wait_end(); if (close(fd)) ereport(PANIC, @@ -4453,10 +4469,12 @@ ReadControlFile(void) errmsg("could not open control file \"%s\": %m", XLOG_CONTROL_FILE))); + pgstat_report_wait_start(WAIT_EVENT_CONTROL_FILE_READ); if (read(fd, ControlFile, sizeof(ControlFileData)) != sizeof(ControlFileData)) ereport(PANIC, (errcode_for_file_access(), errmsg("could not read from control file: %m"))); + pgstat_report_wait_end(); close(fd); @@ -4634,6 +4652,7 @@ UpdateControlFile(void) XLOG_CONTROL_FILE))); errno = 0; + pgstat_report_wait_start(WAIT_EVENT_CONTROL_FILE_WRITE_UPDATE); if (write(fd, ControlFile, sizeof(ControlFileData)) != sizeof(ControlFileData)) { /* if write didn't set errno, assume problem is no disk space */ @@ -4643,11 +4662,14 @@ UpdateControlFile(void) (errcode_for_file_access(), errmsg("could not write to control file: %m"))); } + pgstat_report_wait_end(); + pgstat_report_wait_start(WAIT_EVENT_CONTROL_FILE_SYNC_UPDATE); if (pg_fsync(fd) != 0) ereport(PANIC, (errcode_for_file_access(), errmsg("could not fsync control file: %m"))); + pgstat_report_wait_end(); if (close(fd)) ereport(PANIC, @@ -5036,6 +5058,7 @@ BootStrapXLOG(void) /* Write the first page with the initial record */ errno = 0; + pgstat_report_wait_start(WAIT_EVENT_WAL_BOOTSTRAP_WRITE); if (write(openLogFile, page, XLOG_BLCKSZ) != XLOG_BLCKSZ) { /* if write didn't set errno, assume problem is no disk space */ @@ -5045,11 +5068,14 @@ BootStrapXLOG(void) (errcode_for_file_access(), errmsg("could not write bootstrap transaction log file: %m"))); } + pgstat_report_wait_end(); + pgstat_report_wait_start(WAIT_EVENT_WAL_BOOTSTRAP_SYNC); if (pg_fsync(openLogFile) != 0) ereport(PANIC, (errcode_for_file_access(), errmsg("could not fsync bootstrap transaction log file: %m"))); + pgstat_report_wait_end(); if (close(openLogFile)) ereport(PANIC, @@ -9999,11 +10025,13 @@ assign_xlog_sync_method(int new_sync_method, void *extra) */ if (openLogFile >= 0) { + pgstat_report_wait_start(WAIT_EVENT_WAL_SYNC_METHOD_ASSIGN); if (pg_fsync(openLogFile) != 0) ereport(PANIC, (errcode_for_file_access(), errmsg("could not fsync log segment %s: %m", XLogFileNameP(ThisTimeLineID, openLogSegNo)))); + pgstat_report_wait_end(); if (get_sync_bit(sync_method) != get_sync_bit(new_sync_method)) XLogFileClose(); } @@ -11456,10 +11484,12 @@ retry: goto next_record_is_invalid; } + pgstat_report_wait_start(WAIT_EVENT_WAL_READ); if (read(readFile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; + pgstat_report_wait_end(); XLogFileName(fname, curFileTLI, readSegNo); ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), @@ -11467,6 +11497,7 @@ retry: fname, readOff))); goto next_record_is_invalid; } + pgstat_report_wait_end(); Assert(targetSegNo == readSegNo); Assert(targetPageOff == readOff); diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c index 8b99b78249..b2b9fcbebb 100644 --- a/src/backend/access/transam/xlogutils.c +++ b/src/backend/access/transam/xlogutils.c @@ -24,6 +24,7 @@ #include "access/xlogutils.h" #include "catalog/catalog.h" #include "miscadmin.h" +#include "pgstat.h" #include "storage/smgr.h" #include "utils/guc.h" #include "utils/hsearch.h" @@ -728,7 +729,9 @@ XLogRead(char *buf, TimeLineID tli, XLogRecPtr startptr, Size count) else segbytes = nbytes; + pgstat_report_wait_start(WAIT_EVENT_WAL_READ); readbytes = read(sendFile, p, segbytes); + pgstat_report_wait_end(); if (readbytes <= 0) { char path[MAXPGPATH]; diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index 7cacb1e9b2..5fe9f35199 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -280,6 +280,7 @@ static const char *pgstat_get_wait_activity(WaitEventActivity w); static const char *pgstat_get_wait_client(WaitEventClient w); static const char *pgstat_get_wait_ipc(WaitEventIPC w); static const char *pgstat_get_wait_timeout(WaitEventTimeout w); +static const char *pgstat_get_wait_io(WaitEventIO w); static void pgstat_setheader(PgStat_MsgHdr *hdr, StatMsgType mtype); static void pgstat_send(void *msg, int len); @@ -3176,6 +3177,9 @@ pgstat_get_wait_event_type(uint32 wait_event_info) case PG_WAIT_TIMEOUT: event_type = "Timeout"; break; + case PG_WAIT_IO: + event_type = "IO"; + break; default: event_type = "???"; break; @@ -3246,6 +3250,13 @@ pgstat_get_wait_event(uint32 wait_event_info) event_name = pgstat_get_wait_timeout(w); break; } + case PG_WAIT_IO: + { + WaitEventIO w = (WaitEventIO) wait_event_info; + + event_name = pgstat_get_wait_io(w); + break; + } default: event_name = "unknown wait event"; break; @@ -3438,6 +3449,228 @@ pgstat_get_wait_timeout(WaitEventTimeout w) return event_name; } +/* ---------- + * pgstat_get_wait_io() - + * + * Convert WaitEventIO to string. + * ---------- + */ +static const char * +pgstat_get_wait_io(WaitEventIO w) +{ + const char *event_name = "unknown wait event"; + + switch (w) + { + case WAIT_EVENT_BUFFILE_READ: + event_name = "BufFileRead"; + break; + case WAIT_EVENT_BUFFILE_WRITE: + event_name = "BufFileWrite"; + break; + case WAIT_EVENT_CONTROL_FILE_READ: + event_name = "ControlFileRead"; + break; + case WAIT_EVENT_CONTROL_FILE_SYNC: + event_name = "ControlFileSync"; + break; + case WAIT_EVENT_CONTROL_FILE_SYNC_UPDATE: + event_name = "ControlFileSyncUpdate"; + break; + case WAIT_EVENT_CONTROL_FILE_WRITE: + event_name = "ControlFileWrite"; + break; + case WAIT_EVENT_CONTROL_FILE_WRITE_UPDATE: + event_name = "ControlFileWriteUpdate"; + break; + case WAIT_EVENT_COPY_FILE_READ: + event_name = "CopyFileRead"; + break; + case WAIT_EVENT_COPY_FILE_WRITE: + event_name = "CopyFileWrite"; + break; + case WAIT_EVENT_DATA_FILE_EXTEND: + event_name = "DataFileExtend"; + break; + case WAIT_EVENT_DATA_FILE_FLUSH: + event_name = "DataFileFlush"; + break; + case WAIT_EVENT_DATA_FILE_IMMEDIATE_SYNC: + event_name = "DataFileImmediateSync"; + break; + case WAIT_EVENT_DATA_FILE_PREFETCH: + event_name = "DataFilePrefetch"; + break; + case WAIT_EVENT_DATA_FILE_READ: + event_name = "DataFileRead"; + break; + case WAIT_EVENT_DATA_FILE_SYNC: + event_name = "DataFileSync"; + break; + case WAIT_EVENT_DATA_FILE_TRUNCATE: + event_name = "DataFileTruncate"; + break; + case WAIT_EVENT_DATA_FILE_WRITE: + event_name = "DataFileWrite"; + break; + case WAIT_EVENT_DSM_FILL_ZERO_WRITE: + event_name = "DSMFillZeroWrite"; + break; + case WAIT_EVENT_LOCK_FILE_ADDTODATADIR_READ: + event_name = "LockFileAddToDataDirRead"; + break; + case WAIT_EVENT_LOCK_FILE_ADDTODATADIR_SYNC: + event_name = "LockFileAddToDataDirSync"; + break; + case WAIT_EVENT_LOCK_FILE_ADDTODATADIR_WRITE: + event_name = "LockFileAddToDataDirWrite"; + break; + case WAIT_EVENT_LOCK_FILE_CREATE_READ: + event_name = "LockFileCreateRead"; + break; + case WAIT_EVENT_LOCK_FILE_CREATE_SYNC: + event_name = "LockFileCreateSync"; + break; + case WAIT_EVENT_LOCK_FILE_CREATE_WRITE: + event_name = "LockFileCreateWRITE"; + break; + case WAIT_EVENT_LOCK_FILE_RECHECKDATADIR_READ: + event_name = "LockFileReCheckDataDirRead"; + break; + case WAIT_EVENT_LOGICAL_REWRITE_CHECKPOINT_SYNC: + event_name = "LogicalRewriteCheckpointSync"; + break; + case WAIT_EVENT_LOGICAL_REWRITE_MAPPING_SYNC: + event_name = "LogicalRewriteMappingSync"; + break; + case WAIT_EVENT_LOGICAL_REWRITE_MAPPING_WRITE: + event_name = "LogicalRewriteMappingWrite"; + break; + case WAIT_EVENT_LOGICAL_REWRITE_SYNC: + event_name = "LogicalRewriteSync"; + break; + case WAIT_EVENT_LOGICAL_REWRITE_TRUNCATE: + event_name = "LogicalRewriteTruncate"; + break; + case WAIT_EVENT_LOGICAL_REWRITE_WRITE: + event_name = "LogicalRewriteWrite"; + break; + case WAIT_EVENT_RELATION_MAP_READ: + event_name = "RelationMapRead"; + break; + case WAIT_EVENT_RELATION_MAP_SYNC: + event_name = "RelationMapSync"; + break; + case WAIT_EVENT_RELATION_MAP_WRITE: + event_name = "RelationMapWrite"; + break; + case WAIT_EVENT_REORDER_BUFFER_READ: + event_name = "ReorderBufferRead"; + break; + case WAIT_EVENT_REORDER_BUFFER_WRITE: + event_name = "ReorderBufferWrite"; + break; + case WAIT_EVENT_REORDER_LOGICAL_MAPPING_READ: + event_name = "ReorderLogicalMappingRead"; + break; + case WAIT_EVENT_REPLICATION_SLOT_READ: + event_name = "ReplicationSlotRead"; + break; + case WAIT_EVENT_REPLICATION_SLOT_RESTORE_SYNC: + event_name = "ReplicationSlotRestoreSync"; + break; + case WAIT_EVENT_REPLICATION_SLOT_SYNC: + event_name = "ReplicationSlotSync"; + break; + case WAIT_EVENT_REPLICATION_SLOT_WRITE: + event_name = "ReplicationSlotWrite"; + break; + case WAIT_EVENT_SLRU_FLUSH_SYNC: + event_name = "SLRUFlushSync"; + break; + case WAIT_EVENT_SLRU_READ: + event_name = "SLRURead"; + break; + case WAIT_EVENT_SLRU_SYNC: + event_name = "SLRUSync"; + break; + case WAIT_EVENT_SLRU_WRITE: + event_name = "SLRUWrite"; + break; + case WAIT_EVENT_SNAPBUILD_READ: + event_name = "SnapbuildRead"; + break; + case WAIT_EVENT_SNAPBUILD_SYNC: + event_name = "SnapbuildSync"; + break; + case WAIT_EVENT_SNAPBUILD_WRITE: + event_name = "SnapbuildWrite"; + break; + case WAIT_EVENT_TIMELINE_HISTORY_FILE_SYNC: + event_name = "TimelineHistoryFileSync"; + break; + case WAIT_EVENT_TIMELINE_HISTORY_FILE_WRITE: + event_name = "TimelineHistoryFileWrite"; + break; + case WAIT_EVENT_TIMELINE_HISTORY_READ: + event_name = "TimelineHistoryRead"; + break; + case WAIT_EVENT_TIMELINE_HISTORY_SYNC: + event_name = "TimelineHistorySync"; + break; + case WAIT_EVENT_TIMELINE_HISTORY_WRITE: + event_name = "TimelineHistoryWrite"; + break; + case WAIT_EVENT_TWOPHASE_FILE_READ: + event_name = "TwophaseFileRead"; + break; + case WAIT_EVENT_TWOPHASE_FILE_SYNC: + event_name = "TwophaseFileSync"; + break; + case WAIT_EVENT_TWOPHASE_FILE_WRITE: + event_name = "TwophaseFileWrite"; + break; + case WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ: + event_name = "WALSenderTimelineHistoryRead"; + break; + case WAIT_EVENT_WAL_BOOTSTRAP_SYNC: + event_name = "WALBootstrapSync"; + break; + case WAIT_EVENT_WAL_BOOTSTRAP_WRITE: + event_name = "WALBootstrapWrite"; + break; + case WAIT_EVENT_WAL_COPY_READ: + event_name = "WALCopyRead"; + break; + case WAIT_EVENT_WAL_COPY_SYNC: + event_name = "WALCopySync"; + break; + case WAIT_EVENT_WAL_COPY_WRITE: + event_name = "WALCopyWrite"; + break; + case WAIT_EVENT_WAL_INIT_SYNC: + event_name = "WALInitSync"; + break; + case WAIT_EVENT_WAL_INIT_WRITE: + event_name = "WALInitWrite"; + break; + case WAIT_EVENT_WAL_READ: + event_name = "WALRead"; + break; + case WAIT_EVENT_WAL_SYNC_METHOD_ASSIGN: + event_name = "WALSyncMethodAssign"; + break; + case WAIT_EVENT_WAL_WRITE: + event_name = "WALWrite"; + break; + + /* no default case, so that compiler will warn */ + } + + return event_name; +} + + /* ---------- * pgstat_get_backend_current_activity() - * diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c index 8aac670bd4..b437799c5f 100644 --- a/src/backend/replication/logical/reorderbuffer.c +++ b/src/backend/replication/logical/reorderbuffer.c @@ -58,6 +58,7 @@ #include "catalog/catalog.h" #include "lib/binaryheap.h" #include "miscadmin.h" +#include "pgstat.h" #include "replication/logical.h" #include "replication/reorderbuffer.h" #include "replication/slot.h" @@ -2275,6 +2276,7 @@ ReorderBufferSerializeChange(ReorderBuffer *rb, ReorderBufferTXN *txn, ondisk->size = sz; + pgstat_report_wait_start(WAIT_EVENT_REORDER_BUFFER_WRITE); if (write(fd, rb->outbuf, ondisk->size) != ondisk->size) { int save_errno = errno; @@ -2286,6 +2288,7 @@ ReorderBufferSerializeChange(ReorderBuffer *rb, ReorderBufferTXN *txn, errmsg("could not write to data file for XID %u: %m", txn->xid))); } + pgstat_report_wait_end(); Assert(ondisk->change.action == change->action); } @@ -2366,7 +2369,9 @@ ReorderBufferRestoreChanges(ReorderBuffer *rb, ReorderBufferTXN *txn, * end of this file. */ ReorderBufferSerializeReserve(rb, sizeof(ReorderBufferDiskChange)); + pgstat_report_wait_start(WAIT_EVENT_REORDER_BUFFER_READ); readBytes = read(*fd, rb->outbuf, sizeof(ReorderBufferDiskChange)); + pgstat_report_wait_end(); /* eof */ if (readBytes == 0) @@ -2393,8 +2398,10 @@ ReorderBufferRestoreChanges(ReorderBuffer *rb, ReorderBufferTXN *txn, sizeof(ReorderBufferDiskChange) + ondisk->size); ondisk = (ReorderBufferDiskChange *) rb->outbuf; + pgstat_report_wait_start(WAIT_EVENT_REORDER_BUFFER_READ); readBytes = read(*fd, rb->outbuf + sizeof(ReorderBufferDiskChange), ondisk->size - sizeof(ReorderBufferDiskChange)); + pgstat_report_wait_end(); if (readBytes < 0) ereport(ERROR, @@ -3047,7 +3054,9 @@ ApplyLogicalMappingFile(HTAB *tuplecid_data, Oid relid, const char *fname) memset(&key, 0, sizeof(ReorderBufferTupleCidKey)); /* read all mappings till the end of the file */ + pgstat_report_wait_start(WAIT_EVENT_REORDER_LOGICAL_MAPPING_READ); readBytes = read(fd, &map, sizeof(LogicalRewriteMappingData)); + pgstat_report_wait_end(); if (readBytes < 0) ereport(ERROR, diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c index e129a6b8e4..3f242a8ed7 100644 --- a/src/backend/replication/logical/snapbuild.c +++ b/src/backend/replication/logical/snapbuild.c @@ -115,6 +115,8 @@ #include "access/transam.h" #include "access/xact.h" +#include "pgstat.h" + #include "replication/logical.h" #include "replication/reorderbuffer.h" #include "replication/snapbuild.h" @@ -1580,6 +1582,7 @@ SnapBuildSerialize(SnapBuild *builder, XLogRecPtr lsn) ereport(ERROR, (errmsg("could not open file \"%s\": %m", path))); + pgstat_report_wait_start(WAIT_EVENT_SNAPBUILD_WRITE); if ((write(fd, ondisk, needed_length)) != needed_length) { CloseTransientFile(fd); @@ -1587,6 +1590,7 @@ SnapBuildSerialize(SnapBuild *builder, XLogRecPtr lsn) (errcode_for_file_access(), errmsg("could not write to file \"%s\": %m", tmppath))); } + pgstat_report_wait_end(); /* * fsync the file before renaming so that even if we crash after this we @@ -1596,6 +1600,7 @@ SnapBuildSerialize(SnapBuild *builder, XLogRecPtr lsn) * some noticeable overhead since it's performed synchronously during * decoding? */ + pgstat_report_wait_start(WAIT_EVENT_SNAPBUILD_SYNC); if (pg_fsync(fd) != 0) { CloseTransientFile(fd); @@ -1603,6 +1608,7 @@ SnapBuildSerialize(SnapBuild *builder, XLogRecPtr lsn) (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", tmppath))); } + pgstat_report_wait_end(); CloseTransientFile(fd); fsync_fname("pg_logical/snapshots", true); @@ -1677,7 +1683,9 @@ SnapBuildRestore(SnapBuild *builder, XLogRecPtr lsn) /* read statically sized portion of snapshot */ + pgstat_report_wait_start(WAIT_EVENT_SNAPBUILD_READ); readBytes = read(fd, &ondisk, SnapBuildOnDiskConstantSize); + pgstat_report_wait_end(); if (readBytes != SnapBuildOnDiskConstantSize) { CloseTransientFile(fd); @@ -1703,7 +1711,9 @@ SnapBuildRestore(SnapBuild *builder, XLogRecPtr lsn) SnapBuildOnDiskConstantSize - SnapBuildOnDiskNotChecksummedSize); /* read SnapBuild */ + pgstat_report_wait_start(WAIT_EVENT_SNAPBUILD_READ); readBytes = read(fd, &ondisk.builder, sizeof(SnapBuild)); + pgstat_report_wait_end(); if (readBytes != sizeof(SnapBuild)) { CloseTransientFile(fd); @@ -1717,7 +1727,9 @@ SnapBuildRestore(SnapBuild *builder, XLogRecPtr lsn) /* restore running xacts information */ sz = sizeof(TransactionId) * ondisk.builder.running.xcnt_space; ondisk.builder.running.xip = MemoryContextAllocZero(builder->context, sz); + pgstat_report_wait_start(WAIT_EVENT_SNAPBUILD_READ); readBytes = read(fd, ondisk.builder.running.xip, sz); + pgstat_report_wait_end(); if (readBytes != sz) { CloseTransientFile(fd); @@ -1731,7 +1743,9 @@ SnapBuildRestore(SnapBuild *builder, XLogRecPtr lsn) /* restore committed xacts information */ sz = sizeof(TransactionId) * ondisk.builder.committed.xcnt; ondisk.builder.committed.xip = MemoryContextAllocZero(builder->context, sz); + pgstat_report_wait_start(WAIT_EVENT_SNAPBUILD_READ); readBytes = read(fd, ondisk.builder.committed.xip, sz); + pgstat_report_wait_end(); if (readBytes != sz) { CloseTransientFile(fd); diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c index 10d69d0427..5237a9fb07 100644 --- a/src/backend/replication/slot.c +++ b/src/backend/replication/slot.c @@ -43,6 +43,7 @@ #include "access/xlog_internal.h" #include "common/string.h" #include "miscadmin.h" +#include "pgstat.h" #include "replication/slot.h" #include "storage/fd.h" #include "storage/proc.h" @@ -1100,10 +1101,12 @@ SaveSlotToPath(ReplicationSlot *slot, const char *dir, int elevel) SnapBuildOnDiskChecksummedSize); FIN_CRC32C(cp.checksum); + pgstat_report_wait_start(WAIT_EVENT_REPLICATION_SLOT_WRITE); if ((write(fd, &cp, sizeof(cp))) != sizeof(cp)) { int save_errno = errno; + pgstat_report_wait_end(); CloseTransientFile(fd); errno = save_errno; ereport(elevel, @@ -1112,12 +1115,15 @@ SaveSlotToPath(ReplicationSlot *slot, const char *dir, int elevel) tmppath))); return; } + pgstat_report_wait_end(); /* fsync the temporary file */ + pgstat_report_wait_start(WAIT_EVENT_REPLICATION_SLOT_SYNC); if (pg_fsync(fd) != 0) { int save_errno = errno; + pgstat_report_wait_end(); CloseTransientFile(fd); errno = save_errno; ereport(elevel, @@ -1126,6 +1132,7 @@ SaveSlotToPath(ReplicationSlot *slot, const char *dir, int elevel) tmppath))); return; } + pgstat_report_wait_end(); CloseTransientFile(fd); @@ -1202,6 +1209,7 @@ RestoreSlotFromDisk(const char *name) * Sync state file before we're reading from it. We might have crashed * while it wasn't synced yet and we shouldn't continue on that basis. */ + pgstat_report_wait_start(WAIT_EVENT_REPLICATION_SLOT_RESTORE_SYNC); if (pg_fsync(fd) != 0) { CloseTransientFile(fd); @@ -1210,6 +1218,7 @@ RestoreSlotFromDisk(const char *name) errmsg("could not fsync file \"%s\": %m", path))); } + pgstat_report_wait_end(); /* Also sync the parent directory */ START_CRIT_SECTION(); @@ -1217,7 +1226,9 @@ RestoreSlotFromDisk(const char *name) END_CRIT_SECTION(); /* read part of statefile that's guaranteed to be version independent */ + pgstat_report_wait_start(WAIT_EVENT_REPLICATION_SLOT_READ); readBytes = read(fd, &cp, ReplicationSlotOnDiskConstantSize); + pgstat_report_wait_end(); if (readBytes != ReplicationSlotOnDiskConstantSize) { int saved_errno = errno; @@ -1253,9 +1264,11 @@ RestoreSlotFromDisk(const char *name) path, cp.length))); /* Now that we know the size, read the entire file */ + pgstat_report_wait_start(WAIT_EVENT_REPLICATION_SLOT_READ); readBytes = read(fd, (char *) &cp + ReplicationSlotOnDiskConstantSize, cp.length); + pgstat_report_wait_end(); if (readBytes != cp.length) { int saved_errno = errno; diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 127efecb27..0f6b828336 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -463,7 +463,9 @@ SendTimeLineHistory(TimeLineHistoryCmd *cmd) char rbuf[BLCKSZ]; int nread; + pgstat_report_wait_start(WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ); nread = read(fd, rbuf, sizeof(rbuf)); + pgstat_report_wait_end(); if (nread <= 0) ereport(ERROR, (errcode_for_file_access(), @@ -2126,7 +2128,9 @@ retry: else segbytes = nbytes; + pgstat_report_wait_start(WAIT_EVENT_WAL_READ); readbytes = read(sendFile, p, segbytes); + pgstat_report_wait_end(); if (readbytes <= 0) { ereport(ERROR, diff --git a/src/backend/storage/file/buffile.c b/src/backend/storage/file/buffile.c index 7ebd6360a8..4ca0ea4f2a 100644 --- a/src/backend/storage/file/buffile.c +++ b/src/backend/storage/file/buffile.c @@ -37,6 +37,7 @@ #include "postgres.h" #include "executor/instrument.h" +#include "pgstat.h" #include "storage/fd.h" #include "storage/buffile.h" #include "storage/buf_internals.h" @@ -254,7 +255,10 @@ BufFileLoadBuffer(BufFile *file) /* * Read whatever we can get, up to a full bufferload. */ - file->nbytes = FileRead(thisfile, file->buffer, sizeof(file->buffer)); + file->nbytes = FileRead(thisfile, + file->buffer, + sizeof(file->buffer), + WAIT_EVENT_BUFFILE_READ); if (file->nbytes < 0) file->nbytes = 0; file->offsets[file->curFile] += file->nbytes; @@ -317,7 +321,10 @@ BufFileDumpBuffer(BufFile *file) return; /* seek failed, give up */ file->offsets[file->curFile] = file->curOffset; } - bytestowrite = FileWrite(thisfile, file->buffer + wpos, bytestowrite); + bytestowrite = FileWrite(thisfile, + file->buffer + wpos, + bytestowrite, + WAIT_EVENT_BUFFILE_WRITE); if (bytestowrite <= 0) return; /* failed to write */ file->offsets[file->curFile] += bytestowrite; diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c index 101da47dac..dffe28376b 100644 --- a/src/backend/storage/file/copydir.c +++ b/src/backend/storage/file/copydir.c @@ -25,7 +25,7 @@ #include "storage/copydir.h" #include "storage/fd.h" #include "miscadmin.h" - +#include "pgstat.h" /* * copydir: copy a directory @@ -169,7 +169,9 @@ copy_file(char *fromfile, char *tofile) /* If we got a cancel signal during the copy of the file, quit */ CHECK_FOR_INTERRUPTS(); + pgstat_report_wait_start(WAIT_EVENT_COPY_FILE_READ); nbytes = read(srcfd, buffer, COPY_BUF_SIZE); + pgstat_report_wait_end(); if (nbytes < 0) ereport(ERROR, (errcode_for_file_access(), @@ -177,8 +179,10 @@ copy_file(char *fromfile, char *tofile) if (nbytes == 0) break; errno = 0; + pgstat_report_wait_start(WAIT_EVENT_COPY_FILE_WRITE); if ((int) write(dstfd, buffer, nbytes) != nbytes) { + pgstat_report_wait_end(); /* if write didn't set errno, assume problem is no disk space */ if (errno == 0) errno = ENOSPC; @@ -186,6 +190,7 @@ copy_file(char *fromfile, char *tofile) (errcode_for_file_access(), errmsg("could not write to file \"%s\": %m", tofile))); } + pgstat_report_wait_end(); /* * We fsync the files later but first flush them to avoid spamming the diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c index fd02fc019f..f0ed2e9b5f 100644 --- a/src/backend/storage/file/fd.c +++ b/src/backend/storage/file/fd.c @@ -1550,7 +1550,7 @@ FileClose(File file) * to read into. */ int -FilePrefetch(File file, off_t offset, int amount) +FilePrefetch(File file, off_t offset, int amount, uint32 wait_event_info) { #if defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED) int returnCode; @@ -1565,8 +1565,10 @@ FilePrefetch(File file, off_t offset, int amount) if (returnCode < 0) return returnCode; + pgstat_report_wait_start(wait_event_info); returnCode = posix_fadvise(VfdCache[file].fd, offset, amount, POSIX_FADV_WILLNEED); + pgstat_report_wait_end(); return returnCode; #else @@ -1576,7 +1578,7 @@ FilePrefetch(File file, off_t offset, int amount) } void -FileWriteback(File file, off_t offset, off_t nbytes) +FileWriteback(File file, off_t offset, off_t nbytes, uint32 wait_event_info) { int returnCode; @@ -1597,11 +1599,13 @@ FileWriteback(File file, off_t offset, off_t nbytes) if (returnCode < 0) return; + pgstat_report_wait_start(wait_event_info); pg_flush_data(VfdCache[file].fd, offset, nbytes); + pgstat_report_wait_end(); } int -FileRead(File file, char *buffer, int amount) +FileRead(File file, char *buffer, int amount, uint32 wait_event_info) { int returnCode; Vfd *vfdP; @@ -1620,7 +1624,9 @@ FileRead(File file, char *buffer, int amount) vfdP = &VfdCache[file]; retry: + pgstat_report_wait_start(wait_event_info); returnCode = read(vfdP->fd, buffer, amount); + pgstat_report_wait_end(); if (returnCode >= 0) { @@ -1663,7 +1669,7 @@ retry: } int -FileWrite(File file, char *buffer, int amount) +FileWrite(File file, char *buffer, int amount, uint32 wait_event_info) { int returnCode; Vfd *vfdP; @@ -1721,7 +1727,9 @@ FileWrite(File file, char *buffer, int amount) retry: errno = 0; + pgstat_report_wait_start(wait_event_info); returnCode = write(vfdP->fd, buffer, amount); + pgstat_report_wait_end(); /* if write didn't set errno, assume problem is no disk space */ if (returnCode != amount && errno == 0) @@ -1782,7 +1790,7 @@ retry: } int -FileSync(File file) +FileSync(File file, uint32 wait_event_info) { int returnCode; @@ -1795,7 +1803,11 @@ FileSync(File file) if (returnCode < 0) return returnCode; - return pg_fsync(VfdCache[file].fd); + pgstat_report_wait_start(wait_event_info); + returnCode = pg_fsync(VfdCache[file].fd); + pgstat_report_wait_end(); + + return returnCode; } off_t @@ -1887,7 +1899,7 @@ FileTell(File file) #endif int -FileTruncate(File file, off_t offset) +FileTruncate(File file, off_t offset, uint32 wait_event_info) { int returnCode; @@ -1900,7 +1912,9 @@ FileTruncate(File file, off_t offset) if (returnCode < 0) return returnCode; + pgstat_report_wait_start(wait_event_info); returnCode = ftruncate(VfdCache[file].fd, offset); + pgstat_report_wait_end(); if (returnCode == 0 && VfdCache[file].fileSize > offset) { diff --git a/src/backend/storage/ipc/dsm_impl.c b/src/backend/storage/ipc/dsm_impl.c index b2c9cdc6ed..e0eaefeeb3 100644 --- a/src/backend/storage/ipc/dsm_impl.c +++ b/src/backend/storage/ipc/dsm_impl.c @@ -60,6 +60,7 @@ #ifdef HAVE_SYS_SHM_H #include #endif +#include "pgstat.h" #include "portability/mem.h" #include "storage/dsm_impl.h" @@ -911,10 +912,12 @@ dsm_impl_mmap(dsm_op op, dsm_handle handle, Size request_size, if (goal > ZBUFFER_SIZE) goal = ZBUFFER_SIZE; + pgstat_report_wait_start(WAIT_EVENT_DSM_FILL_ZERO_WRITE); if (write(fd, zbuffer, goal) == goal) remaining -= goal; else success = false; + pgstat_report_wait_end(); } if (!success) diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c index 6c17b54f0d..b0b596d6d9 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -28,6 +28,7 @@ #include "miscadmin.h" #include "access/xlog.h" #include "catalog/catalog.h" +#include "pgstat.h" #include "portability/instr_time.h" #include "postmaster/bgwriter.h" #include "storage/fd.h" @@ -536,7 +537,7 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, errmsg("could not seek to block %u in file \"%s\": %m", blocknum, FilePathName(v->mdfd_vfd)))); - if ((nbytes = FileWrite(v->mdfd_vfd, buffer, BLCKSZ)) != BLCKSZ) + if ((nbytes = FileWrite(v->mdfd_vfd, buffer, BLCKSZ, WAIT_EVENT_DATA_FILE_EXTEND)) != BLCKSZ) { if (nbytes < 0) ereport(ERROR, @@ -667,7 +668,7 @@ mdprefetch(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum) Assert(seekpos < (off_t) BLCKSZ * RELSEG_SIZE); - (void) FilePrefetch(v->mdfd_vfd, seekpos, BLCKSZ); + (void) FilePrefetch(v->mdfd_vfd, seekpos, BLCKSZ, WAIT_EVENT_DATA_FILE_PREFETCH); #endif /* USE_PREFETCH */ } @@ -716,7 +717,7 @@ mdwriteback(SMgrRelation reln, ForkNumber forknum, seekpos = (off_t) BLCKSZ *(blocknum % ((BlockNumber) RELSEG_SIZE)); - FileWriteback(v->mdfd_vfd, seekpos, (off_t) BLCKSZ * nflush); + FileWriteback(v->mdfd_vfd, seekpos, (off_t) BLCKSZ * nflush, WAIT_EVENT_DATA_FILE_FLUSH); nblocks -= nflush; blocknum += nflush; @@ -753,7 +754,7 @@ mdread(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, errmsg("could not seek to block %u in file \"%s\": %m", blocknum, FilePathName(v->mdfd_vfd)))); - nbytes = FileRead(v->mdfd_vfd, buffer, BLCKSZ); + nbytes = FileRead(v->mdfd_vfd, buffer, BLCKSZ, WAIT_EVENT_DATA_FILE_READ); TRACE_POSTGRESQL_SMGR_MD_READ_DONE(forknum, blocknum, reln->smgr_rnode.node.spcNode, @@ -829,7 +830,7 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, errmsg("could not seek to block %u in file \"%s\": %m", blocknum, FilePathName(v->mdfd_vfd)))); - nbytes = FileWrite(v->mdfd_vfd, buffer, BLCKSZ); + nbytes = FileWrite(v->mdfd_vfd, buffer, BLCKSZ, WAIT_EVENT_DATA_FILE_WRITE); TRACE_POSTGRESQL_SMGR_MD_WRITE_DONE(forknum, blocknum, reln->smgr_rnode.node.spcNode, @@ -967,7 +968,7 @@ mdtruncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks) * This segment is no longer active. We truncate the file, but do * not delete it, for reasons explained in the header comments. */ - if (FileTruncate(v->mdfd_vfd, 0) < 0) + if (FileTruncate(v->mdfd_vfd, 0, WAIT_EVENT_DATA_FILE_TRUNCATE) < 0) ereport(ERROR, (errcode_for_file_access(), errmsg("could not truncate file \"%s\": %m", @@ -993,7 +994,7 @@ mdtruncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks) */ BlockNumber lastsegblocks = nblocks - priorblocks; - if (FileTruncate(v->mdfd_vfd, (off_t) lastsegblocks * BLCKSZ) < 0) + if (FileTruncate(v->mdfd_vfd, (off_t) lastsegblocks * BLCKSZ, WAIT_EVENT_DATA_FILE_TRUNCATE) < 0) ereport(ERROR, (errcode_for_file_access(), errmsg("could not truncate file \"%s\" to %u blocks: %m", @@ -1037,7 +1038,7 @@ mdimmedsync(SMgrRelation reln, ForkNumber forknum) { MdfdVec *v = &reln->md_seg_fds[forknum][segno - 1]; - if (FileSync(v->mdfd_vfd) < 0) + if (FileSync(v->mdfd_vfd, WAIT_EVENT_DATA_FILE_IMMEDIATE_SYNC) < 0) ereport(ERROR, (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", @@ -1232,7 +1233,7 @@ mdsync(void) INSTR_TIME_SET_CURRENT(sync_start); if (seg != NULL && - FileSync(seg->mdfd_vfd) >= 0) + FileSync(seg->mdfd_vfd, WAIT_EVENT_DATA_FILE_SYNC) >= 0) { /* Success; update statistics about sync timing */ INSTR_TIME_SET_CURRENT(sync_end); @@ -1443,7 +1444,7 @@ register_dirty_segment(SMgrRelation reln, ForkNumber forknum, MdfdVec *seg) ereport(DEBUG1, (errmsg("could not forward fsync request because request queue is full"))); - if (FileSync(seg->mdfd_vfd) < 0) + if (FileSync(seg->mdfd_vfd, WAIT_EVENT_DATA_FILE_SYNC) < 0) ereport(ERROR, (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", diff --git a/src/backend/utils/cache/relmapper.c b/src/backend/utils/cache/relmapper.c index c9d6e44d9f..047c5b40e8 100644 --- a/src/backend/utils/cache/relmapper.c +++ b/src/backend/utils/cache/relmapper.c @@ -50,6 +50,7 @@ #include "catalog/pg_tablespace.h" #include "catalog/storage.h" #include "miscadmin.h" +#include "pgstat.h" #include "storage/fd.h" #include "storage/lwlock.h" #include "utils/inval.h" @@ -658,11 +659,13 @@ load_relmap_file(bool shared) * look, the sinval signaling mechanism will make us re-read it before we * are able to access any relation that's affected by the change. */ + pgstat_report_wait_start(WAIT_EVENT_RELATION_MAP_READ); if (read(fd, map, sizeof(RelMapFile)) != sizeof(RelMapFile)) ereport(FATAL, (errcode_for_file_access(), errmsg("could not read relation mapping file \"%s\": %m", mapfilename))); + pgstat_report_wait_end(); CloseTransientFile(fd); @@ -774,6 +777,7 @@ write_relmap_file(bool shared, RelMapFile *newmap, } errno = 0; + pgstat_report_wait_start(WAIT_EVENT_RELATION_MAP_WRITE); if (write(fd, newmap, sizeof(RelMapFile)) != sizeof(RelMapFile)) { /* if write didn't set errno, assume problem is no disk space */ @@ -784,6 +788,7 @@ write_relmap_file(bool shared, RelMapFile *newmap, errmsg("could not write to relation mapping file \"%s\": %m", mapfilename))); } + pgstat_report_wait_end(); /* * We choose to fsync the data to disk before considering the task done. @@ -791,11 +796,13 @@ write_relmap_file(bool shared, RelMapFile *newmap, * issue, but it would complicate checkpointing --- see notes for * CheckPointRelationMap. */ + pgstat_report_wait_start(WAIT_EVENT_RELATION_MAP_SYNC); if (pg_fsync(fd) != 0) ereport(ERROR, (errcode_for_file_access(), errmsg("could not fsync relation mapping file \"%s\": %m", mapfilename))); + pgstat_report_wait_end(); if (CloseTransientFile(fd)) ereport(ERROR, diff --git a/src/backend/utils/init/miscinit.c b/src/backend/utils/init/miscinit.c index e0298ee35f..8d149bf272 100644 --- a/src/backend/utils/init/miscinit.c +++ b/src/backend/utils/init/miscinit.c @@ -35,6 +35,7 @@ #include "libpq/libpq.h" #include "mb/pg_wchar.h" #include "miscadmin.h" +#include "pgstat.h" #include "postmaster/autovacuum.h" #include "postmaster/postmaster.h" #include "storage/fd.h" @@ -856,11 +857,13 @@ CreateLockFile(const char *filename, bool amPostmaster, errmsg("could not open lock file \"%s\": %m", filename))); } + pgstat_report_wait_start(WAIT_EVENT_LOCK_FILE_CREATE_READ); if ((len = read(fd, buffer, sizeof(buffer) - 1)) < 0) ereport(FATAL, (errcode_for_file_access(), errmsg("could not read lock file \"%s\": %m", filename))); + pgstat_report_wait_end(); close(fd); if (len == 0) @@ -1009,6 +1012,7 @@ CreateLockFile(const char *filename, bool amPostmaster, strlcat(buffer, "\n", sizeof(buffer)); errno = 0; + pgstat_report_wait_start(WAIT_EVENT_LOCK_FILE_CREATE_WRITE); if (write(fd, buffer, strlen(buffer)) != strlen(buffer)) { int save_errno = errno; @@ -1021,6 +1025,9 @@ CreateLockFile(const char *filename, bool amPostmaster, (errcode_for_file_access(), errmsg("could not write lock file \"%s\": %m", filename))); } + pgstat_report_wait_end(); + + pgstat_report_wait_start(WAIT_EVENT_LOCK_FILE_CREATE_SYNC); if (pg_fsync(fd) != 0) { int save_errno = errno; @@ -1032,6 +1039,7 @@ CreateLockFile(const char *filename, bool amPostmaster, (errcode_for_file_access(), errmsg("could not write lock file \"%s\": %m", filename))); } + pgstat_report_wait_end(); if (close(fd) != 0) { int save_errno = errno; @@ -1164,7 +1172,9 @@ AddToDataDirLockFile(int target_line, const char *str) DIRECTORY_LOCK_FILE))); return; } + pgstat_report_wait_start(WAIT_EVENT_LOCK_FILE_ADDTODATADIR_READ); len = read(fd, srcbuffer, sizeof(srcbuffer) - 1); + pgstat_report_wait_end(); if (len < 0) { ereport(LOG, @@ -1217,9 +1227,11 @@ AddToDataDirLockFile(int target_line, const char *str) */ len = strlen(destbuffer); errno = 0; + pgstat_report_wait_start(WAIT_EVENT_LOCK_FILE_ADDTODATADIR_WRITE); if (lseek(fd, (off_t) 0, SEEK_SET) != 0 || (int) write(fd, destbuffer, len) != len) { + pgstat_report_wait_end(); /* if write didn't set errno, assume problem is no disk space */ if (errno == 0) errno = ENOSPC; @@ -1230,6 +1242,8 @@ AddToDataDirLockFile(int target_line, const char *str) close(fd); return; } + pgstat_report_wait_end(); + pgstat_report_wait_start(WAIT_EVENT_LOCK_FILE_ADDTODATADIR_SYNC); if (pg_fsync(fd) != 0) { ereport(LOG, @@ -1237,6 +1251,7 @@ AddToDataDirLockFile(int target_line, const char *str) errmsg("could not write to file \"%s\": %m", DIRECTORY_LOCK_FILE))); } + pgstat_report_wait_end(); if (close(fd) != 0) { ereport(LOG, @@ -1293,7 +1308,9 @@ RecheckDataDirLockFile(void) return true; } } + pgstat_report_wait_start(WAIT_EVENT_LOCK_FILE_RECHECKDATADIR_READ); len = read(fd, buffer, sizeof(buffer) - 1); + pgstat_report_wait_end(); if (len < 0) { ereport(LOG, diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 60c78d118f..9d2e1fe19f 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -723,6 +723,7 @@ typedef enum BackendState #define PG_WAIT_EXTENSION 0x07000000U #define PG_WAIT_IPC 0x08000000U #define PG_WAIT_TIMEOUT 0x09000000U +#define PG_WAIT_IO 0x0A000000U /* ---------- * Wait Events - Activity @@ -805,6 +806,83 @@ typedef enum WAIT_EVENT_RECOVERY_APPLY_DELAY } WaitEventTimeout; +/* ---------- + * Wait Events - IO + * + * Use this category when a process is waiting for a IO. + * ---------- + */ +typedef enum +{ + WAIT_EVENT_BUFFILE_READ, + WAIT_EVENT_BUFFILE_WRITE, + WAIT_EVENT_CONTROL_FILE_READ, + WAIT_EVENT_CONTROL_FILE_SYNC, + WAIT_EVENT_CONTROL_FILE_SYNC_UPDATE, + WAIT_EVENT_CONTROL_FILE_WRITE, + WAIT_EVENT_CONTROL_FILE_WRITE_UPDATE, + WAIT_EVENT_COPY_FILE_READ, + WAIT_EVENT_COPY_FILE_WRITE, + WAIT_EVENT_DATA_FILE_EXTEND, + WAIT_EVENT_DATA_FILE_FLUSH, + WAIT_EVENT_DATA_FILE_IMMEDIATE_SYNC, + WAIT_EVENT_DATA_FILE_PREFETCH, + WAIT_EVENT_DATA_FILE_READ = PG_WAIT_IO, + WAIT_EVENT_DATA_FILE_SYNC, + WAIT_EVENT_DATA_FILE_TRUNCATE, + WAIT_EVENT_DATA_FILE_WRITE, + WAIT_EVENT_DSM_FILL_ZERO_WRITE, + WAIT_EVENT_LOCK_FILE_ADDTODATADIR_READ, + WAIT_EVENT_LOCK_FILE_ADDTODATADIR_SYNC, + WAIT_EVENT_LOCK_FILE_ADDTODATADIR_WRITE, + WAIT_EVENT_LOCK_FILE_CREATE_READ, + WAIT_EVENT_LOCK_FILE_CREATE_SYNC, + WAIT_EVENT_LOCK_FILE_CREATE_WRITE, + WAIT_EVENT_LOCK_FILE_RECHECKDATADIR_READ, + WAIT_EVENT_LOGICAL_REWRITE_CHECKPOINT_SYNC, + WAIT_EVENT_LOGICAL_REWRITE_MAPPING_SYNC, + WAIT_EVENT_LOGICAL_REWRITE_MAPPING_WRITE, + WAIT_EVENT_LOGICAL_REWRITE_SYNC, + WAIT_EVENT_LOGICAL_REWRITE_TRUNCATE, + WAIT_EVENT_LOGICAL_REWRITE_WRITE, + WAIT_EVENT_RELATION_MAP_READ, + WAIT_EVENT_RELATION_MAP_SYNC, + WAIT_EVENT_RELATION_MAP_WRITE, + WAIT_EVENT_REORDER_BUFFER_READ, + WAIT_EVENT_REORDER_BUFFER_WRITE, + WAIT_EVENT_REORDER_LOGICAL_MAPPING_READ, + WAIT_EVENT_REPLICATION_SLOT_READ, + WAIT_EVENT_REPLICATION_SLOT_RESTORE_SYNC, + WAIT_EVENT_REPLICATION_SLOT_SYNC, + WAIT_EVENT_REPLICATION_SLOT_WRITE, + WAIT_EVENT_SLRU_FLUSH_SYNC, + WAIT_EVENT_SLRU_READ, + WAIT_EVENT_SLRU_SYNC, + WAIT_EVENT_SLRU_WRITE, + WAIT_EVENT_SNAPBUILD_READ, + WAIT_EVENT_SNAPBUILD_SYNC, + WAIT_EVENT_SNAPBUILD_WRITE, + WAIT_EVENT_TIMELINE_HISTORY_FILE_SYNC, + WAIT_EVENT_TIMELINE_HISTORY_FILE_WRITE, + WAIT_EVENT_TIMELINE_HISTORY_READ, + WAIT_EVENT_TIMELINE_HISTORY_SYNC, + WAIT_EVENT_TIMELINE_HISTORY_WRITE, + WAIT_EVENT_TWOPHASE_FILE_READ, + WAIT_EVENT_TWOPHASE_FILE_SYNC, + WAIT_EVENT_TWOPHASE_FILE_WRITE, + WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ, + WAIT_EVENT_WAL_BOOTSTRAP_SYNC, + WAIT_EVENT_WAL_BOOTSTRAP_WRITE, + WAIT_EVENT_WAL_COPY_READ, + WAIT_EVENT_WAL_COPY_SYNC, + WAIT_EVENT_WAL_COPY_WRITE, + WAIT_EVENT_WAL_INIT_SYNC, + WAIT_EVENT_WAL_INIT_WRITE, + WAIT_EVENT_WAL_READ, + WAIT_EVENT_WAL_SYNC_METHOD_ASSIGN, + WAIT_EVENT_WAL_WRITE +} WaitEventIO; + /* ---------- * Command type for progress reporting purposes * ---------- diff --git a/src/include/storage/fd.h b/src/include/storage/fd.h index 1a43a2c844..ac37502928 100644 --- a/src/include/storage/fd.h +++ b/src/include/storage/fd.h @@ -68,13 +68,13 @@ extern int max_safe_fds; extern File PathNameOpenFile(FileName fileName, int fileFlags, int fileMode); extern File OpenTemporaryFile(bool interXact); extern void FileClose(File file); -extern int FilePrefetch(File file, off_t offset, int amount); -extern int FileRead(File file, char *buffer, int amount); -extern int FileWrite(File file, char *buffer, int amount); -extern int FileSync(File file); +extern int FilePrefetch(File file, off_t offset, int amount, uint32 wait_event_info); +extern int FileRead(File file, char *buffer, int amount, uint32 wait_event_info); +extern int FileWrite(File file, char *buffer, int amount, uint32 wait_event_info); +extern int FileSync(File file, uint32 wait_event_info); extern off_t FileSeek(File file, off_t offset, int whence); -extern int FileTruncate(File file, off_t offset); -extern void FileWriteback(File file, off_t offset, off_t nbytes); +extern int FileTruncate(File file, off_t offset, uint32 wait_event_info); +extern void FileWriteback(File file, off_t offset, off_t nbytes, uint32 wait_event_info); extern char *FilePathName(File file); extern int FileGetRawDesc(File file); extern int FileGetRawFlags(File file); -- cgit v1.2.3 From 164bdbe9c2412d90b62d6f145a43f9e41a90003b Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Sat, 18 Mar 2017 09:42:26 -0400 Subject: doc: Correct row count in table The incorrect count causes FOP to error out. --- doc/src/sgml/monitoring.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 9b50fb9c13..dcb2d3303c 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -1279,7 +1279,7 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser Waiting to apply WAL at recovery because it is delayed. - IO + IO BufFileRead Waiting for a read from a buffered file. -- cgit v1.2.3 From cb5fa4130ed5bdd37a51ce8cade3c2a2461097b9 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Sat, 18 Mar 2017 14:17:28 -0400 Subject: doc: Tweak reference page header style Change the header style of references pages in HTML and PDF output to be more like the old style under DSSSL. In particular, the page should start with a header containing the command name, instead of just "Name". --- doc/src/sgml/stylesheet-common.xsl | 2 ++ 1 file changed, 2 insertions(+) (limited to 'doc/src') diff --git a/doc/src/sgml/stylesheet-common.xsl b/doc/src/sgml/stylesheet-common.xsl index c23d38f128..3d0651a823 100644 --- a/doc/src/sgml/stylesheet-common.xsl +++ b/doc/src/sgml/stylesheet-common.xsl @@ -32,6 +32,8 @@ + + -- cgit v1.2.3 From 5e86bb51eaeeb5c70f64102f46c4431cb12999e9 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Sat, 18 Mar 2017 14:25:41 -0400 Subject: doc: Update compatibility claim Update outdated claim that TRUNCATE is a PostgreSQL extension. Add cross-links between DELETE and TRUNCATE references pages. --- doc/src/sgml/ref/delete.sgml | 11 +++++++++-- doc/src/sgml/ref/truncate.sgml | 8 ++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/delete.sgml b/doc/src/sgml/ref/delete.sgml index 74ea90787b..20417a1391 100644 --- a/doc/src/sgml/ref/delete.sgml +++ b/doc/src/sgml/ref/delete.sgml @@ -41,8 +41,7 @@ DELETE FROM [ ONLY ] table_name [ * - is a - PostgreSQL extension that provides a + provides a faster mechanism to remove all rows from a table. @@ -279,4 +278,12 @@ DELETE FROM tasks WHERE CURRENT OF c_tasks; to use WITH with DELETE. + + + See Also + + + + + diff --git a/doc/src/sgml/ref/truncate.sgml b/doc/src/sgml/ref/truncate.sgml index a78e47c095..e9c8a03a63 100644 --- a/doc/src/sgml/ref/truncate.sgml +++ b/doc/src/sgml/ref/truncate.sgml @@ -220,4 +220,12 @@ TRUNCATE othertable CASCADE; considered and compared with other implementations if necessary. + + + See Also + + + + + -- cgit v1.2.3 From a72188a35bbd71a8cc29f10eab27d4140d296b9d Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Sat, 18 Mar 2017 14:41:47 -0400 Subject: doc: Add markup --- doc/src/sgml/installation.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml index 568995c9f2..79201b78e3 100644 --- a/doc/src/sgml/installation.sgml +++ b/doc/src/sgml/installation.sgml @@ -1080,7 +1080,7 @@ su - postgres protocols, as well as some routines in the ]]> - module. --disable-strong-random disables functionality that + module. disables functionality that requires cryptographically strong random numbers, and substitutes a weak pseudo-random-number-generator for the generation of authentication salt values and query cancel keys. It may make -- cgit v1.2.3 From 27f1f585fd7ee749cacd3de8c2c77a457ef4c288 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Sat, 18 Mar 2017 17:40:47 -0400 Subject: doc: Improve wording Link to CREATE and ALTER ROLE consistently, instead of mixing CREATE USER and ALTER ROLE. --- doc/src/sgml/config.sgml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 9963cd61a1..32c39a0516 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -1187,7 +1187,7 @@ include_dir 'conf.d' - When a password is specified in or + When a password is specified in or without writing either ENCRYPTED or UNENCRYPTED, this parameter determines whether the password is to be encrypted. The default value is md5, which @@ -1196,7 +1196,7 @@ include_dir 'conf.d' aliases for md5 and plain, respectively. Setting this parameter to scram will encrypt the password with SCRAM-SHA-256. - + -- cgit v1.2.3 From 767ce36ff36747e334ce3e163d57d5376403db37 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Sat, 18 Mar 2017 23:43:47 -0400 Subject: doc: Fix a few typos and awkward links --- doc/src/sgml/backup.sgml | 4 ++-- doc/src/sgml/config.sgml | 4 ++-- doc/src/sgml/ecpg.sgml | 2 +- doc/src/sgml/high-availability.sgml | 2 +- doc/src/sgml/protocol.sgml | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml index 12f2efe4e2..2d67521775 100644 --- a/doc/src/sgml/backup.sgml +++ b/doc/src/sgml/backup.sgml @@ -878,7 +878,7 @@ SELECT pg_start_backup('label', false, false); pg_dump or pg_dumpall). It is neither necessary nor desirable to stop normal operation of the database - while you do this. See section + while you do this. See for things to consider during this backup. @@ -988,7 +988,7 @@ SELECT pg_start_backup('label', true); pg_dump or pg_dumpall). It is neither necessary nor desirable to stop normal operation of the database - while you do this. See section + while you do this. See for things to consider during this backup. diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 32c39a0516..b379b67b30 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -532,10 +532,10 @@ include_dir 'conf.d' - Specifies the configuration file for - user name mapping + Specifies the configuration file for user name mapping (customarily called pg_ident.conf). This parameter can only be set at server start. + See also . diff --git a/doc/src/sgml/ecpg.sgml b/doc/src/sgml/ecpg.sgml index 508406823d..89d4c57da7 100644 --- a/doc/src/sgml/ecpg.sgml +++ b/doc/src/sgml/ecpg.sgml @@ -1189,7 +1189,7 @@ EXEC SQL END DECLARE SECTION; There are two use cases for arrays as host variables. The first is a way to store some text string in char[] or VARCHAR[], as - explained . The second use case is to + explained in . The second use case is to retrieve multiple rows from a query result without using a cursor. Without an array, to process a query result consisting of multiple rows, it is required to use a cursor and diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml index cc84b911b0..51359d6236 100644 --- a/doc/src/sgml/high-availability.sgml +++ b/doc/src/sgml/high-availability.sgml @@ -207,7 +207,7 @@ protocol to make nodes agree on a serializable transactional order. middleware. Care must also be taken that all transactions either commit or abort on all servers, perhaps using two-phase commit ( - and . + and ). Pgpool-II and Continuent Tungsten are examples of this type of replication. diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index 7c82b48845..244e381de9 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -2296,8 +2296,8 @@ The commands accepted in walsender mode are: The individual protocol messages are discussed in the following - subsections. Individual messages are describer in - section. + subsections. Individual messages are described in + . -- cgit v1.2.3 From b6fb534f10e1dea17dc5641f44cc651b8d60d8f0 Mon Sep 17 00:00:00 2001 From: Andrew Dunstan Date: Mon, 20 Mar 2017 16:40:45 -0400 Subject: Add IF NOT EXISTS for CREATE SERVER and CREATE USER MAPPING There is still some inconsistency with the error messages surrounding foreign servers. Some use the word "foreign" and some don't. My inclination is to remove all such uses of "foreign" on the basis that the CREATE/ALTER/DROP SERVER commands don't use the word. However, that is left for another day. In this patch I have kept to the existing usage in the affected commands, which omits "foreign". Anastasia Lubennikova, reviewed by Arthur Zakirov and Ashtosh Bapat. Discussion: http://postgr.es/m/7c2ab9b8-388a-1ce0-23a3-7acf2a0ed3c6@postgrespro.ru --- doc/src/sgml/ref/create_server.sgml | 14 ++++++++++- doc/src/sgml/ref/create_user_mapping.sgml | 14 ++++++++++- src/backend/commands/foreigncmds.c | 38 ++++++++++++++++++++++++++---- src/backend/parser/gram.y | 23 ++++++++++++++++++ src/include/nodes/parsenodes.h | 2 ++ src/test/regress/expected/foreign_data.out | 6 +++++ src/test/regress/sql/foreign_data.sql | 3 +++ 7 files changed, 93 insertions(+), 7 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/create_server.sgml b/doc/src/sgml/ref/create_server.sgml index 734c6c9fe8..7318481487 100644 --- a/doc/src/sgml/ref/create_server.sgml +++ b/doc/src/sgml/ref/create_server.sgml @@ -21,7 +21,7 @@ PostgreSQL documentation -CREATE SERVER server_name [ TYPE 'server_type' ] [ VERSION 'server_version' ] +CREATE SERVER [IF NOT EXISTS] server_name [ TYPE 'server_type' ] [ VERSION 'server_version' ] FOREIGN DATA WRAPPER fdw_name [ OPTIONS ( option 'value' [, ... ] ) ] @@ -56,6 +56,18 @@ CREATE SERVER server_name [ TYPE '< Parameters + + IF NOT EXISTS + + + Do not throw an error if a server with the same name already exists. + A notice is issued in this case. Note that there is no guarantee that + the existing server is anything like the one that would have been + created. + + + + server_name diff --git a/doc/src/sgml/ref/create_user_mapping.sgml b/doc/src/sgml/ref/create_user_mapping.sgml index 44fe302fb5..1c44679a98 100644 --- a/doc/src/sgml/ref/create_user_mapping.sgml +++ b/doc/src/sgml/ref/create_user_mapping.sgml @@ -21,7 +21,7 @@ PostgreSQL documentation -CREATE USER MAPPING FOR { user_name | USER | CURRENT_USER | PUBLIC } +CREATE USER MAPPING [IF NOT EXISTS] FOR { user_name | USER | CURRENT_USER | PUBLIC } SERVER server_name [ OPTIONS ( option 'value' [ , ... ] ) ] @@ -50,6 +50,18 @@ CREATE USER MAPPING FOR { user_name Parameters + + IF NOT EXISTS + + + Do not throw an error if a mapping of the given user to the given foreign + server already exists. A notice is issued in this case. Note that there + is no guarantee that the existing user mapping is anything like the one + that would have been created. + + + + user_name diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c index 7f551149d2..68100df083 100644 --- a/src/backend/commands/foreigncmds.c +++ b/src/backend/commands/foreigncmds.c @@ -879,12 +879,25 @@ CreateForeignServer(CreateForeignServerStmt *stmt) /* * Check that there is no other foreign server by this name. + * Do nothing if IF NOT EXISTS was enforced. */ if (GetForeignServerByName(stmt->servername, true) != NULL) - ereport(ERROR, - (errcode(ERRCODE_DUPLICATE_OBJECT), - errmsg("server \"%s\" already exists", - stmt->servername))); + { + if (stmt->if_not_exists) + { + ereport(NOTICE, + (errcode(ERRCODE_DUPLICATE_OBJECT), + errmsg("server \"%s\" already exists, skipping", + stmt->servername))); + heap_close(rel, RowExclusiveLock); + return InvalidObjectAddress; + } + else + ereport(ERROR, + (errcode(ERRCODE_DUPLICATE_OBJECT), + errmsg("server \"%s\" already exists", + stmt->servername))); + } /* * Check that the FDW exists and that we have USAGE on it. Also get the @@ -1152,12 +1165,27 @@ CreateUserMapping(CreateUserMappingStmt *stmt) umId = GetSysCacheOid2(USERMAPPINGUSERSERVER, ObjectIdGetDatum(useId), ObjectIdGetDatum(srv->serverid)); + if (OidIsValid(umId)) - ereport(ERROR, + { + if (stmt->if_not_exists) + { + ereport(NOTICE, + (errcode(ERRCODE_DUPLICATE_OBJECT), + errmsg("user mapping for \"%s\" already exists for server %s, skipping", + MappingUserName(useId), + stmt->servername))); + + heap_close(rel, RowExclusiveLock); + return InvalidObjectAddress; + } + else + ereport(ERROR, (errcode(ERRCODE_DUPLICATE_OBJECT), errmsg("user mapping for \"%s\" already exists for server %s", MappingUserName(useId), stmt->servername))); + } fdw = GetForeignDataWrapper(srv->fdwid); diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 6316688a88..d0d45a557b 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -4621,6 +4621,19 @@ CreateForeignServerStmt: CREATE SERVER name opt_type opt_foreign_server_version n->version = $5; n->fdwname = $9; n->options = $10; + n->if_not_exists = false; + $$ = (Node *) n; + } + | CREATE SERVER IF_P NOT EXISTS name opt_type opt_foreign_server_version + FOREIGN DATA_P WRAPPER name create_generic_options + { + CreateForeignServerStmt *n = makeNode(CreateForeignServerStmt); + n->servername = $6; + n->servertype = $7; + n->version = $8; + n->fdwname = $12; + n->options = $13; + n->if_not_exists = true; $$ = (Node *) n; } ; @@ -4853,6 +4866,16 @@ CreateUserMappingStmt: CREATE USER MAPPING FOR auth_ident SERVER name create_gen n->user = $5; n->servername = $7; n->options = $8; + n->if_not_exists = false; + $$ = (Node *) n; + } + | CREATE USER MAPPING IF_P NOT EXISTS FOR auth_ident SERVER name create_generic_options + { + CreateUserMappingStmt *n = makeNode(CreateUserMappingStmt); + n->user = $8; + n->servername = $10; + n->options = $11; + n->if_not_exists = true; $$ = (Node *) n; } ; diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index d576523f6a..a15df229a4 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -2154,6 +2154,7 @@ typedef struct CreateForeignServerStmt char *servertype; /* optional server type */ char *version; /* optional server version */ char *fdwname; /* FDW name */ + bool if_not_exists; /* just do nothing if it already exists? */ List *options; /* generic options to server */ } CreateForeignServerStmt; @@ -2188,6 +2189,7 @@ typedef struct CreateUserMappingStmt NodeTag type; RoleSpec *user; /* user role */ char *servername; /* server name */ + bool if_not_exists; /* just do nothing if it already exists? */ List *options; /* generic options to server */ } CreateUserMappingStmt; diff --git a/src/test/regress/expected/foreign_data.out b/src/test/regress/expected/foreign_data.out index 328366bcfc..1c7a7593f9 100644 --- a/src/test/regress/expected/foreign_data.out +++ b/src/test/regress/expected/foreign_data.out @@ -221,6 +221,10 @@ CREATE FOREIGN DATA WRAPPER foo; CREATE SERVER s1 FOREIGN DATA WRAPPER foo; COMMENT ON SERVER s1 IS 'foreign server'; CREATE USER MAPPING FOR current_user SERVER s1; +CREATE USER MAPPING FOR current_user SERVER s1; -- ERROR +ERROR: user mapping for "regress_foreign_data_user" already exists for server s1 +CREATE USER MAPPING IF NOT EXISTS FOR current_user SERVER s1; -- NOTICE +NOTICE: user mapping for "regress_foreign_data_user" already exists for server s1, skipping \dew+ List of foreign-data wrappers Name | Owner | Handler | Validator | Access privileges | FDW Options | Description @@ -284,6 +288,8 @@ CREATE FOREIGN DATA WRAPPER foo OPTIONS ("test wrapper" 'true'); CREATE SERVER s1 FOREIGN DATA WRAPPER foo; CREATE SERVER s1 FOREIGN DATA WRAPPER foo; -- ERROR ERROR: server "s1" already exists +CREATE SERVER IF NOT EXISTS s1 FOREIGN DATA WRAPPER foo; -- No ERROR, just NOTICE +NOTICE: server "s1" already exists, skipping CREATE SERVER s2 FOREIGN DATA WRAPPER foo OPTIONS (host 'a', dbname 'b'); CREATE SERVER s3 TYPE 'oracle' FOREIGN DATA WRAPPER foo; CREATE SERVER s4 TYPE 'oracle' FOREIGN DATA WRAPPER foo OPTIONS (host 'a', dbname 'b'); diff --git a/src/test/regress/sql/foreign_data.sql b/src/test/regress/sql/foreign_data.sql index c13d5ffbe9..aaf079cf52 100644 --- a/src/test/regress/sql/foreign_data.sql +++ b/src/test/regress/sql/foreign_data.sql @@ -104,6 +104,8 @@ CREATE FOREIGN DATA WRAPPER foo; CREATE SERVER s1 FOREIGN DATA WRAPPER foo; COMMENT ON SERVER s1 IS 'foreign server'; CREATE USER MAPPING FOR current_user SERVER s1; +CREATE USER MAPPING FOR current_user SERVER s1; -- ERROR +CREATE USER MAPPING IF NOT EXISTS FOR current_user SERVER s1; -- NOTICE \dew+ \des+ \deu+ @@ -121,6 +123,7 @@ CREATE SERVER s1 FOREIGN DATA WRAPPER foo; -- ERROR CREATE FOREIGN DATA WRAPPER foo OPTIONS ("test wrapper" 'true'); CREATE SERVER s1 FOREIGN DATA WRAPPER foo; CREATE SERVER s1 FOREIGN DATA WRAPPER foo; -- ERROR +CREATE SERVER IF NOT EXISTS s1 FOREIGN DATA WRAPPER foo; -- No ERROR, just NOTICE CREATE SERVER s2 FOREIGN DATA WRAPPER foo OPTIONS (host 'a', dbname 'b'); CREATE SERVER s3 TYPE 'oracle' FOREIGN DATA WRAPPER foo; CREATE SERVER s4 TYPE 'oracle' FOREIGN DATA WRAPPER foo OPTIONS (host 'a', dbname 'b'); -- cgit v1.2.3 From c709b113e8f5433ea2a66b2fb08d9324dd596642 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Mon, 20 Mar 2017 21:23:56 -0400 Subject: doc: improve createdb example The previous example could error out due to encoding mismatches; use -T/template instead. Reported-by: Jason O'Donnell --- doc/src/sgml/ref/createdb.sgml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/createdb.sgml b/doc/src/sgml/ref/createdb.sgml index c363bd4a56..9fc4c16a81 100644 --- a/doc/src/sgml/ref/createdb.sgml +++ b/doc/src/sgml/ref/createdb.sgml @@ -363,11 +363,11 @@ PostgreSQL documentation To create the database demo using the server on host eden, port 5000, using the - LATIN1 encoding scheme with a look at the - underlying command: + template0 template database, here is the + command-line command and the underlying SQL command: -$ createdb -p 5000 -h eden -E LATIN1 -e demo -CREATE DATABASE demo ENCODING 'LATIN1'; +$ createdb -p 5000 -h eden -T template0 -e demo +CREATE DATABASE demo TEMPLATE template0; -- cgit v1.2.3 From 692ed0567d0a17013b8bc631ad6cab29470a944d Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Mon, 20 Mar 2017 22:22:11 -0400 Subject: doc: adjust 'Infinity' example to include minus This clarifies that quoted infinity values must have the negative signs inside single quotes. Reported-by: Don Morrison --- doc/src/sgml/datatype.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/datatype.sgml b/doc/src/sgml/datatype.sgml index e2f8dee7b6..7a546a0399 100644 --- a/doc/src/sgml/datatype.sgml +++ b/doc/src/sgml/datatype.sgml @@ -756,7 +756,7 @@ FROM generate_series(-3.5, 3.5, 1) as x; floating-point arithmetic does not follow IEEE 754, these values will probably not work as expected.) When writing these values as constants in an SQL command, you must put quotes around them, - for example UPDATE table SET x = 'Infinity'. On input, + for example UPDATE table SET x = '-Infinity'. On input, these strings are recognized in a case-insensitive manner. -- cgit v1.2.3 From 17fa3e834f288a2a8f0b3927d3f7f02451126686 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Mon, 20 Mar 2017 22:33:26 -0400 Subject: doc: clarify that function "ownership" that controls permission It used to say the creation user. Reported-by: Nathan Wagner --- doc/src/sgml/ref/create_function.sgml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/create_function.sgml b/doc/src/sgml/ref/create_function.sgml index e7057789d3..9d0d2f4beb 100644 --- a/doc/src/sgml/ref/create_function.sgml +++ b/doc/src/sgml/ref/create_function.sgml @@ -401,7 +401,7 @@ CREATE [ OR REPLACE ] FUNCTION is to be executed with the privileges of the user that calls it. That is the default. SECURITY DEFINER specifies that the function is to be executed with the - privileges of the user that created it. + privileges of the user that owns it. @@ -747,7 +747,7 @@ SELECT * FROM dup(42); Because a SECURITY DEFINER function is executed - with the privileges of the user that created it, care is needed to + with the privileges of the user that owns it, care is needed to ensure that the function cannot be misused. For security, should be set to exclude any schemas writable by untrusted users. This prevents -- cgit v1.2.3 From 010505546a343820f291e0e298673108e436c696 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 21 Mar 2017 08:33:32 -0400 Subject: doc: Improve markup --- doc/src/sgml/ref/alter_table.sgml | 2 +- doc/src/sgml/ref/create_table.sgml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/alter_table.sgml b/doc/src/sgml/ref/alter_table.sgml index 077c00373d..767ea321da 100644 --- a/doc/src/sgml/ref/alter_table.sgml +++ b/doc/src/sgml/ref/alter_table.sgml @@ -722,7 +722,7 @@ ALTER TABLE [ IF EXISTS ] name - ATTACH PARTITION partition_name FOR VALUES partition_bound_spec + ATTACH PARTITION partition_name FOR VALUES partition_bound_spec This form attaches an existing table (which might itself be partitioned) diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml index bb081ff86f..9ed25c05da 100644 --- a/doc/src/sgml/ref/create_table.sgml +++ b/doc/src/sgml/ref/create_table.sgml @@ -248,7 +248,7 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI - PARTITION OF parent_table FOR VALUES partition_bound_spec + PARTITION OF parent_table FOR VALUES partition_bound_spec Creates the table as partition of the specified -- cgit v1.2.3 From f7946a92b689199cba64e7406a1c12d12637168a Mon Sep 17 00:00:00 2001 From: Andrew Dunstan Date: Tue, 21 Mar 2017 10:19:03 -0400 Subject: Add btree_gist support for enum types. This will allow enums to be used in exclusion constraints. The code uses the new CallerFInfoFunctionCall infrastructure in fmgr, and the support for it added to btree_gist in commit 393bb504d7. Reviewed by Tom Lane and Anastasia Lubennikova Discussion: http://postgr.es/m/56EA8A71.8060107@dunslane.net --- contrib/btree_gist/Makefile | 9 +- contrib/btree_gist/btree_enum.c | 187 +++++++++ contrib/btree_gist/btree_gist--1.4--1.5.sql | 69 ++++ contrib/btree_gist/btree_gist.control | 2 +- contrib/btree_gist/btree_gist.h | 3 +- contrib/btree_gist/btree_utils_num.c | 2 + contrib/btree_gist/data/enum.data | 595 ++++++++++++++++++++++++++++ contrib/btree_gist/expected/enum.out | 91 +++++ contrib/btree_gist/sql/enum.sql | 38 ++ doc/src/sgml/btree-gist.sgml | 2 +- 10 files changed, 991 insertions(+), 7 deletions(-) create mode 100644 contrib/btree_gist/btree_enum.c create mode 100644 contrib/btree_gist/btree_gist--1.4--1.5.sql create mode 100644 contrib/btree_gist/data/enum.data create mode 100644 contrib/btree_gist/expected/enum.out create mode 100644 contrib/btree_gist/sql/enum.sql (limited to 'doc/src') diff --git a/contrib/btree_gist/Makefile b/contrib/btree_gist/Makefile index c70f17869a..af651205ec 100644 --- a/contrib/btree_gist/Makefile +++ b/contrib/btree_gist/Makefile @@ -5,18 +5,19 @@ MODULE_big = btree_gist OBJS = btree_gist.o btree_utils_num.o btree_utils_var.o btree_int2.o \ btree_int4.o btree_int8.o btree_float4.o btree_float8.o btree_cash.o \ btree_oid.o btree_ts.o btree_time.o btree_date.o btree_interval.o \ - btree_macaddr.o btree_macaddr8.o btree_inet.o btree_text.o btree_bytea.o \ - btree_bit.o btree_numeric.o btree_uuid.o $(WIN32RES) + btree_macaddr.o btree_macaddr8.o btree_inet.o btree_text.o \ + btree_bytea.o btree_bit.o btree_numeric.o btree_uuid.o \ + btree_enum.o $(WIN32RES) EXTENSION = btree_gist DATA = btree_gist--unpackaged--1.0.sql btree_gist--1.0--1.1.sql \ btree_gist--1.1--1.2.sql btree_gist--1.2.sql btree_gist--1.2--1.3.sql \ - btree_gist--1.3--1.4.sql + btree_gist--1.3--1.4.sql btree_gist--1.4--1.5.sql PGFILEDESC = "btree_gist - B-tree equivalent GiST operator classes" REGRESS = init int2 int4 int8 float4 float8 cash oid timestamp timestamptz \ time timetz date interval macaddr macaddr8 inet cidr text varchar char \ - bytea bit varbit numeric uuid not_equal + bytea bit varbit numeric uuid not_equal enum SHLIB_LINK += $(filter -lm, $(LIBS)) diff --git a/contrib/btree_gist/btree_enum.c b/contrib/btree_gist/btree_enum.c new file mode 100644 index 0000000000..5e46e782be --- /dev/null +++ b/contrib/btree_gist/btree_enum.c @@ -0,0 +1,187 @@ +/* + * contrib/btree_gist/btree_enum.c + */ +#include "postgres.h" +#include "fmgr.h" +#include "utils/builtins.h" + +#include "btree_gist.h" +#include "btree_utils_num.h" + +/* enums are really Oids, so we just use the same structure */ + +typedef struct +{ + Oid lower; + Oid upper; +} oidKEY; + +/* +** enum ops +*/ +PG_FUNCTION_INFO_V1(gbt_enum_compress); +PG_FUNCTION_INFO_V1(gbt_enum_fetch); +PG_FUNCTION_INFO_V1(gbt_enum_union); +PG_FUNCTION_INFO_V1(gbt_enum_picksplit); +PG_FUNCTION_INFO_V1(gbt_enum_consistent); +PG_FUNCTION_INFO_V1(gbt_enum_penalty); +PG_FUNCTION_INFO_V1(gbt_enum_same); + + +static bool +gbt_enumgt(const void *a, const void *b, FmgrInfo *flinfo) +{ + return DatumGetBool( + CallerFInfoFunctionCall2(enum_gt, flinfo, InvalidOid, ObjectIdGetDatum(*((const Oid *) a)), ObjectIdGetDatum(*((const Oid *) b))) + ); +} +static bool +gbt_enumge(const void *a, const void *b, FmgrInfo *flinfo) +{ + return DatumGetBool( + CallerFInfoFunctionCall2(enum_ge, flinfo, InvalidOid, ObjectIdGetDatum(*((const Oid *) a)), ObjectIdGetDatum(*((const Oid *) b))) + ); +} +static bool +gbt_enumeq(const void *a, const void *b, FmgrInfo *flinfo) +{ + return (*((const Oid *) a) == *((const Oid *) b)); +} +static bool +gbt_enumle(const void *a, const void *b, FmgrInfo *flinfo) +{ + return DatumGetBool( + CallerFInfoFunctionCall2(enum_le, flinfo, InvalidOid, ObjectIdGetDatum(*((const Oid *) a)), ObjectIdGetDatum(*((const Oid *) b))) + ); +} +static bool +gbt_enumlt(const void *a, const void *b, FmgrInfo *flinfo) +{ + return DatumGetBool( + CallerFInfoFunctionCall2(enum_lt, flinfo, InvalidOid, ObjectIdGetDatum(*((const Oid *) a)), ObjectIdGetDatum(*((const Oid *) b))) + ); +} + +static int +gbt_enumkey_cmp(const void *a, const void *b, FmgrInfo *flinfo) +{ + oidKEY *ia = (oidKEY *) (((const Nsrt *) a)->t); + oidKEY *ib = (oidKEY *) (((const Nsrt *) b)->t); + + if (ia->lower == ib->lower) + { + if (ia->upper == ib->upper) + return 0; + + return DatumGetInt32( + CallerFInfoFunctionCall2(enum_cmp, flinfo, InvalidOid, ObjectIdGetDatum(ia->upper), ObjectIdGetDatum(ib->upper)) + ); + } + + return DatumGetInt32( + CallerFInfoFunctionCall2(enum_cmp, flinfo, InvalidOid, ObjectIdGetDatum(ia->lower), ObjectIdGetDatum(ib->lower)) + ); +} + +static const gbtree_ninfo tinfo = +{ + gbt_t_enum, + sizeof(Oid), + 8, /* sizeof(gbtreekey8) */ + gbt_enumgt, + gbt_enumge, + gbt_enumeq, + gbt_enumle, + gbt_enumlt, + gbt_enumkey_cmp, + NULL /* no KNN support at least for now */ +}; + + +/************************************************** + * Enum ops + **************************************************/ + + +Datum +gbt_enum_compress(PG_FUNCTION_ARGS) +{ + GISTENTRY *entry = (GISTENTRY *) PG_GETARG_POINTER(0); + + PG_RETURN_POINTER(gbt_num_compress(entry, &tinfo)); +} + +Datum +gbt_enum_fetch(PG_FUNCTION_ARGS) +{ + GISTENTRY *entry = (GISTENTRY *) PG_GETARG_POINTER(0); + + PG_RETURN_POINTER(gbt_num_fetch(entry, &tinfo)); +} + +Datum +gbt_enum_consistent(PG_FUNCTION_ARGS) +{ + GISTENTRY *entry = (GISTENTRY *) PG_GETARG_POINTER(0); + Oid query = PG_GETARG_OID(1); + StrategyNumber strategy = (StrategyNumber) PG_GETARG_UINT16(2); + + /* Oid subtype = PG_GETARG_OID(3); */ + bool *recheck = (bool *) PG_GETARG_POINTER(4); + oidKEY *kkk = (oidKEY *) DatumGetPointer(entry->key); + GBT_NUMKEY_R key; + + /* All cases served by this function are exact */ + *recheck = false; + + key.lower = (GBT_NUMKEY *) &kkk->lower; + key.upper = (GBT_NUMKEY *) &kkk->upper; + + PG_RETURN_BOOL( + gbt_num_consistent(&key, (void *) &query, &strategy, GIST_LEAF(entry), &tinfo, fcinfo->flinfo) + ); +} + +Datum +gbt_enum_union(PG_FUNCTION_ARGS) +{ + GistEntryVector *entryvec = (GistEntryVector *) PG_GETARG_POINTER(0); + void *out = palloc(sizeof(oidKEY)); + + *(int *) PG_GETARG_POINTER(1) = sizeof(oidKEY); + PG_RETURN_POINTER(gbt_num_union((void *) out, entryvec, &tinfo, fcinfo->flinfo)); +} + + +Datum +gbt_enum_penalty(PG_FUNCTION_ARGS) +{ + oidKEY *origentry = (oidKEY *) DatumGetPointer(((GISTENTRY *) PG_GETARG_POINTER(0))->key); + oidKEY *newentry = (oidKEY *) DatumGetPointer(((GISTENTRY *) PG_GETARG_POINTER(1))->key); + float *result = (float *) PG_GETARG_POINTER(2); + + penalty_num(result, origentry->lower, origentry->upper, newentry->lower, newentry->upper); + + PG_RETURN_POINTER(result); +} + +Datum +gbt_enum_picksplit(PG_FUNCTION_ARGS) +{ + PG_RETURN_POINTER(gbt_num_picksplit( + (GistEntryVector *) PG_GETARG_POINTER(0), + (GIST_SPLITVEC *) PG_GETARG_POINTER(1), + &tinfo, fcinfo->flinfo + )); +} + +Datum +gbt_enum_same(PG_FUNCTION_ARGS) +{ + oidKEY *b1 = (oidKEY *) PG_GETARG_POINTER(0); + oidKEY *b2 = (oidKEY *) PG_GETARG_POINTER(1); + bool *result = (bool *) PG_GETARG_POINTER(2); + + *result = gbt_num_same((void *) b1, (void *) b2, &tinfo, fcinfo->flinfo); + PG_RETURN_POINTER(result); +} diff --git a/contrib/btree_gist/btree_gist--1.4--1.5.sql b/contrib/btree_gist/btree_gist--1.4--1.5.sql new file mode 100644 index 0000000000..cf974c2f53 --- /dev/null +++ b/contrib/btree_gist/btree_gist--1.4--1.5.sql @@ -0,0 +1,69 @@ +/* contrib/btree_gist/btree_gist--1.4--1.5.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION btree_gist UPDATE TO '1.5'" to load this file. \quit + +-- +-- +-- +-- enum ops +-- +-- +-- +-- define the GiST support methods +CREATE FUNCTION gbt_enum_consistent(internal,anyenum,int2,oid,internal) +RETURNS bool +AS 'MODULE_PATHNAME' +LANGUAGE C IMMUTABLE STRICT; + +CREATE FUNCTION gbt_enum_compress(internal) +RETURNS internal +AS 'MODULE_PATHNAME' +LANGUAGE C IMMUTABLE STRICT; + +CREATE FUNCTION gbt_enum_fetch(internal) +RETURNS internal +AS 'MODULE_PATHNAME' +LANGUAGE C IMMUTABLE STRICT; + +CREATE FUNCTION gbt_enum_penalty(internal,internal,internal) +RETURNS internal +AS 'MODULE_PATHNAME' +LANGUAGE C IMMUTABLE STRICT; + +CREATE FUNCTION gbt_enum_picksplit(internal, internal) +RETURNS internal +AS 'MODULE_PATHNAME' +LANGUAGE C IMMUTABLE STRICT; + +CREATE FUNCTION gbt_enum_union(internal, internal) +RETURNS gbtreekey8 +AS 'MODULE_PATHNAME' +LANGUAGE C IMMUTABLE STRICT; + +CREATE FUNCTION gbt_enum_same(gbtreekey8, gbtreekey8, internal) +RETURNS internal +AS 'MODULE_PATHNAME' +LANGUAGE C IMMUTABLE STRICT; + +-- Create the operator class +CREATE OPERATOR CLASS gist_enum_ops +DEFAULT FOR TYPE anyenum USING gist +AS + OPERATOR 1 < , + OPERATOR 2 <= , + OPERATOR 3 = , + OPERATOR 4 >= , + OPERATOR 5 > , + FUNCTION 1 gbt_enum_consistent (internal, anyenum, int2, oid, internal), + FUNCTION 2 gbt_enum_union (internal, internal), + FUNCTION 3 gbt_enum_compress (internal), + FUNCTION 4 gbt_decompress (internal), + FUNCTION 5 gbt_enum_penalty (internal, internal, internal), + FUNCTION 6 gbt_enum_picksplit (internal, internal), + FUNCTION 7 gbt_enum_same (gbtreekey8, gbtreekey8, internal), + STORAGE gbtreekey8; + +ALTER OPERATOR FAMILY gist_enum_ops USING gist ADD + OPERATOR 6 <> (anyenum, anyenum) , + FUNCTION 9 (anyenum, anyenum) gbt_enum_fetch (internal) ; diff --git a/contrib/btree_gist/btree_gist.control b/contrib/btree_gist/btree_gist.control index fdf0e6ad9e..81c850905c 100644 --- a/contrib/btree_gist/btree_gist.control +++ b/contrib/btree_gist/btree_gist.control @@ -1,5 +1,5 @@ # btree_gist extension comment = 'support for indexing common datatypes in GiST' -default_version = '1.4' +default_version = '1.5' module_pathname = '$libdir/btree_gist' relocatable = true diff --git a/contrib/btree_gist/btree_gist.h b/contrib/btree_gist/btree_gist.h index f759299bb2..011285abce 100644 --- a/contrib/btree_gist/btree_gist.h +++ b/contrib/btree_gist/btree_gist.h @@ -33,7 +33,8 @@ enum gbtree_type gbt_t_bytea, gbt_t_bit, gbt_t_inet, - gbt_t_uuid + gbt_t_uuid, + gbt_t_enum }; #endif diff --git a/contrib/btree_gist/btree_utils_num.c b/contrib/btree_gist/btree_utils_num.c index e30924ba1d..d4fee91ee1 100644 --- a/contrib/btree_gist/btree_utils_num.c +++ b/contrib/btree_gist/btree_utils_num.c @@ -48,6 +48,7 @@ gbt_num_compress(GISTENTRY *entry, const gbtree_ninfo *tinfo) leaf = &v.i8; break; case gbt_t_oid: + case gbt_t_enum: v.i4 = DatumGetObjectId(entry->key); leaf = &v.i4; break; @@ -122,6 +123,7 @@ gbt_num_fetch(GISTENTRY *entry, const gbtree_ninfo *tinfo) datum = Int64GetDatum(*(int64 *) entry->key); break; case gbt_t_oid: + case gbt_t_enum: datum = ObjectIdGetDatum(*(Oid *) entry->key); break; case gbt_t_float4: diff --git a/contrib/btree_gist/data/enum.data b/contrib/btree_gist/data/enum.data new file mode 100644 index 0000000000..d03bfced98 --- /dev/null +++ b/contrib/btree_gist/data/enum.data @@ -0,0 +1,595 @@ +r +v +i +b +r +\N +y +v +g +o +y +b +o +o +o +o +v +r +i +o +b +r +g +b +i +o +r +r +r +\N +o +b +v +y +o +\N +i +o +o +g +g +b +y +v +g +g +\N +v +g +i +i +\N +v +y +i +r +\N +r +\N +g +\N +g +\N +v +g +y +v +r +v +r +v +y +i +i +v +y +v +i +b +i +i +r +r +\N +\N +y +r +g +i +y +i +i +r +g +y +\N +i +o +r +y +y +g +o +o +g +y +r +g +v +r +i +r +i +r +y +v +b +i +o +r +\N +o +i +v +o +b +\N +b +g +y +o +v +b +i +v +v +o +y +i +i +i +g +b +b +g +r +i +y +o +\N +r +\N +i +i +g +v +o +y +y +o +i +b +r +y +y +o +g +g +g +\N +y +o +v +g +y +g +v +\N +i +o +v +b +b +\N +y +v +\N +v +\N +i +\N +r +b +r +o +r +b +o +g +i +r +b +g +g +y +b +b +g +y +g +v +v +b +\N +i +v +y +b +b +o +g +b +v +g +g +b +\N +y +r +r +b +\N +r +g +i +o +v +\N +o +r +b +o +b +i +\N +\N +y +b +y +\N +i +i +i +o +y +o +i +b +o +g +r +\N +b +y +\N +g +b +y +y +o +o +b +g +i +i +v +b +o +o +v +i +g +i +o +r +o +i +i +r +b +g +o +o +y +v +g +g +g +r +o +i +i +g +\N +o +v +b +b +v +i +g +y +i +i +g +r +y +i +b +\N +g +y +o +\N +i +i +b +v +o +b +v +r +g +o +v +v +y +r +v +g +\N +v +v +b +y +o +g +i +o +b +r +y +r +v +b +b +\N +i +v +y +r +b +i +y +g +\N +g +r +y +y +g +b +o +v +r +i +g +r +b +b +b +\N +y +y +y +i +o +r +g +g +i +y +g +y +v +o +o +g +\N +b +v +o +y +r +\N +o +i +g +\N +i +i +i +o +b +\N +\N +b +\N +v +v +r +\N +o +b +r +o +b +o +r +y +\N +r +i +b +b +y +v +r +g +r +r +\N +g +\N +v +v +y +r +o +r +o +i +o +\N +r +\N +i +v +b +v +\N +b +r +v +o +\N +i +r +b +g +o +\N +o +g +r +v +y +g +v +r +b +r +v +o +g +i +i +g +i +y +b +i +y +r +y +o +r +b +y +y +b +y +g +b +\N +r +g +b +o +y +o +g +r +g +b +\N +v +v +v +g +b +y +v +o +v +g +o +g +i +b +v +i +r +r +i +b +i +b +o +\N +\N +y +r +g +v +o +y +\N +g +v +o +b +v +v +\N +r +v +y +g +b +o +v +b +v +b +r +r +i +r +v +y +v +y +o +v +g +i +r +o +o +i +y +r +\N +y +r +b +y +y +\N +b +\N +\N +i +v diff --git a/contrib/btree_gist/expected/enum.out b/contrib/btree_gist/expected/enum.out new file mode 100644 index 0000000000..c4b769dd4b --- /dev/null +++ b/contrib/btree_gist/expected/enum.out @@ -0,0 +1,91 @@ +-- enum check +create type rainbow as enum ('r','o','y','g','b','i','v'); +CREATE TABLE enumtmp (a rainbow); +\copy enumtmp from 'data/enum.data' +SET enable_seqscan=on; +select a, count(*) from enumtmp group by a order by 1; + a | count +---+------- + r | 76 + o | 78 + y | 73 + g | 75 + b | 77 + i | 78 + v | 75 + | 63 +(8 rows) + +SELECT count(*) FROM enumtmp WHERE a < 'g'::rainbow; + count +------- + 227 +(1 row) + +SELECT count(*) FROM enumtmp WHERE a <= 'g'::rainbow; + count +------- + 302 +(1 row) + +SELECT count(*) FROM enumtmp WHERE a = 'g'::rainbow; + count +------- + 75 +(1 row) + +SELECT count(*) FROM enumtmp WHERE a >= 'g'::rainbow; + count +------- + 305 +(1 row) + +SELECT count(*) FROM enumtmp WHERE a > 'g'::rainbow; + count +------- + 230 +(1 row) + +CREATE INDEX enumidx ON enumtmp USING gist ( a ); +SET enable_seqscan=off; +SELECT count(*) FROM enumtmp WHERE a < 'g'::rainbow; + count +------- + 227 +(1 row) + +SELECT count(*) FROM enumtmp WHERE a <= 'g'::rainbow; + count +------- + 302 +(1 row) + +SELECT count(*) FROM enumtmp WHERE a = 'g'::rainbow; + count +------- + 75 +(1 row) + +SELECT count(*) FROM enumtmp WHERE a >= 'g'::rainbow; + count +------- + 305 +(1 row) + +SELECT count(*) FROM enumtmp WHERE a > 'g'::rainbow; + count +------- + 230 +(1 row) + +EXPLAIN (COSTS OFF) +SELECT count(*) FROM enumtmp WHERE a >= 'g'::rainbow; + QUERY PLAN +----------------------------------------------- + Aggregate + -> Bitmap Heap Scan on enumtmp + Recheck Cond: (a >= 'g'::rainbow) + -> Bitmap Index Scan on enumidx + Index Cond: (a >= 'g'::rainbow) +(5 rows) + diff --git a/contrib/btree_gist/sql/enum.sql b/contrib/btree_gist/sql/enum.sql new file mode 100644 index 0000000000..476211e979 --- /dev/null +++ b/contrib/btree_gist/sql/enum.sql @@ -0,0 +1,38 @@ +-- enum check + +create type rainbow as enum ('r','o','y','g','b','i','v'); + +CREATE TABLE enumtmp (a rainbow); + +\copy enumtmp from 'data/enum.data' + +SET enable_seqscan=on; + +select a, count(*) from enumtmp group by a order by 1; + +SELECT count(*) FROM enumtmp WHERE a < 'g'::rainbow; + +SELECT count(*) FROM enumtmp WHERE a <= 'g'::rainbow; + +SELECT count(*) FROM enumtmp WHERE a = 'g'::rainbow; + +SELECT count(*) FROM enumtmp WHERE a >= 'g'::rainbow; + +SELECT count(*) FROM enumtmp WHERE a > 'g'::rainbow; + +CREATE INDEX enumidx ON enumtmp USING gist ( a ); + +SET enable_seqscan=off; + +SELECT count(*) FROM enumtmp WHERE a < 'g'::rainbow; + +SELECT count(*) FROM enumtmp WHERE a <= 'g'::rainbow; + +SELECT count(*) FROM enumtmp WHERE a = 'g'::rainbow; + +SELECT count(*) FROM enumtmp WHERE a >= 'g'::rainbow; + +SELECT count(*) FROM enumtmp WHERE a > 'g'::rainbow; + +EXPLAIN (COSTS OFF) +SELECT count(*) FROM enumtmp WHERE a >= 'g'::rainbow; diff --git a/doc/src/sgml/btree-gist.sgml b/doc/src/sgml/btree-gist.sgml index cfdd5be84a..f3c639c2f3 100644 --- a/doc/src/sgml/btree-gist.sgml +++ b/doc/src/sgml/btree-gist.sgml @@ -17,7 +17,7 @@ oid, money, char, varchar, text, bytea, bit, varbit, macaddr, macaddr8, inet, - cidr and uuid. + cidr, uuid, and all enum types. -- cgit v1.2.3 From 80275561fffb58e48da2d3fd72b72a3bfbee8669 Mon Sep 17 00:00:00 2001 From: Andrew Dunstan Date: Tue, 21 Mar 2017 11:11:42 -0400 Subject: Document btree_gin support for enums --- doc/src/sgml/btree-gin.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/btree-gin.sgml b/doc/src/sgml/btree-gin.sgml index 0de8eb5c30..375e7ec4be 100644 --- a/doc/src/sgml/btree-gin.sgml +++ b/doc/src/sgml/btree-gin.sgml @@ -17,7 +17,7 @@ oid, money, "char", varchar, text, bytea, bit, varbit, macaddr, macaddr8, inet, - and cidr. + cidr, and all enum types. -- cgit v1.2.3 From 9212810c4f4a596fc398644e96925a8a0f0332e8 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 21 Mar 2017 22:09:40 -0400 Subject: doc: Update tool sets documentation for modern FreeBSD For several operating systems, we give handy package manager one-liners to install all the requirements for building our documentation. All current production FreeBSD releases have a friendly new package manager a bit like apt/yum, so give a one line command here. Also, add a brief note about gmake vs make in the doc subdirectory. Author: Thomas Munro --- doc/src/sgml/docguide.sgml | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'doc/src') diff --git a/doc/src/sgml/docguide.sgml b/doc/src/sgml/docguide.sgml index 48828aff37..57b67137b1 100644 --- a/doc/src/sgml/docguide.sgml +++ b/doc/src/sgml/docguide.sgml @@ -254,6 +254,19 @@ yum install docbook-dtds docbook-style-dsssl docbook-style-xsl libxslt openjade also be of interest. + + To install the required packages with pkg, use: + +pkg install docbook-sgml docbook-xml docbook-xsl dsssl-docbook-modular libxslt openjade + + + + + When building the documentation from the doc + directory you'll need to use gmake, because the + makefile provided is not suitable for FreeBSD's make. + + More information about the FreeBSD documentation tools can be found in the -- cgit v1.2.3 From 96a7128b7b4c9ce4fb51df8c8b216dfab6340766 Mon Sep 17 00:00:00 2001 From: Andrew Dunstan Date: Wed, 22 Mar 2017 10:00:30 -0400 Subject: Sync pg_dump and pg_dumpall output Before exiting any files are fsync'ed. A --no-sync option is also provided for a faster exit if desired. Michael Paquier. Reviewed by Albe Laurenz Discussion: https://postgr.es/m/CAB7nPqS1uZ=Ov+UruW6jr3vB-S_DLVMPc0dQpV-fTDjmm0ZQMg@mail.gmail.com --- doc/src/sgml/ref/pg_dump.sgml | 14 ++++++++++++++ doc/src/sgml/ref/pg_dumpall.sgml | 14 ++++++++++++++ src/bin/pg_dump/pg_backup.h | 2 +- src/bin/pg_dump/pg_backup_archiver.c | 15 ++++++++++----- src/bin/pg_dump/pg_backup_archiver.h | 1 + src/bin/pg_dump/pg_backup_custom.c | 5 +++++ src/bin/pg_dump/pg_backup_directory.c | 8 ++++++++ src/bin/pg_dump/pg_backup_tar.c | 5 +++++ src/bin/pg_dump/pg_dump.c | 12 ++++++++++-- src/bin/pg_dump/pg_dumpall.c | 15 +++++++++++++++ src/common/file_utils.c | 19 +++++++++++++++++++ src/include/common/file_utils.h | 1 + 12 files changed, 103 insertions(+), 8 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml index a1e03c481d..bb32fb12e0 100644 --- a/doc/src/sgml/ref/pg_dump.sgml +++ b/doc/src/sgml/ref/pg_dump.sgml @@ -859,6 +859,20 @@ PostgreSQL documentation + + + + + By default, pg_dump will wait for all files + to be written safely to disk. This option causes + pg_dump to return without waiting, which is + faster, but means that a subsequent operating system crash can leave + the dump corrupt. Generally, this option is useful for testing + but should not be used when dumping data from production installation. + + + + diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml index afbadce247..070b902487 100644 --- a/doc/src/sgml/ref/pg_dumpall.sgml +++ b/doc/src/sgml/ref/pg_dumpall.sgml @@ -354,6 +354,20 @@ PostgreSQL documentation + + + + + By default, pg_dumpall will wait for all files + to be written safely to disk. This option causes + pg_dumpall to return without waiting, which is + faster, but means that a subsequent operating system crash can leave + the dump corrupt. Generally, this option is useful for testing + but should not be used when dumping data from production installation. + + + + diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 983a999fcd..610bed531c 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -276,7 +276,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt); /* Create a new archive */ extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, ArchiveMode mode, + const int compression, bool dosync, ArchiveMode mode, SetupWorkerPtr setupDumpWorker); /* The --list option */ diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 734373beaa..dd0892539a 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -54,7 +54,8 @@ static const char *modulename = gettext_noop("archiver"); static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, ArchiveMode mode, SetupWorkerPtr setupWorkerPtr); + const int compression, bool dosync, ArchiveMode mode, + SetupWorkerPtr setupWorkerPtr); static void _getObjectDescription(PQExpBuffer buf, TocEntry *te, ArchiveHandle *AH); static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData, bool acl_pass); @@ -202,10 +203,12 @@ setupRestoreWorker(Archive *AHX) /* Public */ Archive * CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, ArchiveMode mode, SetupWorkerPtr setupDumpWorker) + const int compression, bool dosync, ArchiveMode mode, + SetupWorkerPtr setupDumpWorker) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, compression, mode, setupDumpWorker); + ArchiveHandle *AH = _allocAH(FileSpec, fmt, compression, dosync, + mode, setupDumpWorker); return (Archive *) AH; } @@ -215,7 +218,7 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt, Archive * OpenArchive(const char *FileSpec, const ArchiveFormat fmt) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, archModeRead, setupRestoreWorker); + ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker); return (Archive *) AH; } @@ -2269,7 +2272,8 @@ _discoverArchiveFormat(ArchiveHandle *AH) */ static ArchiveHandle * _allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, ArchiveMode mode, SetupWorkerPtr setupWorkerPtr) + const int compression, bool dosync, ArchiveMode mode, + SetupWorkerPtr setupWorkerPtr) { ArchiveHandle *AH; @@ -2323,6 +2327,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, AH->mode = mode; AH->compression = compression; + AH->dosync = dosync; memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse)); diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index a44e16ee45..b00a7ede97 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -312,6 +312,7 @@ struct _archiveHandle * values for compression: -1 * Z_DEFAULT_COMPRESSION 0 COMPRESSION_NONE * 1-9 levels for gzip compression */ + bool dosync; /* data requested to be synced on sight */ ArchiveMode mode; /* File mode - r or w */ void *formatData; /* Header data specific to file format */ diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 5737608f9e..a1f4cb1fea 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -28,6 +28,7 @@ #include "compress_io.h" #include "parallel.h" #include "pg_backup_utils.h" +#include "common/file_utils.h" /*-------- * Routines in the format interface @@ -721,6 +722,10 @@ _CloseArchive(ArchiveHandle *AH) if (fclose(AH->FH) != 0) exit_horribly(modulename, "could not close archive file: %s\n", strerror(errno)); + /* Sync the output file if one is defined */ + if (AH->dosync && AH->mode == archModeWrite && AH->fSpec) + (void) fsync_fname(AH->fSpec, false, progname); + AH->FH = NULL; } diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index 0d7322f73a..79922da8ba 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -37,6 +37,7 @@ #include "compress_io.h" #include "parallel.h" #include "pg_backup_utils.h" +#include "common/file_utils.h" #include #include @@ -593,6 +594,13 @@ _CloseArchive(ArchiveHandle *AH) WriteDataChunks(AH, ctx->pstate); ParallelBackupEnd(AH, ctx->pstate); + + /* + * In directory mode, there is no need to sync all the entries + * individually. Just recurse once through all the files generated. + */ + if (AH->dosync) + fsync_dir_recurse(ctx->directory, progname); } AH->FH = NULL; } diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index 9cadd0c4a4..a2b320f371 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -33,6 +33,7 @@ #include "pg_backup_tar.h" #include "pg_backup_utils.h" #include "pgtar.h" +#include "common/file_utils.h" #include "fe_utils/string_utils.h" #include @@ -901,6 +902,10 @@ _CloseArchive(ArchiveHandle *AH) if (fputc(0, ctx->tarFH) == EOF) WRITE_ERROR_EXIT; } + + /* Sync the output file if one is defined */ + if (AH->dosync && AH->fSpec) + (void) fsync_fname(AH->fSpec, false, progname); } AH->FH = NULL; diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 52fa6f33e3..2b5a52656c 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -89,6 +89,8 @@ typedef enum OidOptions /* global decls */ bool g_verbose; /* User wants verbose narration of our * activities. */ +static bool dosync = true; /* Issue fsync() to make dump durable + * on disk. */ /* subquery used to convert user ID (eg, datdba) to user name */ static const char *username_subquery; @@ -353,6 +355,7 @@ main(int argc, char **argv) {"no-security-labels", no_argument, &dopt.no_security_labels, 1}, {"no-synchronized-snapshots", no_argument, &dopt.no_synchronized_snapshots, 1}, {"no-unlogged-table-data", no_argument, &dopt.no_unlogged_table_data, 1}, + {"no-sync", no_argument, NULL, 7}, {NULL, 0, NULL, 0} }; @@ -533,6 +536,10 @@ main(int argc, char **argv) dumpsnapshot = pg_strdup(optarg); break; + case 7: /* no-sync */ + dosync = false; + break; + default: fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); exit_nicely(1); @@ -632,8 +639,8 @@ main(int argc, char **argv) exit_horribly(NULL, "parallel backup only supported by the directory format\n"); /* Open the output file */ - fout = CreateArchive(filename, archiveFormat, compressLevel, archiveMode, - setupDumpWorker); + fout = CreateArchive(filename, archiveFormat, compressLevel, dosync, + archiveMode, setupDumpWorker); /* Make dump options accessible right away */ SetArchiveOptions(fout, &dopt, NULL); @@ -914,6 +921,7 @@ help(const char *progname) printf(_(" -V, --version output version information, then exit\n")); printf(_(" -Z, --compress=0-9 compression level for compressed formats\n")); printf(_(" --lock-wait-timeout=TIMEOUT fail after waiting TIMEOUT for a table lock\n")); + printf(_(" --no-sync do not wait for changes to be written safely to disk\n")); printf(_(" -?, --help show this help, then exit\n")); printf(_("\nOptions controlling the output content:\n")); diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index 81ed924b9f..d598d10016 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -22,6 +22,7 @@ #include "dumputils.h" #include "pg_backup.h" +#include "common/file_utils.h" #include "fe_utils/string_utils.h" /* version string we expect back from pg_dump */ @@ -63,6 +64,7 @@ static PQExpBuffer pgdumpopts; static char *connstr = ""; static bool skip_acls = false; static bool verbose = false; +static bool dosync = true; static int binary_upgrade = 0; static int column_inserts = 0; @@ -127,6 +129,7 @@ main(int argc, char *argv[]) {"role", required_argument, NULL, 3}, {"use-set-session-authorization", no_argument, &use_setsessauth, 1}, {"no-security-labels", no_argument, &no_security_labels, 1}, + {"no-sync", no_argument, NULL, 4}, {"no-unlogged-table-data", no_argument, &no_unlogged_table_data, 1}, {"no-role-passwords", no_argument, &no_role_passwords, 1}, @@ -297,6 +300,11 @@ main(int argc, char *argv[]) appendShellString(pgdumpopts, use_role); break; + case 4: + dosync = false; + appendPQExpBufferStr(pgdumpopts, " --no-sync"); + break; + default: fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); exit_nicely(1); @@ -549,8 +557,14 @@ main(int argc, char *argv[]) fprintf(OPF, "--\n-- PostgreSQL database cluster dump complete\n--\n\n"); if (filename) + { fclose(OPF); + /* sync the resulting file, errors are not fatal */ + if (dosync) + (void) fsync_fname(filename, false, progname); + } + exit_nicely(0); } @@ -586,6 +600,7 @@ help(void) printf(_(" --if-exists use IF EXISTS when dropping objects\n")); printf(_(" --inserts dump data as INSERT commands, rather than COPY\n")); printf(_(" --no-security-labels do not dump security label assignments\n")); + printf(_(" --no-sync do not wait for changes to be written safely to disk\n")); printf(_(" --no-tablespaces do not dump tablespace assignments\n")); printf(_(" --no-unlogged-table-data do not dump unlogged table data\n")); printf(_(" --no-role-passwords do not dump passwords for roles\n")); diff --git a/src/common/file_utils.c b/src/common/file_utils.c index a978e64f5a..72b0565c71 100644 --- a/src/common/file_utils.c +++ b/src/common/file_utils.c @@ -115,6 +115,25 @@ fsync_pgdata(const char *pg_data, walkdir(pg_tblspc, fsync_fname, true, progname); } +/* + * Issue fsync recursively on the given directory and all its contents. + * + * This is a convenient wrapper on top of walkdir(). + */ +void +fsync_dir_recurse(const char *dir, const char *progname) +{ + /* + * If possible, hint to the kernel that we're soon going to fsync the data + * directory and its contents. + */ +#ifdef PG_FLUSH_DATA_WORKS + walkdir(dir, pre_sync_fname, false, progname); +#endif + + walkdir(dir, fsync_fname, false, progname); +} + /* * walkdir: recursively walk a directory, applying the action to each * regular file and directory (including the named directory itself). diff --git a/src/include/common/file_utils.h b/src/include/common/file_utils.h index 07c25c244d..48cc97a409 100644 --- a/src/include/common/file_utils.h +++ b/src/include/common/file_utils.h @@ -19,6 +19,7 @@ extern int fsync_fname(const char *fname, bool isdir, const char *progname); extern void fsync_pgdata(const char *pg_data, const char *progname, int serverVersion); +extern void fsync_dir_recurse(const char *dir, const char *progname); extern int durable_rename(const char *oldfile, const char *newfile, const char *progname); extern int fsync_parent_path(const char *fname, const char *progname); -- cgit v1.2.3 From 4cfc9484d4effb0a3aa2c8742bdef0c2bc7a3ca5 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Mon, 13 Feb 2017 08:57:45 -0500 Subject: Refine rules for altering publication owner Previously, the new owner had to be a superuser. The new rules are more refined similar to other objects. Reviewed-by: Petr Jelinek --- doc/src/sgml/ref/alter_publication.sgml | 7 +++++-- src/backend/commands/publicationcmds.c | 34 ++++++++++++++++++++++--------- src/test/regress/expected/publication.out | 8 ++++++++ src/test/regress/sql/publication.sql | 4 ++++ 4 files changed, 41 insertions(+), 12 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/alter_publication.sgml b/doc/src/sgml/ref/alter_publication.sgml index 47d83b80be..776661bbeb 100644 --- a/doc/src/sgml/ref/alter_publication.sgml +++ b/doc/src/sgml/ref/alter_publication.sgml @@ -48,8 +48,11 @@ ALTER PUBLICATION name DROP TABLE < - To alter the owner, you must also be a direct or indirect member of the - new owning role. The new owner has to be a superuser + To alter the owner, you must also be a direct or indirect member of the new + owning role. The new owner must have CREATE privilege on + the database. Also, the new owner of a FOR ALL TABLES + publication must be a superuser. However, a superuser can change the + ownership of a publication while circumventing these restrictions. diff --git a/src/backend/commands/publicationcmds.c b/src/backend/commands/publicationcmds.c index 04f83e0a2e..d69e39dc5b 100644 --- a/src/backend/commands/publicationcmds.c +++ b/src/backend/commands/publicationcmds.c @@ -670,17 +670,31 @@ AlterPublicationOwner_internal(Relation rel, HeapTuple tup, Oid newOwnerId) if (form->pubowner == newOwnerId) return; - if (!pg_publication_ownercheck(HeapTupleGetOid(tup), GetUserId())) - aclcheck_error(ACLCHECK_NOT_OWNER, ACL_KIND_PUBLICATION, - NameStr(form->pubname)); + if (!superuser()) + { + AclResult aclresult; - /* New owner must be a superuser */ - if (!superuser_arg(newOwnerId)) - ereport(ERROR, - (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("permission denied to change owner of publication \"%s\"", - NameStr(form->pubname)), - errhint("The owner of a publication must be a superuser."))); + /* Must be owner */ + if (!pg_publication_ownercheck(HeapTupleGetOid(tup), GetUserId())) + aclcheck_error(ACLCHECK_NOT_OWNER, ACL_KIND_PUBLICATION, + NameStr(form->pubname)); + + /* Must be able to become new owner */ + check_is_member_of_role(GetUserId(), newOwnerId); + + /* New owner must have CREATE privilege on database */ + aclresult = pg_database_aclcheck(MyDatabaseId, newOwnerId, ACL_CREATE); + if (aclresult != ACLCHECK_OK) + aclcheck_error(aclresult, ACL_KIND_DATABASE, + get_database_name(MyDatabaseId)); + + if (form->puballtables && !superuser_arg(newOwnerId)) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("permission denied to change owner of publication \"%s\"", + NameStr(form->pubname)), + errhint("The owner of a FOR ALL TABLES publication must be a superuser."))); + } form->pubowner = newOwnerId; CatalogTupleUpdate(rel, &tup->t_self, tup); diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out index 7c4834b213..5a7c0edf7d 100644 --- a/src/test/regress/expected/publication.out +++ b/src/test/regress/expected/publication.out @@ -182,6 +182,14 @@ ALTER PUBLICATION testpub_default RENAME TO testpub_foo; -- rename back to keep the rest simple ALTER PUBLICATION testpub_foo RENAME TO testpub_default; +ALTER PUBLICATION testpub_default OWNER TO regress_publication_user2; +\dRp testpub_default + List of publications + Name | Owner | Inserts | Updates | Deletes +-----------------+---------------------------+---------+---------+--------- + testpub_default | regress_publication_user2 | t | t | t +(1 row) + DROP PUBLICATION testpub_default; DROP PUBLICATION testpib_ins_trunct; DROP PUBLICATION testpub_fortbl; diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql index 46d275acc5..cff9931a77 100644 --- a/src/test/regress/sql/publication.sql +++ b/src/test/regress/sql/publication.sql @@ -108,6 +108,10 @@ ALTER PUBLICATION testpub_default RENAME TO testpub_foo; -- rename back to keep the rest simple ALTER PUBLICATION testpub_foo RENAME TO testpub_default; +ALTER PUBLICATION testpub_default OWNER TO regress_publication_user2; + +\dRp testpub_default + DROP PUBLICATION testpub_default; DROP PUBLICATION testpib_ins_trunct; DROP PUBLICATION testpub_fortbl; -- cgit v1.2.3 From 8df9bd0b445f9bd6134915d4417efde6e85e3add Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Mon, 13 Feb 2017 16:50:29 -0500 Subject: Change logical replication pg_hba.conf use Logical replication no longer uses the "replication" keyword. It just matches database entries in the normal way. The "replication" keyword now only applies to physical replication. Reviewed-by: Petr Jelinek --- doc/src/sgml/client-auth.sgml | 2 +- doc/src/sgml/logical-replication.sgml | 8 +++----- src/backend/libpq/hba.c | 4 ++-- 3 files changed, 6 insertions(+), 8 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/client-auth.sgml b/doc/src/sgml/client-auth.sgml index bbd52a5418..d6b8c04edc 100644 --- a/doc/src/sgml/client-auth.sgml +++ b/doc/src/sgml/client-auth.sgml @@ -193,7 +193,7 @@ hostnossl database user members of the role, directly or indirectly, and not just by virtue of being a superuser. The value replication specifies that the record - matches if a replication connection is requested (note that + matches if a physical replication connection is requested (note that replication connections do not specify any particular database). Otherwise, this is the name of a specific PostgreSQL database. diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml index a6c04e923d..6da39d25e3 100644 --- a/doc/src/sgml/logical-replication.sgml +++ b/doc/src/sgml/logical-replication.sgml @@ -295,11 +295,9 @@ Security - Logical replication connections occur in the same way as with physical streaming - replication. It requires access to be explicitly given using - pg_hba.conf. The role used for the replication - connection must have the REPLICATION attribute. This - gives a role access to both logical and physical replication. + The role used for the replication connection must have + the REPLICATION attribute. Access for the role must be + configured in pg_hba.conf. diff --git a/src/backend/libpq/hba.c b/src/backend/libpq/hba.c index 3817d249c4..7abcae618d 100644 --- a/src/backend/libpq/hba.c +++ b/src/backend/libpq/hba.c @@ -612,9 +612,9 @@ check_db(const char *dbname, const char *role, Oid roleid, List *tokens) foreach(cell, tokens) { tok = lfirst(cell); - if (am_walsender) + if (am_walsender && !am_db_walsender) { - /* walsender connections can only match replication keyword */ + /* physical replication walsender connections can only match replication keyword */ if (token_is_keyword(tok, "replication")) return true; } -- cgit v1.2.3 From 6b76f1bb58f53aec25cfec76391270ea36ad1170 Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Wed, 22 Mar 2017 17:55:16 +0100 Subject: Support multiple RADIUS servers This changes all the RADIUS related parameters (radiusserver, radiussecret, radiusport, radiusidentifier) to be plural and to accept a comma separated list of servers, which will be tried in order. Reviewed by Adam Brightwell --- doc/src/sgml/client-auth.sgml | 26 +++-- src/backend/libpq/auth.c | 204 +++++++++++++++++++++++++-------------- src/backend/libpq/hba.c | 220 +++++++++++++++++++++++++++++++++++------- src/include/libpq/hba.h | 12 ++- 4 files changed, 339 insertions(+), 123 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/client-auth.sgml b/doc/src/sgml/client-auth.sgml index d6b8c04edc..28f5296b5a 100644 --- a/doc/src/sgml/client-auth.sgml +++ b/doc/src/sgml/client-auth.sgml @@ -1621,24 +1621,36 @@ host ... ldap ldapurl="ldap://ldap.example.net/dc=example,dc=net?uid?sub" Access Reject. There is no support for RADIUS accounting. + + Multiple RADIUS servers can be specified, in which case they will + be tried sequentially. If a negative response is received from + a server, the authentication will fail. If no response is received, + the next server in the list will be tried. To specify multiple + servers, put the names within quotes and separate the server names + with a comma. If multiple servers are specified, all other RADIUS + options can also be given as a comma separate list, to apply + individual values to each server. They can also be specified as + a single value, in which case this value will apply to all servers. + + The following configuration options are supported for RADIUS: - radiusserver + radiusservers - The name or IP address of the RADIUS server to connect to. + The name or IP addresses of the RADIUS servers to connect to. This parameter is required. - radiussecret + radiussecrets - The shared secret used when talking securely to the RADIUS + The shared secrets used when talking securely to the RADIUS server. This must have exactly the same value on the PostgreSQL and RADIUS servers. It is recommended that this be a string of at least 16 characters. This parameter is required. @@ -1656,17 +1668,17 @@ host ... ldap ldapurl="ldap://ldap.example.net/dc=example,dc=net?uid?sub" - radiusport + radiusports - The port number on the RADIUS server to connect to. If no port + The port number on the RADIUS servers to connect to. If no port is specified, the default port 1812 will be used. - radiusidentifier + radiusidentifiers The string used as NAS Identifier in the RADIUS diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c index ebf10bbbae..a699a09e9a 100644 --- a/src/backend/libpq/auth.c +++ b/src/backend/libpq/auth.c @@ -197,6 +197,7 @@ static int pg_SSPI_make_upn(char *accountname, *---------------------------------------------------------------- */ static int CheckRADIUSAuth(Port *port); +static int PerformRadiusTransaction(char *server, char *secret, char *portstr, char *identifier, char *user_name, char *passwd); /*---------------------------------------------------------------- @@ -2591,7 +2592,97 @@ static int CheckRADIUSAuth(Port *port) { char *passwd; - char *identifier = "postgresql"; + ListCell *server, + *secrets, + *radiusports, + *identifiers; + + /* Make sure struct alignment is correct */ + Assert(offsetof(radius_packet, vector) == 4); + + /* Verify parameters */ + if (list_length(port->hba->radiusservers) < 1) + { + ereport(LOG, + (errmsg("RADIUS server not specified"))); + return STATUS_ERROR; + } + + if (list_length(port->hba->radiussecrets) < 1) + { + ereport(LOG, + (errmsg("RADIUS secret not specified"))); + return STATUS_ERROR; + } + + /* Send regular password request to client, and get the response */ + sendAuthRequest(port, AUTH_REQ_PASSWORD, NULL, 0); + + passwd = recv_password_packet(port); + if (passwd == NULL) + return STATUS_EOF; /* client wouldn't send password */ + + if (strlen(passwd) == 0) + { + ereport(LOG, + (errmsg("empty password returned by client"))); + return STATUS_ERROR; + } + + if (strlen(passwd) > RADIUS_MAX_PASSWORD_LENGTH) + { + ereport(LOG, + (errmsg("RADIUS authentication does not support passwords longer than %d characters", RADIUS_MAX_PASSWORD_LENGTH))); + return STATUS_ERROR; + } + + /* + * Loop over and try each server in order. + */ + secrets = list_head(port->hba->radiussecrets); + radiusports = list_head(port->hba->radiusports); + identifiers = list_head(port->hba->radiusidentifiers); + foreach(server, port->hba->radiusservers) + { + int ret = PerformRadiusTransaction(lfirst(server), + lfirst(secrets), + radiusports ? lfirst(radiusports) : NULL, + identifiers ? lfirst(identifiers) : NULL, + port->user_name, + passwd); + + /*------ + * STATUS_OK = Login OK + * STATUS_ERROR = Login not OK, but try next server + * STATUS_EOF = Login not OK, and don't try next server + *------ + */ + if (ret == STATUS_OK) + return STATUS_OK; + else if (ret == STATUS_EOF) + return STATUS_ERROR; + + /* + * secret, port and identifiers either have length 0 (use default), + * length 1 (use the same everywhere) or the same length as servers. + * So if the length is >1, we advance one step. In other cases, we + * don't and will then reuse the correct value. + */ + if (list_length(port->hba->radiussecrets) > 1) + secrets = lnext(secrets); + if (list_length(port->hba->radiusports) > 1) + radiusports = lnext(radiusports); + if (list_length(port->hba->radiusidentifiers) > 1) + identifiers = lnext(identifiers); + } + + /* No servers left to try, so give up */ + return STATUS_ERROR; +} + +static int +PerformRadiusTransaction(char *server, char *secret, char *portstr, char *identifier, char *user_name, char *passwd) +{ char radius_buffer[RADIUS_BUFFER_SIZE]; char receive_buffer[RADIUS_BUFFER_SIZE]; radius_packet *packet = (radius_packet *) radius_buffer; @@ -2613,7 +2704,7 @@ CheckRADIUSAuth(Port *port) #endif struct addrinfo hint; struct addrinfo *serveraddrs; - char portstr[128]; + int port; ACCEPT_TYPE_ARG3 addrsize; fd_set fdset; struct timeval endtime; @@ -2621,69 +2712,29 @@ CheckRADIUSAuth(Port *port) j, r; - /* Make sure struct alignment is correct */ - Assert(offsetof(radius_packet, vector) == 4); - - /* Verify parameters */ - if (!port->hba->radiusserver || port->hba->radiusserver[0] == '\0') - { - ereport(LOG, - (errmsg("RADIUS server not specified"))); - return STATUS_ERROR; - } - - if (!port->hba->radiussecret || port->hba->radiussecret[0] == '\0') - { - ereport(LOG, - (errmsg("RADIUS secret not specified"))); - return STATUS_ERROR; - } - - if (port->hba->radiusport == 0) - port->hba->radiusport = 1812; + /* Assign default values */ + if (portstr == NULL) + portstr = "1812"; + if (identifier == NULL) + identifier = "postgresql"; MemSet(&hint, 0, sizeof(hint)); hint.ai_socktype = SOCK_DGRAM; hint.ai_family = AF_UNSPEC; - snprintf(portstr, sizeof(portstr), "%d", port->hba->radiusport); + port = atoi(portstr); - r = pg_getaddrinfo_all(port->hba->radiusserver, portstr, &hint, &serveraddrs); + r = pg_getaddrinfo_all(server, portstr, &hint, &serveraddrs); if (r || !serveraddrs) { ereport(LOG, (errmsg("could not translate RADIUS server name \"%s\" to address: %s", - port->hba->radiusserver, gai_strerror(r)))); + server, gai_strerror(r)))); if (serveraddrs) pg_freeaddrinfo_all(hint.ai_family, serveraddrs); return STATUS_ERROR; } /* XXX: add support for multiple returned addresses? */ - if (port->hba->radiusidentifier && port->hba->radiusidentifier[0]) - identifier = port->hba->radiusidentifier; - - /* Send regular password request to client, and get the response */ - sendAuthRequest(port, AUTH_REQ_PASSWORD, NULL, 0); - - passwd = recv_password_packet(port); - if (passwd == NULL) - return STATUS_EOF; /* client wouldn't send password */ - - if (strlen(passwd) == 0) - { - ereport(LOG, - (errmsg("empty password returned by client"))); - return STATUS_ERROR; - } - - if (strlen(passwd) > RADIUS_MAX_PASSWORD_LENGTH) - { - ereport(LOG, - (errmsg("RADIUS authentication does not support passwords longer than %d characters", RADIUS_MAX_PASSWORD_LENGTH))); - return STATUS_ERROR; - } - - /* Construct RADIUS packet */ packet->code = RADIUS_ACCESS_REQUEST; packet->length = RADIUS_HEADER_LENGTH; @@ -2695,7 +2746,7 @@ CheckRADIUSAuth(Port *port) } packet->id = packet->vector[0]; radius_add_attribute(packet, RADIUS_SERVICE_TYPE, (unsigned char *) &service, sizeof(service)); - radius_add_attribute(packet, RADIUS_USER_NAME, (unsigned char *) port->user_name, strlen(port->user_name)); + radius_add_attribute(packet, RADIUS_USER_NAME, (unsigned char *) user_name, strlen(user_name)); radius_add_attribute(packet, RADIUS_NAS_IDENTIFIER, (unsigned char *) identifier, strlen(identifier)); /* @@ -2705,14 +2756,14 @@ CheckRADIUSAuth(Port *port) * (if necessary) */ encryptedpasswordlen = ((strlen(passwd) + RADIUS_VECTOR_LENGTH - 1) / RADIUS_VECTOR_LENGTH) * RADIUS_VECTOR_LENGTH; - cryptvector = palloc(strlen(port->hba->radiussecret) + RADIUS_VECTOR_LENGTH); - memcpy(cryptvector, port->hba->radiussecret, strlen(port->hba->radiussecret)); + cryptvector = palloc(strlen(secret) + RADIUS_VECTOR_LENGTH); + memcpy(cryptvector, secret, strlen(secret)); /* for the first iteration, we use the Request Authenticator vector */ md5trailer = packet->vector; for (i = 0; i < encryptedpasswordlen; i += RADIUS_VECTOR_LENGTH) { - memcpy(cryptvector + strlen(port->hba->radiussecret), md5trailer, RADIUS_VECTOR_LENGTH); + memcpy(cryptvector + strlen(secret), md5trailer, RADIUS_VECTOR_LENGTH); /* * .. and for subsequent iterations the result of the previous XOR @@ -2720,7 +2771,7 @@ CheckRADIUSAuth(Port *port) */ md5trailer = encryptedpassword + i; - if (!pg_md5_binary(cryptvector, strlen(port->hba->radiussecret) + RADIUS_VECTOR_LENGTH, encryptedpassword + i)) + if (!pg_md5_binary(cryptvector, strlen(secret) + RADIUS_VECTOR_LENGTH, encryptedpassword + i)) { ereport(LOG, (errmsg("could not perform MD5 encryption of password"))); @@ -2812,7 +2863,8 @@ CheckRADIUSAuth(Port *port) if (timeoutval <= 0) { ereport(LOG, - (errmsg("timeout waiting for RADIUS response"))); + (errmsg("timeout waiting for RADIUS response from %s", + server))); closesocket(sock); return STATUS_ERROR; } @@ -2837,7 +2889,8 @@ CheckRADIUSAuth(Port *port) if (r == 0) { ereport(LOG, - (errmsg("timeout waiting for RADIUS response"))); + (errmsg("timeout waiting for RADIUS response from %s", + server))); closesocket(sock); return STATUS_ERROR; } @@ -2864,19 +2917,19 @@ CheckRADIUSAuth(Port *port) } #ifdef HAVE_IPV6 - if (remoteaddr.sin6_port != htons(port->hba->radiusport)) + if (remoteaddr.sin6_port != htons(port)) #else - if (remoteaddr.sin_port != htons(port->hba->radiusport)) + if (remoteaddr.sin_port != htons(port)) #endif { #ifdef HAVE_IPV6 ereport(LOG, - (errmsg("RADIUS response was sent from incorrect port: %d", - ntohs(remoteaddr.sin6_port)))); + (errmsg("RADIUS response from %s was sent from incorrect port: %d", + server, ntohs(remoteaddr.sin6_port)))); #else ereport(LOG, - (errmsg("RADIUS response was sent from incorrect port: %d", - ntohs(remoteaddr.sin_port)))); + (errmsg("RADIUS response from %s was sent from incorrect port: %d", + server, ntohs(remoteaddr.sin_port)))); #endif continue; } @@ -2884,23 +2937,23 @@ CheckRADIUSAuth(Port *port) if (packetlength < RADIUS_HEADER_LENGTH) { ereport(LOG, - (errmsg("RADIUS response too short: %d", packetlength))); + (errmsg("RADIUS response from %s too short: %d", server, packetlength))); continue; } if (packetlength != ntohs(receivepacket->length)) { ereport(LOG, - (errmsg("RADIUS response has corrupt length: %d (actual length %d)", - ntohs(receivepacket->length), packetlength))); + (errmsg("RADIUS response from %s has corrupt length: %d (actual length %d)", + server, ntohs(receivepacket->length), packetlength))); continue; } if (packet->id != receivepacket->id) { ereport(LOG, - (errmsg("RADIUS response is to a different request: %d (should be %d)", - receivepacket->id, packet->id))); + (errmsg("RADIUS response from %s is to a different request: %d (should be %d)", + server, receivepacket->id, packet->id))); continue; } @@ -2908,7 +2961,7 @@ CheckRADIUSAuth(Port *port) * Verify the response authenticator, which is calculated as * MD5(Code+ID+Length+RequestAuthenticator+Attributes+Secret) */ - cryptvector = palloc(packetlength + strlen(port->hba->radiussecret)); + cryptvector = palloc(packetlength + strlen(secret)); memcpy(cryptvector, receivepacket, 4); /* code+id+length */ memcpy(cryptvector + 4, packet->vector, RADIUS_VECTOR_LENGTH); /* request @@ -2917,10 +2970,10 @@ CheckRADIUSAuth(Port *port) if (packetlength > RADIUS_HEADER_LENGTH) /* there may be no * attributes at all */ memcpy(cryptvector + RADIUS_HEADER_LENGTH, receive_buffer + RADIUS_HEADER_LENGTH, packetlength - RADIUS_HEADER_LENGTH); - memcpy(cryptvector + packetlength, port->hba->radiussecret, strlen(port->hba->radiussecret)); + memcpy(cryptvector + packetlength, secret, strlen(secret)); if (!pg_md5_binary(cryptvector, - packetlength + strlen(port->hba->radiussecret), + packetlength + strlen(secret), encryptedpassword)) { ereport(LOG, @@ -2933,7 +2986,8 @@ CheckRADIUSAuth(Port *port) if (memcmp(receivepacket->vector, encryptedpassword, RADIUS_VECTOR_LENGTH) != 0) { ereport(LOG, - (errmsg("RADIUS response has incorrect MD5 signature"))); + (errmsg("RADIUS response from %s has incorrect MD5 signature", + server))); continue; } @@ -2945,13 +2999,13 @@ CheckRADIUSAuth(Port *port) else if (receivepacket->code == RADIUS_ACCESS_REJECT) { closesocket(sock); - return STATUS_ERROR; + return STATUS_EOF; } else { ereport(LOG, - (errmsg("RADIUS response has invalid code (%d) for user \"%s\"", - receivepacket->code, port->user_name))); + (errmsg("RADIUS response from %s has invalid code (%d) for user \"%s\"", + server, receivepacket->code, user_name))); continue; } } /* while (true) */ diff --git a/src/backend/libpq/hba.c b/src/backend/libpq/hba.c index 7abcae618d..49be6638b8 100644 --- a/src/backend/libpq/hba.c +++ b/src/backend/libpq/hba.c @@ -39,6 +39,7 @@ #include "storage/fd.h" #include "utils/acl.h" #include "utils/builtins.h" +#include "utils/varlena.h" #include "utils/guc.h" #include "utils/lsyscache.h" #include "utils/memutils.h" @@ -143,6 +144,8 @@ static List *tokenize_inc_file(List *tokens, const char *outer_filename, const char *inc_filename, int elevel, char **err_msg); static bool parse_hba_auth_opt(char *name, char *val, HbaLine *hbaline, int elevel, char **err_msg); +static bool verify_option_list_length(List *options, char *optionname, + List *masters, char *mastername, int line_num); static ArrayType *gethba_options(HbaLine *hba); static void fill_hba_line(Tuplestorestate *tuple_store, TupleDesc tupdesc, int lineno, HbaLine *hba, const char *err_msg); @@ -1532,8 +1535,52 @@ parse_hba_line(TokenizedLine *tok_line, int elevel) if (parsedline->auth_method == uaRADIUS) { - MANDATORY_AUTH_ARG(parsedline->radiusserver, "radiusserver", "radius"); - MANDATORY_AUTH_ARG(parsedline->radiussecret, "radiussecret", "radius"); + MANDATORY_AUTH_ARG(parsedline->radiusservers, "radiusservers", "radius"); + MANDATORY_AUTH_ARG(parsedline->radiussecrets, "radiussecrets", "radius"); + + if (list_length(parsedline->radiusservers) < 1) + { + ereport(LOG, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("list of RADIUS servers cannot be empty"), + errcontext("line %d of configuration file \"%s\"", + line_num, HbaFileName))); + return NULL; + } + + if (list_length(parsedline->radiussecrets) < 1) + { + ereport(LOG, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("list of RADIUS secrets cannot be empty"), + errcontext("line %d of configuration file \"%s\"", + line_num, HbaFileName))); + return NULL; + } + + /* + * Verify length of option lists - each can be 0 (except for secrets, + * but that's already checked above), 1 (use the same value + * everywhere) or the same as the number of servers. + */ + if (!verify_option_list_length(parsedline->radiussecrets, + "RADIUS secrets", + parsedline->radiusservers, + "RADIUS servers", + line_num)) + return NULL; + if (!verify_option_list_length(parsedline->radiusports, + "RADIUS ports", + parsedline->radiusservers, + "RADIUS servers", + line_num)) + return NULL; + if (!verify_option_list_length(parsedline->radiusidentifiers, + "RADIUS identifiers", + parsedline->radiusservers, + "RADIUS servers", + line_num)) + return NULL; } /* @@ -1547,6 +1594,28 @@ parse_hba_line(TokenizedLine *tok_line, int elevel) return parsedline; } + +static bool +verify_option_list_length(List *options, char *optionname, List *masters, char *mastername, int line_num) +{ + if (list_length(options) == 0 || + list_length(options) == 1 || + list_length(options) == list_length(masters)) + return true; + + ereport(LOG, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("the number of %s (%i) must be 1 or the same as the number of %s (%i)", + optionname, + list_length(options), + mastername, + list_length(masters) + ), + errcontext("line %d of configuration file \"%s\"", + line_num, HbaFileName))); + return false; +} + /* * Parse one name-value pair as an authentication option into the given * HbaLine. Return true if we successfully parse the option, false if we @@ -1766,60 +1835,137 @@ parse_hba_auth_opt(char *name, char *val, HbaLine *hbaline, else hbaline->upn_username = false; } - else if (strcmp(name, "radiusserver") == 0) + else if (strcmp(name, "radiusservers") == 0) { struct addrinfo *gai_result; struct addrinfo hints; int ret; + List *parsed_servers; + ListCell *l; + char *dupval = pstrdup(val); - REQUIRE_AUTH_OPTION(uaRADIUS, "radiusserver", "radius"); - - MemSet(&hints, 0, sizeof(hints)); - hints.ai_socktype = SOCK_DGRAM; - hints.ai_family = AF_UNSPEC; + REQUIRE_AUTH_OPTION(uaRADIUS, "radiusservers", "radius"); - ret = pg_getaddrinfo_all(val, NULL, &hints, &gai_result); - if (ret || !gai_result) + if (!SplitIdentifierString(dupval, ',', &parsed_servers)) { + /* syntax error in list */ ereport(elevel, (errcode(ERRCODE_CONFIG_FILE_ERROR), - errmsg("could not translate RADIUS server name \"%s\" to address: %s", - val, gai_strerror(ret)), + errmsg("could not parse RADIUS server list \"%s\"", + val), errcontext("line %d of configuration file \"%s\"", line_num, HbaFileName))); - *err_msg = psprintf("could not translate RADIUS server name \"%s\" to address: %s", - val, gai_strerror(ret)); - if (gai_result) - pg_freeaddrinfo_all(hints.ai_family, gai_result); return false; } - pg_freeaddrinfo_all(hints.ai_family, gai_result); - hbaline->radiusserver = pstrdup(val); + + /* For each entry in the list, translate it */ + foreach(l, parsed_servers) + { + MemSet(&hints, 0, sizeof(hints)); + hints.ai_socktype = SOCK_DGRAM; + hints.ai_family = AF_UNSPEC; + + ret = pg_getaddrinfo_all((char *) lfirst(l), NULL, &hints, &gai_result); + if (ret || !gai_result) + { + ereport(elevel, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("could not translate RADIUS server name \"%s\" to address: %s", + (char *) lfirst(l), gai_strerror(ret)), + errcontext("line %d of configuration file \"%s\"", + line_num, HbaFileName))); + if (gai_result) + pg_freeaddrinfo_all(hints.ai_family, gai_result); + + list_free(parsed_servers); + return false; + } + pg_freeaddrinfo_all(hints.ai_family, gai_result); + } + + /* All entries are OK, so store them */ + hbaline->radiusservers = parsed_servers; + hbaline->radiusservers_s = pstrdup(val); } - else if (strcmp(name, "radiusport") == 0) + else if (strcmp(name, "radiusports") == 0) { - REQUIRE_AUTH_OPTION(uaRADIUS, "radiusport", "radius"); - hbaline->radiusport = atoi(val); - if (hbaline->radiusport == 0) + List *parsed_ports; + ListCell *l; + char *dupval = pstrdup(val); + + REQUIRE_AUTH_OPTION(uaRADIUS, "radiusports", "radius"); + + if (!SplitIdentifierString(dupval, ',', &parsed_ports)) { ereport(elevel, (errcode(ERRCODE_CONFIG_FILE_ERROR), - errmsg("invalid RADIUS port number: \"%s\"", val), + errmsg("could not parse RADIUS port list \"%s\"", + val), errcontext("line %d of configuration file \"%s\"", line_num, HbaFileName))); *err_msg = psprintf("invalid RADIUS port number: \"%s\"", val); return false; } + + foreach(l, parsed_ports) + { + if (atoi(lfirst(l)) == 0) + { + ereport(elevel, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("invalid RADIUS port number: \"%s\"", val), + errcontext("line %d of configuration file \"%s\"", + line_num, HbaFileName))); + + return false; + } + } + hbaline->radiusports = parsed_ports; + hbaline->radiusports_s = pstrdup(val); } - else if (strcmp(name, "radiussecret") == 0) + else if (strcmp(name, "radiussecrets") == 0) { - REQUIRE_AUTH_OPTION(uaRADIUS, "radiussecret", "radius"); - hbaline->radiussecret = pstrdup(val); + List *parsed_secrets; + char *dupval = pstrdup(val); + + REQUIRE_AUTH_OPTION(uaRADIUS, "radiussecrets", "radius"); + + if (!SplitIdentifierString(dupval, ',', &parsed_secrets)) + { + /* syntax error in list */ + ereport(elevel, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("could not parse RADIUS secret list \"%s\"", + val), + errcontext("line %d of configuration file \"%s\"", + line_num, HbaFileName))); + return false; + } + + hbaline->radiussecrets = parsed_secrets; + hbaline->radiussecrets_s = pstrdup(val); } - else if (strcmp(name, "radiusidentifier") == 0) + else if (strcmp(name, "radiusidentifiers") == 0) { - REQUIRE_AUTH_OPTION(uaRADIUS, "radiusidentifier", "radius"); - hbaline->radiusidentifier = pstrdup(val); + List *parsed_identifiers; + char *dupval = pstrdup(val); + + REQUIRE_AUTH_OPTION(uaRADIUS, "radiusidentifiers", "radius"); + + if (!SplitIdentifierString(dupval, ',', &parsed_identifiers)) + { + /* syntax error in list */ + ereport(elevel, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("could not parse RADIUS identifiers list \"%s\"", + val), + errcontext("line %d of configuration file \"%s\"", + line_num, HbaFileName))); + return false; + } + + hbaline->radiusidentifiers = parsed_identifiers; + hbaline->radiusidentifiers_s = pstrdup(val); } else { @@ -2124,21 +2270,21 @@ gethba_options(HbaLine *hba) if (hba->auth_method == uaRADIUS) { - if (hba->radiusserver) + if (hba->radiusservers_s) options[noptions++] = - CStringGetTextDatum(psprintf("radiusserver=%s", hba->radiusserver)); + CStringGetTextDatum(psprintf("radiusservers=%s", hba->radiusservers_s)); - if (hba->radiussecret) + if (hba->radiussecrets_s) options[noptions++] = - CStringGetTextDatum(psprintf("radiussecret=%s", hba->radiussecret)); + CStringGetTextDatum(psprintf("radiussecrets=%s", hba->radiussecrets_s)); - if (hba->radiusidentifier) + if (hba->radiusidentifiers_s) options[noptions++] = - CStringGetTextDatum(psprintf("radiusidentifier=%s", hba->radiusidentifier)); + CStringGetTextDatum(psprintf("radiusidentifiers=%s", hba->radiusidentifiers_s)); - if (hba->radiusport) + if (hba->radiusports_s) options[noptions++] = - CStringGetTextDatum(psprintf("radiusport=%d", hba->radiusport)); + CStringGetTextDatum(psprintf("radiusports=%s", hba->radiusports_s)); } Assert(noptions <= MAX_HBA_OPTIONS); diff --git a/src/include/libpq/hba.h b/src/include/libpq/hba.h index 8f55edb16a..6c7382e67f 100644 --- a/src/include/libpq/hba.h +++ b/src/include/libpq/hba.h @@ -89,10 +89,14 @@ typedef struct HbaLine bool include_realm; bool compat_realm; bool upn_username; - char *radiusserver; - char *radiussecret; - char *radiusidentifier; - int radiusport; + List *radiusservers; + char *radiusservers_s; + List *radiussecrets; + char *radiussecrets_s; + List *radiusidentifiers; + char *radiusidentifiers_s; + List *radiusports; + char *radiusports_s; } HbaLine; typedef struct IdentLine -- cgit v1.2.3 From a2760915b3a88f8e1b447dd5343549a1ebbb0f78 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 22 Mar 2017 15:11:13 -0400 Subject: doc: Markup and formatting improvements --- doc/src/sgml/ref/alter_subscription.sgml | 6 +++--- doc/src/sgml/ref/create_subscription.sgml | 7 +++++-- doc/src/sgml/ref/drop_subscription.sgml | 4 ++-- 3 files changed, 10 insertions(+), 7 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/alter_subscription.sgml b/doc/src/sgml/ref/alter_subscription.sgml index 032ecbb885..5e18e2ff6c 100644 --- a/doc/src/sgml/ref/alter_subscription.sgml +++ b/doc/src/sgml/ref/alter_subscription.sgml @@ -25,11 +25,11 @@ ALTER SUBSCRIPTION name WITH ( where option can be: - SLOT NAME = slot_name + SLOT NAME = slot_name ALTER SUBSCRIPTION name OWNER TO { new_owner | CURRENT_USER | SESSION_USER } -ALTER SUBSCRIPTION name CONNECTION 'conninfo' -ALTER SUBSCRIPTION name SET PUBLICATION publication_name [, ...] +ALTER SUBSCRIPTION name CONNECTION 'conninfo' +ALTER SUBSCRIPTION name SET PUBLICATION publication_name [, ...] ALTER SUBSCRIPTION name ENABLE ALTER SUBSCRIPTION name DISABLE diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml index 9bed26219c..e200076700 100644 --- a/doc/src/sgml/ref/create_subscription.sgml +++ b/doc/src/sgml/ref/create_subscription.sgml @@ -21,13 +21,16 @@ PostgreSQL documentation -CREATE SUBSCRIPTION subscription_name CONNECTION 'conninfo' PUBLICATION { publication_name [, ...] } [ WITH ( option [, ... ] ) ] +CREATE SUBSCRIPTION subscription_name + CONNECTION 'conninfo' + PUBLICATION { publication_name [, ...] } + [ WITH ( option [, ... ] ) ] where option can be: | ENABLED | DISABLED | CREATE SLOT | NOCREATE SLOT - | SLOT NAME = slot_name + | SLOT NAME = slot_name diff --git a/doc/src/sgml/ref/drop_subscription.sgml b/doc/src/sgml/ref/drop_subscription.sgml index f5e449a9a5..f1ac125057 100644 --- a/doc/src/sgml/ref/drop_subscription.sgml +++ b/doc/src/sgml/ref/drop_subscription.sgml @@ -57,8 +57,8 @@ DROP SUBSCRIPTION [ IF EXISTS ] name - DROP SLOT - NODROP SLOT + DROP SLOT + NODROP SLOT Specifies whether to drop the replication slot on the publisher. The -- cgit v1.2.3 From b9418911900ff5b68a5457b1542668077664e897 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 22 Mar 2017 15:26:04 -0400 Subject: doc: Fix logical replication setup instructions The pg_hba.conf rules were changed in 8df9bd0b445f9bd6134915d4417efde6e85e3add. --- doc/src/sgml/logical-replication.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml index 6da39d25e3..44cd78563d 100644 --- a/doc/src/sgml/logical-replication.sgml +++ b/doc/src/sgml/logical-replication.sgml @@ -374,7 +374,7 @@ wal_level = logical (the values here depend on your actual network configuration and user you want to use for connecting): -host replication repuser 0.0.0.0/0 md5 +host all repuser 0.0.0.0/0 md5 -- cgit v1.2.3 From cbf7ed51b8c0c3e506b91765769f76d5c2e1e9ac Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 22 Mar 2017 15:26:59 -0400 Subject: doc: Improve CREATE PUBLICATION examples --- doc/src/sgml/ref/create_publication.sgml | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/create_publication.sgml b/doc/src/sgml/ref/create_publication.sgml index b00a91ef8a..3cdde801fa 100644 --- a/doc/src/sgml/ref/create_publication.sgml +++ b/doc/src/sgml/ref/create_publication.sgml @@ -163,7 +163,7 @@ CREATE PUBLICATION name - TRUNCATE and other DDL operations + TRUNCATE and DDL operations are not published. @@ -172,16 +172,25 @@ CREATE PUBLICATION name Examples - Create a simple publication that just publishes all DML for tables in it: + Create a publication that publishes all changes in two tables: -CREATE PUBLICATION mypublication; +CREATE PUBLICATION mypublication FOR TABLE users, departments; - Create an insert-only publication: + Create a publication that publishes all changes in all tables: -CREATE PUBLICATION insert_only WITH (NOPUBLISH UPDATE, NOPUBLISH DELETE); +CREATE PUBLICATION alltables FOR ALL TABLES; + + + + + Create a publication that only publishes INSERT + operations in one table: + +CREATE PUBLICATION insert_only FOR TABLE mydata + WITH (NOPUBLISH UPDATE, NOPUBLISH DELETE); -- cgit v1.2.3 From 017e4f2588a7fc6f2d1fbb6fff8afa1ff6e31f2b Mon Sep 17 00:00:00 2001 From: Stephen Frost Date: Wed, 22 Mar 2017 23:44:58 -0400 Subject: Expose waitforarchive option through pg_stop_backup() Internally, we have supported the option to either wait for all of the WAL associated with a backup to be archived, or to return immediately. This option is useful to users of pg_stop_backup() as well, when they are reading the stop backup record position and checking that the WAL they need has been archived independently. This patch adds an additional, optional, argument to pg_stop_backup() which allows the user to indicate if they wish to wait for the WAL to be archived or not. The default matches current behavior, which is to wait. Author: David Steele, with some minor changes, doc updates by me. Reviewed by: Takayuki Tsunakawa, Fujii Masao Discussion: https://postgr.es/m/758e3fd1-45b4-5e28-75cd-e9e7f93a4c02@pgmasters.net --- doc/src/sgml/backup.sgml | 13 ++++++++++++- doc/src/sgml/func.sgml | 10 ++++++++-- src/backend/access/transam/xlog.c | 3 ++- src/backend/access/transam/xlogfuncs.c | 12 ++++++++++-- src/backend/catalog/system_views.sql | 8 +++++++- src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_proc.h | 2 +- 7 files changed, 41 insertions(+), 9 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml index 2d67521775..8cf63e478a 100644 --- a/doc/src/sgml/backup.sgml +++ b/doc/src/sgml/backup.sgml @@ -887,7 +887,7 @@ SELECT pg_start_backup('label', false, false); In the same connection as before, issue the command: -SELECT * FROM pg_stop_backup(false); +SELECT * FROM pg_stop_backup(false, true); This terminates the backup mode and performs an automatic switch to the next WAL segment. The reason for the switch is to arrange for @@ -924,6 +924,17 @@ SELECT * FROM pg_stop_backup(false); pg_stop_backup terminates because of this your backup may not be valid. + + If the backup process monitors and ensures that all WAL segment files + required for the backup are successfully archived then the second + parameter (which defaults to true) can be set to false to have + pg_stop_backup return as soon as the stop backup record is + written to the WAL. By default, pg_stop_backup will wait + until all WAL has been archived, which can take some time. This option + must be used with caution: if WAL archiving is not monitored correctly + then the backup might not include all of the WAL files and will + therefore be incomplete and not able to be restored. + diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 9408a255dc..4dc30caccb 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -18355,7 +18355,7 @@ SELECT set_config('log_statement_stats', 'off', false); - pg_stop_backup(exclusive boolean) + pg_stop_backup(exclusive boolean , wait_for_archive boolean ) setof record Finish performing exclusive or non-exclusive on-line backup (restricted to superusers by default, but other users can be granted EXECUTE to run the function) @@ -18439,7 +18439,13 @@ postgres=# select pg_start_backup('label_goes_here'); pg_start_backup. In a non-exclusive backup, the contents of the backup_label and tablespace_map are returned in the result of the function, and should be written to files in the - backup (and not in the data directory). + backup (and not in the data directory). There is an optional second + parameter of type boolean. If false, the pg_stop_backup + will return immediately after the backup is completed without waiting for + WAL to be archived. This behavior is only useful for backup + software which independently monitors WAL archiving. Otherwise, WAL + required to make the backup consistent might be missing and make the backup + useless. diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index ff4cf3a810..3924738a33 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -10944,7 +10944,8 @@ do_pg_stop_backup(char *labelfile, bool waitforarchive, TimeLineID *stoptli_p) * * We wait forever, since archive_command is supposed to work and we * assume the admin wanted his backup to work completely. If you don't - * wish to wait, you can set statement_timeout. Also, some notices are + * wish to wait, then either waitforarchive should be passed in as false, + * or you can set statement_timeout. Also, some notices are * issued to clue in anyone who might be doing this interactively. */ if (waitforarchive && XLogArchivingActive()) diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c index 96aa15e9cc..5073aaca84 100644 --- a/src/backend/access/transam/xlogfuncs.c +++ b/src/backend/access/transam/xlogfuncs.c @@ -175,6 +175,13 @@ pg_stop_backup(PG_FUNCTION_ARGS) * the backup label and tablespace map files as text fields in as part of the * resultset. * + * The first parameter (variable 'exclusive') allows the user to tell us if + * this is an exclusive or a non-exclusive backup. + * + * The second paramter (variable 'waitforarchive'), which is optional, + * allows the user to choose if they want to wait for the WAL to be archived + * or if we should just return as soon as the WAL record is written. + * * Permission checking for this function is managed through the normal * GRANT system. */ @@ -190,6 +197,7 @@ pg_stop_backup_v2(PG_FUNCTION_ARGS) bool nulls[3]; bool exclusive = PG_GETARG_BOOL(0); + bool waitforarchive = PG_GETARG_BOOL(1); XLogRecPtr stoppoint; /* check to see if caller supports us returning a tuplestore */ @@ -232,7 +240,7 @@ pg_stop_backup_v2(PG_FUNCTION_ARGS) * Stop the exclusive backup, and since we're in an exclusive backup * return NULL for both backup_label and tablespace_map. */ - stoppoint = do_pg_stop_backup(NULL, true, NULL); + stoppoint = do_pg_stop_backup(NULL, waitforarchive, NULL); exclusive_backup_running = false; nulls[1] = true; @@ -250,7 +258,7 @@ pg_stop_backup_v2(PG_FUNCTION_ARGS) * Stop the non-exclusive backup. Return a copy of the backup label * and tablespace map so they can be written to disk by the caller. */ - stoppoint = do_pg_stop_backup(label_file->data, true, NULL); + stoppoint = do_pg_stop_backup(label_file->data, waitforarchive, NULL); nonexclusive_backup_running = false; cancel_before_shmem_exit(nonexclusive_base_backup_cleanup, (Datum) 0); diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index b6552da4b0..c2b0bedc1d 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -988,6 +988,12 @@ CREATE OR REPLACE FUNCTION RETURNS pg_lsn STRICT VOLATILE LANGUAGE internal AS 'pg_start_backup' PARALLEL RESTRICTED; +CREATE OR REPLACE FUNCTION pg_stop_backup ( + exclusive boolean, wait_for_archive boolean DEFAULT true, + OUT lsn pg_lsn, OUT labelfile text, OUT spcmapfile text) + RETURNS SETOF record STRICT VOLATILE LANGUAGE internal as 'pg_stop_backup_v2' + PARALLEL RESTRICTED; + -- legacy definition for compatibility with 9.3 CREATE OR REPLACE FUNCTION json_populate_record(base anyelement, from_json json, use_json_as_text boolean DEFAULT false) @@ -1088,7 +1094,7 @@ AS 'jsonb_insert'; -- available to superuser / cluster owner, if they choose. REVOKE EXECUTE ON FUNCTION pg_start_backup(text, boolean, boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_stop_backup() FROM public; -REVOKE EXECUTE ON FUNCTION pg_stop_backup(boolean) FROM public; +REVOKE EXECUTE ON FUNCTION pg_stop_backup(boolean, boolean) FROM public; REVOKE EXECUTE ON FUNCTION pg_create_restore_point(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_switch_wal() FROM public; REVOKE EXECUTE ON FUNCTION pg_wal_replay_pause() FROM public; diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index b4f1b9a6c2..315f155b64 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201703161 +#define CATALOG_VERSION_NO 201703221 #endif diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index 836d6ff0b2..22635655f5 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -3172,7 +3172,7 @@ DATA(insert OID = 2172 ( pg_start_backup PGNSP PGUID 12 1 0 0 0 f f f f t f v r DESCR("prepare for taking an online backup"); DATA(insert OID = 2173 ( pg_stop_backup PGNSP PGUID 12 1 0 0 0 f f f f t f v r 0 0 3220 "" _null_ _null_ _null_ _null_ _null_ pg_stop_backup _null_ _null_ _null_ )); DESCR("finish taking an online backup"); -DATA(insert OID = 2739 ( pg_stop_backup PGNSP PGUID 12 1 1 0 0 f f f f t t v r 1 0 2249 "16" "{16,3220,25,25}" "{i,o,o,o}" "{exclusive,lsn,labelfile,spcmapfile}" _null_ _null_ pg_stop_backup_v2 _null_ _null_ _null_ )); +DATA(insert OID = 2739 ( pg_stop_backup PGNSP PGUID 12 1 1 0 0 f f f f t t v r 2 0 2249 "16 16" "{16,16,3220,25,25}" "{i,i,o,o,o}" "{exclusive,wait_for_archive,lsn,labelfile,spcmapfile}" _null_ _null_ pg_stop_backup_v2 _null_ _null_ _null_ )); DESCR("finish taking an online backup"); DATA(insert OID = 3813 ( pg_is_in_backup PGNSP PGUID 12 1 0 0 0 f f f f t f v s 0 0 16 "" _null_ _null_ _null_ _null_ _null_ pg_is_in_backup _null_ _null_ _null_ )); DESCR("true if server is in online backup"); -- cgit v1.2.3 From 7c4f52409a8c7d85ed169bbbc1f6092274d03920 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 23 Mar 2017 08:36:36 -0400 Subject: Logical replication support for initial data copy Add functionality for a new subscription to copy the initial data in the tables and then sync with the ongoing apply process. For the copying, add a new internal COPY option to have the COPY source data provided by a callback function. The initial data copy works on the subscriber by receiving COPY data from the publisher and then providing it locally into a COPY that writes to the destination table. A WAL receiver can now execute full SQL commands. This is used here to obtain information about tables and publications. Several new options were added to CREATE and ALTER SUBSCRIPTION to control whether and when initial table syncing happens. Change pg_dump option --no-create-subscription-slots to --no-subscription-connect and use the new CREATE SUBSCRIPTION ... NOCONNECT option for that. Author: Petr Jelinek Tested-by: Erik Rijkers --- contrib/file_fdw/file_fdw.c | 5 +- doc/src/sgml/catalogs.sgml | 78 ++ doc/src/sgml/config.sgml | 25 + doc/src/sgml/logical-replication.sgml | 55 +- doc/src/sgml/monitoring.sgml | 9 +- doc/src/sgml/protocol.sgml | 9 +- doc/src/sgml/ref/alter_subscription.sgml | 50 +- doc/src/sgml/ref/create_subscription.sgml | 38 + doc/src/sgml/ref/pg_dump.sgml | 15 +- src/backend/catalog/Makefile | 1 + src/backend/catalog/heap.c | 6 + src/backend/catalog/pg_publication.c | 4 +- src/backend/catalog/pg_subscription.c | 282 +++++++ src/backend/catalog/system_views.sql | 1 + src/backend/commands/copy.c | 23 +- src/backend/commands/subscriptioncmds.c | 468 ++++++++++-- src/backend/parser/gram.y | 39 +- src/backend/postmaster/pgstat.c | 6 + .../libpqwalreceiver/libpqwalreceiver.c | 203 ++++- src/backend/replication/logical/Makefile | 2 +- src/backend/replication/logical/launcher.c | 130 +++- src/backend/replication/logical/relation.c | 7 + src/backend/replication/logical/snapbuild.c | 85 ++- src/backend/replication/logical/tablesync.c | 840 +++++++++++++++++++++ src/backend/replication/logical/worker.c | 203 +++-- src/backend/replication/repl_gram.y | 32 +- src/backend/replication/repl_scanner.l | 5 +- src/backend/replication/walsender.c | 104 ++- src/backend/tcop/postgres.c | 5 +- src/backend/utils/adt/misc.c | 20 + src/backend/utils/cache/syscache.c | 14 +- src/backend/utils/misc/guc.c | 12 + src/bin/pg_dump/pg_backup.h | 2 +- src/bin/pg_dump/pg_dump.c | 9 +- src/bin/pg_dump/t/002_pg_dump.pl | 2 +- src/include/catalog/catversion.h | 2 +- src/include/catalog/indexing.h | 7 +- src/include/catalog/pg_proc.h | 5 +- src/include/catalog/pg_subscription_rel.h | 78 ++ src/include/commands/copy.h | 5 +- src/include/nodes/nodes.h | 1 + src/include/nodes/parsenodes.h | 13 + src/include/nodes/replnodes.h | 9 + src/include/parser/kwlist.h | 1 + src/include/pgstat.h | 4 +- src/include/replication/logical.h | 13 +- src/include/replication/logicallauncher.h | 1 + src/include/replication/snapbuild.h | 1 + src/include/replication/walreceiver.h | 67 +- src/include/replication/walsender.h | 12 +- src/include/replication/worker_internal.h | 30 +- src/include/utils/syscache.h | 1 + src/test/regress/expected/object_address.out | 3 +- src/test/regress/expected/rules.out | 3 +- src/test/regress/expected/sanity_check.out | 1 + src/test/regress/expected/subscription.out | 45 +- src/test/regress/sql/object_address.sql | 2 +- src/test/regress/sql/subscription.sql | 11 +- src/test/subscription/t/001_rep_changes.pl | 36 +- src/test/subscription/t/002_types.pl | 6 + src/test/subscription/t/003_constraints.pl | 2 +- src/test/subscription/t/004_sync.pl | 159 ++++ 62 files changed, 2966 insertions(+), 341 deletions(-) create mode 100644 src/backend/replication/logical/tablesync.c create mode 100644 src/include/catalog/pg_subscription_rel.h create mode 100644 src/test/subscription/t/004_sync.pl (limited to 'doc/src') diff --git a/contrib/file_fdw/file_fdw.c b/contrib/file_fdw/file_fdw.c index 735b79484c..277639f6e9 100644 --- a/contrib/file_fdw/file_fdw.c +++ b/contrib/file_fdw/file_fdw.c @@ -662,6 +662,7 @@ fileBeginForeignScan(ForeignScanState *node, int eflags) node->ss.ss_currentRelation, filename, is_program, + NULL, NIL, options); @@ -737,6 +738,7 @@ fileReScanForeignScan(ForeignScanState *node) node->ss.ss_currentRelation, festate->filename, festate->is_program, + NULL, NIL, festate->options); } @@ -1100,7 +1102,8 @@ file_acquire_sample_rows(Relation onerel, int elevel, /* * Create CopyState from FDW options. */ - cstate = BeginCopyFrom(NULL, onerel, filename, is_program, NIL, options); + cstate = BeginCopyFrom(NULL, onerel, filename, is_program, NULL, NIL, + options); /* * Use per-tuple memory context to prevent leak of memory used to read diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index df0435c3f0..228ec78031 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -300,6 +300,11 @@ logical replication subscriptions + + pg_subscription_rel + relation state for subscriptions + + pg_tablespace tablespaces within this database cluster @@ -6418,6 +6423,79 @@ + + <structname>pg_subscription_rel</structname> + + + pg_subscription_rel + + + + The catalog pg_subscription_rel contains the + state for each replicated relation in each subscription. This is a + many-to-many mapping. + + + + This catalog only contains tables known to the subscription after running + either CREATE SUBSCRIPTION or + ALTER SUBSCRIPTION ... REFRESH. + + + + <structname>pg_subscription_rel</structname> Columns + + + + + Name + Type + References + Description + + + + + + srsubid + oid + pg_subscription.oid + Reference to subscription + + + + srrelid + oid + pg_class.oid + Reference to relation + + + + srsubstate + char + + + State code: + i = initialize, + d = data is being copied, + s = synchronized, + r = ready (normal replication) + + + + + srsublsn + pg_lsn + + + End LSN for s and r states. + + + + +
    +     + <structname>pg_tablespace</structname> diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index b379b67b30..2de3540def 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3449,6 +3449,31 @@ ANY num_sync ( + max_sync_workers_per_subscription (integer) + + max_sync_workers_per_subscription configuration parameter + + + + + Maximum number of synchronization workers per subscription. This + parameter controls the amount of paralelism of the initial data copy + during the subscription initialization or when new tables are added. + + + Currently, there can be only one synchronization worker per table. + + + The synchronization workers are taken from the pool defined by + max_logical_replication_workers. + + + The default value is 2. + + + + diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml index 44cd78563d..48db9cd08b 100644 --- a/doc/src/sgml/logical-replication.sgml +++ b/doc/src/sgml/logical-replication.sgml @@ -24,9 +24,11 @@     - Logical replication sends changes on the publisher to the subscriber as - they occur in real-time. The subscriber applies the data in the same order - as the publisher so that transactional consistency is guaranteed for + Logical replication of a table typically starts with a taking a snapshot + of the data on the publisher database and copying that to the subscriber. + Once that is done, the changes on the publisher are sent to the subscriber + as they occur in real-time. The subscriber applies the data in the same + order as the publisher so that transactional consistency is guaranteed for publications within a single subscription. This method of data replication is sometimes referred to as transactional replication. @@ -159,7 +161,9 @@ Each subscription will receive changes via one replication slot (see - ). + ). Additional temporary + replication slots may be required for the initial data synchronization + of pre-existing table data. @@ -264,9 +268,25 @@ to replica, which produces the usual effects on triggers and constraints. + + + Initial Snapshot + + The initial data in existing subscribed tables are snapshotted and + copied in a parallel instance of a special kind of apply process. + This process will create its own temporary replication slot and + copy the existing data. Once existing data is copied, the worker + enters synchronization mode, which ensures that the table is brought + up to a synchronized state with the main apply process by streaming + any changes that happened during the initial data copy using standard + logical replication. Once the synchronization is done, the control + of the replication of the table is given back to the main apply + process where the replication continues as normal. + + - + Monitoring @@ -287,7 +307,9 @@ Normally, there is a single apply process running for an enabled subscription. A disabled subscription or a crashed subscription will have - zero rows in this view. + zero rows in this view. If the initial data synchronization of any + table is in progress, there will be additional workers for the tables + being synchronized. @@ -337,10 +359,11 @@ On the publisher side, wal_level must be set to logical, and max_replication_slots - must be set to at least the number of subscriptions expected to connect. - And max_wal_senders should be set to at least the same - as max_replication_slots plus the number of physical replicas - that are connected at the same time. + must be set to at least the number of subscriptions expected to connect, + plus some reserve for table synchronization. And + max_wal_senders should be set to at least the same as + max_replication_slots plus the number of physical + replicas that are connected at the same time. @@ -348,9 +371,9 @@ to be set. In this case it should be set to at least the number of subscriptions that will be added to the subscriber. max_logical_replication_workers must be set to at - least the number of subscriptions. Additionally the - max_worker_processes may need to be adjusted to - accommodate for replication workers, at least + least the number of subscriptions, again plus some reserve for the table + synchronization. Additionally the max_worker_processes + may need to be adjusted to accommodate for replication workers, at least (max_logical_replication_workers + 1). Note that some extensions and parallel queries also take worker slots from max_worker_processes. @@ -393,8 +416,10 @@ CREATE SUBSCRIPTION mysub CONNECTION 'dbname=foo host=bar user=repuser' PUBLICAT - The above will start the replication process of changes to - users and departments tables. + The above will start the replication process, which synchronizes the + initial table contents of the tables users and + departments and then starts replicating + incremental changes to those tables. diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index dcb2d3303c..eb6f486677 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -1863,6 +1863,12 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i integer Process ID of the subscription worker process + + relid + Oid + OID of the relation that the worker is synchronizing; null for the + main apply worker + received_lsn pg_lsn @@ -1899,7 +1905,8 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i The pg_stat_subscription view will contain one row per subscription for main worker (with null PID if the worker is - not running). + not running), and additional rows for workers handling the initial data + copy of the subscribed tables. diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index 244e381de9..48ca414031 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -1487,7 +1487,7 @@ The commands accepted in walsender mode are: - CREATE_REPLICATION_SLOT slot_name [ TEMPORARY ] { PHYSICAL [ RESERVE_WAL ] | LOGICAL output_plugin [ EXPORT_SNAPSHOT | NOEXPORT_SNAPSHOT ] } + CREATE_REPLICATION_SLOT slot_name [ TEMPORARY ] { PHYSICAL [ RESERVE_WAL ] | LOGICAL output_plugin [ EXPORT_SNAPSHOT | NOEXPORT_SNAPSHOT | USE_SNAPSHOT ] } CREATE_REPLICATION_SLOT @@ -1542,12 +1542,17 @@ The commands accepted in walsender mode are: EXPORT_SNAPSHOT NOEXPORT_SNAPSHOT + USE_SNAPSHOT Decides what to do with the snapshot created during logical slot initialization. EXPORT_SNAPSHOT, which is the default, will export the snapshot for use in other sessions. This option can't - be used inside a transaction. NOEXPORT_SNAPSHOT will + be used inside a transaction. USE_SNAPSHOT will use the + snapshot for the current transaction executing the command. This + option must be used in a transaction, and + CREATE_REPLICATION_SLOT must be the first command + run in that transaction. Finally, NOEXPORT_SNAPSHOT will just use the snapshot for logical decoding as normal but won't do anything else with it. diff --git a/doc/src/sgml/ref/alter_subscription.sgml b/doc/src/sgml/ref/alter_subscription.sgml index 5e18e2ff6c..6f94247b92 100644 --- a/doc/src/sgml/ref/alter_subscription.sgml +++ b/doc/src/sgml/ref/alter_subscription.sgml @@ -21,15 +21,21 @@ PostgreSQL documentation -ALTER SUBSCRIPTION name WITH ( option [, ... ] ) ] +ALTER SUBSCRIPTION name WITH ( suboption [, ... ] ) ] -where option can be: +where suboption can be: - SLOT NAME = slot_name + SLOT NAME = slot_name + +ALTER SUBSCRIPTION name SET PUBLICATION publication_name [, ...] { REFRESH WITH ( puboption [, ... ] ) | NOREFRESH } +ALTER SUBSCRIPTION name REFRESH PUBLICATION WITH ( puboption [, ... ] ) + +where puboption can be: + + COPY DATA | NOCOPY DATA ALTER SUBSCRIPTION name OWNER TO { new_owner | CURRENT_USER | SESSION_USER } ALTER SUBSCRIPTION name CONNECTION 'conninfo' -ALTER SUBSCRIPTION name SET PUBLICATION publication_name [, ...] ALTER SUBSCRIPTION name ENABLE ALTER SUBSCRIPTION name DISABLE @@ -65,7 +71,6 @@ ALTER SUBSCRIPTION name DISABLE CONNECTION 'conninfo' - SET PUBLICATION publication_name SLOT NAME = slot_name @@ -76,6 +81,40 @@ ALTER SUBSCRIPTION name DISABLE + + SET PUBLICATION publication_name + + + Changes list of subscribed publications. See + for more information. + + + When REFRESH is specified, this command will also + act like REFRESH PUBLICATION. When + NOREFRESH is specified, the comamnd will not try to + refresh table information. + + + + + + REFRESH PUBLICATION + + + Fetch missing table information from publisher. This will start + replication of tables that were added to the subscribed-to publications + since the last invocation of REFRESH PUBLICATION or + since CREATE SUBSCRIPTION. + + + The COPY DATA and NOCOPY DATA + options specify if the existing data in the publications that are being + subscribed to should be copied. COPY DATA is the + default. + + + + ENABLE @@ -95,6 +134,7 @@ ALTER SUBSCRIPTION name DISABLE + diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml index e200076700..8f3c30b9b0 100644 --- a/doc/src/sgml/ref/create_subscription.sgml +++ b/doc/src/sgml/ref/create_subscription.sgml @@ -31,6 +31,8 @@ CREATE SUBSCRIPTION subscription_nameslot_name + | COPY DATA | NOCOPY DATA + | NOCONNECT @@ -132,6 +134,42 @@ CREATE SUBSCRIPTION subscription_name + + + COPY DATA + NOCOPY DATA + + + Specifies if the existing data in the publications that are being + subscribed to should be copied once the replication starts. + COPY DATA is the default. + + + + + + NOCONNECT + + + Instructs CREATE SUBSCRIPTION to skip the initial + connection to the provider. This will change default values of other + options to DISABLED, + NOCREATE SLOT, and NOCOPY DATA. + + + It's not allowed to combine NOCONNECT and + ENABLED, CREATE SLOT, or + COPY DATA. + + + Since no connection is made when this option is specified, the tables + are not subscribed, so after you enable the subscription nothing will + be replicated. It is required to run + ALTER SUBSCRIPTION ... REFRESH PUBLICATION in order for + tables to be subscribed. + + + diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml index bb32fb12e0..4f19b89232 100644 --- a/doc/src/sgml/ref/pg_dump.sgml +++ b/doc/src/sgml/ref/pg_dump.sgml @@ -799,22 +799,23 @@ PostgreSQL documentation - + - When dumping logical replication subscriptions, - generate CREATE SUBSCRIPTION commands that do not - create the remote replication slot. That way, the dump can be - restored without requiring network access to the remote servers. + Do not dump security labels. - + - Do not dump security labels. + When dumping logical replication subscriptions, + generate CREATE SUBSCRIPTION commands that do not + make remote connections for creating replication slot or initial table + copy. That way, the dump can be restored without requiring network + access to the remote servers. diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile index 31368585d2..159cab5c18 100644 --- a/src/backend/catalog/Makefile +++ b/src/backend/catalog/Makefile @@ -44,6 +44,7 @@ POSTGRES_BKI_SRCS = $(addprefix $(top_srcdir)/src/include/catalog/,\ pg_default_acl.h pg_init_privs.h pg_seclabel.h pg_shseclabel.h \ pg_collation.h pg_partitioned_table.h pg_range.h pg_transform.h \ pg_sequence.h pg_publication.h pg_publication_rel.h pg_subscription.h \ + pg_subscription_rel.h toasting.h indexing.h \ toasting.h indexing.h \ ) diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index 41c0056556..d49dcdc015 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -52,6 +52,7 @@ #include "catalog/pg_opclass.h" #include "catalog/pg_partitioned_table.h" #include "catalog/pg_statistic.h" +#include "catalog/pg_subscription_rel.h" #include "catalog/pg_tablespace.h" #include "catalog/pg_type.h" #include "catalog/pg_type_fn.h" @@ -1831,6 +1832,11 @@ heap_drop_with_catalog(Oid relid) */ relation_close(rel, NoLock); + /* + * Remove any associated relation synchronization states. + */ + RemoveSubscriptionRel(InvalidOid, relid); + /* * Forget any ON COMMIT action for the rel */ diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c index 0f784690ce..9330e2380a 100644 --- a/src/backend/catalog/pg_publication.c +++ b/src/backend/catalog/pg_publication.c @@ -221,8 +221,8 @@ GetPublicationRelations(Oid pubid) BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(pubid)); - scan = systable_beginscan(pubrelsrel, PublicationRelMapIndexId, true, - NULL, 1, &scankey); + scan = systable_beginscan(pubrelsrel, PublicationRelPrrelidPrpubidIndexId, + true, NULL, 1, &scankey); result = NIL; while (HeapTupleIsValid(tup = systable_getnext(scan))) diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c index 20fdd6a54f..e420ec14d2 100644 --- a/src/backend/catalog/pg_subscription.c +++ b/src/backend/catalog/pg_subscription.c @@ -19,15 +19,20 @@ #include "access/genam.h" #include "access/heapam.h" #include "access/htup_details.h" +#include "access/xact.h" +#include "catalog/indexing.h" #include "catalog/pg_type.h" #include "catalog/pg_subscription.h" +#include "catalog/pg_subscription_rel.h" #include "nodes/makefuncs.h" #include "utils/array.h" #include "utils/builtins.h" #include "utils/fmgroids.h" +#include "utils/pg_lsn.h" +#include "utils/rel.h" #include "utils/syscache.h" @@ -206,3 +211,280 @@ textarray_to_stringlist(ArrayType *textarray) return res; } + +/* + * Set the state of a subscription table. + */ +Oid +SetSubscriptionRelState(Oid subid, Oid relid, char state, + XLogRecPtr sublsn) +{ + Relation rel; + HeapTuple tup; + Oid subrelid; + bool nulls[Natts_pg_subscription_rel]; + Datum values[Natts_pg_subscription_rel]; + + /* Prevent concurrent changes. */ + rel = heap_open(SubscriptionRelRelationId, ShareRowExclusiveLock); + + /* Try finding existing mapping. */ + tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP, + ObjectIdGetDatum(relid), + ObjectIdGetDatum(subid)); + + /* + * If the record for given table does not exist yet create new + * record, otherwise update the existing one. + */ + if (!HeapTupleIsValid(tup)) + { + /* Form the tuple. */ + memset(values, 0, sizeof(values)); + memset(nulls, false, sizeof(nulls)); + values[Anum_pg_subscription_rel_srsubid - 1] = ObjectIdGetDatum(subid); + values[Anum_pg_subscription_rel_srrelid - 1] = ObjectIdGetDatum(relid); + values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state); + if (sublsn != InvalidXLogRecPtr) + values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn); + else + nulls[Anum_pg_subscription_rel_srsublsn - 1] = true; + + tup = heap_form_tuple(RelationGetDescr(rel), values, nulls); + + /* Insert tuple into catalog. */ + subrelid = CatalogTupleInsert(rel, tup); + + heap_freetuple(tup); + } + else + { + bool replaces[Natts_pg_subscription_rel]; + + /* Update the tuple. */ + memset(values, 0, sizeof(values)); + memset(nulls, false, sizeof(nulls)); + memset(replaces, false, sizeof(replaces)); + + replaces[Anum_pg_subscription_rel_srsubstate - 1] = true; + values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state); + + replaces[Anum_pg_subscription_rel_srsublsn - 1] = true; + if (sublsn != InvalidXLogRecPtr) + values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn); + else + nulls[Anum_pg_subscription_rel_srsublsn - 1] = true; + + tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls, + replaces); + + /* Update the catalog. */ + CatalogTupleUpdate(rel, &tup->t_self, tup); + + subrelid = HeapTupleGetOid(tup); + } + + /* Cleanup. */ + heap_close(rel, NoLock); + + return subrelid; +} + +/* + * Get state of subscription table. + * + * Returns SUBREL_STATE_UNKNOWN when not found and missing_ok is true. + */ +char +GetSubscriptionRelState(Oid subid, Oid relid, XLogRecPtr *sublsn, + bool missing_ok) +{ + Relation rel; + HeapTuple tup; + char substate; + bool isnull; + Datum d; + + rel = heap_open(SubscriptionRelRelationId, AccessShareLock); + + /* Try finding the mapping. */ + tup = SearchSysCache2(SUBSCRIPTIONRELMAP, + ObjectIdGetDatum(relid), + ObjectIdGetDatum(subid)); + + if (!HeapTupleIsValid(tup)) + { + if (missing_ok) + { + heap_close(rel, AccessShareLock); + *sublsn = InvalidXLogRecPtr; + return SUBREL_STATE_UNKNOWN; + } + + elog(ERROR, "subscription table %u in subscription %u does not exist", + relid, subid); + } + + /* Get the state. */ + d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup, + Anum_pg_subscription_rel_srsubstate, &isnull); + Assert(!isnull); + substate = DatumGetChar(d); + d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup, + Anum_pg_subscription_rel_srsublsn, &isnull); + if (isnull) + *sublsn = InvalidXLogRecPtr; + else + *sublsn = DatumGetLSN(d); + + /* Cleanup */ + ReleaseSysCache(tup); + heap_close(rel, AccessShareLock); + + return substate; +} + +/* + * Drop subscription relation mapping. These can be for a particular + * subscription, or for a particular relation, or both. + */ +void +RemoveSubscriptionRel(Oid subid, Oid relid) +{ + Relation rel; + HeapScanDesc scan; + ScanKeyData skey[2]; + HeapTuple tup; + int nkeys = 0; + + /* Prevent concurrent changes (see SetSubscriptionRelState()). */ + rel = heap_open(SubscriptionRelRelationId, ShareRowExclusiveLock); + + if (OidIsValid(subid)) + { + ScanKeyInit(&skey[nkeys++], + Anum_pg_subscription_rel_srsubid, + BTEqualStrategyNumber, + F_OIDEQ, + ObjectIdGetDatum(subid)); + } + + if (OidIsValid(relid)) + { + ScanKeyInit(&skey[nkeys++], + Anum_pg_subscription_rel_srrelid, + BTEqualStrategyNumber, + F_OIDEQ, + ObjectIdGetDatum(relid)); + } + + /* Do the search and delete what we found. */ + scan = heap_beginscan_catalog(rel, nkeys, skey); + while (HeapTupleIsValid(tup = heap_getnext(scan, ForwardScanDirection))) + { + simple_heap_delete(rel, &tup->t_self); + } + heap_endscan(scan); + + heap_close(rel, ShareRowExclusiveLock); +} + + +/* + * Get all relations for subscription. + * + * Returned list is palloced in current memory context. + */ +List * +GetSubscriptionRelations(Oid subid) +{ + List *res = NIL; + Relation rel; + HeapTuple tup; + int nkeys = 0; + ScanKeyData skey[2]; + SysScanDesc scan; + + rel = heap_open(SubscriptionRelRelationId, AccessShareLock); + + ScanKeyInit(&skey[nkeys++], + Anum_pg_subscription_rel_srsubid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(subid)); + + scan = systable_beginscan(rel, InvalidOid, false, + NULL, nkeys, skey); + + while (HeapTupleIsValid(tup = systable_getnext(scan))) + { + Form_pg_subscription_rel subrel; + SubscriptionRelState *relstate; + + subrel = (Form_pg_subscription_rel) GETSTRUCT(tup); + + relstate = (SubscriptionRelState *)palloc(sizeof(SubscriptionRelState)); + relstate->relid = subrel->srrelid; + relstate->state = subrel->srsubstate; + relstate->lsn = subrel->srsublsn; + + res = lappend(res, relstate); + } + + /* Cleanup */ + systable_endscan(scan); + heap_close(rel, AccessShareLock); + + return res; +} + +/* + * Get all relations for subscription that are not in a ready state. + * + * Returned list is palloced in current memory context. + */ +List * +GetSubscriptionNotReadyRelations(Oid subid) +{ + List *res = NIL; + Relation rel; + HeapTuple tup; + int nkeys = 0; + ScanKeyData skey[2]; + SysScanDesc scan; + + rel = heap_open(SubscriptionRelRelationId, AccessShareLock); + + ScanKeyInit(&skey[nkeys++], + Anum_pg_subscription_rel_srsubid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(subid)); + + ScanKeyInit(&skey[nkeys++], + Anum_pg_subscription_rel_srsubstate, + BTEqualStrategyNumber, F_CHARNE, + CharGetDatum(SUBREL_STATE_READY)); + + scan = systable_beginscan(rel, InvalidOid, false, + NULL, nkeys, skey); + + while (HeapTupleIsValid(tup = systable_getnext(scan))) + { + Form_pg_subscription_rel subrel; + SubscriptionRelState *relstate; + + subrel = (Form_pg_subscription_rel) GETSTRUCT(tup); + + relstate = (SubscriptionRelState *)palloc(sizeof(SubscriptionRelState)); + relstate->relid = subrel->srrelid; + relstate->state = subrel->srsubstate; + relstate->lsn = subrel->srsublsn; + + res = lappend(res, relstate); + } + + /* Cleanup */ + systable_endscan(scan); + heap_close(rel, AccessShareLock); + + return res; +} diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index c2b0bedc1d..5723714fb9 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -733,6 +733,7 @@ CREATE VIEW pg_stat_subscription AS su.oid AS subid, su.subname, st.pid, + st.relid, st.received_lsn, st.last_msg_send_time, st.last_msg_receipt_time, diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index ba89b292d1..b0fd09f458 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -60,7 +60,8 @@ typedef enum CopyDest { COPY_FILE, /* to/from file (or a piped program) */ COPY_OLD_FE, /* to/from frontend (2.0 protocol) */ - COPY_NEW_FE /* to/from frontend (3.0 protocol) */ + COPY_NEW_FE, /* to/from frontend (3.0 protocol) */ + COPY_CALLBACK /* to/from callback function */ } CopyDest; /* @@ -109,6 +110,7 @@ typedef struct CopyStateData List *attnumlist; /* integer list of attnums to copy */ char *filename; /* filename, or NULL for STDIN/STDOUT */ bool is_program; /* is 'filename' a program to popen? */ + copy_data_source_cb data_source_cb; /* function for reading data*/ bool binary; /* binary format? */ bool oids; /* include OIDs? */ bool freeze; /* freeze rows on loading? */ @@ -299,7 +301,6 @@ static uint64 DoCopyTo(CopyState cstate); static uint64 CopyTo(CopyState cstate); static void CopyOneRowTo(CopyState cstate, Oid tupleOid, Datum *values, bool *nulls); -static uint64 CopyFrom(CopyState cstate); static void CopyFromInsertBatch(CopyState cstate, EState *estate, CommandId mycid, int hi_options, ResultRelInfo *resultRelInfo, TupleTableSlot *myslot, @@ -529,6 +530,9 @@ CopySendEndOfRow(CopyState cstate) /* Dump the accumulated row as one CopyData message */ (void) pq_putmessage('d', fe_msgbuf->data, fe_msgbuf->len); break; + case COPY_CALLBACK: + Assert(false); /* Not yet supported. */ + break; } resetStringInfo(fe_msgbuf); @@ -643,6 +647,9 @@ CopyGetData(CopyState cstate, void *databuf, int minread, int maxread) bytesread += avail; } break; + case COPY_CALLBACK: + bytesread = cstate->data_source_cb(databuf, minread, maxread); + break; } return bytesread; @@ -969,7 +976,7 @@ DoCopy(ParseState *pstate, const CopyStmt *stmt, PreventCommandIfParallelMode("COPY FROM"); cstate = BeginCopyFrom(pstate, rel, stmt->filename, stmt->is_program, - stmt->attlist, stmt->options); + NULL, stmt->attlist, stmt->options); cstate->range_table = range_table; *processed = CopyFrom(cstate); /* copy from file to database */ EndCopyFrom(cstate); @@ -2286,7 +2293,7 @@ limit_printout_length(const char *str) /* * Copy FROM file to relation. */ -static uint64 +uint64 CopyFrom(CopyState cstate) { HeapTuple tuple; @@ -2878,6 +2885,7 @@ BeginCopyFrom(ParseState *pstate, Relation rel, const char *filename, bool is_program, + copy_data_source_cb data_source_cb, List *attnamelist, List *options) { @@ -2992,7 +3000,12 @@ BeginCopyFrom(ParseState *pstate, cstate->num_defaults = num_defaults; cstate->is_program = is_program; - if (pipe) + if (data_source_cb) + { + cstate->copy_dest = COPY_CALLBACK; + cstate->data_source_cb = data_source_cb; + } + else if (pipe) { Assert(!is_program); /* the grammar does not allow this */ if (whereToSendOutput == DestRemote) diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c index 0198e6d75b..0784ca7951 100644 --- a/src/backend/commands/subscriptioncmds.c +++ b/src/backend/commands/subscriptioncmds.c @@ -20,27 +20,36 @@ #include "access/htup_details.h" #include "access/xact.h" +#include "catalog/dependency.h" #include "catalog/indexing.h" +#include "catalog/namespace.h" #include "catalog/objectaccess.h" #include "catalog/objectaddress.h" #include "catalog/pg_type.h" #include "catalog/pg_subscription.h" +#include "catalog/pg_subscription_rel.h" #include "commands/defrem.h" #include "commands/event_trigger.h" #include "commands/subscriptioncmds.h" +#include "nodes/makefuncs.h" + #include "replication/logicallauncher.h" #include "replication/origin.h" #include "replication/walreceiver.h" +#include "replication/walsender.h" #include "replication/worker_internal.h" #include "storage/lmgr.h" #include "utils/builtins.h" +#include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/syscache.h" +static List *fetch_table_list(WalReceiverConn *wrconn, List *publications); + /* * Common option parsing function for CREATE and ALTER SUBSCRIPTION commands. * @@ -49,17 +58,17 @@ * accomodate that. */ static void -parse_subscription_options(List *options, char **conninfo, - List **publications, bool *enabled_given, - bool *enabled, bool *create_slot, char **slot_name) +parse_subscription_options(List *options, bool *connect, bool *enabled_given, + bool *enabled, bool *create_slot, char **slot_name, + bool *copy_data) { ListCell *lc; + bool connect_given = false; bool create_slot_given = false; + bool copy_data_given = false; - if (conninfo) - *conninfo = NULL; - if (publications) - *publications = NIL; + if (connect) + *connect = true; if (enabled) { *enabled_given = false; @@ -69,29 +78,23 @@ parse_subscription_options(List *options, char **conninfo, *create_slot = true; if (slot_name) *slot_name = NULL; + if (copy_data) + *copy_data = true; /* Parse options */ foreach (lc, options) { DefElem *defel = (DefElem *) lfirst(lc); - if (strcmp(defel->defname, "conninfo") == 0 && conninfo) - { - if (*conninfo) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("conflicting or redundant options"))); - - *conninfo = defGetString(defel); - } - else if (strcmp(defel->defname, "publication") == 0 && publications) + if (strcmp(defel->defname, "noconnect") == 0 && connect) { - if (*publications) + if (connect_given) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("conflicting or redundant options"))); - *publications = defGetStringList(defel); + connect_given = true; + *connect = !defGetBoolean(defel); } else if (strcmp(defel->defname, "enabled") == 0 && enabled) { @@ -142,9 +145,57 @@ parse_subscription_options(List *options, char **conninfo, *slot_name = defGetString(defel); } + else if (strcmp(defel->defname, "copy data") == 0 && copy_data) + { + if (copy_data_given) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"))); + + copy_data_given = true; + *copy_data = defGetBoolean(defel); + } + else if (strcmp(defel->defname, "nocopy data") == 0 && copy_data) + { + if (copy_data_given) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"))); + + copy_data_given = true; + *copy_data = !defGetBoolean(defel); + } else elog(ERROR, "unrecognized option: %s", defel->defname); } + + /* + * We've been explicitly asked to not connect, that requires some + * additional processing. + */ + if (connect && !*connect) + { + /* Check for incompatible options from the user. */ + if (*enabled_given && *enabled) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("noconnect and enabled are mutually exclusive options"))); + + if (create_slot_given && *create_slot) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("noconnect and create slot are mutually exclusive options"))); + + if (copy_data_given && *copy_data) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("noconnect and copy data are mutually exclusive options"))); + + /* Change the defaults of other options. */ + *enabled = false; + *create_slot = false; + *copy_data = false; + } } /* @@ -214,8 +265,10 @@ CreateSubscription(CreateSubscriptionStmt *stmt, bool isTopLevel) Datum values[Natts_pg_subscription]; Oid owner = GetUserId(); HeapTuple tup; + bool connect; bool enabled_given; bool enabled; + bool copy_data; char *conninfo; char *slotname; char originname[NAMEDATALEN]; @@ -226,9 +279,8 @@ CreateSubscription(CreateSubscriptionStmt *stmt, bool isTopLevel) * Parse and check options. * Connection and publication should not be specified here. */ - parse_subscription_options(stmt->options, NULL, NULL, - &enabled_given, &enabled, - &create_slot, &slotname); + parse_subscription_options(stmt->options, &connect, &enabled_given, + &enabled, &create_slot, &slotname, ©_data); /* * Since creating a replication slot is not transactional, rolling back @@ -297,14 +349,17 @@ CreateSubscription(CreateSubscriptionStmt *stmt, bool isTopLevel) replorigin_create(originname); /* - * If requested, create the replication slot on remote side for our - * newly created subscription. + * Connect to remote side to execute requested commands and fetch table + * info. */ - if (create_slot) + if (connect) { XLogRecPtr lsn; char *err; WalReceiverConn *wrconn; + List *tables; + ListCell *lc; + char table_state; /* Try to connect to the publisher. */ wrconn = walrcv_connect(conninfo, true, stmt->subname, &err); @@ -315,13 +370,43 @@ CreateSubscription(CreateSubscriptionStmt *stmt, bool isTopLevel) PG_TRY(); { /* - * Create permanent slot for the subscription. We won't use the - * initial snapshot for anything, so no need to export it. + * If requested, create permanent slot for the subscription. + * We won't use the initial snapshot for anything, so no need + * to export it. + */ + if (create_slot) + { + walrcv_create_slot(wrconn, slotname, false, + CRS_NOEXPORT_SNAPSHOT, &lsn); + ereport(NOTICE, + (errmsg("created replication slot \"%s\" on publisher", + slotname))); + } + + /* + * Set sync state based on if we were asked to do data copy or + * not. */ - walrcv_create_slot(wrconn, slotname, false, false, &lsn); + table_state = copy_data ? SUBREL_STATE_INIT : SUBREL_STATE_READY; + + /* + * Get the table list from publisher and build local table status + * info. + */ + tables = fetch_table_list(wrconn, publications); + foreach (lc, tables) + { + RangeVar *rv = (RangeVar *) lfirst(lc); + Oid relid; + + relid = RangeVarGetRelid(rv, AccessShareLock, true); + + SetSubscriptionRelState(subid, relid, table_state, + InvalidXLogRecPtr); + } + ereport(NOTICE, - (errmsg("created replication slot \"%s\" on publisher", - slotname))); + (errmsg("synchronized table states"))); } PG_CATCH(); { @@ -334,6 +419,11 @@ CreateSubscription(CreateSubscriptionStmt *stmt, bool isTopLevel) /* And we are done with the remote side. */ walrcv_disconnect(wrconn); } + else + ereport(WARNING, + (errmsg("tables were not subscribed, you will have to run " + "ALTER SUBSCRIPTION ... REFRESH PUBLICATION to " + "subscribe the tables"))); heap_close(rel, RowExclusiveLock); @@ -346,6 +436,108 @@ CreateSubscription(CreateSubscriptionStmt *stmt, bool isTopLevel) return myself; } +static void +AlterSubscription_refresh(Subscription *sub, bool copy_data) +{ + char *err; + List *pubrel_names; + List *subrel_states; + Oid *subrel_local_oids; + Oid *pubrel_local_oids; + ListCell *lc; + int off; + + /* Load the library providing us libpq calls. */ + load_file("libpqwalreceiver", false); + + /* Try to connect to the publisher. */ + wrconn = walrcv_connect(sub->conninfo, true, sub->name, &err); + if (!wrconn) + ereport(ERROR, + (errmsg("could not connect to the publisher: %s", err))); + + /* Get the table list from publisher. */ + pubrel_names = fetch_table_list(wrconn, sub->publications); + + /* We are done with the remote side, close connection. */ + walrcv_disconnect(wrconn); + + /* Get local table list. */ + subrel_states = GetSubscriptionRelations(sub->oid); + + /* + * Build qsorted array of local table oids for faster lookup. + * This can potentially contain all tables in the database so + * speed of lookup is important. + */ + subrel_local_oids = palloc(list_length(subrel_states) * sizeof(Oid)); + off = 0; + foreach(lc, subrel_states) + { + SubscriptionRelState *relstate = (SubscriptionRelState *) lfirst(lc); + subrel_local_oids[off++] = relstate->relid; + } + qsort(subrel_local_oids, list_length(subrel_states), + sizeof(Oid), oid_cmp); + + /* + * Walk over the remote tables and try to match them to locally + * known tables. If the table is not known locally create a new state + * for it. + * + * Also builds array of local oids of remote tables for the next step. + */ + off = 0; + pubrel_local_oids = palloc(list_length(pubrel_names) * sizeof(Oid)); + + foreach (lc, pubrel_names) + { + RangeVar *rv = (RangeVar *) lfirst(lc); + Oid relid; + + relid = RangeVarGetRelid(rv, AccessShareLock, false); + pubrel_local_oids[off++] = relid; + + if (!bsearch(&relid, subrel_local_oids, + list_length(subrel_states), sizeof(Oid), oid_cmp)) + { + SetSubscriptionRelState(sub->oid, relid, + copy_data ? SUBREL_STATE_INIT : SUBREL_STATE_READY, + InvalidXLogRecPtr); + ereport(NOTICE, + (errmsg("added subscription for table %s.%s", + quote_identifier(rv->schemaname), + quote_identifier(rv->relname)))); + } + } + + /* + * Next remove state for tables we should not care about anymore using + * the data we collected above + */ + qsort(pubrel_local_oids, list_length(pubrel_names), + sizeof(Oid), oid_cmp); + + for (off = 0; off < list_length(subrel_states); off++) + { + Oid relid = subrel_local_oids[off]; + + if (!bsearch(&relid, pubrel_local_oids, + list_length(pubrel_names), sizeof(Oid), oid_cmp)) + { + char *namespace; + + RemoveSubscriptionRel(sub->oid, relid); + + namespace = get_namespace_name(get_rel_namespace(relid)); + ereport(NOTICE, + (errmsg("removed subscription for table %s.%s", + quote_identifier(namespace), + quote_identifier(get_rel_name(relid))))); + } + } +} + /* * Alter the existing subscription. */ @@ -359,11 +551,7 @@ AlterSubscription(AlterSubscriptionStmt *stmt) Datum values[Natts_pg_subscription]; HeapTuple tup; Oid subid; - bool enabled_given; - bool enabled; - char *conninfo; - char *slot_name; - List *publications; + bool update_tuple = false; rel = heap_open(SubscriptionRelationId, RowExclusiveLock); @@ -384,52 +572,113 @@ AlterSubscription(AlterSubscriptionStmt *stmt) subid = HeapTupleGetOid(tup); - /* Parse options. */ - parse_subscription_options(stmt->options, &conninfo, &publications, - &enabled_given, &enabled, - NULL, &slot_name); - /* Form a new tuple. */ memset(values, 0, sizeof(values)); memset(nulls, false, sizeof(nulls)); memset(replaces, false, sizeof(replaces)); - if (enabled_given) - { - values[Anum_pg_subscription_subenabled - 1] = BoolGetDatum(enabled); - replaces[Anum_pg_subscription_subenabled - 1] = true; - } - if (conninfo) - { - values[Anum_pg_subscription_subconninfo - 1] = - CStringGetTextDatum(conninfo); - replaces[Anum_pg_subscription_subconninfo - 1] = true; - } - if (slot_name) - { - values[Anum_pg_subscription_subslotname - 1] = - DirectFunctionCall1(namein, CStringGetDatum(slot_name)); - replaces[Anum_pg_subscription_subslotname - 1] = true; - } - if (publications != NIL) + switch (stmt->kind) { - values[Anum_pg_subscription_subpublications - 1] = - publicationListToArray(publications); - replaces[Anum_pg_subscription_subpublications - 1] = true; + case ALTER_SUBSCRIPTION_OPTIONS: + { + char *slot_name; + + parse_subscription_options(stmt->options, NULL, NULL, NULL, + NULL, &slot_name, NULL); + + values[Anum_pg_subscription_subslotname - 1] = + DirectFunctionCall1(namein, CStringGetDatum(slot_name)); + replaces[Anum_pg_subscription_subslotname - 1] = true; + + update_tuple = true; + break; + } + + case ALTER_SUBSCRIPTION_ENABLED: + { + bool enabled, + enabled_given; + + parse_subscription_options(stmt->options, NULL, + &enabled_given, &enabled, NULL, + NULL, NULL); + Assert(enabled_given); + + values[Anum_pg_subscription_subenabled - 1] = + BoolGetDatum(enabled); + replaces[Anum_pg_subscription_subenabled - 1] = true; + + update_tuple = true; + break; + } + + case ALTER_SUBSCRIPTION_CONNECTION: + values[Anum_pg_subscription_subconninfo - 1] = + CStringGetTextDatum(stmt->conninfo); + replaces[Anum_pg_subscription_subconninfo - 1] = true; + update_tuple = true; + break; + + case ALTER_SUBSCRIPTION_PUBLICATION: + case ALTER_SUBSCRIPTION_PUBLICATION_REFRESH: + { + bool copy_data; + Subscription *sub = GetSubscription(subid, false); + + parse_subscription_options(stmt->options, NULL, NULL, NULL, + NULL, NULL, ©_data); + + values[Anum_pg_subscription_subpublications - 1] = + publicationListToArray(stmt->publication); + replaces[Anum_pg_subscription_subpublications - 1] = true; + + update_tuple = true; + + /* Refresh if user asked us to. */ + if (stmt->kind == ALTER_SUBSCRIPTION_PUBLICATION_REFRESH) + { + /* Make sure refresh sees the new list of publications. */ + sub->publications = stmt->publication; + + AlterSubscription_refresh(sub, copy_data); + } + + break; + } + + case ALTER_SUBSCRIPTION_REFRESH: + { + bool copy_data; + Subscription *sub = GetSubscription(subid, false); + + parse_subscription_options(stmt->options, NULL, NULL, NULL, + NULL, NULL, ©_data); + + AlterSubscription_refresh(sub, copy_data); + + break; + } + + default: + elog(ERROR, "unrecognized ALTER SUBSCRIPTION kind %d", + stmt->kind); } - tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls, - replaces); + /* Update the catalog if needed. */ + if (update_tuple) + { + tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls, + replaces); - /* Update the catalog. */ - CatalogTupleUpdate(rel, &tup->t_self, tup); + CatalogTupleUpdate(rel, &tup->t_self, tup); - ObjectAddressSet(myself, SubscriptionRelationId, subid); + heap_freetuple(tup); + } - /* Cleanup. */ - heap_freetuple(tup); heap_close(rel, RowExclusiveLock); + ObjectAddressSet(myself, SubscriptionRelationId, subid); + InvokeObjectPostAlterHook(SubscriptionRelationId, subid, 0); return myself; @@ -537,8 +786,11 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel) /* Clean up dependencies */ deleteSharedDependencyRecordsFor(SubscriptionRelationId, subid, 0); + /* Remove any associated relation synchronization states. */ + RemoveSubscriptionRel(subid, InvalidOid); + /* Kill the apply worker so that the slot becomes accessible. */ - logicalrep_worker_stop(subid); + logicalrep_worker_stop(subid, InvalidOid); /* Remove the origin tracking if exists. */ snprintf(originname, sizeof(originname), "pg_%u", subid); @@ -571,15 +823,20 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel) PG_TRY(); { - if (!walrcv_command(wrconn, cmd.data, &err)) + WalRcvExecResult *res; + res = walrcv_exec(wrconn, cmd.data, 0, NULL); + + if (res->status != WALRCV_OK_COMMAND) ereport(ERROR, (errmsg("could not drop the replication slot \"%s\" on publisher", slotname), - errdetail("The error was: %s", err))); + errdetail("The error was: %s", res->err))); else ereport(NOTICE, (errmsg("dropped replication slot \"%s\" on publisher", slotname))); + + walrcv_clear_result(res); } PG_CATCH(); { @@ -691,3 +948,72 @@ AlterSubscriptionOwner_oid(Oid subid, Oid newOwnerId) heap_close(rel, RowExclusiveLock); } + +/* + * Get the list of tables which belong to specified publications on the + * publisher connection. + */ +static List * +fetch_table_list(WalReceiverConn *wrconn, List *publications) +{ + WalRcvExecResult *res; + StringInfoData cmd; + TupleTableSlot *slot; + Oid tableRow[2] = {TEXTOID, TEXTOID}; + ListCell *lc; + bool first; + List *tablelist = NIL; + + Assert(list_length(publications) > 0); + + initStringInfo(&cmd); + appendStringInfo(&cmd, "SELECT DISTINCT t.schemaname, t.tablename\n" + " FROM pg_catalog.pg_publication_tables t\n" + " WHERE t.pubname IN ("); + first = true; + foreach (lc, publications) + { + char *pubname = strVal(lfirst(lc)); + + if (first) + first = false; + else + appendStringInfoString(&cmd, ", "); + + appendStringInfo(&cmd, "%s", quote_literal_cstr(pubname)); + } + appendStringInfoString(&cmd, ")"); + + res = walrcv_exec(wrconn, cmd.data, 2, tableRow); + pfree(cmd.data); + + if (res->status != WALRCV_OK_TUPLES) + ereport(ERROR, + (errmsg("could not receive list of replicated tables from the publisher: %s", + res->err))); + + /* Process tables. */ + slot = MakeSingleTupleTableSlot(res->tupledesc); + while (tuplestore_gettupleslot(res->tuplestore, true, false, slot)) + { + char *nspname; + char *relname; + bool isnull; + RangeVar *rv; + + nspname = TextDatumGetCString(slot_getattr(slot, 1, &isnull)); + Assert(!isnull); + relname = TextDatumGetCString(slot_getattr(slot, 2, &isnull)); + Assert(!isnull); + + rv = makeRangeVar(pstrdup(nspname), pstrdup(relname), -1); + tablelist = lappend(tablelist, rv); + + ExecClearTuple(slot); + } + ExecDropSingleTupleTableSlot(slot); + + walrcv_clear_result(res); + + return tablelist; +} diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index d0d45a557b..50126baacf 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -651,7 +651,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); MAPPING MATCH MATERIALIZED MAXVALUE METHOD MINUTE_P MINVALUE MODE MONTH_P MOVE NAME_P NAMES NATIONAL NATURAL NCHAR NEW NEXT NO NONE - NOT NOTHING NOTIFY NOTNULL NOWAIT NULL_P NULLIF + NOREFRESH NOT NOTHING NOTIFY NOTNULL NOWAIT NULL_P NULLIF NULLS_P NUMERIC OBJECT_P OF OFF OFFSET OIDS OLD ON ONLY OPERATOR OPTION OPTIONS OR @@ -9095,6 +9095,7 @@ AlterSubscriptionStmt: { AlterSubscriptionStmt *n = makeNode(AlterSubscriptionStmt); + n->kind = ALTER_SUBSCRIPTION_OPTIONS; n->subname = $3; n->options = $5; $$ = (Node *)n; @@ -9103,24 +9104,45 @@ AlterSubscriptionStmt: { AlterSubscriptionStmt *n = makeNode(AlterSubscriptionStmt); + n->kind = ALTER_SUBSCRIPTION_CONNECTION; n->subname = $3; - n->options = list_make1(makeDefElem("conninfo", - (Node *)makeString($5), @1)); + n->conninfo = $5; + $$ = (Node *)n; + } + | ALTER SUBSCRIPTION name REFRESH PUBLICATION opt_definition + { + AlterSubscriptionStmt *n = + makeNode(AlterSubscriptionStmt); + n->kind = ALTER_SUBSCRIPTION_REFRESH; + n->subname = $3; + n->options = $6; + $$ = (Node *)n; + } + | ALTER SUBSCRIPTION name SET PUBLICATION publication_name_list REFRESH opt_definition + { + AlterSubscriptionStmt *n = + makeNode(AlterSubscriptionStmt); + n->kind = ALTER_SUBSCRIPTION_PUBLICATION_REFRESH; + n->subname = $3; + n->publication = $6; + n->options = $8; $$ = (Node *)n; } - | ALTER SUBSCRIPTION name SET PUBLICATION publication_name_list + | ALTER SUBSCRIPTION name SET PUBLICATION publication_name_list NOREFRESH { AlterSubscriptionStmt *n = makeNode(AlterSubscriptionStmt); + n->kind = ALTER_SUBSCRIPTION_PUBLICATION; n->subname = $3; - n->options = list_make1(makeDefElem("publication", - (Node *)$6, @1)); + n->publication = $6; + n->options = NIL; $$ = (Node *)n; } | ALTER SUBSCRIPTION name ENABLE_P { AlterSubscriptionStmt *n = makeNode(AlterSubscriptionStmt); + n->kind = ALTER_SUBSCRIPTION_ENABLED; n->subname = $3; n->options = list_make1(makeDefElem("enabled", (Node *)makeInteger(TRUE), @1)); @@ -9130,11 +9152,13 @@ AlterSubscriptionStmt: { AlterSubscriptionStmt *n = makeNode(AlterSubscriptionStmt); + n->kind = ALTER_SUBSCRIPTION_ENABLED; n->subname = $3; n->options = list_make1(makeDefElem("enabled", (Node *)makeInteger(FALSE), @1)); $$ = (Node *)n; - } ; + } + ; /***************************************************************************** * @@ -14548,6 +14572,7 @@ unreserved_keyword: | NEW | NEXT | NO + | NOREFRESH | NOTHING | NOTIFY | NOWAIT diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index 3a50488db3..b704788eb5 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -3415,6 +3415,12 @@ pgstat_get_wait_ipc(WaitEventIPC w) case WAIT_EVENT_SYNC_REP: event_name = "SyncRep"; break; + case WAIT_EVENT_LOGICAL_SYNC_DATA: + event_name = "LogicalSyncData"; + break; + case WAIT_EVENT_LOGICAL_SYNC_STATE_CHANGE: + event_name = "LogicalSyncStateChange"; + break; /* no default case, so that compiler will warn */ } diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c index 65a9e6c81c..4dd8eef1f9 100644 --- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c +++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c @@ -22,14 +22,16 @@ #include "libpq-fe.h" #include "pqexpbuffer.h" #include "access/xlog.h" +#include "catalog/pg_type.h" +#include "funcapi.h" #include "mb/pg_wchar.h" #include "miscadmin.h" #include "pgstat.h" -#include "replication/logicalproto.h" #include "replication/walreceiver.h" -#include "storage/proc.h" #include "utils/builtins.h" +#include "utils/memutils.h" #include "utils/pg_lsn.h" +#include "utils/tuplestore.h" PG_MODULE_MAGIC; @@ -68,10 +70,12 @@ static void libpqrcv_send(WalReceiverConn *conn, const char *buffer, static char *libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, bool temporary, - bool export_snapshot, + CRSSnapshotAction snapshot_action, XLogRecPtr *lsn); -static bool libpqrcv_command(WalReceiverConn *conn, - const char *cmd, char **err); +static WalRcvExecResult *libpqrcv_exec(WalReceiverConn *conn, + const char *query, + const int nRetTypes, + const Oid *retTypes); static void libpqrcv_disconnect(WalReceiverConn *conn); static WalReceiverFunctionsType PQWalReceiverFunctions = { @@ -85,7 +89,7 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = { libpqrcv_receive, libpqrcv_send, libpqrcv_create_slot, - libpqrcv_command, + libpqrcv_exec, libpqrcv_disconnect }; @@ -431,10 +435,8 @@ libpqrcv_endstreaming(WalReceiverConn *conn, TimeLineID *next_tli) * next timeline's ID, or just CommandComplete if the server was shut * down. * - * If we had not yet received CopyDone from the backend, PGRES_COPY_IN - * would also be possible. However, at the moment this function is only - * called after receiving CopyDone from the backend - the walreceiver - * never terminates replication on its own initiative. + * If we had not yet received CopyDone from the backend, PGRES_COPY_OUT + * is also possible in case we aborted the copy in mid-stream. */ res = PQgetResult(conn->streamConn); if (PQresultStatus(res) == PGRES_TUPLES_OK) @@ -531,7 +533,7 @@ libpqrcv_readtimelinehistoryfile(WalReceiverConn *conn, * Windows. * * The function is modeled on PQexec() in libpq, but only implements - * those parts that are in use in the walreceiver. + * those parts that are in use in the walreceiver api. * * Queries are always executed on the connection in streamConn. */ @@ -543,8 +545,9 @@ libpqrcv_PQexec(PGconn *streamConn, const char *query) /* * PQexec() silently discards any prior query results on the connection. - * This is not required for walreceiver since it's expected that walsender - * won't generate any such junk results. + * This is not required for this function as it's expected that the + * caller (which is this library in all cases) will behave correctly and + * we don't have to be backwards compatible with old libpq. */ /* @@ -593,8 +596,7 @@ libpqrcv_PQexec(PGconn *streamConn, const char *query) /* * Emulate the PQexec()'s behavior of returning the last result when - * there are many. Since walsender will never generate multiple - * results, we skip the concatenation of error messages. + * there are many. We are fine with returning just last error message. */ result = PQgetResult(streamConn); if (result == NULL) @@ -675,8 +677,19 @@ libpqrcv_receive(WalReceiverConn *conn, char **buffer, PGresult *res; res = PQgetResult(conn->streamConn); - if (PQresultStatus(res) == PGRES_COMMAND_OK || - PQresultStatus(res) == PGRES_COPY_IN) + if (PQresultStatus(res) == PGRES_COMMAND_OK) + { + PQclear(res); + + /* Verify that there are no more results */ + res = PQgetResult(conn->streamConn); + if (res != NULL) + ereport(ERROR, + (errmsg("unexpected result after CommandComplete: %s", + PQerrorMessage(conn->streamConn)))); + return -1; + } + else if (PQresultStatus(res) == PGRES_COPY_IN) { PQclear(res); return -1; @@ -721,7 +734,8 @@ libpqrcv_send(WalReceiverConn *conn, const char *buffer, int nbytes) */ static char * libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, - bool temporary, bool export_snapshot, XLogRecPtr *lsn) + bool temporary, CRSSnapshotAction snapshot_action, + XLogRecPtr *lsn) { PGresult *res; StringInfoData cmd; @@ -737,10 +751,18 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, if (conn->logical) { appendStringInfo(&cmd, " LOGICAL pgoutput"); - if (export_snapshot) - appendStringInfo(&cmd, " EXPORT_SNAPSHOT"); - else - appendStringInfo(&cmd, " NOEXPORT_SNAPSHOT"); + switch (snapshot_action) + { + case CRS_EXPORT_SNAPSHOT: + appendStringInfo(&cmd, " EXPORT_SNAPSHOT"); + break; + case CRS_NOEXPORT_SNAPSHOT: + appendStringInfo(&cmd, " NOEXPORT_SNAPSHOT"); + break; + case CRS_USE_SNAPSHOT: + appendStringInfo(&cmd, " USE_SNAPSHOT"); + break; + } } res = libpqrcv_PQexec(conn->streamConn, cmd.data); @@ -767,28 +789,139 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, } /* - * Run command. + * Convert tuple query result to tuplestore. + */ +static void +libpqrcv_processTuples(PGresult *pgres, WalRcvExecResult *walres, + const int nRetTypes, const Oid *retTypes) +{ + int tupn; + int coln; + int nfields = PQnfields(pgres); + HeapTuple tuple; + AttInMetadata *attinmeta; + MemoryContext rowcontext; + MemoryContext oldcontext; + + /* No point in doing anything here if there were no tuples returned. */ + if (PQntuples(pgres) == 0) + return; + + /* Make sure we got expected number of fields. */ + if (nfields != nRetTypes) + ereport(ERROR, + (errmsg("invalid query responser"), + errdetail("Expected %d fields, got %d fields.", + nRetTypes, nfields))); + + + walres->tuplestore = tuplestore_begin_heap(true, false, work_mem); + + /* Create tuple descriptor corresponding to expected result. */ + walres->tupledesc = CreateTemplateTupleDesc(nRetTypes, false); + for (coln = 0; coln < nRetTypes; coln++) + TupleDescInitEntry(walres->tupledesc, (AttrNumber) coln + 1, + PQfname(pgres, coln), retTypes[coln], -1, 0); + attinmeta = TupleDescGetAttInMetadata(walres->tupledesc); + + /* Create temporary context for local allocations. */ + rowcontext = AllocSetContextCreate(CurrentMemoryContext, + "libpqrcv query result context", + ALLOCSET_DEFAULT_SIZES); + + /* Process returned rows. */ + for (tupn = 0; tupn < PQntuples(pgres); tupn++) + { + char *cstrs[MaxTupleAttributeNumber]; + + CHECK_FOR_INTERRUPTS(); + + /* Do the allocations in temporary context. */ + oldcontext = MemoryContextSwitchTo(rowcontext); + + /* + * Fill cstrs with null-terminated strings of column values. + */ + for (coln = 0; coln < nfields; coln++) + { + if (PQgetisnull(pgres, tupn, coln)) + cstrs[coln] = NULL; + else + cstrs[coln] = PQgetvalue(pgres, tupn, coln); + } + + /* Convert row to a tuple, and add it to the tuplestore */ + tuple = BuildTupleFromCStrings(attinmeta, cstrs); + tuplestore_puttuple(walres->tuplestore, tuple); + + /* Clean up */ + MemoryContextSwitchTo(oldcontext); + MemoryContextReset(rowcontext); + } + + MemoryContextDelete(rowcontext); +} + +/* + * Public interface for sending generic queries (and commands). * - * Returns if the command has succeeded and fills the err with palloced - * error message if not. + * This can only be called from process connected to database. */ -static bool -libpqrcv_command(WalReceiverConn *conn, const char *cmd, char **err) +static WalRcvExecResult * +libpqrcv_exec(WalReceiverConn *conn, const char *query, + const int nRetTypes, const Oid *retTypes) { - PGresult *res; + PGresult *pgres = NULL; + WalRcvExecResult *walres = palloc0(sizeof(WalRcvExecResult)); - res = libpqrcv_PQexec(conn->streamConn, cmd); + if (MyDatabaseId == InvalidOid) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("the query interface requires a database connection"))); - if (PQresultStatus(res) != PGRES_COMMAND_OK) + pgres = libpqrcv_PQexec(conn->streamConn, query); + + switch (PQresultStatus(pgres)) { - PQclear(res); - *err = pchomp(PQerrorMessage(conn->streamConn)); - return false; + case PGRES_SINGLE_TUPLE: + case PGRES_TUPLES_OK: + walres->status = WALRCV_OK_TUPLES; + libpqrcv_processTuples(pgres, walres, nRetTypes, retTypes); + break; + + case PGRES_COPY_IN: + walres->status = WALRCV_OK_COPY_IN; + break; + + case PGRES_COPY_OUT: + walres->status = WALRCV_OK_COPY_OUT; + break; + + case PGRES_COPY_BOTH: + walres->status = WALRCV_OK_COPY_BOTH; + break; + + case PGRES_COMMAND_OK: + walres->status = WALRCV_OK_COMMAND; + break; + + /* Empty query is considered error. */ + case PGRES_EMPTY_QUERY: + walres->status = WALRCV_ERROR; + walres->err = _("empty query"); + break; + + case PGRES_NONFATAL_ERROR: + case PGRES_FATAL_ERROR: + case PGRES_BAD_RESPONSE: + walres->status = WALRCV_ERROR; + walres->err = pchomp(PQerrorMessage(conn->streamConn)); + break; } - PQclear(res); + PQclear(pgres); - return true; + return walres; } /* diff --git a/src/backend/replication/logical/Makefile b/src/backend/replication/logical/Makefile index 259befa4e6..bb417b042e 100644 --- a/src/backend/replication/logical/Makefile +++ b/src/backend/replication/logical/Makefile @@ -15,6 +15,6 @@ include $(top_builddir)/src/Makefile.global override CPPFLAGS := -I$(srcdir) $(CPPFLAGS) OBJS = decode.o launcher.o logical.o logicalfuncs.o message.o origin.o \ - proto.o relation.o reorderbuffer.o snapbuild.o worker.o + proto.o relation.o reorderbuffer.o snapbuild.o tablesync.o worker.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c index 20b43626dd..255b22597b 100644 --- a/src/backend/replication/logical/launcher.c +++ b/src/backend/replication/logical/launcher.c @@ -27,6 +27,7 @@ #include "access/xact.h" #include "catalog/pg_subscription.h" +#include "catalog/pg_subscription_rel.h" #include "libpq/pqsignal.h" @@ -56,6 +57,8 @@ #define DEFAULT_NAPTIME_PER_CYCLE 180000L int max_logical_replication_workers = 4; +int max_sync_workers_per_subscription = 2; + LogicalRepWorker *MyLogicalRepWorker = NULL; typedef struct LogicalRepCtxStruct @@ -198,20 +201,22 @@ WaitForReplicationWorkerAttach(LogicalRepWorker *worker, /* * Walks the workers array and searches for one that matches given - * subscription id. + * subscription id and relid. */ LogicalRepWorker * -logicalrep_worker_find(Oid subid) +logicalrep_worker_find(Oid subid, Oid relid, bool only_running) { int i; LogicalRepWorker *res = NULL; Assert(LWLockHeldByMe(LogicalRepWorkerLock)); + /* Search for attached worker for a given subscription id. */ for (i = 0; i < max_logical_replication_workers; i++) { LogicalRepWorker *w = &LogicalRepCtx->workers[i]; - if (w->subid == subid && w->proc && IsBackendPid(w->proc->pid)) + if (w->subid == subid && w->relid == relid && + (!only_running || (w->proc && IsBackendPid(w->proc->pid)))) { res = w; break; @@ -225,7 +230,8 @@ logicalrep_worker_find(Oid subid) * Start new apply background worker. */ void -logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid) +logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid, + Oid relid) { BackgroundWorker bgw; BackgroundWorkerHandle *bgw_handle; @@ -270,10 +276,18 @@ logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid) } /* Prepare the worker info. */ - memset(worker, 0, sizeof(LogicalRepWorker)); + worker->proc = NULL; worker->dbid = dbid; worker->userid = userid; worker->subid = subid; + worker->relid = relid; + worker->relstate = SUBREL_STATE_UNKNOWN; + worker->relstate_lsn = InvalidXLogRecPtr; + worker->last_lsn = InvalidXLogRecPtr; + TIMESTAMP_NOBEGIN(worker->last_send_time); + TIMESTAMP_NOBEGIN(worker->last_recv_time); + worker->reply_lsn = InvalidXLogRecPtr; + TIMESTAMP_NOBEGIN(worker->reply_time); LWLockRelease(LogicalRepWorkerLock); @@ -282,8 +296,12 @@ logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid) BGWORKER_BACKEND_DATABASE_CONNECTION; bgw.bgw_start_time = BgWorkerStart_RecoveryFinished; bgw.bgw_main = ApplyWorkerMain; - snprintf(bgw.bgw_name, BGW_MAXLEN, - "logical replication worker for subscription %u", subid); + if (OidIsValid(relid)) + snprintf(bgw.bgw_name, BGW_MAXLEN, + "logical replication worker for subscription %u sync %u", subid, relid); + else + snprintf(bgw.bgw_name, BGW_MAXLEN, + "logical replication worker for subscription %u", subid); bgw.bgw_restart_time = BGW_NEVER_RESTART; bgw.bgw_notify_pid = MyProcPid; @@ -307,13 +325,13 @@ logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid) * slot. */ void -logicalrep_worker_stop(Oid subid) +logicalrep_worker_stop(Oid subid, Oid relid) { LogicalRepWorker *worker; LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); - worker = logicalrep_worker_find(subid); + worker = logicalrep_worker_find(subid, relid, false); /* No worker, nothing to do. */ if (!worker) @@ -395,6 +413,31 @@ logicalrep_worker_stop(Oid subid) } } +/* + * Wake up (using latch) the logical replication worker. + */ +void +logicalrep_worker_wakeup(Oid subid, Oid relid) +{ + LogicalRepWorker *worker; + + LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + worker = logicalrep_worker_find(subid, relid, true); + LWLockRelease(LogicalRepWorkerLock); + + if (worker) + logicalrep_worker_wakeup_ptr(worker); +} + +/* + * Wake up (using latch) the logical replication worker. + */ +void +logicalrep_worker_wakeup_ptr(LogicalRepWorker *worker) +{ + SetLatch(&worker->proc->procLatch); +} + /* * Attach to a slot. */ @@ -457,6 +500,29 @@ logicalrep_worker_sigterm(SIGNAL_ARGS) SetLatch(MyLatch); } +/* + * Count the number of registered (not necessarily running) sync workers + * for a subscription. + */ +int +logicalrep_sync_worker_count(Oid subid) +{ + int i; + int res = 0; + + Assert(LWLockHeldByMe(LogicalRepWorkerLock)); + + /* Search for attached worker for a given subscription id. */ + for (i = 0; i < max_logical_replication_workers; i++) + { + LogicalRepWorker *w = &LogicalRepCtx->workers[i]; + if (w->subid == subid && OidIsValid(w->relid)) + res++; + } + + return res; +} + /* * ApplyLauncherShmemSize * Compute space needed for replication launcher shared memory @@ -512,7 +578,20 @@ ApplyLauncherShmemInit(void) &found); if (!found) + { + int slot; + memset(LogicalRepCtx, 0, ApplyLauncherShmemSize()); + + /* Initialize memory and spin locks for each worker slot. */ + for (slot = 0; slot < max_logical_replication_workers; slot++) + { + LogicalRepWorker *worker = &LogicalRepCtx->workers[slot]; + + memset(worker, 0, sizeof(LogicalRepWorker)); + SpinLockInit(&worker->relmutex); + } + } } /* @@ -607,12 +686,13 @@ ApplyLauncherMain(Datum main_arg) LogicalRepWorker *w; LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); - w = logicalrep_worker_find(sub->oid); + w = logicalrep_worker_find(sub->oid, InvalidOid, false); LWLockRelease(LogicalRepWorkerLock); if (sub->enabled && w == NULL) { - logicalrep_worker_launch(sub->dbid, sub->oid, sub->name, sub->owner); + logicalrep_worker_launch(sub->dbid, sub->oid, sub->name, + sub->owner, InvalidOid); last_start_time = now; wait_time = wal_retrieve_retry_interval; /* Limit to one worker per mainloop cycle. */ @@ -664,7 +744,7 @@ ApplyLauncherMain(Datum main_arg) Datum pg_stat_get_subscription(PG_FUNCTION_ARGS) { -#define PG_STAT_GET_SUBSCRIPTION_COLS 7 +#define PG_STAT_GET_SUBSCRIPTION_COLS 8 Oid subid = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); int i; ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; @@ -723,27 +803,31 @@ pg_stat_get_subscription(PG_FUNCTION_ARGS) MemSet(nulls, 0, sizeof(nulls)); values[0] = ObjectIdGetDatum(worker.subid); - values[1] = Int32GetDatum(worker_pid); + if (OidIsValid(worker.relid)) + values[1] = ObjectIdGetDatum(worker.relid); + else + nulls[1] = true; + values[2] = Int32GetDatum(worker_pid); if (XLogRecPtrIsInvalid(worker.last_lsn)) - nulls[2] = true; + nulls[3] = true; else - values[2] = LSNGetDatum(worker.last_lsn); + values[3] = LSNGetDatum(worker.last_lsn); if (worker.last_send_time == 0) - nulls[3] = true; + nulls[4] = true; else - values[3] = TimestampTzGetDatum(worker.last_send_time); + values[4] = TimestampTzGetDatum(worker.last_send_time); if (worker.last_recv_time == 0) - nulls[4] = true; + nulls[5] = true; else - values[4] = TimestampTzGetDatum(worker.last_recv_time); + values[5] = TimestampTzGetDatum(worker.last_recv_time); if (XLogRecPtrIsInvalid(worker.reply_lsn)) - nulls[5] = true; + nulls[6] = true; else - values[5] = LSNGetDatum(worker.reply_lsn); + values[6] = LSNGetDatum(worker.reply_lsn); if (worker.reply_time == 0) - nulls[6] = true; + nulls[7] = true; else - values[6] = TimestampTzGetDatum(worker.reply_time); + values[7] = TimestampTzGetDatum(worker.reply_time); tuplestore_putvalues(tupstore, tupdesc, values, nulls); diff --git a/src/backend/replication/logical/relation.c b/src/backend/replication/logical/relation.c index d8dc0c7194..875a08185a 100644 --- a/src/backend/replication/logical/relation.c +++ b/src/backend/replication/logical/relation.c @@ -19,6 +19,7 @@ #include "access/heapam.h" #include "access/sysattr.h" #include "catalog/namespace.h" +#include "catalog/pg_subscription_rel.h" #include "nodes/makefuncs.h" #include "replication/logicalrelation.h" #include "replication/worker_internal.h" @@ -357,6 +358,12 @@ logicalrep_rel_open(LogicalRepRelId remoteid, LOCKMODE lockmode) else entry->localrel = heap_open(entry->localreloid, lockmode); + if (entry->state != SUBREL_STATE_READY) + entry->state = GetSubscriptionRelState(MySubscription->oid, + entry->localreloid, + &entry->statelsn, + true); + return entry; } diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c index 3f242a8ed7..a73a7b98f9 100644 --- a/src/backend/replication/logical/snapbuild.c +++ b/src/backend/replication/logical/snapbuild.c @@ -499,51 +499,32 @@ SnapBuildBuildSnapshot(SnapBuild *builder, TransactionId xid) } /* - * Export a snapshot so it can be set in another session with SET TRANSACTION - * SNAPSHOT. - * - * For that we need to start a transaction in the current backend as the - * importing side checks whether the source transaction is still open to make - * sure the xmin horizon hasn't advanced since then. + * Build the initial slot snapshot and convert it to normal snapshot that + * is understood by HeapTupleSatisfiesMVCC. * - * After that we convert a locally built snapshot into the normal variant - * understood by HeapTupleSatisfiesMVCC et al. + * The snapshot will be usable directly in current transaction or exported + * for loading in different transaction. */ -const char * -SnapBuildExportSnapshot(SnapBuild *builder) +Snapshot +SnapBuildInitalSnapshot(SnapBuild *builder) { Snapshot snap; - char *snapname; TransactionId xid; TransactionId *newxip; int newxcnt = 0; + Assert(!FirstSnapshotSet); + Assert(XactIsoLevel = XACT_REPEATABLE_READ); + if (builder->state != SNAPBUILD_CONSISTENT) - elog(ERROR, "cannot export a snapshot before reaching a consistent state"); + elog(ERROR, "cannot build an initial slot snapshot before reaching a consistent state"); if (!builder->committed.includes_all_transactions) - elog(ERROR, "cannot export a snapshot, not all transactions are monitored anymore"); + elog(ERROR, "cannot build an initial slot snapshot, not all transactions are monitored anymore"); /* so we don't overwrite the existing value */ if (TransactionIdIsValid(MyPgXact->xmin)) - elog(ERROR, "cannot export a snapshot when MyPgXact->xmin already is valid"); - - if (IsTransactionOrTransactionBlock()) - elog(ERROR, "cannot export a snapshot from within a transaction"); - - if (SavedResourceOwnerDuringExport) - elog(ERROR, "can only export one snapshot at a time"); - - SavedResourceOwnerDuringExport = CurrentResourceOwner; - ExportInProgress = true; - - StartTransactionCommand(); - - Assert(!FirstSnapshotSet); - - /* There doesn't seem to a nice API to set these */ - XactIsoLevel = XACT_REPEATABLE_READ; - XactReadOnly = true; + elog(ERROR, "cannot build an initial slot snapshot when MyPgXact->xmin already is valid"); snap = SnapBuildBuildSnapshot(builder, GetTopTransactionId()); @@ -578,7 +559,9 @@ SnapBuildExportSnapshot(SnapBuild *builder) if (test == NULL) { if (newxcnt >= GetMaxSnapshotXidCount()) - elog(ERROR, "snapshot too large"); + ereport(ERROR, + (errcode(ERRCODE_T_R_SERIALIZATION_FAILURE), + errmsg("initial slot snapshot too large"))); newxip[newxcnt++] = xid; } @@ -589,9 +572,43 @@ SnapBuildExportSnapshot(SnapBuild *builder) snap->xcnt = newxcnt; snap->xip = newxip; + return snap; +} + +/* + * Export a snapshot so it can be set in another session with SET TRANSACTION + * SNAPSHOT. + * + * For that we need to start a transaction in the current backend as the + * importing side checks whether the source transaction is still open to make + * sure the xmin horizon hasn't advanced since then. + */ +const char * +SnapBuildExportSnapshot(SnapBuild *builder) +{ + Snapshot snap; + char *snapname; + + if (IsTransactionOrTransactionBlock()) + elog(ERROR, "cannot export a snapshot from within a transaction"); + + if (SavedResourceOwnerDuringExport) + elog(ERROR, "can only export one snapshot at a time"); + + SavedResourceOwnerDuringExport = CurrentResourceOwner; + ExportInProgress = true; + + StartTransactionCommand(); + + /* There doesn't seem to a nice API to set these */ + XactIsoLevel = XACT_REPEATABLE_READ; + XactReadOnly = true; + + snap = SnapBuildInitalSnapshot(builder); + /* - * now that we've built a plain snapshot, use the normal mechanisms for - * exporting it + * now that we've built a plain snapshot, make it active and use the + * normal mechanisms for exporting it */ snapname = ExportSnapshot(snap); diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c new file mode 100644 index 0000000000..3e16b0d576 --- /dev/null +++ b/src/backend/replication/logical/tablesync.c @@ -0,0 +1,840 @@ +/*------------------------------------------------------------------------- + * tablesync.c + * PostgreSQL logical replication + * + * Copyright (c) 2012-2016, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/replication/logical/tablesync.c + * + * NOTES + * This file contains code for initial table data synchronization for + * logical replication. + * + * The initial data synchronization is done separately for each table, + * in separate apply worker that only fetches the initial snapshot data + * from the publisher and then synchronizes the position in stream with + * the main apply worker. + * + * The are several reasons for doing the synchronization this way: + * - It allows us to parallelize the initial data synchronization + * which lowers the time needed for it to happen. + * - The initial synchronization does not have to hold the xid and LSN + * for the time it takes to copy data of all tables, causing less + * bloat and lower disk consumption compared to doing the + * synchronization in single process for whole database. + * - It allows us to synchronize the tables added after the initial + * synchronization has finished. + * + * The stream position synchronization works in multiple steps. + * - Sync finishes copy and sets table state as SYNCWAIT and waits + * for state to change in a loop. + * - Apply periodically checks tables that are synchronizing for SYNCWAIT. + * When the desired state appears it will compare its position in the + * stream with the SYNCWAIT position and based on that changes the + * state to based on following rules: + * - if the apply is in front of the sync in the wal stream the new + * state is set to CATCHUP and apply loops until the sync process + * catches up to the same LSN as apply + * - if the sync is in front of the apply in the wal stream the new + * state is set to SYNCDONE + * - if both apply and sync are at the same position in the wal stream + * the state of the table is set to READY + * - If the state was set to CATCHUP sync will read the stream and + * apply changes until it catches up to the specified stream + * position and then sets state to READY and signals apply that it + * can stop waiting and exits, if the state was set to something + * else than CATCHUP the sync process will simply end. + * - If the state was set to SYNCDONE by apply, the apply will + * continue tracking the table until it reaches the SYNCDONE stream + * position at which point it sets state to READY and stops tracking. + * + * The catalog pg_subscription_rel is used to keep information about + * subscribed tables and their state and some transient state during + * data synchronization is kept in shared memory. + * + * Example flows look like this: + * - Apply is in front: + * sync:8 + * -> set SYNCWAIT + * apply:10 + * -> set CATCHUP + * -> enter wait-loop + * sync:10 + * -> set READY + * -> exit + * apply:10 + * -> exit wait-loop + * -> continue rep + * - Sync in front: + * sync:10 + * -> set SYNCWAIT + * apply:8 + * -> set SYNCDONE + * -> continue per-table filtering + * sync:10 + * -> exit + * apply:10 + * -> set READY + * -> stop per-table filtering + * -> continue rep + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "miscadmin.h" +#include "pgstat.h" + +#include "access/xact.h" + +#include "catalog/pg_subscription_rel.h" +#include "catalog/pg_type.h" + +#include "commands/copy.h" + +#include "replication/logicallauncher.h" +#include "replication/logicalrelation.h" +#include "replication/walreceiver.h" +#include "replication/worker_internal.h" + +#include "storage/ipc.h" + +#include "utils/builtins.h" +#include "utils/lsyscache.h" +#include "utils/memutils.h" + +static bool table_states_valid = false; + +StringInfo copybuf = NULL; + +/* + * Exit routine for synchronization worker. + */ +static void pg_attribute_noreturn() +finish_sync_worker(void) +{ + /* Commit any outstanding transaction. */ + if (IsTransactionState()) + CommitTransactionCommand(); + + /* And flush all writes. */ + XLogFlush(GetXLogWriteRecPtr()); + + /* Find the main apply worker and signal it. */ + logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid); + + ereport(LOG, + (errmsg("logical replication synchronization worker finished processing"))); + + /* Stop gracefully */ + walrcv_disconnect(wrconn); + proc_exit(0); +} + +/* + * Wait until the table synchronization change. + * + * Returns false if the relation subscription state disappeared. + */ +static bool +wait_for_sync_status_change(Oid relid, char origstate) +{ + int rc; + char state = origstate; + + while (!got_SIGTERM) + { + LogicalRepWorker *worker; + + LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + worker = logicalrep_worker_find(MyLogicalRepWorker->subid, + relid, false); + if (!worker) + { + LWLockRelease(LogicalRepWorkerLock); + return false; + } + state = worker->relstate; + LWLockRelease(LogicalRepWorkerLock); + + if (state == SUBREL_STATE_UNKNOWN) + return false; + + if (state != origstate) + return true; + + rc = WaitLatch(&MyProc->procLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, + 10000L, WAIT_EVENT_LOGICAL_SYNC_STATE_CHANGE); + + /* emergency bailout if postmaster has died */ + if (rc & WL_POSTMASTER_DEATH) + proc_exit(1); + + ResetLatch(&MyProc->procLatch); + } + + return false; +} + +/* + * Callback from syscache invalidation. + */ +void +invalidate_syncing_table_states(Datum arg, int cacheid, uint32 hashvalue) +{ + table_states_valid = false; +} + +/* + * Handle table synchronization cooperation from the synchronization + * worker. + * + * If the sync worker is in catch up mode and reached the predetermined + * synchronization point in the WAL stream, mark the table as READY and + * finish. If it caught up too far, set to SYNCDONE and finish. Things will + * then proceed in the "sync in front" scenario. + */ +static void +process_syncing_tables_for_sync(XLogRecPtr current_lsn) +{ + Assert(IsTransactionState()); + + SpinLockAcquire(&MyLogicalRepWorker->relmutex); + + if (MyLogicalRepWorker->relstate == SUBREL_STATE_CATCHUP && + current_lsn >= MyLogicalRepWorker->relstate_lsn) + { + TimeLineID tli; + + MyLogicalRepWorker->relstate = + (current_lsn == MyLogicalRepWorker->relstate_lsn) + ? SUBREL_STATE_READY + : SUBREL_STATE_SYNCDONE; + MyLogicalRepWorker->relstate_lsn = current_lsn; + + SpinLockRelease(&MyLogicalRepWorker->relmutex); + + SetSubscriptionRelState(MyLogicalRepWorker->subid, + MyLogicalRepWorker->relid, + MyLogicalRepWorker->relstate, + MyLogicalRepWorker->relstate_lsn); + + walrcv_endstreaming(wrconn, &tli); + finish_sync_worker(); + } + else + SpinLockRelease(&MyLogicalRepWorker->relmutex); +} + +/* + * Handle table synchronization cooperation from the apply worker. + * + * Walk over all subscription tables that are individually tracked by the + * apply process (currently, all that have state other than + * SUBREL_STATE_READY) and manage synchronization for them. + * + * If there are tables that need synchronizing and are not being synchronized + * yet, start sync workers for them (if there are free slots for sync + * workers). + * + * For tables that are being synchronized already, check if sync workers + * either need action from the apply worker or have finished. + * + * The usual scenario is that the apply got ahead of the sync while the sync + * ran, and then the action needed by apply is to mark a table for CATCHUP and + * wait for the catchup to happen. In the less common case that sync worker + * got in front of the apply worker, the table is marked as SYNCDONE but not + * ready yet, as it needs to be tracked until apply reaches the same position + * to which it was synced. + * + * If the synchronization position is reached, then the table can be marked as + * READY and is no longer tracked. + */ +static void +process_syncing_tables_for_apply(XLogRecPtr current_lsn) +{ + static List *table_states = NIL; + ListCell *lc; + + Assert(!IsTransactionState()); + + /* We need up to date sync state info for subscription tables here. */ + if (!table_states_valid) + { + MemoryContext oldctx; + List *rstates; + ListCell *lc; + SubscriptionRelState *rstate; + + /* Clean the old list. */ + list_free_deep(table_states); + table_states = NIL; + + StartTransactionCommand(); + + /* Fetch all non-ready tables. */ + rstates = GetSubscriptionNotReadyRelations(MySubscription->oid); + + /* Allocate the tracking info in a permanent memory context. */ + oldctx = MemoryContextSwitchTo(CacheMemoryContext); + foreach(lc, rstates) + { + rstate = palloc(sizeof(SubscriptionRelState)); + memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState)); + table_states = lappend(table_states, rstate); + } + MemoryContextSwitchTo(oldctx); + + CommitTransactionCommand(); + + table_states_valid = true; + } + + /* Process all tables that are being synchronized. */ + foreach(lc, table_states) + { + SubscriptionRelState *rstate = (SubscriptionRelState *)lfirst(lc); + + if (rstate->state == SUBREL_STATE_SYNCDONE) + { + /* + * Apply has caught up to the position where the table sync + * has finished. Time to mark the table as ready so that + * apply will just continue to replicate it normally. + */ + if (current_lsn >= rstate->lsn) + { + rstate->state = SUBREL_STATE_READY; + rstate->lsn = current_lsn; + StartTransactionCommand(); + SetSubscriptionRelState(MyLogicalRepWorker->subid, + rstate->relid, rstate->state, + rstate->lsn); + CommitTransactionCommand(); + } + } + else + { + LogicalRepWorker *syncworker; + int nsyncworkers = 0; + + LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + syncworker = logicalrep_worker_find(MyLogicalRepWorker->subid, + rstate->relid, false); + if (syncworker) + { + SpinLockAcquire(&syncworker->relmutex); + rstate->state = syncworker->relstate; + rstate->lsn = syncworker->relstate_lsn; + SpinLockRelease(&syncworker->relmutex); + } + else + /* + * If no sync worker for this table yet, could running sync + * workers for this subscription, while we have the lock, for + * later. + */ + nsyncworkers = logicalrep_sync_worker_count(MyLogicalRepWorker->subid); + LWLockRelease(LogicalRepWorkerLock); + + /* + * There is a worker synchronizing the relation and waiting for + * apply to do something. + */ + if (syncworker && rstate->state == SUBREL_STATE_SYNCWAIT) + { + /* + * There are three possible synchronization situations here. + * + * a) Apply is in front of the table sync: We tell the table + * sync to CATCHUP. + * + * b) Apply is behind the table sync: We tell the table sync + * to mark the table as SYNCDONE and finish. + + * c) Apply and table sync are at the same position: We tell + * table sync to mark the table as READY and finish. + * + * In any case we'll need to wait for table sync to change + * the state in catalog and only then continue ourselves. + */ + if (current_lsn > rstate->lsn) + { + rstate->state = SUBREL_STATE_CATCHUP; + rstate->lsn = current_lsn; + } + else if (current_lsn == rstate->lsn) + { + rstate->state = SUBREL_STATE_READY; + rstate->lsn = current_lsn; + } + else + rstate->state = SUBREL_STATE_SYNCDONE; + + SpinLockAcquire(&syncworker->relmutex); + syncworker->relstate = rstate->state; + syncworker->relstate_lsn = rstate->lsn; + SpinLockRelease(&syncworker->relmutex); + + /* Signal the sync worker, as it may be waiting for us. */ + logicalrep_worker_wakeup_ptr(syncworker); + + /* + * Enter busy loop and wait for synchronization status + * change. + */ + wait_for_sync_status_change(rstate->relid, rstate->state); + } + + /* + * If there is no sync worker registered for the table and + * there is some free sync worker slot, start new sync worker + * for the table. + */ + else if (!syncworker && nsyncworkers < max_sync_workers_per_subscription) + { + logicalrep_worker_launch(MyLogicalRepWorker->dbid, + MySubscription->oid, + MySubscription->name, + MyLogicalRepWorker->userid, + rstate->relid); + } + } + } +} + +/* + * Process state possible change(s) of tables that are being synchronized. + */ +void +process_syncing_tables(XLogRecPtr current_lsn) +{ + if (am_tablesync_worker()) + process_syncing_tables_for_sync(current_lsn); + else + process_syncing_tables_for_apply(current_lsn); +} + +/* + * Create list of columns for COPY based on logical relation mapping. + */ +static List * +make_copy_attnamelist(LogicalRepRelMapEntry *rel) +{ + List *attnamelist = NIL; + TupleDesc desc = RelationGetDescr(rel->localrel); + int i; + + for (i = 0; i < desc->natts; i++) + { + int remoteattnum = rel->attrmap[i]; + + /* Skip dropped attributes. */ + if (desc->attrs[i]->attisdropped) + continue; + + /* Skip attributes that are missing on remote side. */ + if (remoteattnum < 0) + continue; + + attnamelist = lappend(attnamelist, + makeString(rel->remoterel.attnames[remoteattnum])); + } + + return attnamelist; +} + +/* + * Data source callback for the COPY FROM, which reads from the remote + * connection and passes the data back to our local COPY. + */ +static int +copy_read_data(void *outbuf, int minread, int maxread) +{ + int bytesread = 0; + int avail; + + /* If there are some leftover data from previous read, use them. */ + avail = copybuf->len - copybuf->cursor; + if (avail) + { + if (avail > maxread) + avail = maxread; + memcpy(outbuf, ©buf->data[copybuf->cursor], avail); + copybuf->cursor += avail; + maxread -= avail; + bytesread += avail; + } + + while (!got_SIGTERM && maxread > 0 && bytesread < minread) + { + pgsocket fd = PGINVALID_SOCKET; + int rc; + int len; + char *buf = NULL; + + for (;;) + { + /* Try read the data. */ + len = walrcv_receive(wrconn, &buf, &fd); + + CHECK_FOR_INTERRUPTS(); + + if (len == 0) + break; + else if (len < 0) + return bytesread; + else + { + /* Process the data */ + copybuf->data = buf; + copybuf->len = len; + copybuf->cursor = 0; + + avail = copybuf->len - copybuf->cursor; + if (avail > maxread) + avail = maxread; + memcpy(outbuf, ©buf->data[copybuf->cursor], avail); + outbuf = (void *) ((char *) outbuf + avail); + copybuf->cursor += avail; + maxread -= avail; + bytesread += avail; + } + + if (maxread <= 0 || bytesread >= minread) + return bytesread; + } + + /* + * Wait for more data or latch. + */ + rc = WaitLatchOrSocket(&MyProc->procLatch, + WL_SOCKET_READABLE | WL_LATCH_SET | + WL_TIMEOUT | WL_POSTMASTER_DEATH, + fd, 1000L, WAIT_EVENT_LOGICAL_SYNC_DATA); + + /* Emergency bailout if postmaster has died */ + if (rc & WL_POSTMASTER_DEATH) + proc_exit(1); + + ResetLatch(&MyProc->procLatch); + } + + /* Check for exit condition. */ + if (got_SIGTERM) + proc_exit(0); + + return bytesread; +} + + +/* + * Get information about remote relation in similar fashion the RELATION + * message provides during replication. + */ +static void +fetch_remote_table_info(char *nspname, char *relname, + LogicalRepRelation *lrel) +{ + WalRcvExecResult *res; + StringInfoData cmd; + TupleTableSlot *slot; + Oid tableRow[2] = {OIDOID, CHAROID}; + Oid attrRow[4] = {TEXTOID, OIDOID, INT4OID, BOOLOID}; + bool isnull; + int natt; + + lrel->nspname = nspname; + lrel->relname = relname; + + /* First fetch Oid and replica identity. */ + initStringInfo(&cmd); + appendStringInfo(&cmd, "SELECT c.oid, c.relreplident" + " FROM pg_catalog.pg_class c," + " pg_catalog.pg_namespace n" + " WHERE n.nspname = %s" + " AND c.relname = %s" + " AND c.relkind = 'r'", + quote_literal_cstr(nspname), + quote_literal_cstr(relname)); + res = walrcv_exec(wrconn, cmd.data, 2, tableRow); + + if (res->status != WALRCV_OK_TUPLES) + ereport(ERROR, + (errmsg("could not fetch table info for table \"%s.%s\" from publisher: %s", + nspname, relname, res->err))); + + slot = MakeSingleTupleTableSlot(res->tupledesc); + if (!tuplestore_gettupleslot(res->tuplestore, true, false, slot)) + ereport(ERROR, + (errmsg("table \"%s.%s\" not found on publisher", + nspname, relname))); + + lrel->remoteid = DatumGetObjectId(slot_getattr(slot, 1, &isnull)); + Assert(!isnull); + lrel->replident = DatumGetChar(slot_getattr(slot, 2, &isnull)); + Assert(!isnull); + + ExecDropSingleTupleTableSlot(slot); + walrcv_clear_result(res); + + /* Now fetch columns. */ + resetStringInfo(&cmd); + appendStringInfo(&cmd, + "SELECT a.attname," + " a.atttypid," + " a.atttypmod," + " a.attnum = ANY(i.indkey)" + " FROM pg_catalog.pg_attribute a" + " LEFT JOIN pg_catalog.pg_index i" + " ON (i.indexrelid = pg_get_replica_identity_index(%u))" + " WHERE a.attnum > 0::pg_catalog.int2" + " AND NOT a.attisdropped" + " AND a.attrelid = %u" + " ORDER BY a.attnum", + lrel->remoteid, lrel->remoteid); + res = walrcv_exec(wrconn, cmd.data, 4, attrRow); + + if (res->status != WALRCV_OK_TUPLES) + ereport(ERROR, + (errmsg("could not fetch table info for table \"%s.%s\": %s", + nspname, relname, res->err))); + + /* We don't know number of rows coming, so allocate enough space. */ + lrel->attnames = palloc0(MaxTupleAttributeNumber * sizeof(char *)); + lrel->atttyps = palloc0(MaxTupleAttributeNumber * sizeof(Oid)); + lrel->attkeys = NULL; + + natt = 0; + slot = MakeSingleTupleTableSlot(res->tupledesc); + while (tuplestore_gettupleslot(res->tuplestore, true, false, slot)) + { + lrel->attnames[natt] = + pstrdup(TextDatumGetCString(slot_getattr(slot, 1, &isnull))); + Assert(!isnull); + lrel->atttyps[natt] = DatumGetObjectId(slot_getattr(slot, 2, &isnull)); + Assert(!isnull); + if (DatumGetBool(slot_getattr(slot, 4, &isnull))) + lrel->attkeys = bms_add_member(lrel->attkeys, natt); + + /* Should never happen. */ + if (++natt >= MaxTupleAttributeNumber) + elog(ERROR, "too many columns in remote table \"%s.%s\"", + nspname, relname); + + ExecClearTuple(slot); + } + ExecDropSingleTupleTableSlot(slot); + + lrel->natts = natt; + + walrcv_clear_result(res); + pfree(cmd.data); +} + +/* + * Copy existing data of a table from publisher. + * + * Caller is responsible for locking the local relation. + */ +static void +copy_table(Relation rel) +{ + LogicalRepRelMapEntry *relmapentry; + LogicalRepRelation lrel; + WalRcvExecResult *res; + StringInfoData cmd; + CopyState cstate; + List *attnamelist; + + /* Get the publisher relation info. */ + fetch_remote_table_info(get_namespace_name(RelationGetNamespace(rel)), + RelationGetRelationName(rel), &lrel); + + /* Put the relation into relmap. */ + logicalrep_relmap_update(&lrel); + + /* Map the publisher relation to local one. */ + relmapentry = logicalrep_rel_open(lrel.remoteid, NoLock); + Assert(rel == relmapentry->localrel); + + /* Start copy on the publisher. */ + initStringInfo(&cmd); + appendStringInfo(&cmd, "COPY %s TO STDOUT", + quote_qualified_identifier(lrel.nspname, lrel.relname)); + res = walrcv_exec(wrconn, cmd.data, 0, NULL); + pfree(cmd.data); + if (res->status != WALRCV_OK_COPY_OUT) + ereport(ERROR, + (errmsg("could not start initial contents copy for table \"%s.%s\": %s", + lrel.nspname, lrel.relname, res->err))); + walrcv_clear_result(res); + + copybuf = makeStringInfo(); + + /* Create CopyState for ingestion of the data from publisher. */ + attnamelist = make_copy_attnamelist(relmapentry); + cstate = BeginCopyFrom(NULL, rel, NULL, false, copy_read_data, attnamelist, NIL); + + /* Do the copy */ + (void) CopyFrom(cstate); + + logicalrep_rel_close(relmapentry, NoLock); +} + +/* + * Start syncing the table in the sync worker. + * + * The returned slot name is palloced in current memory context. + */ +char * +LogicalRepSyncTableStart(XLogRecPtr *origin_startpos) +{ + char *slotname; + char *err; + + /* Check the state of the table synchronization. */ + StartTransactionCommand(); + SpinLockAcquire(&MyLogicalRepWorker->relmutex); + MyLogicalRepWorker->relstate = + GetSubscriptionRelState(MyLogicalRepWorker->subid, + MyLogicalRepWorker->relid, + &MyLogicalRepWorker->relstate_lsn, + false); + SpinLockRelease(&MyLogicalRepWorker->relmutex); + CommitTransactionCommand(); + + /* + * To build a slot name for the sync work, we are limited to NAMEDATALEN - + * 1 characters. We cut the original slot name to NAMEDATALEN - 28 chars + * and append _%u_sync_%u (1 + 10 + 6 + 10 + '\0'). (It's actually the + * NAMEDATALEN on the remote that matters, but this scheme will also work + * reasonably if that is different.) + */ + StaticAssertStmt(NAMEDATALEN >= 32, "NAMEDATALEN too small"); /* for sanity */ + slotname = psprintf("%.*s_%u_sync_%u", + NAMEDATALEN - 28, + MySubscription->slotname, + MySubscription->oid, + MyLogicalRepWorker->relid); + + wrconn = walrcv_connect(MySubscription->conninfo, true, slotname, &err); + if (wrconn == NULL) + ereport(ERROR, + (errmsg("could not connect to the publisher: %s", err))); + + switch (MyLogicalRepWorker->relstate) + { + case SUBREL_STATE_INIT: + case SUBREL_STATE_DATASYNC: + { + Relation rel; + WalRcvExecResult *res; + + SpinLockAcquire(&MyLogicalRepWorker->relmutex); + MyLogicalRepWorker->relstate = SUBREL_STATE_DATASYNC; + MyLogicalRepWorker->relstate_lsn = InvalidXLogRecPtr; + SpinLockRelease(&MyLogicalRepWorker->relmutex); + + /* Update the state and make it visible to others. */ + StartTransactionCommand(); + SetSubscriptionRelState(MyLogicalRepWorker->subid, + MyLogicalRepWorker->relid, + MyLogicalRepWorker->relstate, + MyLogicalRepWorker->relstate_lsn); + CommitTransactionCommand(); + + /* + * We want to do the table data sync in single + * transaction. + */ + StartTransactionCommand(); + + /* + * Use standard write lock here. It might be better to + * disallow access to table while it's being synchronized. + * But we don't want to block the main apply process from + * working and it has to open relation in RowExclusiveLock + * when remapping remote relation id to local one. + */ + rel = heap_open(MyLogicalRepWorker->relid, RowExclusiveLock); + + /* + * Create temporary slot for the sync process. + * We do this inside transaction so that we can use the + * snapshot made by the slot to get existing data. + */ + res = walrcv_exec(wrconn, + "BEGIN READ ONLY ISOLATION LEVEL " + "REPEATABLE READ", 0, NULL); + if (res->status != WALRCV_OK_COMMAND) + ereport(ERROR, + (errmsg("table copy could not start transaction on publisher"), + errdetail("The error was: %s", res->err))); + walrcv_clear_result(res); + + /* + * Create new temporary logical decoding slot. + * + * We'll use slot for data copy so make sure the snapshot + * is used for the transaction, that way the COPY will get + * data that is consistent with the lsn used by the slot + * to start decoding. + */ + walrcv_create_slot(wrconn, slotname, true, + CRS_USE_SNAPSHOT, origin_startpos); + + copy_table(rel); + + res = walrcv_exec(wrconn, "COMMIT", 0, NULL); + if (res->status != WALRCV_OK_COMMAND) + ereport(ERROR, + (errmsg("table copy could not finish transaction on publisher"), + errdetail("The error was: %s", res->err))); + walrcv_clear_result(res); + + heap_close(rel, NoLock); + + /* Make the copy visible. */ + CommandCounterIncrement(); + + /* + * We are done with the initial data synchronization, + * update the state. + */ + SpinLockAcquire(&MyLogicalRepWorker->relmutex); + MyLogicalRepWorker->relstate = SUBREL_STATE_SYNCWAIT; + MyLogicalRepWorker->relstate_lsn = *origin_startpos; + SpinLockRelease(&MyLogicalRepWorker->relmutex); + + /* + * Wait for main apply worker to either tell us to + * catchup or that we are done. + */ + wait_for_sync_status_change(MyLogicalRepWorker->relid, + MyLogicalRepWorker->relstate); + if (MyLogicalRepWorker->relstate != SUBREL_STATE_CATCHUP) + { + /* Update the new state. */ + SetSubscriptionRelState(MyLogicalRepWorker->subid, + MyLogicalRepWorker->relid, + MyLogicalRepWorker->relstate, + MyLogicalRepWorker->relstate_lsn); + finish_sync_worker(); + } + break; + } + case SUBREL_STATE_SYNCDONE: + case SUBREL_STATE_READY: + /* Nothing to do here but finish. */ + finish_sync_worker(); + break; + default: + elog(ERROR, "unknown relation state \"%c\"", + MyLogicalRepWorker->relstate); + } + + return slotname; +} diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index c3e54af259..bbf3506be0 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -32,6 +32,7 @@ #include "catalog/namespace.h" #include "catalog/pg_subscription.h" +#include "catalog/pg_subscription_rel.h" #include "commands/trigger.h" @@ -101,7 +102,7 @@ typedef struct SlotErrCallbackArg } SlotErrCallbackArg; static MemoryContext ApplyContext = NULL; -static MemoryContext ApplyCacheContext = NULL; +MemoryContext ApplyCacheContext = NULL; WalReceiverConn *wrconn = NULL; @@ -109,6 +110,7 @@ Subscription *MySubscription = NULL; bool MySubscriptionValid = false; bool in_remote_transaction = false; +static XLogRecPtr remote_final_lsn = InvalidXLogRecPtr; static void send_feedback(XLogRecPtr recvpos, bool force, bool requestReply); @@ -116,6 +118,30 @@ static void store_flush_position(XLogRecPtr remote_lsn); static void reread_subscription(void); +/* + * Should this worker apply changes for given relation. + * + * This is mainly needed for initial relation data sync as that runs in + * separate worker process running in parallel and we need some way to skip + * changes coming to the main apply worker during the sync of a table. + * + * Note we need to do smaller or equals comparison for SYNCDONE state because + * it might hold position of end of intitial slot consistent point WAL + * record + 1 (ie start of next record) and next record can be COMMIT of + * transaction we are now processing (which is what we set remote_final_lsn + * to in apply_handle_begin). + */ +static bool +should_apply_changes_for_rel(LogicalRepRelMapEntry *rel) +{ + if (am_tablesync_worker()) + return MyLogicalRepWorker->relid == rel->localreloid; + else + return (rel->state == SUBREL_STATE_READY || + (rel->state == SUBREL_STATE_SYNCDONE && + rel->statelsn <= remote_final_lsn)); +} + /* * Make sure that we started local transaction. * @@ -398,6 +424,8 @@ apply_handle_begin(StringInfo s) replorigin_session_origin_timestamp = begin_data.committime; replorigin_session_origin_lsn = begin_data.final_lsn; + remote_final_lsn = begin_data.final_lsn; + in_remote_transaction = true; pgstat_report_activity(STATE_RUNNING, NULL); @@ -418,7 +446,10 @@ apply_handle_commit(StringInfo s) Assert(commit_data.commit_lsn == replorigin_session_origin_lsn); Assert(commit_data.committime == replorigin_session_origin_timestamp); - if (IsTransactionState()) + Assert(commit_data.commit_lsn == remote_final_lsn); + + /* The synchronization worker runs in single transaction. */ + if (IsTransactionState() && !am_tablesync_worker()) { CommitTransactionCommand(); @@ -427,6 +458,9 @@ apply_handle_commit(StringInfo s) in_remote_transaction = false; + /* Process any tables that are being synchronized in parallel. */ + process_syncing_tables(commit_data.end_lsn); + pgstat_report_activity(STATE_IDLE, NULL); } @@ -442,7 +476,8 @@ apply_handle_origin(StringInfo s) * ORIGIN message can only come inside remote transaction and before * any actual writes. */ - if (!in_remote_transaction || IsTransactionState()) + if (!in_remote_transaction || + (IsTransactionState() && !am_tablesync_worker())) ereport(ERROR, (errcode(ERRCODE_PROTOCOL_VIOLATION), errmsg("ORIGIN message sent out of order"))); @@ -515,6 +550,15 @@ apply_handle_insert(StringInfo s) relid = logicalrep_read_insert(s, &newtup); rel = logicalrep_rel_open(relid, RowExclusiveLock); + if (!should_apply_changes_for_rel(rel)) + { + /* + * The relation can't become interesting in the middle of the + * transaction so it's safe to unlock it. + */ + logicalrep_rel_close(rel, RowExclusiveLock); + return; + } /* Initialize the executor state. */ estate = create_estate_for_relation(rel); @@ -607,6 +651,15 @@ apply_handle_update(StringInfo s) relid = logicalrep_read_update(s, &has_oldtup, &oldtup, &newtup); rel = logicalrep_rel_open(relid, RowExclusiveLock); + if (!should_apply_changes_for_rel(rel)) + { + /* + * The relation can't become interesting in the middle of the + * transaction so it's safe to unlock it. + */ + logicalrep_rel_close(rel, RowExclusiveLock); + return; + } /* Check if we can do the update. */ check_relation_updatable(rel); @@ -716,6 +769,15 @@ apply_handle_delete(StringInfo s) relid = logicalrep_read_delete(s, &oldtup); rel = logicalrep_rel_open(relid, RowExclusiveLock); + if (!should_apply_changes_for_rel(rel)) + { + /* + * The relation can't become interesting in the middle of the + * transaction so it's safe to unlock it. + */ + logicalrep_rel_close(rel, RowExclusiveLock); + return; + } /* Check if we can do the delete. */ check_relation_updatable(rel); @@ -927,10 +989,8 @@ UpdateWorkerStats(XLogRecPtr last_lsn, TimestampTz send_time, bool reply) * Apply main loop. */ static void -ApplyLoop(void) +LogicalRepApplyLoop(XLogRecPtr last_received) { - XLogRecPtr last_received = InvalidXLogRecPtr; - /* Init the ApplyContext which we use for easier cleanup. */ ApplyContext = AllocSetContextCreate(TopMemoryContext, "ApplyContext", @@ -1014,15 +1074,18 @@ ApplyLoop(void) } else if (c == 'k') { - XLogRecPtr endpos; + XLogRecPtr end_lsn; TimestampTz timestamp; bool reply_requested; - endpos = pq_getmsgint64(&s); + end_lsn = pq_getmsgint64(&s); timestamp = pq_getmsgint64(&s); reply_requested = pq_getmsgbyte(&s); - send_feedback(endpos, reply_requested, false); + if (last_received < end_lsn) + last_received = end_lsn; + + send_feedback(last_received, reply_requested, false); UpdateWorkerStats(last_received, timestamp, true); } /* other message types are purposefully ignored */ @@ -1030,6 +1093,9 @@ ApplyLoop(void) len = walrcv_receive(wrconn, &buf, &fd); } + + /* confirm all writes at once */ + send_feedback(last_received, false, false); } if (!in_remote_transaction) @@ -1038,15 +1104,13 @@ ApplyLoop(void) * If we didn't get any transactions for a while there might be * unconsumed invalidation messages in the queue, consume them now. */ - StartTransactionCommand(); - /* Check for subscription change */ + AcceptInvalidationMessages(); if (!MySubscriptionValid) reread_subscription(); - CommitTransactionCommand(); - } - /* confirm all writes at once */ - send_feedback(last_received, false, false); + /* Process any table synchronization changes. */ + process_syncing_tables(last_received); + } /* Cleanup the memory. */ MemoryContextResetAndDeleteChildren(ApplyContext); @@ -1054,7 +1118,11 @@ ApplyLoop(void) /* Check if we need to exit the streaming loop. */ if (endofstream) + { + TimeLineID tli; + walrcv_endstreaming(wrconn, &tli); break; + } /* * Wait for more data or latch. @@ -1222,6 +1290,14 @@ reread_subscription(void) { MemoryContext oldctx; Subscription *newsub; + bool started_tx = false; + + /* This function might be called inside or outside of transaction. */ + if (!IsTransactionState()) + { + StartTransactionCommand(); + started_tx = true; + } /* Ensure allocations in permanent context. */ oldctx = MemoryContextSwitchTo(ApplyCacheContext); @@ -1319,6 +1395,9 @@ reread_subscription(void) MemoryContextSwitchTo(oldctx); + if (started_tx) + CommitTransactionCommand(); + MySubscriptionValid = true; } @@ -1339,11 +1418,8 @@ ApplyWorkerMain(Datum main_arg) int worker_slot = DatumGetObjectId(main_arg); MemoryContext oldctx; char originname[NAMEDATALEN]; - RepOriginId originid; XLogRecPtr origin_startpos; - char *err; - int server_version; - TimeLineID startpointTLI; + char *myslotname; WalRcvStreamOptions options; /* Attach to slot */ @@ -1402,49 +1478,90 @@ ApplyWorkerMain(Datum main_arg) subscription_change_cb, (Datum) 0); - ereport(LOG, - (errmsg("logical replication apply for subscription \"%s\" has started", - MySubscription->name))); - - /* Setup replication origin tracking. */ - snprintf(originname, sizeof(originname), "pg_%u", MySubscription->oid); - originid = replorigin_by_name(originname, true); - if (!OidIsValid(originid)) - originid = replorigin_create(originname); - replorigin_session_setup(originid); - replorigin_session_origin = originid; - origin_startpos = replorigin_session_get_progress(false); + if (am_tablesync_worker()) + elog(LOG, "logical replication sync for subscription %s, table %s started", + MySubscription->name, get_rel_name(MyLogicalRepWorker->relid)); + else + elog(LOG, "logical replication apply for subscription %s started", + MySubscription->name); CommitTransactionCommand(); /* Connect to the origin and start the replication. */ elog(DEBUG1, "connecting to publisher using connection string \"%s\"", MySubscription->conninfo); - wrconn = walrcv_connect(MySubscription->conninfo, true, - MySubscription->name, &err); - if (wrconn == NULL) - ereport(ERROR, - (errmsg("could not connect to the publisher: %s", err))); + + if (am_tablesync_worker()) + { + char *syncslotname; + + /* This is table synchroniation worker, call initial sync. */ + syncslotname = LogicalRepSyncTableStart(&origin_startpos); + + /* The slot name needs to be allocated in permanent memory context. */ + oldctx = MemoryContextSwitchTo(ApplyCacheContext); + myslotname = pstrdup(syncslotname); + MemoryContextSwitchTo(oldctx); + + pfree(syncslotname); + } + else + { + /* This is main apply worker */ + RepOriginId originid; + TimeLineID startpointTLI; + char *err; + int server_version; + + myslotname = MySubscription->slotname; + + /* Setup replication origin tracking. */ + StartTransactionCommand(); + snprintf(originname, sizeof(originname), "pg_%u", MySubscription->oid); + originid = replorigin_by_name(originname, true); + if (!OidIsValid(originid)) + originid = replorigin_create(originname); + replorigin_session_setup(originid); + replorigin_session_origin = originid; + origin_startpos = replorigin_session_get_progress(false); + CommitTransactionCommand(); + + wrconn = walrcv_connect(MySubscription->conninfo, true, myslotname, + &err); + if (wrconn == NULL) + ereport(ERROR, + (errmsg("could not connect to the publisher: %s", err))); + + /* + * We don't really use the output identify_system for anything + * but it does some initializations on the upstream so let's still + * call it. + */ + (void) walrcv_identify_system(wrconn, &startpointTLI, + &server_version); + + } /* - * We don't really use the output identify_system for anything - * but it does some initializations on the upstream so let's still - * call it. + * Setup callback for syscache so that we know when something + * changes in the subscription relation state. */ - (void) walrcv_identify_system(wrconn, &startpointTLI, &server_version); + CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP, + invalidate_syncing_table_states, + (Datum) 0); /* Build logical replication streaming options. */ options.logical = true; options.startpoint = origin_startpos; - options.slotname = MySubscription->slotname; + options.slotname = myslotname; options.proto.logical.proto_version = LOGICALREP_PROTO_VERSION_NUM; options.proto.logical.publication_names = MySubscription->publications; - /* Start streaming from the slot. */ + /* Start normal logical streaming replication. */ walrcv_startstreaming(wrconn, &options); /* Run the main loop. */ - ApplyLoop(); + LogicalRepApplyLoop(origin_startpos); walrcv_disconnect(wrconn); diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y index f1e43bc9f3..ec047c827c 100644 --- a/src/backend/replication/repl_gram.y +++ b/src/backend/replication/repl_gram.y @@ -25,6 +25,8 @@ /* Result of the parsing is returned here */ Node *replication_parse_result; +static SQLCmd *make_sqlcmd(void); + /* * Bison doesn't allocate anything that needs to live across parser calls, @@ -57,6 +59,7 @@ Node *replication_parse_result; %token SCONST IDENT %token UCONST %token RECPTR +%token T_WORD /* Keyword tokens. */ %token K_BASE_BACKUP @@ -81,11 +84,12 @@ Node *replication_parse_result; %token K_TEMPORARY %token K_EXPORT_SNAPSHOT %token K_NOEXPORT_SNAPSHOT +%token K_USE_SNAPSHOT %type command %type base_backup start_replication start_logical_replication create_replication_slot drop_replication_slot identify_system - timeline_history show + timeline_history show sql_cmd %type base_backup_opt_list %type base_backup_opt %type opt_timeline @@ -118,6 +122,7 @@ command: | drop_replication_slot | timeline_history | show + | sql_cmd ; /* @@ -248,6 +253,11 @@ create_slot_opt: $$ = makeDefElem("export_snapshot", (Node *)makeInteger(FALSE), -1); } + | K_USE_SNAPSHOT + { + $$ = makeDefElem("use_snapshot", + (Node *)makeInteger(TRUE), -1); + } | K_RESERVE_WAL { $$ = makeDefElem("reserve_wal", @@ -373,6 +383,26 @@ plugin_opt_arg: SCONST { $$ = (Node *) makeString($1); } | /* EMPTY */ { $$ = NULL; } ; + +sql_cmd: + IDENT { $$ = (Node *) make_sqlcmd(); } + ; %% +static SQLCmd * +make_sqlcmd(void) +{ + SQLCmd *cmd = makeNode(SQLCmd); + int tok; + + /* Just move lexer to the end of command. */ + for (;;) + { + tok = yylex(); + if (tok == ';' || tok == 0) + break; + } + return cmd; +} + #include "repl_scanner.c" diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l index f56d41d59c..52ae7b343f 100644 --- a/src/backend/replication/repl_scanner.l +++ b/src/backend/replication/repl_scanner.l @@ -102,6 +102,7 @@ SLOT { return K_SLOT; } TEMPORARY { return K_TEMPORARY; } EXPORT_SNAPSHOT { return K_EXPORT_SNAPSHOT; } NOEXPORT_SNAPSHOT { return K_NOEXPORT_SNAPSHOT; } +USE_SNAPSHOT { return K_USE_SNAPSHOT; } "," { return ','; } ";" { return ';'; } @@ -180,9 +181,7 @@ NOEXPORT_SNAPSHOT { return K_NOEXPORT_SNAPSHOT; } } . { - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("syntax error: unexpected character \"%s\"", yytext))); + return T_WORD; } %% diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 75617709ec..c6ba916c49 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -753,7 +753,7 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req static void parseCreateReplSlotOptions(CreateReplicationSlotCmd *cmd, bool *reserve_wal, - bool *export_snapshot) + CRSSnapshotAction *snapshot_action) { ListCell *lc; bool snapshot_action_given = false; @@ -772,7 +772,18 @@ parseCreateReplSlotOptions(CreateReplicationSlotCmd *cmd, errmsg("conflicting or redundant options"))); snapshot_action_given = true; - *export_snapshot = defGetBoolean(defel); + *snapshot_action = defGetBoolean(defel) ? CRS_EXPORT_SNAPSHOT : + CRS_NOEXPORT_SNAPSHOT; + } + else if (strcmp(defel->defname, "use_snapshot") == 0) + { + if (snapshot_action_given || cmd->kind != REPLICATION_KIND_LOGICAL) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"))); + + snapshot_action_given = true; + *snapshot_action = CRS_USE_SNAPSHOT; } else if (strcmp(defel->defname, "reserve_wal") == 0) { @@ -799,7 +810,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) char xpos[MAXFNAMELEN]; char *slot_name; bool reserve_wal = false; - bool export_snapshot = true; + CRSSnapshotAction snapshot_action = CRS_EXPORT_SNAPSHOT; DestReceiver *dest; TupOutputState *tstate; TupleDesc tupdesc; @@ -808,7 +819,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) Assert(!MyReplicationSlot); - parseCreateReplSlotOptions(cmd, &reserve_wal, &export_snapshot); + parseCreateReplSlotOptions(cmd, &reserve_wal, &snapshot_action); /* setup state for XLogReadPage */ sendTimeLineIsHistoric = false; @@ -838,6 +849,40 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) { LogicalDecodingContext *ctx; + /* + * Do options check early so that we can bail before calling the + * DecodingContextFindStartpoint which can take long time. + */ + if (snapshot_action == CRS_EXPORT_SNAPSHOT) + { + if (IsTransactionBlock()) + ereport(ERROR, + (errmsg("CREATE_REPLICATION_SLOT ... EXPORT_SNAPSHOT " + "must not be called inside a transaction"))); + } + else if (snapshot_action == CRS_USE_SNAPSHOT) + { + if (!IsTransactionBlock()) + ereport(ERROR, + (errmsg("CREATE_REPLICATION_SLOT ... USE_SNAPSHOT " + "must be called inside a transaction"))); + + if (XactIsoLevel != XACT_REPEATABLE_READ) + ereport(ERROR, + (errmsg("CREATE_REPLICATION_SLOT ... USE_SNAPSHOT " + "must be called in REPEATABLE READ isolation mode transaction"))); + + if (FirstSnapshotSet) + ereport(ERROR, + (errmsg("CREATE_REPLICATION_SLOT ... USE_SNAPSHOT " + "must be called before any query"))); + + if (IsSubTransaction()) + ereport(ERROR, + (errmsg("CREATE_REPLICATION_SLOT ... USE_SNAPSHOT " + "must not be called in a subtransaction"))); + } + ctx = CreateInitDecodingContext(cmd->plugin, NIL, logical_read_xlog_page, WalSndPrepareWrite, WalSndWriteData); @@ -855,13 +900,22 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) DecodingContextFindStartpoint(ctx); /* - * Export the snapshot if we've been asked to do so. + * Export or use the snapshot if we've been asked to do so. * * NB. We will convert the snapbuild.c kind of snapshot to normal * snapshot when doing this. */ - if (export_snapshot) + if (snapshot_action == CRS_EXPORT_SNAPSHOT) + { snapshot_name = SnapBuildExportSnapshot(ctx->snapshot_builder); + } + else if (snapshot_action == CRS_USE_SNAPSHOT) + { + Snapshot snap; + + snap = SnapBuildInitalSnapshot(ctx->snapshot_builder); + RestoreTransactionSnapshot(snap, MyProc); + } /* don't need the decoding context anymore */ FreeDecodingContext(ctx); @@ -1277,8 +1331,11 @@ WalSndWaitForWal(XLogRecPtr loc) /* * Execute an incoming replication command. + * + * Returns true if the cmd_string was recognized as WalSender command, false + * if not. */ -void +bool exec_replication_command(const char *cmd_string) { int parse_rc; @@ -1317,6 +1374,25 @@ exec_replication_command(const char *cmd_string) cmd_node = replication_parse_result; + /* + * CREATE_REPLICATION_SLOT ... LOGICAL exports a snapshot. If it was + * called outside of transaction the snapshot should be cleared here. + */ + if (!IsTransactionBlock()) + SnapBuildClearExportedSnapshot(); + + /* + * For aborted transactions, don't allow anything except pure SQL, + * the exec_simple_query() will handle it correctly. + */ + if (IsAbortedTransactionBlockState() && !IsA(cmd_node, SQLCmd)) + ereport(ERROR, + (errcode(ERRCODE_IN_FAILED_SQL_TRANSACTION), + errmsg("current transaction is aborted, " + "commands ignored until end of transaction block"))); + + CHECK_FOR_INTERRUPTS(); + /* * Allocate buffers that will be used for each outgoing and incoming * message. We do this just once per command to reduce palloc overhead. @@ -1332,6 +1408,7 @@ exec_replication_command(const char *cmd_string) break; case T_BaseBackupCmd: + PreventTransactionChain(true, "BASE_BACKUP"); SendBaseBackup((BaseBackupCmd *) cmd_node); break; @@ -1347,6 +1424,8 @@ exec_replication_command(const char *cmd_string) { StartReplicationCmd *cmd = (StartReplicationCmd *) cmd_node; + PreventTransactionChain(true, "START_REPLICATION"); + if (cmd->kind == REPLICATION_KIND_PHYSICAL) StartReplication(cmd); else @@ -1355,6 +1434,7 @@ exec_replication_command(const char *cmd_string) } case T_TimeLineHistoryCmd: + PreventTransactionChain(true, "TIMELINE_HISTORY"); SendTimeLineHistory((TimeLineHistoryCmd *) cmd_node); break; @@ -1367,6 +1447,14 @@ exec_replication_command(const char *cmd_string) } break; + case T_SQLCmd: + if (MyDatabaseId == InvalidOid) + ereport(ERROR, + (errmsg("not connected to database"))); + + /* Tell the caller that this wasn't a WalSender command. */ + return false; + default: elog(ERROR, "unrecognized replication command node tag: %u", cmd_node->type); @@ -1378,6 +1466,8 @@ exec_replication_command(const char *cmd_string) /* Send CommandComplete message */ EndCommand("SELECT", DestRemote); + + return true; } /* diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index b07d6c6cb9..ba41f90712 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -4061,7 +4061,10 @@ PostgresMain(int argc, char *argv[], pq_getmsgend(&input_message); if (am_walsender) - exec_replication_command(query_string); + { + if (!exec_replication_command(query_string)) + exec_simple_query(query_string); + } else exec_simple_query(query_string); diff --git a/src/backend/utils/adt/misc.c b/src/backend/utils/adt/misc.c index 1ec7f32470..7dcecb2f0f 100644 --- a/src/backend/utils/adt/misc.c +++ b/src/backend/utils/adt/misc.c @@ -982,3 +982,23 @@ pg_current_logfile_1arg(PG_FUNCTION_ARGS) { return pg_current_logfile(fcinfo); } + +/* + * SQL wrapper around RelationGetReplicaIndex(). + */ +Datum +pg_get_replica_identity_index(PG_FUNCTION_ARGS) +{ + Oid reloid = PG_GETARG_OID(0); + Oid idxoid; + Relation rel; + + rel = heap_open(reloid, AccessShareLock); + idxoid = RelationGetReplicaIndex(rel); + heap_close(rel, AccessShareLock); + + if (OidIsValid(idxoid)) + PG_RETURN_OID(idxoid); + else + PG_RETURN_NULL(); +} diff --git a/src/backend/utils/cache/syscache.c b/src/backend/utils/cache/syscache.c index b1c0b4b1be..d5a376406f 100644 --- a/src/backend/utils/cache/syscache.c +++ b/src/backend/utils/cache/syscache.c @@ -62,6 +62,7 @@ #include "catalog/pg_replication_origin.h" #include "catalog/pg_statistic.h" #include "catalog/pg_subscription.h" +#include "catalog/pg_subscription_rel.h" #include "catalog/pg_tablespace.h" #include "catalog/pg_transform.h" #include "catalog/pg_ts_config.h" @@ -693,7 +694,7 @@ static const struct cachedesc cacheinfo[] = { 64 }, {PublicationRelRelationId, /* PUBLICATIONRELMAP */ - PublicationRelMapIndexId, + PublicationRelPrrelidPrpubidIndexId, 2, { Anum_pg_publication_rel_prrelid, @@ -758,6 +759,17 @@ static const struct cachedesc cacheinfo[] = { }, 4 }, + {SubscriptionRelRelationId, /* SUBSCRIPTIONRELMAP */ + SubscriptionRelSrrelidSrsubidIndexId, + 2, + { + Anum_pg_subscription_rel_srrelid, + Anum_pg_subscription_rel_srsubid, + 0, + 0 + }, + 64 + }, {TableSpaceRelationId, /* TABLESPACEOID */ TablespaceOidIndexId, 1, diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 4feb26aa7a..291bf7631d 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -2497,6 +2497,18 @@ static struct config_int ConfigureNamesInt[] = NULL, NULL, NULL }, + { + {"max_sync_workers_per_subscription", + PGC_SIGHUP, + RESOURCES_ASYNCHRONOUS, + gettext_noop("Maximum number of table synchronization workers per subscription."), + NULL, + }, + &max_sync_workers_per_subscription, + 2, 0, MAX_BACKENDS, + NULL, NULL, NULL + }, + { {"log_rotation_age", PGC_SIGHUP, LOGGING_WHERE, gettext_noop("Automatic log file rotation will occur after N minutes."), diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 610bed531c..98bc1a586a 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -155,7 +155,7 @@ typedef struct _dumpOptions int use_setsessauth; int enable_row_security; int include_subscriptions; - int no_create_subscription_slots; + int no_subscription_connect; /* default, if no "inclusion" switches appear, is to dump everything */ bool include_everything; diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 2b5a52656c..a98747d89a 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -351,8 +351,8 @@ main(int argc, char **argv) {"snapshot", required_argument, NULL, 6}, {"strict-names", no_argument, &strict_names, 1}, {"use-set-session-authorization", no_argument, &dopt.use_setsessauth, 1}, - {"no-create-subscription-slots", no_argument, &dopt.no_create_subscription_slots, 1}, {"no-security-labels", no_argument, &dopt.no_security_labels, 1}, + {"no-subscription-connect", no_argument, &dopt.no_subscription_connect, 1}, {"no-synchronized-snapshots", no_argument, &dopt.no_synchronized_snapshots, 1}, {"no-unlogged-table-data", no_argument, &dopt.no_unlogged_table_data, 1}, {"no-sync", no_argument, NULL, 7}, @@ -951,9 +951,8 @@ help(const char *progname) printf(_(" --if-exists use IF EXISTS when dropping objects\n")); printf(_(" --include-subscriptions dump logical replication subscriptions\n")); printf(_(" --inserts dump data as INSERT commands, rather than COPY\n")); - printf(_(" --no-create-subscription-slots\n" - " do not create replication slots for subscriptions\n")); printf(_(" --no-security-labels do not dump security label assignments\n")); + printf(_(" --no-subscription-connect dump subscriptions so they don't connect on restore\n")); printf(_(" --no-synchronized-snapshots do not use synchronized snapshots in parallel jobs\n")); printf(_(" --no-tablespaces do not dump tablespace assignments\n")); printf(_(" --no-unlogged-table-data do not dump unlogged table data\n")); @@ -3774,8 +3773,8 @@ dumpSubscription(Archive *fout, SubscriptionInfo *subinfo) appendPQExpBufferStr(query, ", SLOT NAME = "); appendStringLiteralAH(query, subinfo->subslotname, fout); - if (dopt->no_create_subscription_slots) - appendPQExpBufferStr(query, ", NOCREATE SLOT"); + if (dopt->no_subscription_connect) + appendPQExpBufferStr(query, ", NOCONNECT"); appendPQExpBufferStr(query, ");\n"); diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl index a46dcdbcd7..021f4bf081 100644 --- a/src/bin/pg_dump/t/002_pg_dump.pl +++ b/src/bin/pg_dump/t/002_pg_dump.pl @@ -4224,7 +4224,7 @@ qr/CREATE TRANSFORM FOR integer LANGUAGE sql \(FROM SQL WITH FUNCTION pg_catalog create_order => 50, create_sql => 'CREATE SUBSCRIPTION sub1 CONNECTION \'dbname=doesnotexist\' PUBLICATION pub1 - WITH (DISABLED, NOCREATE SLOT);', + WITH (DISABLED, NOCONNECT);', regexp => qr/^ \QCREATE SUBSCRIPTION sub1 CONNECTION 'dbname=doesnotexist' PUBLICATION pub1 WITH (DISABLED, SLOT NAME = 'sub1');\E /xm, diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 315f155b64..d8679f5f59 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201703221 +#define CATALOG_VERSION_NO 201703231 #endif diff --git a/src/include/catalog/indexing.h b/src/include/catalog/indexing.h index 6bce7328a2..5d4190c05e 100644 --- a/src/include/catalog/indexing.h +++ b/src/include/catalog/indexing.h @@ -340,8 +340,8 @@ DECLARE_UNIQUE_INDEX(pg_publication_pubname_index, 6111, on pg_publication using DECLARE_UNIQUE_INDEX(pg_publication_rel_oid_index, 6112, on pg_publication_rel using btree(oid oid_ops)); #define PublicationRelObjectIndexId 6112 -DECLARE_UNIQUE_INDEX(pg_publication_rel_map_index, 6113, on pg_publication_rel using btree(prrelid oid_ops, prpubid oid_ops)); -#define PublicationRelMapIndexId 6113 +DECLARE_UNIQUE_INDEX(pg_publication_rel_prrelid_prpubid_index, 6113, on pg_publication_rel using btree(prrelid oid_ops, prpubid oid_ops)); +#define PublicationRelPrrelidPrpubidIndexId 6113 DECLARE_UNIQUE_INDEX(pg_subscription_oid_index, 6114, on pg_subscription using btree(oid oid_ops)); #define SubscriptionObjectIndexId 6114 @@ -349,6 +349,9 @@ DECLARE_UNIQUE_INDEX(pg_subscription_oid_index, 6114, on pg_subscription using b DECLARE_UNIQUE_INDEX(pg_subscription_subname_index, 6115, on pg_subscription using btree(subdbid oid_ops, subname name_ops)); #define SubscriptionNameIndexId 6115 +DECLARE_UNIQUE_INDEX(pg_subscription_rel_srrelid_srsubid_index, 6117, on pg_subscription_rel using btree(srrelid oid_ops, srsubid oid_ops)); +#define SubscriptionRelSrrelidSrsubidIndexId 6117 + /* last step of initialization script: build the indexes declared above */ BUILD_INDICES diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index 22635655f5..78c23e3f5d 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -2021,6 +2021,9 @@ DESCR("is a relation insertable/updatable/deletable"); DATA(insert OID = 3843 ( pg_column_is_updatable PGNSP PGUID 12 10 0 0 0 f f f f t f s s 3 0 16 "2205 21 16" _null_ _null_ _null_ _null_ _null_ pg_column_is_updatable _null_ _null_ _null_ )); DESCR("is a column updatable"); +DATA(insert OID = 6120 ( pg_get_replica_identity_index PGNSP PGUID 12 10 0 0 0 f f f f t f s s 1 0 2205 "2205" _null_ _null_ _null_ _null_ _null_ pg_get_replica_identity_index _null_ _null_ _null_ )); +DESCR("oid of replica identity index if any"); + /* Deferrable unique constraint trigger */ DATA(insert OID = 1250 ( unique_key_recheck PGNSP PGUID 12 1 0 0 0 f f f f t f v s 0 0 2279 "" _null_ _null_ _null_ _null_ _null_ unique_key_recheck _null_ _null_ _null_ )); DESCR("deferred UNIQUE constraint check"); @@ -2805,7 +2808,7 @@ DATA(insert OID = 3099 ( pg_stat_get_wal_senders PGNSP PGUID 12 1 10 0 0 f f f DESCR("statistics: information about currently active replication"); DATA(insert OID = 3317 ( pg_stat_get_wal_receiver PGNSP PGUID 12 1 0 0 0 f f f f f f s r 0 0 2249 "" "{23,25,3220,23,3220,23,1184,1184,3220,1184,25,25}" "{o,o,o,o,o,o,o,o,o,o,o,o}" "{pid,status,receive_start_lsn,receive_start_tli,received_lsn,received_tli,last_msg_send_time,last_msg_receipt_time,latest_end_lsn,latest_end_time,slot_name,conninfo}" _null_ _null_ pg_stat_get_wal_receiver _null_ _null_ _null_ )); DESCR("statistics: information about WAL receiver"); -DATA(insert OID = 6118 ( pg_stat_get_subscription PGNSP PGUID 12 1 0 0 0 f f f f f f s r 1 0 2249 "26" "{26,26,23,3220,1184,1184,3220,1184}" "{i,o,o,o,o,o,o,o}" "{subid,subid,pid,received_lsn,last_msg_send_time,last_msg_receipt_time,latest_end_lsn,latest_end_time}" _null_ _null_ pg_stat_get_subscription _null_ _null_ _null_ )); +DATA(insert OID = 6118 ( pg_stat_get_subscription PGNSP PGUID 12 1 0 0 0 f f f f f f s r 1 0 2249 "26" "{26,26,26,23,3220,1184,1184,3220,1184}" "{i,o,o,o,o,o,o,o,o}" "{subid,subid,relid,pid,received_lsn,last_msg_send_time,last_msg_receipt_time,latest_end_lsn,latest_end_time}" _null_ _null_ pg_stat_get_subscription _null_ _null_ _null_ )); DESCR("statistics: information about subscription"); DATA(insert OID = 2026 ( pg_backend_pid PGNSP PGUID 12 1 0 0 0 f f f f t f s r 0 0 23 "" _null_ _null_ _null_ _null_ _null_ pg_backend_pid _null_ _null_ _null_ )); DESCR("statistics: current backend PID"); diff --git a/src/include/catalog/pg_subscription_rel.h b/src/include/catalog/pg_subscription_rel.h new file mode 100644 index 0000000000..129aa99f29 --- /dev/null +++ b/src/include/catalog/pg_subscription_rel.h @@ -0,0 +1,78 @@ +/* ------------------------------------------------------------------------- + * + * pg_subscription_rel.h + * Local info about tables that come from the publisher of a + * subscription (pg_subscription_rel). + * + * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * ------------------------------------------------------------------------- + */ +#ifndef PG_SUBSCRIPTION_REL_H +#define PG_SUBSCRIPTION_REL_H + +#include "catalog/genbki.h" + +/* ---------------- + * pg_subscription_rel definition. cpp turns this into + * typedef struct FormData_pg_subscription_rel + * ---------------- + */ +#define SubscriptionRelRelationId 6102 + +/* Workaround for genbki not knowing about XLogRecPtr */ +#define pg_lsn XLogRecPtr + +CATALOG(pg_subscription_rel,6102) BKI_WITHOUT_OIDS +{ + Oid srsubid; /* Oid of subscription */ + Oid srrelid; /* Oid of relation */ + char srsubstate; /* state of the relation in subscription */ + pg_lsn srsublsn; /* remote lsn of the state change + * used for synchronization coordination */ +} FormData_pg_subscription_rel; + +typedef FormData_pg_subscription_rel *Form_pg_subscription_rel; + +/* ---------------- + * compiler constants for pg_subscription_rel + * ---------------- + */ +#define Natts_pg_subscription_rel 4 +#define Anum_pg_subscription_rel_srsubid 1 +#define Anum_pg_subscription_rel_srrelid 2 +#define Anum_pg_subscription_rel_srsubstate 3 +#define Anum_pg_subscription_rel_srsublsn 4 + +/* ---------------- + * substate constants + * ---------------- + */ +#define SUBREL_STATE_INIT 'i' /* initializing (sublsn NULL) */ +#define SUBREL_STATE_DATASYNC 'd' /* data is being synchronized (sublsn NULL) */ +#define SUBREL_STATE_SYNCDONE 's' /* synchronization finished infront of apply (sublsn set) */ +#define SUBREL_STATE_READY 'r' /* ready (sublsn set) */ + +/* These are never stored in the catalog, we only use them for IPC. */ +#define SUBREL_STATE_UNKNOWN '\0' /* unknown state */ +#define SUBREL_STATE_SYNCWAIT 'w' /* waiting for sync */ +#define SUBREL_STATE_CATCHUP 'c' /* catching up with apply */ + +typedef struct SubscriptionRelState +{ + Oid relid; + XLogRecPtr lsn; + char state; +} SubscriptionRelState; + +extern Oid SetSubscriptionRelState(Oid subid, Oid relid, char state, + XLogRecPtr sublsn); +extern char GetSubscriptionRelState(Oid subid, Oid relid, + XLogRecPtr *sublsn, bool missing_ok); +extern void RemoveSubscriptionRel(Oid subid, Oid relid); + +extern List *GetSubscriptionRelations(Oid subid); +extern List *GetSubscriptionNotReadyRelations(Oid subid); + +#endif /* PG_SUBSCRIPTION_REL_H */ diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index d63ca0f5e9..f081f2219f 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -21,6 +21,7 @@ /* CopyStateData is private in commands/copy.c */ typedef struct CopyStateData *CopyState; +typedef int (*copy_data_source_cb) (void *outbuf, int minread, int maxread); extern void DoCopy(ParseState *state, const CopyStmt *stmt, int stmt_location, int stmt_len, @@ -28,7 +29,7 @@ extern void DoCopy(ParseState *state, const CopyStmt *stmt, extern void ProcessCopyOptions(ParseState *pstate, CopyState cstate, bool is_from, List *options); extern CopyState BeginCopyFrom(ParseState *pstate, Relation rel, const char *filename, - bool is_program, List *attnamelist, List *options); + bool is_program, copy_data_source_cb data_source_cb, List *attnamelist, List *options); extern void EndCopyFrom(CopyState cstate); extern bool NextCopyFrom(CopyState cstate, ExprContext *econtext, Datum *values, bool *nulls, Oid *tupleOid); @@ -36,6 +37,8 @@ extern bool NextCopyFromRawFields(CopyState cstate, char ***fields, int *nfields); extern void CopyFromErrorCallback(void *arg); +extern uint64 CopyFrom(CopyState cstate); + extern DestReceiver *CreateCopyDestReceiver(void); #endif /* COPY_H */ diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index 2cbd6d77b8..9a4221a9e7 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -488,6 +488,7 @@ typedef enum NodeTag T_DropReplicationSlotCmd, T_StartReplicationCmd, T_TimeLineHistoryCmd, + T_SQLCmd, /* * TAGS FOR RANDOM OTHER STUFF diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index a15df229a4..582e0e0ebe 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -3319,10 +3319,23 @@ typedef struct CreateSubscriptionStmt List *options; /* List of DefElem nodes */ } CreateSubscriptionStmt; +typedef enum AlterSubscriptionType +{ + ALTER_SUBSCRIPTION_OPTIONS, + ALTER_SUBSCRIPTION_CONNECTION, + ALTER_SUBSCRIPTION_PUBLICATION, + ALTER_SUBSCRIPTION_PUBLICATION_REFRESH, + ALTER_SUBSCRIPTION_REFRESH, + ALTER_SUBSCRIPTION_ENABLED +} AlterSubscriptionType; + typedef struct AlterSubscriptionStmt { NodeTag type; + AlterSubscriptionType kind; /* ALTER_SUBSCRIPTION_OPTIONS, etc */ char *subname; /* Name of of the subscription */ + char *conninfo; /* Connection string to publisher */ + List *publication; /* One or more publication to subscribe to */ List *options; /* List of DefElem nodes */ } AlterSubscriptionStmt; diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h index 996da3c02e..92ada41b6d 100644 --- a/src/include/nodes/replnodes.h +++ b/src/include/nodes/replnodes.h @@ -96,4 +96,13 @@ typedef struct TimeLineHistoryCmd TimeLineID timeline; } TimeLineHistoryCmd; +/* ---------------------- + * SQL commands + * ---------------------- + */ +typedef struct SQLCmd +{ + NodeTag type; +} SQLCmd; + #endif /* REPLNODES_H */ diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h index 28c4dab258..6cd36c7fe3 100644 --- a/src/include/parser/kwlist.h +++ b/src/include/parser/kwlist.h @@ -258,6 +258,7 @@ PG_KEYWORD("new", NEW, UNRESERVED_KEYWORD) PG_KEYWORD("next", NEXT, UNRESERVED_KEYWORD) PG_KEYWORD("no", NO, UNRESERVED_KEYWORD) PG_KEYWORD("none", NONE, COL_NAME_KEYWORD) +PG_KEYWORD("norefresh", NOREFRESH, UNRESERVED_KEYWORD) PG_KEYWORD("not", NOT, RESERVED_KEYWORD) PG_KEYWORD("nothing", NOTHING, UNRESERVED_KEYWORD) PG_KEYWORD("notify", NOTIFY, UNRESERVED_KEYWORD) diff --git a/src/include/pgstat.h b/src/include/pgstat.h index f2daf32e1a..a675242971 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -790,7 +790,9 @@ typedef enum WAIT_EVENT_PARALLEL_FINISH, WAIT_EVENT_PARALLEL_BITMAP_SCAN, WAIT_EVENT_SAFE_SNAPSHOT, - WAIT_EVENT_SYNC_REP + WAIT_EVENT_SYNC_REP, + WAIT_EVENT_LOGICAL_SYNC_DATA, + WAIT_EVENT_LOGICAL_SYNC_STATE_CHANGE } WaitEventIPC; /* ---------- diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h index fd34964bad..d10dd2c90a 100644 --- a/src/include/replication/logical.h +++ b/src/include/replication/logical.h @@ -31,9 +31,11 @@ typedef struct LogicalDecodingContext /* memory context this is all allocated in */ MemoryContext context; - /* infrastructure pieces */ - XLogReaderState *reader; + /* The associated replication slot */ ReplicationSlot *slot; + + /* infrastructure pieces for decoding */ + XLogReaderState *reader; struct ReorderBuffer *reorder; struct SnapBuild *snapshot_builder; @@ -75,6 +77,7 @@ typedef struct LogicalDecodingContext TransactionId write_xid; } LogicalDecodingContext; + extern void CheckLogicalDecodingRequirements(void); extern LogicalDecodingContext *CreateInitDecodingContext(char *plugin, @@ -92,6 +95,12 @@ extern void DecodingContextFindStartpoint(LogicalDecodingContext *ctx); extern bool DecodingContextReady(LogicalDecodingContext *ctx); extern void FreeDecodingContext(LogicalDecodingContext *ctx); +extern LogicalDecodingContext *CreateCopyDecodingContext( + List *output_plugin_options, + LogicalOutputPluginWriterPrepareWrite prepare_write, + LogicalOutputPluginWriterWrite do_write); +extern List *DecodingContextGetTableList(LogicalDecodingContext *ctx); + extern void LogicalIncreaseXminForSlot(XLogRecPtr lsn, TransactionId xmin); extern void LogicalIncreaseRestartDecodingForSlot(XLogRecPtr current_lsn, XLogRecPtr restart_lsn); diff --git a/src/include/replication/logicallauncher.h b/src/include/replication/logicallauncher.h index cfe3db10dd..060946a096 100644 --- a/src/include/replication/logicallauncher.h +++ b/src/include/replication/logicallauncher.h @@ -13,6 +13,7 @@ #define LOGICALLAUNCHER_H extern int max_logical_replication_workers; +extern int max_sync_workers_per_subscription; extern void ApplyLauncherRegister(void); extern void ApplyLauncherMain(Datum main_arg); diff --git a/src/include/replication/snapbuild.h b/src/include/replication/snapbuild.h index 5e824ae6fc..091a9f91e3 100644 --- a/src/include/replication/snapbuild.h +++ b/src/include/replication/snapbuild.h @@ -59,6 +59,7 @@ extern void FreeSnapshotBuilder(SnapBuild *cache); extern void SnapBuildSnapDecRefcount(Snapshot snap); +extern Snapshot SnapBuildInitalSnapshot(SnapBuild *builder); extern const char *SnapBuildExportSnapshot(SnapBuild *snapstate); extern void SnapBuildClearExportedSnapshot(void); diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h index 78e577c89b..fb55c30fa1 100644 --- a/src/include/replication/walreceiver.h +++ b/src/include/replication/walreceiver.h @@ -15,9 +15,12 @@ #include "access/xlog.h" #include "access/xlogdefs.h" #include "fmgr.h" +#include "replication/logicalproto.h" +#include "replication/walsender.h" #include "storage/latch.h" #include "storage/spin.h" #include "pgtime.h" +#include "utils/tuplestore.h" /* user-settable parameters */ extern int wal_receiver_status_interval; @@ -160,6 +163,33 @@ typedef struct struct WalReceiverConn; typedef struct WalReceiverConn WalReceiverConn; +/* + * Status of walreceiver query execution. + * + * We only define statuses that are currently used. + */ +typedef enum +{ + WALRCV_ERROR, /* There was error when executing the query. */ + WALRCV_OK_COMMAND, /* Query executed utility or replication command. */ + WALRCV_OK_TUPLES, /* Query returned tuples. */ + WALRCV_OK_COPY_IN, /* Query started COPY FROM. */ + WALRCV_OK_COPY_OUT, /* Query started COPY TO. */ + WALRCV_OK_COPY_BOTH, /* Query started COPY BOTH replication protocol. */ +} WalRcvExecStatus; + +/* + * Return value for walrcv_query, returns the status of the execution and + * tuples if any. + */ +typedef struct WalRcvExecResult +{ + WalRcvExecStatus status; + char *err; + Tuplestorestate *tuplestore; + TupleDesc tupledesc; +} WalRcvExecResult; + /* libpqwalreceiver hooks */ typedef WalReceiverConn *(*walrcv_connect_fn) (const char *conninfo, bool logical, const char *appname, @@ -183,9 +213,12 @@ typedef void (*walrcv_send_fn) (WalReceiverConn *conn, const char *buffer, int nbytes); typedef char *(*walrcv_create_slot_fn) (WalReceiverConn *conn, const char *slotname, bool temporary, - bool export_snapshot, XLogRecPtr *lsn); -typedef bool (*walrcv_command_fn) (WalReceiverConn *conn, const char *cmd, - char **err); + CRSSnapshotAction snapshot_action, + XLogRecPtr *lsn); +typedef WalRcvExecResult *(*walrcv_exec_fn) (WalReceiverConn *conn, + const char *query, + const int nRetTypes, + const Oid *retTypes); typedef void (*walrcv_disconnect_fn) (WalReceiverConn *conn); typedef struct WalReceiverFunctionsType @@ -200,7 +233,7 @@ typedef struct WalReceiverFunctionsType walrcv_receive_fn walrcv_receive; walrcv_send_fn walrcv_send; walrcv_create_slot_fn walrcv_create_slot; - walrcv_command_fn walrcv_command; + walrcv_exec_fn walrcv_exec; walrcv_disconnect_fn walrcv_disconnect; } WalReceiverFunctionsType; @@ -224,13 +257,31 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions; WalReceiverFunctions->walrcv_receive(conn, buffer, wait_fd) #define walrcv_send(conn, buffer, nbytes) \ WalReceiverFunctions->walrcv_send(conn, buffer, nbytes) -#define walrcv_create_slot(conn, slotname, temporary, export_snapshot, lsn) \ - WalReceiverFunctions->walrcv_create_slot(conn, slotname, temporary, export_snapshot, lsn) -#define walrcv_command(conn, cmd, err) \ - WalReceiverFunctions->walrcv_command(conn, cmd, err) +#define walrcv_create_slot(conn, slotname, temporary, snapshot_action, lsn) \ + WalReceiverFunctions->walrcv_create_slot(conn, slotname, temporary, snapshot_action, lsn) +#define walrcv_exec(conn, exec, nRetTypes, retTypes) \ + WalReceiverFunctions->walrcv_exec(conn, exec, nRetTypes, retTypes) #define walrcv_disconnect(conn) \ WalReceiverFunctions->walrcv_disconnect(conn) +static inline void +walrcv_clear_result(WalRcvExecResult *walres) +{ + if (!walres) + return; + + if (walres->err) + pfree(walres->err); + + if (walres->tuplestore) + tuplestore_end(walres->tuplestore); + + if (walres->tupledesc) + FreeTupleDesc(walres->tupledesc); + + pfree(walres); +} + /* prototypes for functions in walreceiver.c */ extern void WalReceiverMain(void) pg_attribute_noreturn(); diff --git a/src/include/replication/walsender.h b/src/include/replication/walsender.h index fe23f6619f..2ca903872e 100644 --- a/src/include/replication/walsender.h +++ b/src/include/replication/walsender.h @@ -16,6 +16,16 @@ #include "fmgr.h" +/* + * What to do with a snapshot in create replication slot command. + */ +typedef enum +{ + CRS_EXPORT_SNAPSHOT, + CRS_NOEXPORT_SNAPSHOT, + CRS_USE_SNAPSHOT +} CRSSnapshotAction; + /* global state */ extern bool am_walsender; extern bool am_cascading_walsender; @@ -28,7 +38,7 @@ extern int wal_sender_timeout; extern bool log_replication_commands; extern void InitWalSender(void); -extern void exec_replication_command(const char *query_string); +extern bool exec_replication_command(const char *query_string); extern void WalSndErrorCleanup(void); extern void WalSndSignals(void); extern Size WalSndShmemSize(void); diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h index 8cbf2687a9..bf96d340ca 100644 --- a/src/include/replication/worker_internal.h +++ b/src/include/replication/worker_internal.h @@ -33,6 +33,9 @@ typedef struct LogicalRepWorker /* Used for initial table synchronization. */ Oid relid; + char relstate; + XLogRecPtr relstate_lsn; + slock_t relmutex; /* Stats. */ XLogRecPtr last_lsn; @@ -42,6 +45,9 @@ typedef struct LogicalRepWorker TimestampTz reply_time; } LogicalRepWorker; +/* Memory context for cached variables in apply worker. */ +MemoryContext ApplyCacheContext; + /* libpqreceiver connection */ extern struct WalReceiverConn *wrconn; @@ -53,12 +59,26 @@ extern bool in_remote_transaction; extern bool got_SIGTERM; extern void logicalrep_worker_attach(int slot); -extern LogicalRepWorker *logicalrep_worker_find(Oid subid); -extern int logicalrep_worker_count(Oid subid); -extern void logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid); -extern void logicalrep_worker_stop(Oid subid); -extern void logicalrep_worker_wakeup(Oid subid); +extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid, + bool only_running); +extern void logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, + Oid userid, Oid relid); +extern void logicalrep_worker_stop(Oid subid, Oid relid); +extern void logicalrep_worker_wakeup(Oid subid, Oid relid); +extern void logicalrep_worker_wakeup_ptr(LogicalRepWorker *worker); + +extern int logicalrep_sync_worker_count(Oid subid); extern void logicalrep_worker_sigterm(SIGNAL_ARGS); +extern char *LogicalRepSyncTableStart(XLogRecPtr *origin_startpos); +void process_syncing_tables(XLogRecPtr current_lsn); +void invalidate_syncing_table_states(Datum arg, int cacheid, + uint32 hashvalue); + +static inline bool +am_tablesync_worker(void) +{ + return OidIsValid(MyLogicalRepWorker->relid); +} #endif /* WORKER_INTERNAL_H */ diff --git a/src/include/utils/syscache.h b/src/include/utils/syscache.h index 66f60d271e..b35faf81b9 100644 --- a/src/include/utils/syscache.h +++ b/src/include/utils/syscache.h @@ -89,6 +89,7 @@ enum SysCacheIdentifier STATRELATTINH, SUBSCRIPTIONOID, SUBSCRIPTIONNAME, + SUBSCRIPTIONRELMAP, TABLESPACEOID, TRFOID, TRFTYPELANG, diff --git a/src/test/regress/expected/object_address.out b/src/test/regress/expected/object_address.out index 90c4ba4608..978d9a9a0f 100644 --- a/src/test/regress/expected/object_address.out +++ b/src/test/regress/expected/object_address.out @@ -37,7 +37,8 @@ CREATE TRANSFORM FOR int LANGUAGE SQL ( FROM SQL WITH FUNCTION varchar_transform(internal), TO SQL WITH FUNCTION int4recv(internal)); CREATE PUBLICATION addr_pub FOR TABLE addr_nsp.gentable; -CREATE SUBSCRIPTION addr_sub CONNECTION '' PUBLICATION bar WITH (DISABLED, NOCREATE SLOT); +CREATE SUBSCRIPTION addr_sub CONNECTION '' PUBLICATION bar WITH (DISABLED, NOCONNECT); +WARNING: tables were not subscribed, you will have to run ALTER SUBSCRIPTION ... REFRESH PUBLICATION to subscribe the tables -- test some error cases SELECT pg_get_object_address('stone', '{}', '{}'); ERROR: unrecognized object type "stone" diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index bd13ae6010..f7c3a637b5 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1847,13 +1847,14 @@ pg_stat_ssl| SELECT s.pid, pg_stat_subscription| SELECT su.oid AS subid, su.subname, st.pid, + st.relid, st.received_lsn, st.last_msg_send_time, st.last_msg_receipt_time, st.latest_end_lsn, st.latest_end_time FROM (pg_subscription su - LEFT JOIN pg_stat_get_subscription(NULL::oid) st(subid, pid, received_lsn, last_msg_send_time, last_msg_receipt_time, latest_end_lsn, latest_end_time) ON ((st.subid = su.oid))); + LEFT JOIN pg_stat_get_subscription(NULL::oid) st(subid, relid, pid, received_lsn, last_msg_send_time, last_msg_receipt_time, latest_end_lsn, latest_end_time) ON ((st.subid = su.oid))); pg_stat_sys_indexes| SELECT pg_stat_all_indexes.relid, pg_stat_all_indexes.indexrelid, pg_stat_all_indexes.schemaname, diff --git a/src/test/regress/expected/sanity_check.out b/src/test/regress/expected/sanity_check.out index 88b4c973a1..8e3028edaa 100644 --- a/src/test/regress/expected/sanity_check.out +++ b/src/test/regress/expected/sanity_check.out @@ -143,6 +143,7 @@ pg_shdescription|t pg_shseclabel|t pg_statistic|t pg_subscription|t +pg_subscription_rel|t pg_tablespace|t pg_transform|t pg_trigger|t diff --git a/src/test/regress/expected/subscription.out b/src/test/regress/expected/subscription.out index 3471d88ca7..0912bef657 100644 --- a/src/test/regress/expected/subscription.out +++ b/src/test/regress/expected/subscription.out @@ -14,7 +14,6 @@ CREATE SUBSCRIPTION testsub PUBLICATION foo; ERROR: syntax error at or near "PUBLICATION" LINE 1: CREATE SUBSCRIPTION testsub PUBLICATION foo; ^ -set client_min_messages to error; -- fail - cannot do CREATE SUBSCRIPTION CREATE SLOT inside transaction block BEGIN; CREATE SUBSCRIPTION testsub CONNECTION 'testconn' PUBLICATION testpub WITH (CREATE SLOT); @@ -23,8 +22,8 @@ COMMIT; CREATE SUBSCRIPTION testsub CONNECTION 'testconn' PUBLICATION testpub; ERROR: invalid connection string syntax: missing "=" after "testconn" in connection info string -CREATE SUBSCRIPTION testsub CONNECTION 'dbname=doesnotexist' PUBLICATION testpub WITH (DISABLED, NOCREATE SLOT); -reset client_min_messages; +CREATE SUBSCRIPTION testsub CONNECTION 'dbname=doesnotexist' PUBLICATION testpub WITH (NOCONNECT); +WARNING: tables were not subscribed, you will have to run ALTER SUBSCRIPTION ... REFRESH PUBLICATION to subscribe the tables \dRs+ List of subscriptions Name | Owner | Enabled | Publication | Conninfo @@ -32,38 +31,30 @@ reset client_min_messages; testsub | regress_subscription_user | f | {testpub} | dbname=doesnotexist (1 row) -ALTER SUBSCRIPTION testsub SET PUBLICATION testpub2, testpub3; -\dRs - List of subscriptions - Name | Owner | Enabled | Publication ----------+---------------------------+---------+--------------------- - testsub | regress_subscription_user | f | {testpub2,testpub3} -(1 row) - +ALTER SUBSCRIPTION testsub SET PUBLICATION testpub2, testpub3 NOREFRESH; ALTER SUBSCRIPTION testsub CONNECTION 'dbname=doesnotexist2'; -ALTER SUBSCRIPTION testsub SET PUBLICATION testpub, testpub1; \dRs+ List of subscriptions - Name | Owner | Enabled | Publication | Conninfo ----------+---------------------------+---------+--------------------+---------------------- - testsub | regress_subscription_user | f | {testpub,testpub1} | dbname=doesnotexist2 + Name | Owner | Enabled | Publication | Conninfo +---------+---------------------------+---------+---------------------+---------------------- + testsub | regress_subscription_user | f | {testpub2,testpub3} | dbname=doesnotexist2 (1 row) BEGIN; ALTER SUBSCRIPTION testsub ENABLE; \dRs - List of subscriptions - Name | Owner | Enabled | Publication ----------+---------------------------+---------+-------------------- - testsub | regress_subscription_user | t | {testpub,testpub1} + List of subscriptions + Name | Owner | Enabled | Publication +---------+---------------------------+---------+--------------------- + testsub | regress_subscription_user | t | {testpub2,testpub3} (1 row) ALTER SUBSCRIPTION testsub DISABLE; \dRs - List of subscriptions - Name | Owner | Enabled | Publication ----------+---------------------------+---------+-------------------- - testsub | regress_subscription_user | f | {testpub,testpub1} + List of subscriptions + Name | Owner | Enabled | Publication +---------+---------------------------+---------+--------------------- + testsub | regress_subscription_user | f | {testpub2,testpub3} (1 row) COMMIT; @@ -74,10 +65,10 @@ ERROR: must be owner of subscription testsub RESET ROLE; ALTER SUBSCRIPTION testsub RENAME TO testsub_foo; \dRs - List of subscriptions - Name | Owner | Enabled | Publication --------------+---------------------------+---------+-------------------- - testsub_foo | regress_subscription_user | f | {testpub,testpub1} + List of subscriptions + Name | Owner | Enabled | Publication +-------------+---------------------------+---------+--------------------- + testsub_foo | regress_subscription_user | f | {testpub2,testpub3} (1 row) -- rename back to keep the rest simple diff --git a/src/test/regress/sql/object_address.sql b/src/test/regress/sql/object_address.sql index 6b85fe2949..28476daff1 100644 --- a/src/test/regress/sql/object_address.sql +++ b/src/test/regress/sql/object_address.sql @@ -40,7 +40,7 @@ CREATE TRANSFORM FOR int LANGUAGE SQL ( FROM SQL WITH FUNCTION varchar_transform(internal), TO SQL WITH FUNCTION int4recv(internal)); CREATE PUBLICATION addr_pub FOR TABLE addr_nsp.gentable; -CREATE SUBSCRIPTION addr_sub CONNECTION '' PUBLICATION bar WITH (DISABLED, NOCREATE SLOT); +CREATE SUBSCRIPTION addr_sub CONNECTION '' PUBLICATION bar WITH (DISABLED, NOCONNECT); -- test some error cases SELECT pg_get_object_address('stone', '{}', '{}'); diff --git a/src/test/regress/sql/subscription.sql b/src/test/regress/sql/subscription.sql index 5c05b14f9e..c1199ee629 100644 --- a/src/test/regress/sql/subscription.sql +++ b/src/test/regress/sql/subscription.sql @@ -12,24 +12,19 @@ CREATE SUBSCRIPTION testsub CONNECTION 'foo'; -- fail - no connection CREATE SUBSCRIPTION testsub PUBLICATION foo; -set client_min_messages to error; -- fail - cannot do CREATE SUBSCRIPTION CREATE SLOT inside transaction block BEGIN; CREATE SUBSCRIPTION testsub CONNECTION 'testconn' PUBLICATION testpub WITH (CREATE SLOT); COMMIT; CREATE SUBSCRIPTION testsub CONNECTION 'testconn' PUBLICATION testpub; -CREATE SUBSCRIPTION testsub CONNECTION 'dbname=doesnotexist' PUBLICATION testpub WITH (DISABLED, NOCREATE SLOT); -reset client_min_messages; -\dRs+ - -ALTER SUBSCRIPTION testsub SET PUBLICATION testpub2, testpub3; +CREATE SUBSCRIPTION testsub CONNECTION 'dbname=doesnotexist' PUBLICATION testpub WITH (NOCONNECT); -\dRs +\dRs+ +ALTER SUBSCRIPTION testsub SET PUBLICATION testpub2, testpub3 NOREFRESH; ALTER SUBSCRIPTION testsub CONNECTION 'dbname=doesnotexist2'; -ALTER SUBSCRIPTION testsub SET PUBLICATION testpub, testpub1; \dRs+ diff --git a/src/test/subscription/t/001_rep_changes.pl b/src/test/subscription/t/001_rep_changes.pl index b81028aed1..d1817f57da 100644 --- a/src/test/subscription/t/001_rep_changes.pl +++ b/src/test/subscription/t/001_rep_changes.pl @@ -3,7 +3,7 @@ use strict; use warnings; use PostgresNode; use TestLib; -use Test::More tests => 11; +use Test::More tests => 14; # Initialize publisher node my $node_publisher = get_new_node('publisher'); @@ -19,7 +19,7 @@ $node_subscriber->start; $node_publisher->safe_psql('postgres', "CREATE TABLE tab_notrep AS SELECT generate_series(1,10) AS a"); $node_publisher->safe_psql('postgres', - "CREATE TABLE tab_ins (a int)"); + "CREATE TABLE tab_ins AS SELECT generate_series(1,1002) AS a"); $node_publisher->safe_psql('postgres', "CREATE TABLE tab_full AS SELECT generate_series(1,10) AS a"); $node_publisher->safe_psql('postgres', @@ -56,10 +56,20 @@ my $caughtup_query = $node_publisher->poll_query_until('postgres', $caughtup_query) or die "Timed out while waiting for subscriber to catch up"; +# Also wait for initial table sync to finish +my $synced_query = +"SELECT count(1) = 0 FROM pg_subscription_rel WHERE srsubstate NOT IN ('r', 's');"; +$node_subscriber->poll_query_until('postgres', $synced_query) + or die "Timed out while waiting for subscriber to synchronize data"; + my $result = $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM tab_notrep"); is($result, qq(0), 'check non-replicated table is empty on subscriber'); +$result = + $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM tab_ins"); +is($result, qq(1002), 'check initial data was copied to subscriber'); + $node_publisher->safe_psql('postgres', "INSERT INTO tab_ins SELECT generate_series(1,50)"); $node_publisher->safe_psql('postgres', @@ -79,7 +89,7 @@ $node_publisher->poll_query_until('postgres', $caughtup_query) $result = $node_subscriber->safe_psql('postgres', "SELECT count(*), min(a), max(a) FROM tab_ins"); -is($result, qq(50|1|50), 'check replicated inserts on subscriber'); +is($result, qq(1052|1|1002), 'check replicated inserts on subscriber'); $result = $node_subscriber->safe_psql('postgres', "SELECT count(*), min(a), max(a) FROM tab_rep"); @@ -109,7 +119,7 @@ $node_publisher->poll_query_until('postgres', $caughtup_query) $result = $node_subscriber->safe_psql('postgres', "SELECT count(*), min(a), max(a) FROM tab_full"); -is($result, qq(10|1|100), 'update works with REPLICA IDENTITY FULL and duplicate tuples'); +is($result, qq(20|1|100), 'update works with REPLICA IDENTITY FULL and duplicate tuples'); # check that change of connection string and/or publication list causes # restart of subscription workers. Not all of these are registered as tests @@ -126,7 +136,7 @@ $node_publisher->poll_query_until('postgres', $oldpid = $node_publisher->safe_psql('postgres', "SELECT pid FROM pg_stat_replication WHERE application_name = '$appname';"); $node_subscriber->safe_psql('postgres', - "ALTER SUBSCRIPTION tap_sub SET PUBLICATION tap_pub_ins_only"); + "ALTER SUBSCRIPTION tap_sub SET PUBLICATION tap_pub_ins_only REFRESH WITH (NOCOPY DATA)"); $node_publisher->poll_query_until('postgres', "SELECT pid != $oldpid FROM pg_stat_replication WHERE application_name = '$appname';") or die "Timed out while waiting for apply to restart"; @@ -141,7 +151,7 @@ $node_publisher->poll_query_until('postgres', $caughtup_query) $result = $node_subscriber->safe_psql('postgres', "SELECT count(*), min(a), max(a) FROM tab_ins"); -is($result, qq(150|1|1100), 'check replicated inserts after subscription publication change'); +is($result, qq(1152|1|1100), 'check replicated inserts after subscription publication change'); $result = $node_subscriber->safe_psql('postgres', "SELECT count(*), min(a), max(a) FROM tab_rep"); @@ -154,6 +164,8 @@ $node_publisher->safe_psql('postgres', "ALTER PUBLICATION tap_pub_ins_only ADD TABLE tab_full"); $node_publisher->safe_psql('postgres', "DELETE FROM tab_ins WHERE a > 0"); +$node_subscriber->safe_psql('postgres', + "ALTER SUBSCRIPTION tap_sub REFRESH PUBLICATION WITH (NOCOPY DATA)"); $node_publisher->safe_psql('postgres', "INSERT INTO tab_full VALUES(0)"); @@ -163,11 +175,11 @@ $node_publisher->poll_query_until('postgres', $caughtup_query) # note that data are different on provider and subscriber $result = $node_subscriber->safe_psql('postgres', "SELECT count(*), min(a), max(a) FROM tab_ins"); -is($result, qq(50|1|50), 'check replicated deletes after alter publication'); +is($result, qq(1052|1|1002), 'check replicated deletes after alter publication'); $result = $node_subscriber->safe_psql('postgres', "SELECT count(*), min(a), max(a) FROM tab_full"); -is($result, qq(11|0|100), 'check replicated insert after alter publication'); +is($result, qq(21|0|100), 'check replicated insert after alter publication'); # check restart on rename $oldpid = $node_publisher->safe_psql('postgres', @@ -189,6 +201,14 @@ $result = $node_publisher->safe_psql('postgres', "SELECT count(*) FROM pg_replication_slots"); is($result, qq(0), 'check replication slot was dropped on publisher'); +$result = + $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM pg_subscription_rel"); +is($result, qq(0), 'check subscription relation status was dropped on subscriber'); + +$result = + $node_publisher->safe_psql('postgres', "SELECT count(*) FROM pg_replication_slots"); +is($result, qq(0), 'check replication slot was dropped on publisher'); + $result = $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM pg_replication_origin"); is($result, qq(0), 'check replication origin was dropped on subscriber'); diff --git a/src/test/subscription/t/002_types.pl b/src/test/subscription/t/002_types.pl index f44e1e671d..ad15e85c0c 100644 --- a/src/test/subscription/t/002_types.pl +++ b/src/test/subscription/t/002_types.pl @@ -111,6 +111,12 @@ my $caughtup_query = $node_publisher->poll_query_until('postgres', $caughtup_query) or die "Timed out while waiting for subscriber to catch up"; +# Wait for initial sync to finish as well +my $synced_query = +"SELECT count(1) = 0 FROM pg_subscription_rel WHERE srsubstate NOT IN ('s', 'r');"; +$node_subscriber->poll_query_until('postgres', $synced_query) + or die "Timed out while waiting for subscriber to synchronize data"; + # Insert initial test data $node_publisher->safe_psql('postgres', qq( -- test_tbl_one_array_col diff --git a/src/test/subscription/t/003_constraints.pl b/src/test/subscription/t/003_constraints.pl index b785132f5b..11b8254155 100644 --- a/src/test/subscription/t/003_constraints.pl +++ b/src/test/subscription/t/003_constraints.pl @@ -34,7 +34,7 @@ $node_publisher->safe_psql('postgres', my $appname = 'tap_sub'; $node_subscriber->safe_psql('postgres', - "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub;"); + "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub WITH (NOCOPY DATA)"); # Wait for subscriber to finish initialization my $caughtup_query = diff --git a/src/test/subscription/t/004_sync.pl b/src/test/subscription/t/004_sync.pl new file mode 100644 index 0000000000..87541a0e6e --- /dev/null +++ b/src/test/subscription/t/004_sync.pl @@ -0,0 +1,159 @@ +# Tests for logical replication table syncing +use strict; +use warnings; +use PostgresNode; +use TestLib; +use Test::More tests => 7; + +# Initialize publisher node +my $node_publisher = get_new_node('publisher'); +$node_publisher->init(allows_streaming => 'logical'); +$node_publisher->start; + +# Create subscriber node +my $node_subscriber = get_new_node('subscriber'); +$node_subscriber->init(allows_streaming => 'logical'); +$node_subscriber->start; + +# Create some preexisting content on publisher +$node_publisher->safe_psql('postgres', + "CREATE TABLE tab_rep (a int primary key)"); +$node_publisher->safe_psql('postgres', + "INSERT INTO tab_rep SELECT generate_series(1,10)"); + +# Setup structure on subscriber +$node_subscriber->safe_psql('postgres', + "CREATE TABLE tab_rep (a int primary key)"); + +# Setup logical replication +my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; +$node_publisher->safe_psql('postgres', + "CREATE PUBLICATION tap_pub FOR ALL TABLES"); + +my $appname = 'tap_sub'; +$node_subscriber->safe_psql('postgres', + "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub"); + +# Wait for subscriber to finish initialization +my $caughtup_query = +"SELECT pg_current_wal_location() <= replay_location FROM pg_stat_replication WHERE application_name = '$appname';"; +$node_publisher->poll_query_until('postgres', $caughtup_query) + or die "Timed out while waiting for subscriber to catch up"; + +# Also wait for initial table sync to finish +my $synced_query = +"SELECT count(1) = 0 FROM pg_subscription_rel WHERE srsubstate NOT IN ('r', 's');"; +$node_subscriber->poll_query_until('postgres', $synced_query) + or die "Timed out while waiting for subscriber to synchronize data"; + +my $result = + $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM tab_rep"); +is($result, qq(10), 'initial data synced for first sub'); + +# drop subscription so that there is unreplicated data +$node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION tap_sub"); + +$node_publisher->safe_psql('postgres', + "INSERT INTO tab_rep SELECT generate_series(11,20)"); + +# recreate the subscription, it will try to do initial copy +$node_subscriber->safe_psql('postgres', + "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub"); + +# but it will be stuck on data copy as it will fail on constraint +my $started_query = +"SELECT srsubstate = 'd' FROM pg_subscription_rel;"; +$node_subscriber->poll_query_until('postgres', $started_query) + or die "Timed out while waiting for subscriber to start sync"; + +# remove the conflicting data +$node_subscriber->safe_psql('postgres', + "DELETE FROM tab_rep;"); + +# wait for sync to finish this time +$node_subscriber->poll_query_until('postgres', $synced_query) + or die "Timed out while waiting for subscriber to synchronize data"; + +# check that all data is synced +$result = + $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM tab_rep"); +is($result, qq(20), 'initial data synced for second sub'); + +# now check another subscription for the same node pair +$node_subscriber->safe_psql('postgres', + "CREATE SUBSCRIPTION tap_sub2 CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub WITH (NOCOPY DATA)"); + +# wait for it to start +$node_subscriber->poll_query_until('postgres', "SELECT pid IS NOT NULL FROM pg_stat_subscription WHERE subname = 'tap_sub2' AND relid IS NULL") + or die "Timed out while waiting for subscriber to start"; + +# and drop both subscriptions +$node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION tap_sub"); +$node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION tap_sub2"); + +# check subscriptions are removed +$result = + $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM pg_subscription"); +is($result, qq(0), 'second and third sub are dropped'); + +# remove the conflicting data +$node_subscriber->safe_psql('postgres', + "DELETE FROM tab_rep;"); + +# recreate the subscription again +$node_subscriber->safe_psql('postgres', + "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub"); + +# and wait for data sync to finish again +$node_subscriber->poll_query_until('postgres', $synced_query) + or die "Timed out while waiting for subscriber to synchronize data"; + +# check that all data is synced +$result = + $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM tab_rep"); +is($result, qq(20), 'initial data synced for fourth sub'); + +# add new table on subscriber +$node_subscriber->safe_psql('postgres', + "CREATE TABLE tab_rep_next (a int)"); + +# setup structure with existing data on pubisher +$node_publisher->safe_psql('postgres', + "CREATE TABLE tab_rep_next (a) AS SELECT generate_series(1,10)"); + +# Wait for subscription to catch up +$node_publisher->poll_query_until('postgres', $caughtup_query) + or die "Timed out while waiting for subscriber to catch up"; + +$result = + $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM tab_rep_next"); +is($result, qq(0), 'no data for table added after subscription initialized'); + +# ask for data sync +$node_subscriber->safe_psql('postgres', + "ALTER SUBSCRIPTION tap_sub REFRESH PUBLICATION"); + +# wait for sync to finish +$node_subscriber->poll_query_until('postgres', $synced_query) + or die "Timed out while waiting for subscriber to synchronize data"; + +$result = + $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM tab_rep_next"); +is($result, qq(10), 'data for table added after subscription initialized are now synced'); + +# Add some data +$node_publisher->safe_psql('postgres', + "INSERT INTO tab_rep_next SELECT generate_series(1,10)"); + +# Wait for subscription to catch up +$node_publisher->poll_query_until('postgres', $caughtup_query) + or die "Timed out while waiting for subscriber to catch up"; + +$result = + $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM tab_rep_next"); +is($result, qq(20), 'changes for table added after subscription initialized replicated'); + +$node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION tap_sub"); + +$node_subscriber->stop('fast'); +$node_publisher->stop('fast'); -- cgit v1.2.3 From 6912acc04f0bbcfdb799a120618507601e862490 Mon Sep 17 00:00:00 2001 From: Simon Riggs Date: Thu, 23 Mar 2017 14:05:28 +0000 Subject: Replication lag tracking for walsenders Adds write_lag, flush_lag and replay_lag cols to pg_stat_replication. Implements a lag tracker module that reports the lag times based upon measurements of the time taken for recent WAL to be written, flushed and replayed and for the sender to hear about it. These times represent the commit lag that was (or would have been) introduced by each synchronous commit level, if the remote server was configured as a synchronous standby. For an asynchronous standby, the replay_lag column approximates the delay before recent transactions became visible to queries. If the standby server has entirely caught up with the sending server and there is no more WAL activity, the most recently measured lag times will continue to be displayed for a short time and then show NULL. Physical replication lag tracking is automatic. Logical replication tracking is possible but is the responsibility of the logical decoding plugin. Tracking is a private module operating within each walsender individually, with values reported to shared memory. Module not used outside of walsender. Design and code is good enough now to commit - kudos to the author. In many ways a difficult topic, with important and subtle behaviour so this shoudl be expected to generate discussion and multiple open items: Test now! Author: Thomas Munro, following designs by Fujii Masao and Simon Riggs Review: Simon Riggs, Ian Barwick and Craig Ringer --- doc/src/sgml/monitoring.sgml | 69 +++++++ src/backend/access/transam/xlog.c | 14 ++ src/backend/catalog/system_views.sql | 3 + src/backend/replication/walsender.c | 277 +++++++++++++++++++++++++++- src/include/catalog/pg_proc.h | 2 +- src/include/replication/logical.h | 2 + src/include/replication/walsender_private.h | 5 + src/test/regress/expected/rules.out | 5 +- 8 files changed, 370 insertions(+), 7 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index eb6f486677..356a2f0c4c 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -1695,6 +1695,36 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i Last transaction log position replayed into the database on this standby server + + write_lag + interval + Time elapsed between flushing recent WAL locally and receiving + notification that this standby server has written it (but not yet + flushed it or applied it). This can be used to gauge the delay that + synchronous_commit level + remote_write incurred while committing if this + server was configured as a synchronous standby. + + + flush_lag + interval + Time elapsed between flushing recent WAL locally and receiving + notification that this standby server has written and flushed it + (but not yet applied it). This can be used to gauge the delay that + synchronous_commit level + remote_flush incurred while committing if this + server was configured as a synchronous standby. + + + replay_lag + interval + Time elapsed between flushing recent WAL locally and receiving + notification that this standby server has written, flushed and + applied it. This can be used to gauge the delay that + synchronous_commit level + remote_apply incurred while committing if this + server was configured as a synchronous standby. + sync_priority integer @@ -1745,6 +1775,45 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i listed; no information is available about downstream standby servers. + + The lag times reported in the pg_stat_replication + view are measurements of the time taken for recent WAL to be written, + flushed and replayed and for the sender to know about it. These times + represent the commit delay that was (or would have been) introduced by each + synchronous commit level, if the remote server was configured as a + synchronous standby. For an asynchronous standby, the + replay_lag column approximates the delay + before recent transactions became visible to queries. If the standby + server has entirely caught up with the sending server and there is no more + WAL activity, the most recently measured lag times will continue to be + displayed for a short time and then show NULL. + + + + Lag times work automatically for physical replication. Logical decoding + plugins may optionally emit tracking messages; if they do not, the tracking + mechanism will simply display NULL lag. + + + + + The reported lag times are not predictions of how long it will take for + the standby to catch up with the sending server assuming the current + rate of replay. Such a system would show similar times while new WAL is + being generated, but would differ when the sender becomes idle. In + particular, when the standby has caught up completely, + pg_stat_replication shows the time taken to + write, flush and replay the most recent reported WAL position rather than + zero as some users might expect. This is consistent with the goal of + measuring synchronous commit and transaction visibility delays for + recent write transactions. + To reduce confusion for users expecting a different model of lag, the + lag columns revert to NULL after a short time on a fully replayed idle + system. Monitoring systems should choose whether to represent this + as missing data, zero or continue to display the last known value. + + +
    <structname>pg_stat_wal_receiver</structname> View diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 3924738a33..de1937e013 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -11555,6 +11555,7 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, { static TimestampTz last_fail_time = 0; TimestampTz now; + bool streaming_reply_sent = false; /*------- * Standby mode is implemented by a state machine: @@ -11877,6 +11878,19 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, break; } + /* + * Since we have replayed everything we have received so + * far and are about to start waiting for more WAL, let's + * tell the upstream server our replay location now so + * that pg_stat_replication doesn't show stale + * information. + */ + if (!streaming_reply_sent) + { + WalRcvForceReply(); + streaming_reply_sent = true; + } + /* * Wait for more WAL to arrive. Time out after 5 seconds * to react to a trigger file promptly. diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 5723714fb9..80d14296de 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -705,6 +705,9 @@ CREATE VIEW pg_stat_replication AS W.write_location, W.flush_location, W.replay_location, + W.write_lag, + W.flush_lag, + W.replay_lag, W.sync_priority, W.sync_state FROM pg_stat_get_activity(NULL) AS S diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index c6ba916c49..a29d0e7cf4 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -190,6 +190,26 @@ static volatile sig_atomic_t replication_active = false; static LogicalDecodingContext *logical_decoding_ctx = NULL; static XLogRecPtr logical_startptr = InvalidXLogRecPtr; +/* A sample associating a log position with the time it was written. */ +typedef struct +{ + XLogRecPtr lsn; + TimestampTz time; +} WalTimeSample; + +/* The size of our buffer of time samples. */ +#define LAG_TRACKER_BUFFER_SIZE 8192 + +/* A mechanism for tracking replication lag. */ +static struct +{ + XLogRecPtr last_lsn; + WalTimeSample buffer[LAG_TRACKER_BUFFER_SIZE]; + int write_head; + int read_heads[NUM_SYNC_REP_WAIT_MODE]; + WalTimeSample last_read[NUM_SYNC_REP_WAIT_MODE]; +} LagTracker; + /* Signal handlers */ static void WalSndSigHupHandler(SIGNAL_ARGS); static void WalSndXLogSendHandler(SIGNAL_ARGS); @@ -221,6 +241,7 @@ static long WalSndComputeSleeptime(TimestampTz now); static void WalSndPrepareWrite(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId xid, bool last_write); static void WalSndWriteData(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId xid, bool last_write); static XLogRecPtr WalSndWaitForWal(XLogRecPtr loc); +static TimeOffset LagTrackerRead(int head, XLogRecPtr lsn, TimestampTz now); static void XLogRead(char *buf, XLogRecPtr startptr, Size count); @@ -246,6 +267,9 @@ InitWalSender(void) */ MarkPostmasterChildWalSender(); SendPostmasterSignal(PMSIGNAL_ADVANCE_STATE_MACHINE); + + /* Initialize empty timestamp buffer for lag tracking. */ + memset(&LagTracker, 0, sizeof(LagTracker)); } /* @@ -1646,6 +1670,13 @@ ProcessStandbyReplyMessage(void) flushPtr, applyPtr; bool replyRequested; + TimeOffset writeLag, + flushLag, + applyLag; + bool clearLagTimes; + TimestampTz now; + + static bool fullyAppliedLastTime = false; /* the caller already consumed the msgtype byte */ writePtr = pq_getmsgint64(&reply_message); @@ -1660,6 +1691,30 @@ ProcessStandbyReplyMessage(void) (uint32) (applyPtr >> 32), (uint32) applyPtr, replyRequested ? " (reply requested)" : ""); + /* See if we can compute the round-trip lag for these positions. */ + now = GetCurrentTimestamp(); + writeLag = LagTrackerRead(SYNC_REP_WAIT_WRITE, writePtr, now); + flushLag = LagTrackerRead(SYNC_REP_WAIT_FLUSH, flushPtr, now); + applyLag = LagTrackerRead(SYNC_REP_WAIT_APPLY, applyPtr, now); + + /* + * If the standby reports that it has fully replayed the WAL in two + * consecutive reply messages, then the second such message must result + * from wal_receiver_status_interval expiring on the standby. This is a + * convenient time to forget the lag times measured when it last + * wrote/flushed/applied a WAL record, to avoid displaying stale lag data + * until more WAL traffic arrives. + */ + clearLagTimes = false; + if (applyPtr == sentPtr) + { + if (fullyAppliedLastTime) + clearLagTimes = true; + fullyAppliedLastTime = true; + } + else + fullyAppliedLastTime = false; + /* Send a reply if the standby requested one. */ if (replyRequested) WalSndKeepalive(false); @@ -1675,6 +1730,12 @@ ProcessStandbyReplyMessage(void) walsnd->write = writePtr; walsnd->flush = flushPtr; walsnd->apply = applyPtr; + if (writeLag != -1 || clearLagTimes) + walsnd->writeLag = writeLag; + if (flushLag != -1 || clearLagTimes) + walsnd->flushLag = flushLag; + if (applyLag != -1 || clearLagTimes) + walsnd->applyLag = applyLag; SpinLockRelease(&walsnd->mutex); } @@ -2063,6 +2124,9 @@ InitWalSenderSlot(void) walsnd->write = InvalidXLogRecPtr; walsnd->flush = InvalidXLogRecPtr; walsnd->apply = InvalidXLogRecPtr; + walsnd->writeLag = -1; + walsnd->flushLag = -1; + walsnd->applyLag = -1; walsnd->state = WALSNDSTATE_STARTUP; walsnd->latch = &MyProc->procLatch; SpinLockRelease(&walsnd->mutex); @@ -2389,6 +2453,32 @@ XLogSendPhysical(void) SendRqstPtr = GetFlushRecPtr(); } + /* + * Record the current system time as an approximation of the time at which + * this WAL position was written for the purposes of lag tracking. + * + * In theory we could make XLogFlush() record a time in shmem whenever WAL + * is flushed and we could get that time as well as the LSN when we call + * GetFlushRecPtr() above (and likewise for the cascading standby + * equivalent), but rather than putting any new code into the hot WAL path + * it seems good enough to capture the time here. We should reach this + * after XLogFlush() runs WalSndWakeupProcessRequests(), and although that + * may take some time, we read the WAL flush pointer and take the time + * very close to together here so that we'll get a later position if it + * is still moving. + * + * Because LagTrackerWriter ignores samples when the LSN hasn't advanced, + * this gives us a cheap approximation for the WAL flush time for this + * LSN. + * + * Note that the LSN is not necessarily the LSN for the data contained in + * the present message; it's the end of the the WAL, which might be + * further ahead. All the lag tracking machinery cares about is finding + * out when that arbitrary LSN is eventually reported as written, flushed + * and applied, so that it can measure the elapsed time. + */ + LagTrackerWrite(SendRqstPtr, GetCurrentTimestamp()); + /* * If this is a historic timeline and we've reached the point where we * forked to the next timeline, stop streaming. @@ -2543,6 +2633,11 @@ XLogSendLogical(void) if (record != NULL) { + /* + * Note the lack of any call to LagTrackerWrite() which is the responsibility + * of the logical decoding plugin. Response messages are handled normally, + * so this responsibility does not extend to needing to call LagTrackerRead(). + */ LogicalDecodingProcessRecord(logical_decoding_ctx, logical_decoding_ctx->reader); sentPtr = logical_decoding_ctx->reader->EndRecPtr; @@ -2839,6 +2934,17 @@ WalSndGetStateString(WalSndState state) return "UNKNOWN"; } +static Interval * +offset_to_interval(TimeOffset offset) +{ + Interval *result = palloc(sizeof(Interval)); + + result->month = 0; + result->day = 0; + result->time = offset; + + return result; +} /* * Returns activity of walsenders, including pids and xlog locations sent to @@ -2847,7 +2953,7 @@ WalSndGetStateString(WalSndState state) Datum pg_stat_get_wal_senders(PG_FUNCTION_ARGS) { -#define PG_STAT_GET_WAL_SENDERS_COLS 8 +#define PG_STAT_GET_WAL_SENDERS_COLS 11 ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; TupleDesc tupdesc; Tuplestorestate *tupstore; @@ -2895,6 +3001,9 @@ pg_stat_get_wal_senders(PG_FUNCTION_ARGS) XLogRecPtr write; XLogRecPtr flush; XLogRecPtr apply; + TimeOffset writeLag; + TimeOffset flushLag; + TimeOffset applyLag; int priority; WalSndState state; Datum values[PG_STAT_GET_WAL_SENDERS_COLS]; @@ -2909,6 +3018,9 @@ pg_stat_get_wal_senders(PG_FUNCTION_ARGS) write = walsnd->write; flush = walsnd->flush; apply = walsnd->apply; + writeLag = walsnd->writeLag; + flushLag = walsnd->flushLag; + applyLag = walsnd->applyLag; priority = walsnd->sync_standby_priority; SpinLockRelease(&walsnd->mutex); @@ -2950,7 +3062,22 @@ pg_stat_get_wal_senders(PG_FUNCTION_ARGS) */ priority = XLogRecPtrIsInvalid(walsnd->flush) ? 0 : priority; - values[6] = Int32GetDatum(priority); + if (writeLag < 0) + nulls[6] = true; + else + values[6] = IntervalPGetDatum(offset_to_interval(writeLag)); + + if (flushLag < 0) + nulls[7] = true; + else + values[7] = IntervalPGetDatum(offset_to_interval(flushLag)); + + if (applyLag < 0) + nulls[8] = true; + else + values[8] = IntervalPGetDatum(offset_to_interval(applyLag)); + + values[9] = Int32GetDatum(priority); /* * More easily understood version of standby state. This is purely @@ -2964,12 +3091,12 @@ pg_stat_get_wal_senders(PG_FUNCTION_ARGS) * states. We report just "quorum" for them. */ if (priority == 0) - values[7] = CStringGetTextDatum("async"); + values[10] = CStringGetTextDatum("async"); else if (list_member_int(sync_standbys, i)) - values[7] = SyncRepConfig->syncrep_method == SYNC_REP_PRIORITY ? + values[10] = SyncRepConfig->syncrep_method == SYNC_REP_PRIORITY ? CStringGetTextDatum("sync") : CStringGetTextDatum("quorum"); else - values[7] = CStringGetTextDatum("potential"); + values[10] = CStringGetTextDatum("potential"); } tuplestore_putvalues(tupstore, tupdesc, values, nulls); @@ -3037,3 +3164,143 @@ WalSndKeepaliveIfNecessary(TimestampTz now) WalSndShutdown(); } } + +/* + * Record the end of the WAL and the time it was flushed locally, so that + * LagTrackerRead can compute the elapsed time (lag) when this WAL position is + * eventually reported to have been written, flushed and applied by the + * standby in a reply message. + * Exported to allow logical decoding plugins to call this when they choose. + */ +void +LagTrackerWrite(XLogRecPtr lsn, TimestampTz local_flush_time) +{ + bool buffer_full; + int new_write_head; + int i; + + if (!am_walsender) + return; + + /* + * If the lsn hasn't advanced since last time, then do nothing. This way + * we only record a new sample when new WAL has been written. + */ + if (LagTracker.last_lsn == lsn) + return; + LagTracker.last_lsn = lsn; + + /* + * If advancing the write head of the circular buffer would crash into any + * of the read heads, then the buffer is full. In other words, the + * slowest reader (presumably apply) is the one that controls the release + * of space. + */ + new_write_head = (LagTracker.write_head + 1) % LAG_TRACKER_BUFFER_SIZE; + buffer_full = false; + for (i = 0; i < NUM_SYNC_REP_WAIT_MODE; ++i) + { + if (new_write_head == LagTracker.read_heads[i]) + buffer_full = true; + } + + /* + * If the buffer is full, for now we just rewind by one slot and overwrite + * the last sample, as a simple (if somewhat uneven) way to lower the + * sampling rate. There may be better adaptive compaction algorithms. + */ + if (buffer_full) + { + new_write_head = LagTracker.write_head; + if (LagTracker.write_head > 0) + LagTracker.write_head--; + else + LagTracker.write_head = LAG_TRACKER_BUFFER_SIZE - 1; + } + + /* Store a sample at the current write head position. */ + LagTracker.buffer[LagTracker.write_head].lsn = lsn; + LagTracker.buffer[LagTracker.write_head].time = local_flush_time; + LagTracker.write_head = new_write_head; +} + +/* + * Find out how much time has elapsed between the moment WAL position 'lsn' + * (or the highest known earlier LSN) was flushed locally and the time 'now'. + * We have a separate read head for each of the reported LSN locations we + * receive in replies from standby; 'head' controls which read head is + * used. Whenever a read head crosses an LSN which was written into the + * lag buffer with LagTrackerWrite, we can use the associated timestamp to + * find out the time this LSN (or an earlier one) was flushed locally, and + * therefore compute the lag. + * + * Return -1 if no new sample data is available, and otherwise the elapsed + * time in microseconds. + */ +static TimeOffset +LagTrackerRead(int head, XLogRecPtr lsn, TimestampTz now) +{ + TimestampTz time = 0; + + /* Read all unread samples up to this LSN or end of buffer. */ + while (LagTracker.read_heads[head] != LagTracker.write_head && + LagTracker.buffer[LagTracker.read_heads[head]].lsn <= lsn) + { + time = LagTracker.buffer[LagTracker.read_heads[head]].time; + LagTracker.last_read[head] = + LagTracker.buffer[LagTracker.read_heads[head]]; + LagTracker.read_heads[head] = + (LagTracker.read_heads[head] + 1) % LAG_TRACKER_BUFFER_SIZE; + } + + if (time > now) + { + /* If the clock somehow went backwards, treat as not found. */ + return -1; + } + else if (time == 0) + { + /* + * We didn't cross a time. If there is a future sample that we + * haven't reached yet, and we've already reached at least one sample, + * let's interpolate the local flushed time. This is mainly useful for + * reporting a completely stuck apply position as having increasing + * lag, since otherwise we'd have to wait for it to eventually start + * moving again and cross one of our samples before we can show the + * lag increasing. + */ + if (LagTracker.read_heads[head] != LagTracker.write_head && + LagTracker.last_read[head].time != 0) + { + double fraction; + WalTimeSample prev = LagTracker.last_read[head]; + WalTimeSample next = LagTracker.buffer[LagTracker.read_heads[head]]; + + Assert(lsn >= prev.lsn); + Assert(prev.lsn < next.lsn); + + if (prev.time > next.time) + { + /* If the clock somehow went backwards, treat as not found. */ + return -1; + } + + /* See how far we are between the previous and next samples. */ + fraction = + (double) (lsn - prev.lsn) / (double) (next.lsn - prev.lsn); + + /* Scale the local flush time proportionally. */ + time = (TimestampTz) + ((double) prev.time + (next.time - prev.time) * fraction); + } + else + { + /* Couldn't interpolate due to lack of data. */ + return -1; + } + } + + /* Return the elapsed time since local flush time in microseconds. */ + Assert(time != 0); + return now - time; +} diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index 78c23e3f5d..a5b415346b 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -2804,7 +2804,7 @@ DATA(insert OID = 2022 ( pg_stat_get_activity PGNSP PGUID 12 1 100 0 0 f f f DESCR("statistics: information about currently active backends"); DATA(insert OID = 3318 ( pg_stat_get_progress_info PGNSP PGUID 12 1 100 0 0 f f f f t t s r 1 0 2249 "25" "{25,23,26,26,20,20,20,20,20,20,20,20,20,20}" "{i,o,o,o,o,o,o,o,o,o,o,o,o,o}" "{cmdtype,pid,datid,relid,param1,param2,param3,param4,param5,param6,param7,param8,param9,param10}" _null_ _null_ pg_stat_get_progress_info _null_ _null_ _null_ )); DESCR("statistics: information about progress of backends running maintenance command"); -DATA(insert OID = 3099 ( pg_stat_get_wal_senders PGNSP PGUID 12 1 10 0 0 f f f f f t s r 0 0 2249 "" "{23,25,3220,3220,3220,3220,23,25}" "{o,o,o,o,o,o,o,o}" "{pid,state,sent_location,write_location,flush_location,replay_location,sync_priority,sync_state}" _null_ _null_ pg_stat_get_wal_senders _null_ _null_ _null_ )); +DATA(insert OID = 3099 ( pg_stat_get_wal_senders PGNSP PGUID 12 1 10 0 0 f f f f f t s r 0 0 2249 "" "{23,25,3220,3220,3220,3220,1186,1186,1186,23,25}" "{o,o,o,o,o,o,o,o,o,o,o}" "{pid,state,sent_location,write_location,flush_location,replay_location,write_lag,flush_lag,replay_lag,sync_priority,sync_state}" _null_ _null_ pg_stat_get_wal_senders _null_ _null_ _null_ )); DESCR("statistics: information about currently active replication"); DATA(insert OID = 3317 ( pg_stat_get_wal_receiver PGNSP PGUID 12 1 0 0 0 f f f f f f s r 0 0 2249 "" "{23,25,3220,23,3220,23,1184,1184,3220,1184,25,25}" "{o,o,o,o,o,o,o,o,o,o,o,o}" "{pid,status,receive_start_lsn,receive_start_tli,received_lsn,received_tli,last_msg_send_time,last_msg_receipt_time,latest_end_lsn,latest_end_time,slot_name,conninfo}" _null_ _null_ pg_stat_get_wal_receiver _null_ _null_ _null_ )); DESCR("statistics: information about WAL receiver"); diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h index d10dd2c90a..7d6c88efe3 100644 --- a/src/include/replication/logical.h +++ b/src/include/replication/logical.h @@ -106,6 +106,8 @@ extern void LogicalIncreaseRestartDecodingForSlot(XLogRecPtr current_lsn, XLogRecPtr restart_lsn); extern void LogicalConfirmReceivedLocation(XLogRecPtr lsn); +extern void LagTrackerWrite(XLogRecPtr lsn, TimestampTz local_flush_time); + extern bool filter_by_origin_cb_wrapper(LogicalDecodingContext *ctx, RepOriginId origin_id); #endif diff --git a/src/include/replication/walsender_private.h b/src/include/replication/walsender_private.h index 5e6ccfc57b..2c59056cef 100644 --- a/src/include/replication/walsender_private.h +++ b/src/include/replication/walsender_private.h @@ -47,6 +47,11 @@ typedef struct WalSnd XLogRecPtr flush; XLogRecPtr apply; + /* Measured lag times, or -1 for unknown/none. */ + TimeOffset writeLag; + TimeOffset flushLag; + TimeOffset applyLag; + /* Protects shared variables shown above. */ slock_t mutex; diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index f7c3a637b5..c4c8450b83 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1831,10 +1831,13 @@ pg_stat_replication| SELECT s.pid, w.write_location, w.flush_location, w.replay_location, + w.write_lag, + w.flush_lag, + w.replay_lag, w.sync_priority, w.sync_state FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, ssl, sslversion, sslcipher, sslbits, sslcompression, sslclientdn) - JOIN pg_stat_get_wal_senders() w(pid, state, sent_location, write_location, flush_location, replay_location, sync_priority, sync_state) ON ((s.pid = w.pid))) + JOIN pg_stat_get_wal_senders() w(pid, state, sent_location, write_location, flush_location, replay_location, write_lag, flush_lag, replay_lag, sync_priority, sync_state) ON ((s.pid = w.pid))) LEFT JOIN pg_authid u ON ((s.usesysid = u.oid))); pg_stat_ssl| SELECT s.pid, s.ssl, -- cgit v1.2.3 From 50c956add83963d7bbb367dd0b879fccddebd623 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 23 Mar 2017 14:16:45 -0400 Subject: Remove createlang and droplang They have been deprecated since PostgreSQL 9.1. Reviewed-by: Magnus Hagander Reviewed-by: Daniel Gustafsson --- doc/src/sgml/installation.sgml | 12 +- doc/src/sgml/plperl.sgml | 3 +- doc/src/sgml/plpython.sgml | 3 +- doc/src/sgml/pltcl.sgml | 7 +- doc/src/sgml/ref/allfiles.sgml | 2 - doc/src/sgml/ref/create_function.sgml | 1 - doc/src/sgml/ref/create_language.sgml | 15 +- doc/src/sgml/ref/createlang.sgml | 291 ---------------------------------- doc/src/sgml/ref/drop_language.sgml | 1 - doc/src/sgml/ref/droplang.sgml | 288 --------------------------------- doc/src/sgml/reference.sgml | 2 - doc/src/sgml/release-9.1.sgml | 4 +- doc/src/sgml/xplang.sgml | 9 +- src/bin/scripts/.gitignore | 2 - src/bin/scripts/Makefile | 6 +- src/bin/scripts/createlang.c | 251 ----------------------------- src/bin/scripts/droplang.c | 250 ----------------------------- src/bin/scripts/nls.mk | 4 +- src/bin/scripts/t/030_createlang.pl | 25 --- src/bin/scripts/t/060_droplang.pl | 23 --- src/tools/msvc/Install.pm | 4 +- 21 files changed, 21 insertions(+), 1182 deletions(-) delete mode 100644 doc/src/sgml/ref/createlang.sgml delete mode 100644 doc/src/sgml/ref/droplang.sgml delete mode 100644 src/bin/scripts/createlang.c delete mode 100644 src/bin/scripts/droplang.c delete mode 100644 src/bin/scripts/t/030_createlang.pl delete mode 100644 src/bin/scripts/t/060_droplang.pl (limited to 'doc/src') diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml index 79201b78e3..f8a222e637 100644 --- a/doc/src/sgml/installation.sgml +++ b/doc/src/sgml/installation.sgml @@ -2256,17 +2256,17 @@ hosts=local4,bind4 memory management. You can have a server with many multiples of gigabytes of RAM free, but still get out of memory or address space errors when running applications. One example - is createlang failing with unusual errors. + is loading of extensions failing with unusual errors. For example, running as the owner of the PostgreSQL installation: --bash-3.00$ createlang plperl template1 -createlang: language installation failed: ERROR: could not load library "/opt/dbs/pgsql748/lib/plperl.so": A memory address is not in the address space for the process. +=# CREATE EXTENSION plperl; +ERROR: could not load library "/opt/dbs/pgsql/lib/plperl.so": A memory address is not in the address space for the process. Running as a non-owner in the group possessing the PostgreSQL installation: --bash-3.00$ createlang plperl template1 -createlang: language installation failed: ERROR: could not load library "/opt/dbs/pgsql748/lib/plperl.so": Bad address +=# CREATE EXTENSION plperl; +ERROR: could not load library "/opt/dbs/pgsql/lib/plperl.so": Bad address Another example is out of memory errors in the PostgreSQL server logs, with every memory allocation near or greater than 256 MB @@ -2284,7 +2284,7 @@ createlang: language installation failed: ERROR: could not load library "/opt/d - In the case of the createlang example, above, + In the case of the plperl example, above, check your umask and the permissions of the binaries in your PostgreSQL installation. The binaries involved in that example were 32-bit and installed as mode 750 instead of 755. Due to the diff --git a/doc/src/sgml/plperl.sgml b/doc/src/sgml/plperl.sgml index 9117769125..dd2ffbc6ce 100644 --- a/doc/src/sgml/plperl.sgml +++ b/doc/src/sgml/plperl.sgml @@ -27,8 +27,7 @@ To install PL/Perl in a particular database, use - CREATE EXTENSION plperl, or from the shell command line use - createlang plperl dbname. + CREATE EXTENSION plperl. diff --git a/doc/src/sgml/plpython.sgml b/doc/src/sgml/plpython.sgml index 46397781be..fb5d336efc 100644 --- a/doc/src/sgml/plpython.sgml +++ b/doc/src/sgml/plpython.sgml @@ -14,8 +14,7 @@ To install PL/Python in a particular database, use - CREATE EXTENSION plpythonu, or from the shell command line use - createlang plpythonu dbname (but + CREATE EXTENSION plpythonu (but see also ). diff --git a/doc/src/sgml/pltcl.sgml b/doc/src/sgml/pltcl.sgml index ed745a7481..ba4af2aec5 100644 --- a/doc/src/sgml/pltcl.sgml +++ b/doc/src/sgml/pltcl.sgml @@ -66,10 +66,9 @@ directory if Tcl support is specified in the configuration step of the installation procedure. To install PL/Tcl and/or PL/TclU in a particular database, use the - CREATE EXTENSION command or the - createlang program, for example - createlang pltcl dbname or - createlang pltclu dbname. + CREATE EXTENSION command, for example + CREATE EXTENSION pltcl or + CREATE EXTENSION pltclu. diff --git a/doc/src/sgml/ref/allfiles.sgml b/doc/src/sgml/ref/allfiles.sgml index 2bc4d9fd64..974e1b74e4 100644 --- a/doc/src/sgml/ref/allfiles.sgml +++ b/doc/src/sgml/ref/allfiles.sgml @@ -182,10 +182,8 @@ Complete list of usable sgml source files in this directory. - - diff --git a/doc/src/sgml/ref/create_function.sgml b/doc/src/sgml/ref/create_function.sgml index 9d0d2f4beb..3f86141f80 100644 --- a/doc/src/sgml/ref/create_function.sgml +++ b/doc/src/sgml/ref/create_function.sgml @@ -847,7 +847,6 @@ COMMIT; - diff --git a/doc/src/sgml/ref/create_language.sgml b/doc/src/sgml/ref/create_language.sgml index 41da16d977..75165b677f 100644 --- a/doc/src/sgml/ref/create_language.sgml +++ b/doc/src/sgml/ref/create_language.sgml @@ -230,21 +230,14 @@ CREATE [ OR REPLACE ] [ TRUSTED ] [ PROCEDURAL ] LANGUAGE program is a simple wrapper around - the CREATE LANGUAGE command. It eases - installation of procedural languages from the shell command line. - - - - Use , or better yet the program, to drop procedural languages. + Use to drop procedural languages. The system catalog pg_language (see ) records information about the - currently installed languages. Also, createlang - has an option to list the installed languages. + currently installed languages. Also, the psql + command \dL lists the installed languages. @@ -325,8 +318,6 @@ CREATE LANGUAGE plsample - - diff --git a/doc/src/sgml/ref/createlang.sgml b/doc/src/sgml/ref/createlang.sgml deleted file mode 100644 index e9c95d3278..0000000000 --- a/doc/src/sgml/ref/createlang.sgml +++ /dev/null @@ -1,291 +0,0 @@ - - - - - createlang - - - - createlang - 1 - Application - - - - createlang - install a PostgreSQL procedural language - - - - - createlang - connection-option - langname - dbname - - - - createlang - connection-option - - dbname - - - - - - Description - - - createlang is a utility for adding a - procedural language to a PostgreSQL database. - - - - createlang is just a wrapper around the - SQL command. - - - - - createlang is deprecated and may be removed - in a future PostgreSQL release. Direct use - of the CREATE EXTENSION command is recommended instead. - - - - - - - Options - - - createlang accepts the following command-line arguments: - - - - langname - - - Specifies the name of the procedural language to be - installed. (This name is lower-cased.) - - - - - - - - - - Specifies the database to which the language should be added. - The default is to use the database with the same name as the - current system user. - - - - - - - - - - Display SQL commands as they are executed. - - - - - - - - - - Show a list of already installed languages in the target database. - - - - - - - - - - Print the createlang version and exit. - - - - - - - - - - Show help about createlang command line - arguments, and exit. - - - - - - - - - createlang also accepts - the following command-line arguments for connection parameters: - - - - - - - - Specifies the host name of the machine on which the - server - is running. If the value begins with a slash, it is used - as the directory for the Unix domain socket. - - - - - - - - - - Specifies the TCP port or local Unix domain socket file - extension on which the server - is listening for connections. - - - - - - - - - - User name to connect as. - - - - - - - - - - Never issue a password prompt. If the server requires - password authentication and a password is not available by - other means such as a .pgpass file, the - connection attempt will fail. This option can be useful in - batch jobs and scripts where no user is present to enter a - password. - - - - - - - - - - Force createlang to prompt for a - password before connecting to a database. - - - - This option is never essential, since - createlang will automatically prompt - for a password if the server demands password authentication. - However, createlang will waste a - connection attempt finding out that the server wants a password. - In some cases it is worth typing - - - - - - - - - - Environment - - - - PGDATABASE - PGHOST - PGPORT - PGUSER - - - - Default connection parameters - - - - - - - This utility, like most other PostgreSQL utilities, - also uses the environment variables supported by libpq - (see ). - - - - - - - Diagnostics - - - Most error messages are self-explanatory. If not, run - createlang with the - option and see the respective SQL command - for details. Also, any default connection settings and environment - variables used by the libpq front-end - library will apply. - - - - - - Notes - - - Use to remove a language. - - - - - - Examples - - - To install the language pltcl into the database - template1: - -$ createlang pltcl template1 - - Note that installing the language into template1 - will cause it to be automatically installed into subsequently-created - databases as well. - - - - - See Also - - - - - - - - - diff --git a/doc/src/sgml/ref/drop_language.sgml b/doc/src/sgml/ref/drop_language.sgml index 0facc62876..f014a74d45 100644 --- a/doc/src/sgml/ref/drop_language.sgml +++ b/doc/src/sgml/ref/drop_language.sgml @@ -120,7 +120,6 @@ DROP LANGUAGE plsample; - diff --git a/doc/src/sgml/ref/droplang.sgml b/doc/src/sgml/ref/droplang.sgml deleted file mode 100644 index 86f05d6b05..0000000000 --- a/doc/src/sgml/ref/droplang.sgml +++ /dev/null @@ -1,288 +0,0 @@ - - - - - droplang - - - - droplang - 1 - Application - - - - droplang - remove a PostgreSQL procedural language - - - - - droplang - connection-option - langname - dbname - - - - droplang - connection-option - - dbname - - - - - - Description - - - - droplang is a utility for removing an - existing procedural language from a - PostgreSQL database. - - - - droplang is just a wrapper around the - SQL command. - - - - - droplang is deprecated and may be removed - in a future PostgreSQL release. Direct use - of the DROP EXTENSION command is recommended instead. - - - - - - - Options - - - droplang accepts the following command line arguments: - - - - langname - - - Specifies the name of the procedural language to be removed. - (This name is lower-cased.) - - - - - - - - - - Specifies from which database the language should be removed. - The default is to use the database with the same name as the - current system user. - - - - - - - - - - Display SQL commands as they are executed. - - - - - - - - - - Show a list of already installed languages in the target database. - - - - - - - - - - Print the droplang version and exit. - - - - - - - - - - Show help about droplang command line - arguments, and exit. - - - - - - - - - droplang also accepts - the following command line arguments for connection parameters: - - - - - - - - Specifies the host name of the machine on which the - server - is running. If host begins with a slash, it is used - as the directory for the Unix domain socket. - - - - - - - - - - Specifies the Internet TCP/IP port or local Unix domain socket file - extension on which the server - is listening for connections. - - - - - - - - - - User name to connect as. - - - - - - - - - - Never issue a password prompt. If the server requires - password authentication and a password is not available by - other means such as a .pgpass file, the - connection attempt will fail. This option can be useful in - batch jobs and scripts where no user is present to enter a - password. - - - - - - - - - - Force droplang to prompt for a - password before connecting to a database. - - - - This option is never essential, since - droplang will automatically prompt - for a password if the server demands password authentication. - However, droplang will waste a - connection attempt finding out that the server wants a password. - In some cases it is worth typing - - - - - - - - - - Environment - - - - PGDATABASE - PGHOST - PGPORT - PGUSER - - - - Default connection parameters - - - - - - - This utility, like most other PostgreSQL utilities, - also uses the environment variables supported by libpq - (see ). - - - - - - - Diagnostics - - - Most error messages are self-explanatory. If not, run - droplang with the - option and see under the respective SQL command - for details. Also, any default connection settings and environment - variables used by the libpq front-end - library will apply. - - - - - - Notes - - - Use to add a language. - - - - - - Examples - - - To remove the language pltcl: - -$ droplang pltcl dbname - - - - - See Also - - - - - - - - - diff --git a/doc/src/sgml/reference.sgml b/doc/src/sgml/reference.sgml index c8191de9fe..3d8ad232fa 100644 --- a/doc/src/sgml/reference.sgml +++ b/doc/src/sgml/reference.sgml @@ -233,10 +233,8 @@ &clusterdb; &createdb; - &createlang; &createuser; &dropdb; - &droplang; &dropuser; &ecpgRef; &pgBasebackup; diff --git a/doc/src/sgml/release-9.1.sgml b/doc/src/sgml/release-9.1.sgml index eb2d6ac278..2d0540c39e 100644 --- a/doc/src/sgml/release-9.1.sgml +++ b/doc/src/sgml/release-9.1.sgml @@ -10860,9 +10860,7 @@ SELECT * FROM places ORDER BY location <-> point '(101,456)' LIMIT 10; - Mark createlang - and droplang + Mark createlang and droplang as deprecated now that they just invoke extension commands (Tom Lane) diff --git a/doc/src/sgml/xplang.sgml b/doc/src/sgml/xplang.sgml index 9fa97d4c70..4460c8f361 100644 --- a/doc/src/sgml/xplang.sgml +++ b/doc/src/sgml/xplang.sgml @@ -56,14 +56,7 @@ For the languages supplied with the standard distribution, it is only necessary to execute CREATE EXTENSION language_name to install the language into the - current database. Alternatively, the program can be used to do this from the shell - command line. For example, to install the language - PL/Perl into the database - template1, use: - -createlang plperl template1 - + current database. The manual procedure described below is only recommended for installing languages that have not been packaged as extensions. diff --git a/src/bin/scripts/.gitignore b/src/bin/scripts/.gitignore index 40998d9670..0f23fe0004 100644 --- a/src/bin/scripts/.gitignore +++ b/src/bin/scripts/.gitignore @@ -1,9 +1,7 @@ /clusterdb /createdb -/createlang /createuser /dropdb -/droplang /dropuser /reindexdb /vacuumdb diff --git a/src/bin/scripts/Makefile b/src/bin/scripts/Makefile index 2f911e05ba..45ac492cfd 100644 --- a/src/bin/scripts/Makefile +++ b/src/bin/scripts/Makefile @@ -16,7 +16,7 @@ subdir = src/bin/scripts top_builddir = ../../.. include $(top_builddir)/src/Makefile.global -PROGRAMS = createdb createlang createuser dropdb droplang dropuser clusterdb vacuumdb reindexdb pg_isready +PROGRAMS = createdb createuser dropdb dropuser clusterdb vacuumdb reindexdb pg_isready override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS) LDFLAGS += -L$(top_builddir)/src/fe_utils -lpgfeutils -lpq @@ -27,10 +27,8 @@ all: $(PROGRAMS) $(CC) $(CFLAGS) $^ $(libpq_pgport) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) createdb: createdb.o common.o | submake-libpq submake-libpgport submake-libpgfeutils -createlang: createlang.o common.o | submake-libpq submake-libpgport submake-libpgfeutils createuser: createuser.o common.o | submake-libpq submake-libpgport submake-libpgfeutils dropdb: dropdb.o common.o | submake-libpq submake-libpgport submake-libpgfeutils -droplang: droplang.o common.o | submake-libpq submake-libpgport submake-libpgfeutils dropuser: dropuser.o common.o | submake-libpq submake-libpgport submake-libpgfeutils clusterdb: clusterdb.o common.o | submake-libpq submake-libpgport submake-libpgfeutils vacuumdb: vacuumdb.o common.o | submake-libpq submake-libpgport submake-libpgfeutils @@ -40,8 +38,6 @@ pg_isready: pg_isready.o common.o | submake-libpq submake-libpgport submake-libp install: all installdirs $(INSTALL_PROGRAM) createdb$(X) '$(DESTDIR)$(bindir)'/createdb$(X) $(INSTALL_PROGRAM) dropdb$(X) '$(DESTDIR)$(bindir)'/dropdb$(X) - $(INSTALL_PROGRAM) createlang$(X) '$(DESTDIR)$(bindir)'/createlang$(X) - $(INSTALL_PROGRAM) droplang$(X) '$(DESTDIR)$(bindir)'/droplang$(X) $(INSTALL_PROGRAM) createuser$(X) '$(DESTDIR)$(bindir)'/createuser$(X) $(INSTALL_PROGRAM) dropuser$(X) '$(DESTDIR)$(bindir)'/dropuser$(X) $(INSTALL_PROGRAM) clusterdb$(X) '$(DESTDIR)$(bindir)'/clusterdb$(X) diff --git a/src/bin/scripts/createlang.c b/src/bin/scripts/createlang.c deleted file mode 100644 index 5897bfe747..0000000000 --- a/src/bin/scripts/createlang.c +++ /dev/null @@ -1,251 +0,0 @@ -/*------------------------------------------------------------------------- - * - * createlang - * - * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * src/bin/scripts/createlang.c - * - *------------------------------------------------------------------------- - */ -#include "postgres_fe.h" - -#include "common.h" -#include "fe_utils/print.h" - -static void help(const char *progname); - - -int -main(int argc, char *argv[]) -{ - static struct option long_options[] = { - {"list", no_argument, NULL, 'l'}, - {"host", required_argument, NULL, 'h'}, - {"port", required_argument, NULL, 'p'}, - {"username", required_argument, NULL, 'U'}, - {"no-password", no_argument, NULL, 'w'}, - {"password", no_argument, NULL, 'W'}, - {"dbname", required_argument, NULL, 'd'}, - {"echo", no_argument, NULL, 'e'}, - {NULL, 0, NULL, 0} - }; - - const char *progname; - int optindex; - int c; - - bool listlangs = false; - const char *dbname = NULL; - char *host = NULL; - char *port = NULL; - char *username = NULL; - enum trivalue prompt_password = TRI_DEFAULT; - bool echo = false; - char *langname = NULL; - - char *p; - - PQExpBufferData sql; - - PGconn *conn; - PGresult *result; - - progname = get_progname(argv[0]); - set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pgscripts")); - - handle_help_version_opts(argc, argv, "createlang", help); - - while ((c = getopt_long(argc, argv, "lh:p:U:wWd:e", long_options, &optindex)) != -1) - { - switch (c) - { - case 'l': - listlangs = true; - break; - case 'h': - host = pg_strdup(optarg); - break; - case 'p': - port = pg_strdup(optarg); - break; - case 'U': - username = pg_strdup(optarg); - break; - case 'w': - prompt_password = TRI_NO; - break; - case 'W': - prompt_password = TRI_YES; - break; - case 'd': - dbname = pg_strdup(optarg); - break; - case 'e': - echo = true; - break; - default: - fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); - exit(1); - } - } - - /* - * We set dbname from positional arguments if it is not already set by - * option arguments -d. If not doing listlangs, positional dbname must - * follow positional langname. - */ - - if (argc - optind > 0) - { - if (listlangs) - { - if (dbname == NULL) - dbname = argv[optind++]; - } - else - { - langname = argv[optind++]; - if (argc - optind > 0 && dbname == NULL) - dbname = argv[optind++]; - } - } - - if (argc - optind > 0) - { - fprintf(stderr, _("%s: too many command-line arguments (first is \"%s\")\n"), - progname, argv[optind]); - fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); - exit(1); - } - - if (dbname == NULL) - { - if (getenv("PGDATABASE")) - dbname = getenv("PGDATABASE"); - else if (getenv("PGUSER")) - dbname = getenv("PGUSER"); - else - dbname = get_user_name_or_exit(progname); - } - - initPQExpBuffer(&sql); - - /* - * List option - */ - if (listlangs) - { - printQueryOpt popt; - static const bool translate_columns[] = {false, true}; - - conn = connectDatabase(dbname, host, port, username, prompt_password, - progname, false, false); - - printfPQExpBuffer(&sql, "SELECT lanname as \"%s\", " - "(CASE WHEN lanpltrusted THEN '%s' ELSE '%s' END) as \"%s\" " - "FROM pg_catalog.pg_language WHERE lanispl;", - gettext_noop("Name"), - gettext_noop("yes"), gettext_noop("no"), - gettext_noop("Trusted?")); - result = executeQuery(conn, sql.data, progname, echo); - - memset(&popt, 0, sizeof(popt)); - popt.topt.format = PRINT_ALIGNED; - popt.topt.border = 1; - popt.topt.start_table = true; - popt.topt.stop_table = true; - popt.topt.encoding = PQclientEncoding(conn); - popt.title = _("Procedural Languages"); - popt.translate_header = true; - popt.translate_columns = translate_columns; - popt.n_translate_columns = lengthof(translate_columns); - - printQuery(result, &popt, stdout, false, NULL); - - PQfinish(conn); - exit(0); - } - - if (langname == NULL) - { - fprintf(stderr, _("%s: missing required argument language name\n"), progname); - fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); - exit(1); - } - - /* lower case language name */ - for (p = langname; *p; p++) - if (*p >= 'A' && *p <= 'Z') - *p += ('a' - 'A'); - - conn = connectDatabase(dbname, host, port, username, prompt_password, - progname, false, false); - - /* - * Make sure the language isn't already installed - */ - printfPQExpBuffer(&sql, - "SELECT oid FROM pg_catalog.pg_language WHERE lanname = '%s';", - langname); - result = executeQuery(conn, sql.data, progname, echo); - if (PQntuples(result) > 0) - { - fprintf(stderr, - _("%s: language \"%s\" is already installed in database \"%s\"\n"), - progname, langname, PQdb(conn)); - PQfinish(conn); - /* separate exit status for "already installed" */ - exit(2); - } - PQclear(result); - - /* - * In 9.1 and up, assume that languages should be installed using CREATE - * EXTENSION. However, it's possible this tool could be used against an - * older server, and it's easy enough to continue supporting the old way. - */ - if (PQserverVersion(conn) >= 90100) - printfPQExpBuffer(&sql, "CREATE EXTENSION \"%s\";", langname); - else - printfPQExpBuffer(&sql, "CREATE LANGUAGE \"%s\";", langname); - - if (echo) - printf("%s\n", sql.data); - result = PQexec(conn, sql.data); - if (PQresultStatus(result) != PGRES_COMMAND_OK) - { - fprintf(stderr, _("%s: language installation failed: %s"), - progname, PQerrorMessage(conn)); - PQfinish(conn); - exit(1); - } - - PQclear(result); - PQfinish(conn); - exit(0); -} - - - -static void -help(const char *progname) -{ - printf(_("%s installs a procedural language into a PostgreSQL database.\n\n"), progname); - printf(_("Usage:\n")); - printf(_(" %s [OPTION]... LANGNAME [DBNAME]\n"), progname); - printf(_("\nOptions:\n")); - printf(_(" -d, --dbname=DBNAME database to install language in\n")); - printf(_(" -e, --echo show the commands being sent to the server\n")); - printf(_(" -l, --list show a list of currently installed languages\n")); - printf(_(" -V, --version output version information, then exit\n")); - printf(_(" -?, --help show this help, then exit\n")); - printf(_("\nConnection options:\n")); - printf(_(" -h, --host=HOSTNAME database server host or socket directory\n")); - printf(_(" -p, --port=PORT database server port\n")); - printf(_(" -U, --username=USERNAME user name to connect as\n")); - printf(_(" -w, --no-password never prompt for password\n")); - printf(_(" -W, --password force password prompt\n")); - printf(_("\nReport bugs to .\n")); -} diff --git a/src/bin/scripts/droplang.c b/src/bin/scripts/droplang.c deleted file mode 100644 index 97d1de43ae..0000000000 --- a/src/bin/scripts/droplang.c +++ /dev/null @@ -1,250 +0,0 @@ -/*------------------------------------------------------------------------- - * - * droplang - * - * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * src/bin/scripts/droplang.c - * - *------------------------------------------------------------------------- - */ -#include "postgres_fe.h" - -#include "common.h" -#include "fe_utils/print.h" - - -static void help(const char *progname); - - -int -main(int argc, char *argv[]) -{ - static struct option long_options[] = { - {"list", no_argument, NULL, 'l'}, - {"host", required_argument, NULL, 'h'}, - {"port", required_argument, NULL, 'p'}, - {"username", required_argument, NULL, 'U'}, - {"no-password", no_argument, NULL, 'w'}, - {"password", no_argument, NULL, 'W'}, - {"dbname", required_argument, NULL, 'd'}, - {"echo", no_argument, NULL, 'e'}, - {NULL, 0, NULL, 0} - }; - - const char *progname; - int optindex; - int c; - bool listlangs = false; - const char *dbname = NULL; - char *host = NULL; - char *port = NULL; - char *username = NULL; - enum trivalue prompt_password = TRI_DEFAULT; - bool echo = false; - char *langname = NULL; - char *p; - PQExpBufferData sql; - PGconn *conn; - PGresult *result; - - progname = get_progname(argv[0]); - set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pgscripts")); - - handle_help_version_opts(argc, argv, "droplang", help); - - while ((c = getopt_long(argc, argv, "lh:p:U:wWd:e", long_options, &optindex)) != -1) - { - switch (c) - { - case 'l': - listlangs = true; - break; - case 'h': - host = pg_strdup(optarg); - break; - case 'p': - port = pg_strdup(optarg); - break; - case 'U': - username = pg_strdup(optarg); - break; - case 'w': - prompt_password = TRI_NO; - break; - case 'W': - prompt_password = TRI_YES; - break; - case 'd': - dbname = pg_strdup(optarg); - break; - case 'e': - echo = true; - break; - default: - fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); - exit(1); - } - } - - /* - * We set dbname from positional arguments if it is not already set by - * option arguments -d. If not doing listlangs, positional dbname must - * follow positional langname. - */ - - if (argc - optind > 0) - { - if (listlangs) - { - if (dbname == NULL) - dbname = argv[optind++]; - } - else - { - langname = argv[optind++]; - if (argc - optind > 0 && dbname == NULL) - dbname = argv[optind++]; - } - } - - if (argc - optind > 0) - { - fprintf(stderr, _("%s: too many command-line arguments (first is \"%s\")\n"), - progname, argv[optind]); - fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); - exit(1); - } - - if (dbname == NULL) - { - if (getenv("PGDATABASE")) - dbname = getenv("PGDATABASE"); - else if (getenv("PGUSER")) - dbname = getenv("PGUSER"); - else - dbname = get_user_name_or_exit(progname); - } - - initPQExpBuffer(&sql); - - /* - * List option - */ - if (listlangs) - { - printQueryOpt popt; - static const bool translate_columns[] = {false, true}; - - conn = connectDatabase(dbname, host, port, username, prompt_password, - progname, false, false); - - printfPQExpBuffer(&sql, "SELECT lanname as \"%s\", " - "(CASE WHEN lanpltrusted THEN '%s' ELSE '%s' END) as \"%s\" " - "FROM pg_catalog.pg_language WHERE lanispl;", - gettext_noop("Name"), - gettext_noop("yes"), gettext_noop("no"), - gettext_noop("Trusted?")); - result = executeQuery(conn, sql.data, progname, echo); - - memset(&popt, 0, sizeof(popt)); - popt.topt.format = PRINT_ALIGNED; - popt.topt.border = 1; - popt.topt.start_table = true; - popt.topt.stop_table = true; - popt.topt.encoding = PQclientEncoding(conn); - popt.title = _("Procedural Languages"); - popt.translate_header = true; - popt.translate_columns = translate_columns; - popt.n_translate_columns = lengthof(translate_columns); - - printQuery(result, &popt, stdout, false, NULL); - - PQfinish(conn); - exit(0); - } - - if (langname == NULL) - { - fprintf(stderr, _("%s: missing required argument language name\n"), - progname); - fprintf(stderr, _("Try \"%s --help\" for more information.\n"), - progname); - exit(1); - } - - /* lower case language name */ - for (p = langname; *p; p++) - if (*p >= 'A' && *p <= 'Z') - *p += ('a' - 'A'); - - conn = connectDatabase(dbname, host, port, username, prompt_password, - progname, false, false); - - /* - * Force schema search path to be just pg_catalog, so that we don't have - * to be paranoid about search paths below. - */ - executeCommand(conn, "SET search_path = pg_catalog;", progname, echo); - - /* - * Make sure the language is installed - */ - printfPQExpBuffer(&sql, "SELECT oid " - "FROM pg_language WHERE lanname = '%s' AND lanispl;", - langname); - result = executeQuery(conn, sql.data, progname, echo); - if (PQntuples(result) == 0) - { - fprintf(stderr, _("%s: language \"%s\" is not installed in " - "database \"%s\"\n"), - progname, langname, PQdb(conn)); - PQfinish(conn); - exit(1); - } - PQclear(result); - - /* - * Attempt to drop the language. We do not use CASCADE, so that the drop - * will fail if there are any functions in the language. - */ - printfPQExpBuffer(&sql, "DROP EXTENSION \"%s\";", langname); - - if (echo) - printf("%s\n", sql.data); - result = PQexec(conn, sql.data); - if (PQresultStatus(result) != PGRES_COMMAND_OK) - { - fprintf(stderr, _("%s: language removal failed: %s"), - progname, PQerrorMessage(conn)); - PQfinish(conn); - exit(1); - } - - PQclear(result); - PQfinish(conn); - exit(0); -} - - -static void -help(const char *progname) -{ - printf(_("%s removes a procedural language from a database.\n\n"), progname); - printf(_("Usage:\n")); - printf(_(" %s [OPTION]... LANGNAME [DBNAME]\n"), progname); - printf(_("\nOptions:\n")); - printf(_(" -d, --dbname=DBNAME database from which to remove the language\n")); - printf(_(" -e, --echo show the commands being sent to the server\n")); - printf(_(" -l, --list show a list of currently installed languages\n")); - printf(_(" -V, --version output version information, then exit\n")); - printf(_(" -?, --help show this help, then exit\n")); - printf(_("\nConnection options:\n")); - printf(_(" -h, --host=HOSTNAME database server host or socket directory\n")); - printf(_(" -p, --port=PORT database server port\n")); - printf(_(" -U, --username=USERNAME user name to connect as\n")); - printf(_(" -w, --no-password never prompt for password\n")); - printf(_(" -W, --password force password prompt\n")); - printf(_("\nReport bugs to .\n")); -} diff --git a/src/bin/scripts/nls.mk b/src/bin/scripts/nls.mk index 9156d08a80..ef64083f5f 100644 --- a/src/bin/scripts/nls.mk +++ b/src/bin/scripts/nls.mk @@ -1,8 +1,8 @@ # src/bin/scripts/nls.mk CATALOG_NAME = pgscripts AVAIL_LANGUAGES = cs de es fr it ja ko pl pt_BR ru sv zh_CN -GETTEXT_FILES = createdb.c createlang.c createuser.c \ - dropdb.c droplang.c dropuser.c \ +GETTEXT_FILES = createdb.c createuser.c \ + dropdb.c dropuser.c \ clusterdb.c vacuumdb.c reindexdb.c \ pg_isready.c \ common.c \ diff --git a/src/bin/scripts/t/030_createlang.pl b/src/bin/scripts/t/030_createlang.pl deleted file mode 100644 index ffbd35dcc5..0000000000 --- a/src/bin/scripts/t/030_createlang.pl +++ /dev/null @@ -1,25 +0,0 @@ -use strict; -use warnings; - -use PostgresNode; -use TestLib; -use Test::More tests => 14; - -program_help_ok('createlang'); -program_version_ok('createlang'); -program_options_handling_ok('createlang'); - -my $node = get_new_node('main'); -$node->init; -$node->start; - -$node->command_fails([ 'createlang', 'plpgsql' ], - 'fails if language already exists'); - -$node->safe_psql('postgres', 'DROP EXTENSION plpgsql'); -$node->issues_sql_like( - [ 'createlang', 'plpgsql' ], - qr/statement: CREATE EXTENSION "plpgsql"/, - 'SQL CREATE EXTENSION run'); - -$node->command_like([ 'createlang', '--list' ], qr/plpgsql/, 'list output'); diff --git a/src/bin/scripts/t/060_droplang.pl b/src/bin/scripts/t/060_droplang.pl deleted file mode 100644 index 904cea2d4b..0000000000 --- a/src/bin/scripts/t/060_droplang.pl +++ /dev/null @@ -1,23 +0,0 @@ -use strict; -use warnings; - -use PostgresNode; -use TestLib; -use Test::More tests => 11; - -program_help_ok('droplang'); -program_version_ok('droplang'); -program_options_handling_ok('droplang'); - -my $node = get_new_node('main'); -$node->init; -$node->start; - -$node->issues_sql_like( - [ 'droplang', 'plpgsql', 'postgres' ], - qr/statement: DROP EXTENSION "plpgsql"/, - 'SQL DROP EXTENSION run'); - -$node->command_fails( - [ 'droplang', 'nonexistent', 'postgres' ], - 'fails with nonexistent language'); diff --git a/src/tools/msvc/Install.pm b/src/tools/msvc/Install.pm index 0aa1422b9f..b81f4dd809 100644 --- a/src/tools/msvc/Install.pm +++ b/src/tools/msvc/Install.pm @@ -20,8 +20,8 @@ our (@ISA, @EXPORT_OK); my $insttype; my @client_contribs = ('oid2name', 'pgbench', 'vacuumlo'); my @client_program_files = ( - 'clusterdb', 'createdb', 'createlang', 'createuser', - 'dropdb', 'droplang', 'dropuser', 'ecpg', + 'clusterdb', 'createdb', 'createuser', + 'dropdb', 'dropuser', 'ecpg', 'libecpg', 'libecpg_compat', 'libpgtypes', 'libpq', 'pg_basebackup', 'pg_config', 'pg_dump', 'pg_dumpall', 'pg_isready', 'pg_receivewal', 'pg_recvlogical', 'pg_restore', -- cgit v1.2.3 From ea42cc18c35381f639d45628d792e790ff39e271 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Thu, 23 Mar 2017 14:08:23 -0400 Subject: Track the oldest XID that can be safely looked up in CLOG. This provides infrastructure for looking up arbitrary, user-supplied XIDs without a risk of scary-looking failures from within the clog module. Normally, the oldest XID that can be safely looked up in CLOG is the same as the oldest XID that can reused without causing wraparound, and the latter is already tracked. However, while truncation is in progress, the values are different, so we must keep track of them separately. Craig Ringer, reviewed by Simon Riggs and by me. Discussion: http://postgr.es/m/CAMsr+YHQiWNEi0daCTboS40T+V5s_+dst3PYv_8v2wNVH+Xx4g@mail.gmail.com --- doc/src/sgml/monitoring.sgml | 4 +++ src/backend/access/rmgrdesc/clogdesc.c | 12 +++++++-- src/backend/access/transam/clog.c | 46 +++++++++++++++++++++++++------- src/backend/access/transam/transam.c | 4 +-- src/backend/access/transam/varsup.c | 23 +++++++++++++++- src/backend/access/transam/xlog.c | 11 ++++++++ src/backend/commands/vacuum.c | 2 +- src/backend/storage/lmgr/lwlocknames.txt | 1 + src/include/access/clog.h | 8 +++++- src/include/access/transam.h | 7 +++++ src/include/access/xlog_internal.h | 2 +- 11 files changed, 102 insertions(+), 18 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 356a2f0c4c..e930731b16 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -1017,6 +1017,10 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser OldSnapshotTimeMapLock Waiting to read or update old snapshot control information. + + CLogTruncationLock + Waiting to truncate the transaction log or waiting for transaction log truncation to finish. + clog Waiting for I/O on a clog (transaction status) buffer. diff --git a/src/backend/access/rmgrdesc/clogdesc.c b/src/backend/access/rmgrdesc/clogdesc.c index 352de48dbe..ef268c5ab3 100644 --- a/src/backend/access/rmgrdesc/clogdesc.c +++ b/src/backend/access/rmgrdesc/clogdesc.c @@ -23,12 +23,20 @@ clog_desc(StringInfo buf, XLogReaderState *record) char *rec = XLogRecGetData(record); uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; - if (info == CLOG_ZEROPAGE || info == CLOG_TRUNCATE) + if (info == CLOG_ZEROPAGE) { int pageno; memcpy(&pageno, rec, sizeof(int)); - appendStringInfo(buf, "%d", pageno); + appendStringInfo(buf, "page %d", pageno); + } + else if (info == CLOG_TRUNCATE) + { + xl_clog_truncate xlrec; + + memcpy(&xlrec, rec, sizeof(xl_clog_truncate)); + appendStringInfo(buf, "page %d; oldestXact %u", + xlrec.pageno, xlrec.oldestXact); } } diff --git a/src/backend/access/transam/clog.c b/src/backend/access/transam/clog.c index 5b1d13dac1..2d33510930 100644 --- a/src/backend/access/transam/clog.c +++ b/src/backend/access/transam/clog.c @@ -83,7 +83,8 @@ static SlruCtlData ClogCtlData; static int ZeroCLOGPage(int pageno, bool writeXlog); static bool CLOGPagePrecedes(int page1, int page2); static void WriteZeroPageXlogRec(int pageno); -static void WriteTruncateXlogRec(int pageno); +static void WriteTruncateXlogRec(int pageno, TransactionId oldestXact, + Oid oldestXidDb); static void TransactionIdSetPageStatus(TransactionId xid, int nsubxids, TransactionId *subxids, XidStatus status, XLogRecPtr lsn, int pageno); @@ -640,7 +641,7 @@ ExtendCLOG(TransactionId newestXact) * the XLOG flush unless we have confirmed that there is a removable segment. */ void -TruncateCLOG(TransactionId oldestXact) +TruncateCLOG(TransactionId oldestXact, Oid oldestxid_datoid) { int cutoffPage; @@ -654,8 +655,26 @@ TruncateCLOG(TransactionId oldestXact) if (!SlruScanDirectory(ClogCtl, SlruScanDirCbReportPresence, &cutoffPage)) return; /* nothing to remove */ - /* Write XLOG record and flush XLOG to disk */ - WriteTruncateXlogRec(cutoffPage); + /* + * Advance oldestClogXid before truncating clog, so concurrent xact status + * lookups can ensure they don't attempt to access truncated-away clog. + * + * It's only necessary to do this if we will actually truncate away clog + * pages. + */ + AdvanceOldestClogXid(oldestXact); + + /* vac_truncate_clog already advanced oldestXid */ + Assert(TransactionIdPrecedesOrEquals(oldestXact, + ShmemVariableCache->oldestXid)); + + /* + * Write XLOG record and flush XLOG to disk. We record the oldest xid we're + * keeping information about here so we can ensure that it's always ahead + * of clog truncation in case we crash, and so a standby finds out the new + * valid xid before the next checkpoint. + */ + WriteTruncateXlogRec(cutoffPage, oldestXact, oldestxid_datoid); /* Now we can remove the old CLOG segment(s) */ SimpleLruTruncate(ClogCtl, cutoffPage); @@ -704,12 +723,17 @@ WriteZeroPageXlogRec(int pageno) * in TruncateCLOG(). */ static void -WriteTruncateXlogRec(int pageno) +WriteTruncateXlogRec(int pageno, TransactionId oldestXact, Oid oldestXactDb) { XLogRecPtr recptr; + xl_clog_truncate xlrec; + + xlrec.pageno = pageno; + xlrec.oldestXact = oldestXact; + xlrec.oldestXactDb = oldestXactDb; XLogBeginInsert(); - XLogRegisterData((char *) (&pageno), sizeof(int)); + XLogRegisterData((char *) (&xlrec), sizeof(xl_clog_truncate)); recptr = XLogInsert(RM_CLOG_ID, CLOG_TRUNCATE); XLogFlush(recptr); } @@ -742,17 +766,19 @@ clog_redo(XLogReaderState *record) } else if (info == CLOG_TRUNCATE) { - int pageno; + xl_clog_truncate xlrec; - memcpy(&pageno, XLogRecGetData(record), sizeof(int)); + memcpy(&xlrec, XLogRecGetData(record), sizeof(xl_clog_truncate)); /* * During XLOG replay, latest_page_number isn't set up yet; insert a * suitable value to bypass the sanity test in SimpleLruTruncate. */ - ClogCtl->shared->latest_page_number = pageno; + ClogCtl->shared->latest_page_number = xlrec.pageno; + + AdvanceOldestClogXid(xlrec.oldestXact); - SimpleLruTruncate(ClogCtl, pageno); + SimpleLruTruncate(ClogCtl, xlrec.pageno); } else elog(PANIC, "clog_redo: unknown op code %u", info); diff --git a/src/backend/access/transam/transam.c b/src/backend/access/transam/transam.c index b91a259e80..562b53be9a 100644 --- a/src/backend/access/transam/transam.c +++ b/src/backend/access/transam/transam.c @@ -119,7 +119,7 @@ TransactionLogFetch(TransactionId transactionId) * True iff transaction associated with the identifier did commit. * * Note: - * Assumes transaction identifier is valid. + * Assumes transaction identifier is valid and exists in clog. */ bool /* true if given transaction committed */ TransactionIdDidCommit(TransactionId transactionId) @@ -175,7 +175,7 @@ TransactionIdDidCommit(TransactionId transactionId) * True iff transaction associated with the identifier did abort. * * Note: - * Assumes transaction identifier is valid. + * Assumes transaction identifier is valid and exists in clog. */ bool /* true if given transaction aborted */ TransactionIdDidAbort(TransactionId transactionId) diff --git a/src/backend/access/transam/varsup.c b/src/backend/access/transam/varsup.c index 42fc351f7b..5efbfbd3d6 100644 --- a/src/backend/access/transam/varsup.c +++ b/src/backend/access/transam/varsup.c @@ -259,7 +259,28 @@ ReadNewTransactionId(void) } /* - * Determine the last safe XID to allocate given the currently oldest + * Advance the cluster-wide value for the oldest valid clog entry. + * + * We must acquire CLogTruncationLock to advance the oldestClogXid. It's not + * necessary to hold the lock during the actual clog truncation, only when we + * advance the limit, as code looking up arbitrary xids is required to hold + * CLogTruncationLock from when it tests oldestClogXid through to when it + * completes the clog lookup. + */ +void +AdvanceOldestClogXid(TransactionId oldest_datfrozenxid) +{ + LWLockAcquire(CLogTruncationLock, LW_EXCLUSIVE); + if (TransactionIdPrecedes(ShmemVariableCache->oldestClogXid, + oldest_datfrozenxid)) + { + ShmemVariableCache->oldestClogXid = oldest_datfrozenxid; + } + LWLockRelease(CLogTruncationLock); +} + +/* + * Determine the last safe XID to allocate using the currently oldest * datfrozenxid (ie, the oldest XID that might exist in any database * of our cluster), and the OID of the (or a) database with that value. */ diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index de1937e013..b99ded5df6 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -5016,6 +5016,7 @@ BootStrapXLOG(void) ShmemVariableCache->nextOid = checkPoint.nextOid; ShmemVariableCache->oidCount = 0; MultiXactSetNextMXact(checkPoint.nextMulti, checkPoint.nextMultiOffset); + AdvanceOldestClogXid(checkPoint.oldestXid); SetTransactionIdLimit(checkPoint.oldestXid, checkPoint.oldestXidDB); SetMultiXactIdLimit(checkPoint.oldestMulti, checkPoint.oldestMultiDB, true); SetCommitTsLimit(InvalidTransactionId, InvalidTransactionId); @@ -6622,6 +6623,7 @@ StartupXLOG(void) ShmemVariableCache->nextOid = checkPoint.nextOid; ShmemVariableCache->oidCount = 0; MultiXactSetNextMXact(checkPoint.nextMulti, checkPoint.nextMultiOffset); + AdvanceOldestClogXid(checkPoint.oldestXid); SetTransactionIdLimit(checkPoint.oldestXid, checkPoint.oldestXidDB); SetMultiXactIdLimit(checkPoint.oldestMulti, checkPoint.oldestMultiDB, true); SetCommitTsLimit(checkPoint.oldestCommitTsXid, @@ -8687,6 +8689,11 @@ CreateCheckPoint(int flags) /* * Get the other info we need for the checkpoint record. + * + * We don't need to save oldestClogXid in the checkpoint, it only matters + * for the short period in which clog is being truncated, and if we crash + * during that we'll redo the clog truncation and fix up oldestClogXid + * there. */ LWLockAcquire(XidGenLock, LW_SHARED); checkPoint.nextXid = ShmemVariableCache->nextXid; @@ -9616,6 +9623,10 @@ xlog_redo(XLogReaderState *record) MultiXactAdvanceOldest(checkPoint.oldestMulti, checkPoint.oldestMultiDB); + /* + * No need to set oldestClogXid here as well; it'll be set when we + * redo an xl_clog_truncate if it changed since initialization. + */ SetTransactionIdLimit(checkPoint.oldestXid, checkPoint.oldestXidDB); /* diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index d8ae3e1e43..9fbb0eb4eb 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -1194,7 +1194,7 @@ vac_truncate_clog(TransactionId frozenXID, /* * Truncate CLOG, multixact and CommitTs to the oldest computed value. */ - TruncateCLOG(frozenXID); + TruncateCLOG(frozenXID, oldestxid_datoid); TruncateCommitTs(frozenXID); TruncateMultiXact(minMulti, minmulti_datoid); diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt index cd8b08f50d..e6025ecedb 100644 --- a/src/backend/storage/lmgr/lwlocknames.txt +++ b/src/backend/storage/lmgr/lwlocknames.txt @@ -49,3 +49,4 @@ MultiXactTruncationLock 41 OldSnapshotTimeMapLock 42 BackendRandomLock 43 LogicalRepWorkerLock 44 +CLogTruncationLock 45 diff --git a/src/include/access/clog.h b/src/include/access/clog.h index 2894bd5620..60a9e11a0f 100644 --- a/src/include/access/clog.h +++ b/src/include/access/clog.h @@ -28,6 +28,12 @@ typedef int XidStatus; #define TRANSACTION_STATUS_ABORTED 0x02 #define TRANSACTION_STATUS_SUB_COMMITTED 0x03 +typedef struct xl_clog_truncate +{ + int pageno; + TransactionId oldestXact; + Oid oldestXactDb; +} xl_clog_truncate; extern void TransactionIdSetTreeStatus(TransactionId xid, int nsubxids, TransactionId *subxids, XidStatus status, XLogRecPtr lsn); @@ -42,7 +48,7 @@ extern void TrimCLOG(void); extern void ShutdownCLOG(void); extern void CheckPointCLOG(void); extern void ExtendCLOG(TransactionId newestXact); -extern void TruncateCLOG(TransactionId oldestXact); +extern void TruncateCLOG(TransactionId oldestXact, Oid oldestxid_datoid); /* XLOG stuff */ #define CLOG_ZEROPAGE 0x00 diff --git a/src/include/access/transam.h b/src/include/access/transam.h index 522c104073..d25a2dd207 100644 --- a/src/include/access/transam.h +++ b/src/include/access/transam.h @@ -134,6 +134,12 @@ typedef struct VariableCacheData */ TransactionId latestCompletedXid; /* newest XID that has committed or * aborted */ + + /* + * These fields are protected by CLogTruncationLock + */ + TransactionId oldestClogXid; /* oldest it's safe to look up in clog */ + } VariableCacheData; typedef VariableCacheData *VariableCache; @@ -173,6 +179,7 @@ extern TransactionId GetNewTransactionId(bool isSubXact); extern TransactionId ReadNewTransactionId(void); extern void SetTransactionIdLimit(TransactionId oldest_datfrozenxid, Oid oldest_datoid); +extern void AdvanceOldestClogXid(TransactionId oldest_datfrozenxid); extern bool ForceTransactionIdLimitUpdate(void); extern Oid GetNewObjectId(void); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index 7957cab98c..c09c0f8e5f 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -31,7 +31,7 @@ /* * Each page of XLOG file has a header like this: */ -#define XLOG_PAGE_MAGIC 0xD096 /* can be used as WAL version indicator */ +#define XLOG_PAGE_MAGIC 0xD097 /* can be used as WAL version indicator */ typedef struct XLogPageHeaderData { -- cgit v1.2.3 From eccfef81e1f73ee41f1d8bfe4fa4e80576945048 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 23 Mar 2017 15:25:34 -0400 Subject: ICU support Add a column collprovider to pg_collation that determines which library provides the collation data. The existing choices are default and libc, and this adds an icu choice, which uses the ICU4C library. The pg_locale_t type is changed to a union that contains the provider-specific locale handles. Users of locale information are changed to look into that struct for the appropriate handle to use. Also add a collversion column that records the version of the collation when it is created, and check at run time whether it is still the same. This detects potentially incompatible library upgrades that can corrupt indexes and other structures. This is currently only supported by ICU-provided collations. initdb initializes the default collation set as before from the `locale -a` output but also adds all available ICU locales with a "-x-icu" appended. Currently, ICU-provided collations can only be explicitly named collations. The global database locales are still always libc-provided. ICU support is enabled by configure --with-icu. Reviewed-by: Thomas Munro Reviewed-by: Andreas Karlsson --- aclocal.m4 | 1 + config/pkg.m4 | 275 ++++++ configure | 313 ++++++ configure.in | 35 + doc/src/sgml/catalogs.sgml | 19 + doc/src/sgml/charset.sgml | 177 +++- doc/src/sgml/func.sgml | 17 + doc/src/sgml/installation.sgml | 15 + doc/src/sgml/mvcc.sgml | 3 +- doc/src/sgml/ref/alter_collation.sgml | 55 ++ doc/src/sgml/ref/create_collation.sgml | 37 +- src/Makefile.global.in | 4 + src/backend/Makefile | 2 +- src/backend/catalog/pg_collation.c | 52 +- src/backend/commands/collationcmds.c | 288 +++++- src/backend/common.mk | 2 + src/backend/nodes/copyfuncs.c | 13 + src/backend/nodes/equalfuncs.c | 11 + src/backend/parser/gram.y | 18 +- src/backend/regex/regc_pg_locale.c | 110 ++- src/backend/tcop/utility.c | 8 + src/backend/utils/adt/formatting.c | 453 +++++---- src/backend/utils/adt/like.c | 53 +- src/backend/utils/adt/pg_locale.c | 266 ++++- src/backend/utils/adt/selfuncs.c | 8 +- src/backend/utils/adt/varlena.c | 179 +++- src/backend/utils/mb/encnames.c | 76 ++ src/bin/initdb/initdb.c | 3 +- src/bin/pg_dump/pg_dump.c | 75 +- src/bin/pg_dump/t/002_pg_dump.pl | 2 +- src/bin/psql/describe.c | 7 +- src/include/catalog/pg_collation.h | 25 +- src/include/catalog/pg_collation_fn.h | 2 + src/include/catalog/pg_proc.h | 3 + src/include/commands/collationcmds.h | 1 + src/include/mb/pg_wchar.h | 6 + src/include/nodes/nodes.h | 1 + src/include/nodes/parsenodes.h | 11 + src/include/pg_config.h.in | 6 + src/include/utils/pg_locale.h | 32 +- src/test/regress/GNUmakefile | 3 + src/test/regress/expected/collate.icu.out | 1126 ++++++++++++++++++++++ src/test/regress/expected/collate.linux.utf8.out | 80 +- src/test/regress/sql/collate.icu.sql | 433 +++++++++ src/test/regress/sql/collate.linux.utf8.sql | 32 + 45 files changed, 3929 insertions(+), 409 deletions(-) create mode 100644 config/pkg.m4 create mode 100644 src/test/regress/expected/collate.icu.out create mode 100644 src/test/regress/sql/collate.icu.sql (limited to 'doc/src') diff --git a/aclocal.m4 b/aclocal.m4 index 6f930b6fc1..5ca902b6a2 100644 --- a/aclocal.m4 +++ b/aclocal.m4 @@ -7,6 +7,7 @@ m4_include([config/docbook.m4]) m4_include([config/general.m4]) m4_include([config/libtool.m4]) m4_include([config/perl.m4]) +m4_include([config/pkg.m4]) m4_include([config/programs.m4]) m4_include([config/python.m4]) m4_include([config/tcl.m4]) diff --git a/config/pkg.m4 b/config/pkg.m4 new file mode 100644 index 0000000000..13a8890178 --- /dev/null +++ b/config/pkg.m4 @@ -0,0 +1,275 @@ +# pkg.m4 - Macros to locate and utilise pkg-config. -*- Autoconf -*- +# serial 12 (pkg-config-0.29.2) + +dnl Copyright © 2004 Scott James Remnant . +dnl Copyright © 2012-2015 Dan Nicholson +dnl +dnl This program is free software; you can redistribute it and/or modify +dnl it under the terms of the GNU General Public License as published by +dnl the Free Software Foundation; either version 2 of the License, or +dnl (at your option) any later version. +dnl +dnl This program is distributed in the hope that it will be useful, but +dnl WITHOUT ANY WARRANTY; without even the implied warranty of +dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +dnl General Public License for more details. +dnl +dnl You should have received a copy of the GNU General Public License +dnl along with this program; if not, write to the Free Software +dnl Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA +dnl 02111-1307, USA. +dnl +dnl As a special exception to the GNU General Public License, if you +dnl distribute this file as part of a program that contains a +dnl configuration script generated by Autoconf, you may include it under +dnl the same distribution terms that you use for the rest of that +dnl program. + +dnl PKG_PREREQ(MIN-VERSION) +dnl ----------------------- +dnl Since: 0.29 +dnl +dnl Verify that the version of the pkg-config macros are at least +dnl MIN-VERSION. Unlike PKG_PROG_PKG_CONFIG, which checks the user's +dnl installed version of pkg-config, this checks the developer's version +dnl of pkg.m4 when generating configure. +dnl +dnl To ensure that this macro is defined, also add: +dnl m4_ifndef([PKG_PREREQ], +dnl [m4_fatal([must install pkg-config 0.29 or later before running autoconf/autogen])]) +dnl +dnl See the "Since" comment for each macro you use to see what version +dnl of the macros you require. +m4_defun([PKG_PREREQ], +[m4_define([PKG_MACROS_VERSION], [0.29.2]) +m4_if(m4_version_compare(PKG_MACROS_VERSION, [$1]), -1, + [m4_fatal([pkg.m4 version $1 or higher is required but ]PKG_MACROS_VERSION[ found])]) +])dnl PKG_PREREQ + +dnl PKG_PROG_PKG_CONFIG([MIN-VERSION]) +dnl ---------------------------------- +dnl Since: 0.16 +dnl +dnl Search for the pkg-config tool and set the PKG_CONFIG variable to +dnl first found in the path. Checks that the version of pkg-config found +dnl is at least MIN-VERSION. If MIN-VERSION is not specified, 0.9.0 is +dnl used since that's the first version where most current features of +dnl pkg-config existed. +AC_DEFUN([PKG_PROG_PKG_CONFIG], +[m4_pattern_forbid([^_?PKG_[A-Z_]+$]) +m4_pattern_allow([^PKG_CONFIG(_(PATH|LIBDIR|SYSROOT_DIR|ALLOW_SYSTEM_(CFLAGS|LIBS)))?$]) +m4_pattern_allow([^PKG_CONFIG_(DISABLE_UNINSTALLED|TOP_BUILD_DIR|DEBUG_SPEW)$]) +AC_ARG_VAR([PKG_CONFIG], [path to pkg-config utility]) +AC_ARG_VAR([PKG_CONFIG_PATH], [directories to add to pkg-config's search path]) +AC_ARG_VAR([PKG_CONFIG_LIBDIR], [path overriding pkg-config's built-in search path]) + +if test "x$ac_cv_env_PKG_CONFIG_set" != "xset"; then + AC_PATH_TOOL([PKG_CONFIG], [pkg-config]) +fi +if test -n "$PKG_CONFIG"; then + _pkg_min_version=m4_default([$1], [0.9.0]) + AC_MSG_CHECKING([pkg-config is at least version $_pkg_min_version]) + if $PKG_CONFIG --atleast-pkgconfig-version $_pkg_min_version; then + AC_MSG_RESULT([yes]) + else + AC_MSG_RESULT([no]) + PKG_CONFIG="" + fi +fi[]dnl +])dnl PKG_PROG_PKG_CONFIG + +dnl PKG_CHECK_EXISTS(MODULES, [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND]) +dnl ------------------------------------------------------------------- +dnl Since: 0.18 +dnl +dnl Check to see whether a particular set of modules exists. Similar to +dnl PKG_CHECK_MODULES(), but does not set variables or print errors. +dnl +dnl Please remember that m4 expands AC_REQUIRE([PKG_PROG_PKG_CONFIG]) +dnl only at the first occurence in configure.ac, so if the first place +dnl it's called might be skipped (such as if it is within an "if", you +dnl have to call PKG_CHECK_EXISTS manually +AC_DEFUN([PKG_CHECK_EXISTS], +[AC_REQUIRE([PKG_PROG_PKG_CONFIG])dnl +if test -n "$PKG_CONFIG" && \ + AC_RUN_LOG([$PKG_CONFIG --exists --print-errors "$1"]); then + m4_default([$2], [:]) +m4_ifvaln([$3], [else + $3])dnl +fi]) + +dnl _PKG_CONFIG([VARIABLE], [COMMAND], [MODULES]) +dnl --------------------------------------------- +dnl Internal wrapper calling pkg-config via PKG_CONFIG and setting +dnl pkg_failed based on the result. +m4_define([_PKG_CONFIG], +[if test -n "$$1"; then + pkg_cv_[]$1="$$1" + elif test -n "$PKG_CONFIG"; then + PKG_CHECK_EXISTS([$3], + [pkg_cv_[]$1=`$PKG_CONFIG --[]$2 "$3" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes ], + [pkg_failed=yes]) + else + pkg_failed=untried +fi[]dnl +])dnl _PKG_CONFIG + +dnl _PKG_SHORT_ERRORS_SUPPORTED +dnl --------------------------- +dnl Internal check to see if pkg-config supports short errors. +AC_DEFUN([_PKG_SHORT_ERRORS_SUPPORTED], +[AC_REQUIRE([PKG_PROG_PKG_CONFIG]) +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi[]dnl +])dnl _PKG_SHORT_ERRORS_SUPPORTED + + +dnl PKG_CHECK_MODULES(VARIABLE-PREFIX, MODULES, [ACTION-IF-FOUND], +dnl [ACTION-IF-NOT-FOUND]) +dnl -------------------------------------------------------------- +dnl Since: 0.4.0 +dnl +dnl Note that if there is a possibility the first call to +dnl PKG_CHECK_MODULES might not happen, you should be sure to include an +dnl explicit call to PKG_PROG_PKG_CONFIG in your configure.ac +AC_DEFUN([PKG_CHECK_MODULES], +[AC_REQUIRE([PKG_PROG_PKG_CONFIG])dnl +AC_ARG_VAR([$1][_CFLAGS], [C compiler flags for $1, overriding pkg-config])dnl +AC_ARG_VAR([$1][_LIBS], [linker flags for $1, overriding pkg-config])dnl + +pkg_failed=no +AC_MSG_CHECKING([for $2]) + +_PKG_CONFIG([$1][_CFLAGS], [cflags], [$2]) +_PKG_CONFIG([$1][_LIBS], [libs], [$2]) + +m4_define([_PKG_TEXT], [Alternatively, you may set the environment variables $1[]_CFLAGS +and $1[]_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details.]) + +if test $pkg_failed = yes; then + AC_MSG_RESULT([no]) + _PKG_SHORT_ERRORS_SUPPORTED + if test $_pkg_short_errors_supported = yes; then + $1[]_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "$2" 2>&1` + else + $1[]_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "$2" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$$1[]_PKG_ERRORS" >&AS_MESSAGE_LOG_FD + + m4_default([$4], [AC_MSG_ERROR( +[Package requirements ($2) were not met: + +$$1_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +_PKG_TEXT])[]dnl + ]) +elif test $pkg_failed = untried; then + AC_MSG_RESULT([no]) + m4_default([$4], [AC_MSG_FAILURE( +[The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +_PKG_TEXT + +To get pkg-config, see .])[]dnl + ]) +else + $1[]_CFLAGS=$pkg_cv_[]$1[]_CFLAGS + $1[]_LIBS=$pkg_cv_[]$1[]_LIBS + AC_MSG_RESULT([yes]) + $3 +fi[]dnl +])dnl PKG_CHECK_MODULES + + +dnl PKG_CHECK_MODULES_STATIC(VARIABLE-PREFIX, MODULES, [ACTION-IF-FOUND], +dnl [ACTION-IF-NOT-FOUND]) +dnl --------------------------------------------------------------------- +dnl Since: 0.29 +dnl +dnl Checks for existence of MODULES and gathers its build flags with +dnl static libraries enabled. Sets VARIABLE-PREFIX_CFLAGS from --cflags +dnl and VARIABLE-PREFIX_LIBS from --libs. +dnl +dnl Note that if there is a possibility the first call to +dnl PKG_CHECK_MODULES_STATIC might not happen, you should be sure to +dnl include an explicit call to PKG_PROG_PKG_CONFIG in your +dnl configure.ac. +AC_DEFUN([PKG_CHECK_MODULES_STATIC], +[AC_REQUIRE([PKG_PROG_PKG_CONFIG])dnl +_save_PKG_CONFIG=$PKG_CONFIG +PKG_CONFIG="$PKG_CONFIG --static" +PKG_CHECK_MODULES($@) +PKG_CONFIG=$_save_PKG_CONFIG[]dnl +])dnl PKG_CHECK_MODULES_STATIC + + +dnl PKG_INSTALLDIR([DIRECTORY]) +dnl ------------------------- +dnl Since: 0.27 +dnl +dnl Substitutes the variable pkgconfigdir as the location where a module +dnl should install pkg-config .pc files. By default the directory is +dnl $libdir/pkgconfig, but the default can be changed by passing +dnl DIRECTORY. The user can override through the --with-pkgconfigdir +dnl parameter. +AC_DEFUN([PKG_INSTALLDIR], +[m4_pushdef([pkg_default], [m4_default([$1], ['${libdir}/pkgconfig'])]) +m4_pushdef([pkg_description], + [pkg-config installation directory @<:@]pkg_default[@:>@]) +AC_ARG_WITH([pkgconfigdir], + [AS_HELP_STRING([--with-pkgconfigdir], pkg_description)],, + [with_pkgconfigdir=]pkg_default) +AC_SUBST([pkgconfigdir], [$with_pkgconfigdir]) +m4_popdef([pkg_default]) +m4_popdef([pkg_description]) +])dnl PKG_INSTALLDIR + + +dnl PKG_NOARCH_INSTALLDIR([DIRECTORY]) +dnl -------------------------------- +dnl Since: 0.27 +dnl +dnl Substitutes the variable noarch_pkgconfigdir as the location where a +dnl module should install arch-independent pkg-config .pc files. By +dnl default the directory is $datadir/pkgconfig, but the default can be +dnl changed by passing DIRECTORY. The user can override through the +dnl --with-noarch-pkgconfigdir parameter. +AC_DEFUN([PKG_NOARCH_INSTALLDIR], +[m4_pushdef([pkg_default], [m4_default([$1], ['${datadir}/pkgconfig'])]) +m4_pushdef([pkg_description], + [pkg-config arch-independent installation directory @<:@]pkg_default[@:>@]) +AC_ARG_WITH([noarch-pkgconfigdir], + [AS_HELP_STRING([--with-noarch-pkgconfigdir], pkg_description)],, + [with_noarch_pkgconfigdir=]pkg_default) +AC_SUBST([noarch_pkgconfigdir], [$with_noarch_pkgconfigdir]) +m4_popdef([pkg_default]) +m4_popdef([pkg_description]) +])dnl PKG_NOARCH_INSTALLDIR + + +dnl PKG_CHECK_VAR(VARIABLE, MODULE, CONFIG-VARIABLE, +dnl [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND]) +dnl ------------------------------------------- +dnl Since: 0.28 +dnl +dnl Retrieves the value of the pkg-config variable for the given module. +AC_DEFUN([PKG_CHECK_VAR], +[AC_REQUIRE([PKG_PROG_PKG_CONFIG])dnl +AC_ARG_VAR([$1], [value of $3 for $2, overriding pkg-config])dnl + +_PKG_CONFIG([$1], [variable="][$3]["], [$2]) +AS_VAR_COPY([$1], [pkg_cv_][$1]) + +AS_VAR_IF([$1], [""], [$5], [$4])dnl +])dnl PKG_CHECK_VAR diff --git a/configure b/configure index 9528622421..4b8229e959 100755 --- a/configure +++ b/configure @@ -715,6 +715,12 @@ krb_srvtab with_python with_perl with_tcl +ICU_LIBS +ICU_CFLAGS +PKG_CONFIG_LIBDIR +PKG_CONFIG_PATH +PKG_CONFIG +with_icu enable_thread_safety INCLUDES autodepend @@ -821,6 +827,7 @@ with_CC enable_depend enable_cassert enable_thread_safety +with_icu with_tcl with_tclconfig with_perl @@ -856,6 +863,11 @@ LDFLAGS LIBS CPPFLAGS CPP +PKG_CONFIG +PKG_CONFIG_PATH +PKG_CONFIG_LIBDIR +ICU_CFLAGS +ICU_LIBS LDFLAGS_EX LDFLAGS_SL DOCBOOKSTYLE' @@ -1511,6 +1523,7 @@ Optional Packages: --with-wal-segsize=SEGSIZE set WAL segment size in MB [16] --with-CC=CMD set compiler (deprecated) + --with-icu build with ICU support --with-tcl build Tcl modules (PL/Tcl) --with-tclconfig=DIR tclConfig.sh is in DIR --with-perl build Perl modules (PL/Perl) @@ -1546,6 +1559,13 @@ Some influential environment variables: CPPFLAGS (Objective) C/C++ preprocessor flags, e.g. -I if you have headers in a nonstandard directory CPP C preprocessor + PKG_CONFIG path to pkg-config utility + PKG_CONFIG_PATH + directories to add to pkg-config's search path + PKG_CONFIG_LIBDIR + path overriding pkg-config's built-in search path + ICU_CFLAGS C compiler flags for ICU, overriding pkg-config + ICU_LIBS linker flags for ICU, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only DOCBOOKSTYLE @@ -5361,6 +5381,255 @@ fi $as_echo "$enable_thread_safety" >&6; } +# +# ICU +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with ICU support" >&5 +$as_echo_n "checking whether to build with ICU support... " >&6; } + + + +# Check whether --with-icu was given. +if test "${with_icu+set}" = set; then : + withval=$with_icu; + case $withval in + yes) + +$as_echo "#define USE_ICU 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-icu option" "$LINENO" 5 + ;; + esac + +else + with_icu=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_icu" >&5 +$as_echo "$with_icu" >&6; } + + +if test "$with_icu" = yes; then + + + + + + + +if test "x$ac_cv_env_PKG_CONFIG_set" != "xset"; then + if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}pkg-config", so it can be a program name with args. +set dummy ${ac_tool_prefix}pkg-config; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_PKG_CONFIG+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $PKG_CONFIG in + [\\/]* | ?:[\\/]*) + ac_cv_path_PKG_CONFIG="$PKG_CONFIG" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_PKG_CONFIG="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +PKG_CONFIG=$ac_cv_path_PKG_CONFIG +if test -n "$PKG_CONFIG"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PKG_CONFIG" >&5 +$as_echo "$PKG_CONFIG" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +fi +if test -z "$ac_cv_path_PKG_CONFIG"; then + ac_pt_PKG_CONFIG=$PKG_CONFIG + # Extract the first word of "pkg-config", so it can be a program name with args. +set dummy pkg-config; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_ac_pt_PKG_CONFIG+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $ac_pt_PKG_CONFIG in + [\\/]* | ?:[\\/]*) + ac_cv_path_ac_pt_PKG_CONFIG="$ac_pt_PKG_CONFIG" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_ac_pt_PKG_CONFIG="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +ac_pt_PKG_CONFIG=$ac_cv_path_ac_pt_PKG_CONFIG +if test -n "$ac_pt_PKG_CONFIG"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_pt_PKG_CONFIG" >&5 +$as_echo "$ac_pt_PKG_CONFIG" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + if test "x$ac_pt_PKG_CONFIG" = x; then + PKG_CONFIG="" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + PKG_CONFIG=$ac_pt_PKG_CONFIG + fi +else + PKG_CONFIG="$ac_cv_path_PKG_CONFIG" +fi + +fi +if test -n "$PKG_CONFIG"; then + _pkg_min_version=0.9.0 + { $as_echo "$as_me:${as_lineno-$LINENO}: checking pkg-config is at least version $_pkg_min_version" >&5 +$as_echo_n "checking pkg-config is at least version $_pkg_min_version... " >&6; } + if $PKG_CONFIG --atleast-pkgconfig-version $_pkg_min_version; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + PKG_CONFIG="" + fi +fi + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for icu-uc icu-i18n" >&5 +$as_echo_n "checking for icu-uc icu-i18n... " >&6; } + +if test -n "$ICU_CFLAGS"; then + pkg_cv_ICU_CFLAGS="$ICU_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"icu-uc icu-i18n\""; } >&5 + ($PKG_CONFIG --exists --print-errors "icu-uc icu-i18n") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_ICU_CFLAGS=`$PKG_CONFIG --cflags "icu-uc icu-i18n" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$ICU_LIBS"; then + pkg_cv_ICU_LIBS="$ICU_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"icu-uc icu-i18n\""; } >&5 + ($PKG_CONFIG --exists --print-errors "icu-uc icu-i18n") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_ICU_LIBS=`$PKG_CONFIG --libs "icu-uc icu-i18n" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + ICU_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "icu-uc icu-i18n" 2>&1` + else + ICU_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "icu-uc icu-i18n" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$ICU_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (icu-uc icu-i18n) were not met: + +$ICU_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables ICU_CFLAGS +and ICU_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables ICU_CFLAGS +and ICU_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see . +See \`config.log' for more details" "$LINENO" 5; } +else + ICU_CFLAGS=$pkg_cv_ICU_CFLAGS + ICU_LIBS=$pkg_cv_ICU_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi +fi + # # Optionally build Tcl modules (PL/Tcl) # @@ -13467,6 +13736,50 @@ fi done +if test "$with_icu" = yes; then + # ICU functions are macros, so we need to do this the long way. + + # ucol_strcollUTF8() appeared in ICU 50. + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ucol_strcollUTF8" >&5 +$as_echo_n "checking for ucol_strcollUTF8... " >&6; } +if ${pgac_cv_func_ucol_strcollUTF8+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_save_CPPFLAGS=$CPPFLAGS +CPPFLAGS="$ICU_CFLAGS $CPPFLAGS" +ac_save_LIBS=$LIBS +LIBS="$ICU_LIBS $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include + +int +main () +{ +ucol_strcollUTF8(NULL, NULL, 0, NULL, 0, NULL); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + pgac_cv_func_ucol_strcollUTF8=yes +else + pgac_cv_func_ucol_strcollUTF8=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +CPPFLAGS=$ac_save_CPPFLAGS +LIBS=$ac_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_func_ucol_strcollUTF8" >&5 +$as_echo "$pgac_cv_func_ucol_strcollUTF8" >&6; } + if test "$pgac_cv_func_ucol_strcollUTF8" = yes ; then + +$as_echo "#define HAVE_UCOL_STRCOLLUTF8 1" >>confdefs.h + + fi +fi + # Lastly, restore full LIBS list and check for readline/libedit symbols LIBS="$LIBS_including_readline" diff --git a/configure.in b/configure.in index 7f234f543d..6c74214171 100644 --- a/configure.in +++ b/configure.in @@ -613,6 +613,19 @@ fi AC_MSG_RESULT([$enable_thread_safety]) AC_SUBST(enable_thread_safety) +# +# ICU +# +AC_MSG_CHECKING([whether to build with ICU support]) +PGAC_ARG_BOOL(with, icu, no, [build with ICU support], + [AC_DEFINE([USE_ICU], 1, [Define to build with ICU support. (--with-icu)])]) +AC_MSG_RESULT([$with_icu]) +AC_SUBST(with_icu) + +if test "$with_icu" = yes; then + PKG_CHECK_MODULES(ICU, icu-uc icu-i18n) +fi + # # Optionally build Tcl modules (PL/Tcl) # @@ -1635,6 +1648,28 @@ fi AC_CHECK_FUNCS([strtoll strtoq], [break]) AC_CHECK_FUNCS([strtoull strtouq], [break]) +if test "$with_icu" = yes; then + # ICU functions are macros, so we need to do this the long way. + + # ucol_strcollUTF8() appeared in ICU 50. + AC_CACHE_CHECK([for ucol_strcollUTF8], [pgac_cv_func_ucol_strcollUTF8], +[ac_save_CPPFLAGS=$CPPFLAGS +CPPFLAGS="$ICU_CFLAGS $CPPFLAGS" +ac_save_LIBS=$LIBS +LIBS="$ICU_LIBS $LIBS" +AC_LINK_IFELSE([AC_LANG_PROGRAM( +[#include +], +[ucol_strcollUTF8(NULL, NULL, 0, NULL, 0, NULL);])], +[pgac_cv_func_ucol_strcollUTF8=yes], +[pgac_cv_func_ucol_strcollUTF8=no]) +CPPFLAGS=$ac_save_CPPFLAGS +LIBS=$ac_save_LIBS]) + if test "$pgac_cv_func_ucol_strcollUTF8" = yes ; then + AC_DEFINE([HAVE_UCOL_STRCOLLUTF8], 1, [Define to 1 if you have the `ucol_strcollUTF8' function.]) + fi +fi + # Lastly, restore full LIBS list and check for readline/libedit symbols LIBS="$LIBS_including_readline" diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 228ec78031..c531c73aac 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -2020,6 +2020,14 @@ Owner of the collation + + collprovider + char + + Provider of the collation: d = database + default, c = libc, i = icu + + collencoding int4 @@ -2041,6 +2049,17 @@ LC_CTYPE for this collation object + + + collversion + text + + + Provider-specific version of the collation. This is recorded when the + collation is created and then checked when it is used, to detect + changes in the collation definition that could lead to data corruption. + +
    diff --git a/doc/src/sgml/charset.sgml b/doc/src/sgml/charset.sgml index 2aba0fc528..5c55f397f8 100644 --- a/doc/src/sgml/charset.sgml +++ b/doc/src/sgml/charset.sgml @@ -500,20 +500,46 @@ SELECT * FROM test1 ORDER BY a || b COLLATE "fr_FR"; Managing Collations - A collation is an SQL schema object that maps an SQL name to - operating system locales. In particular, it maps to a combination - of LC_COLLATE and LC_CTYPE. (As + A collation is an SQL schema object that maps an SQL name to locales + provided by libraries installed in the operating system. A collation + definition has a provider that specifies which + library supplies the locale data. One standard provider name + is libc, which uses the locales provided by the + operating system C library. These are the locales that most tools + provided by the operating system use. Another provider + is icu, which uses the external + ICUICU library. Support for ICU has to be + configured when PostgreSQL is built. + + + + A collation object provided by libc maps to a + combination of LC_COLLATE and LC_CTYPE + settings. (As the name would suggest, the main purpose of a collation is to set LC_COLLATE, which controls the sort order. But it is rarely necessary in practice to have an LC_CTYPE setting that is different from LC_COLLATE, so it is more convenient to collect these under one concept than to create another infrastructure for - setting LC_CTYPE per expression.) Also, a collation + setting LC_CTYPE per expression.) Also, + a libc collation is tied to a character set encoding (see ). The same collation name may exist for different encodings. + + A collation provided by icu maps to a named collator + provided by the ICU library. ICU does not support + separate collate and ctype settings, so they + are always the same. Also, ICU collations are independent of the + encoding, so there is always only one ICU collation for a given name in a + database. + + + + Standard Collations + On all platforms, the collations named default, C, and POSIX are available. Additional @@ -527,13 +553,37 @@ SELECT * FROM test1 ORDER BY a || b COLLATE "fr_FR"; code byte values. + + Additionally, the SQL standard collation name ucs_basic + is available for encoding UTF8. It is equivalent + to C and sorts by Unicode code point. + + + + + Predefined Collations + If the operating system provides support for using multiple locales within a single program (newlocale and related functions), + or support for ICU is configured, then when a database cluster is initialized, initdb populates the system catalog pg_collation with collations based on all the locales it finds on the operating - system at the time. For example, the operating system might + system at the time. + + + + To inspect the currently available locales, use the query SELECT + * FROM pg_collation, or the command \dOS+ + in psql. + + + + libc collations + + + For example, the operating system might provide a locale named de_DE.utf8. initdb would then create a collation named de_DE.utf8 for encoding UTF8 @@ -548,13 +598,14 @@ SELECT * FROM test1 ORDER BY a || b COLLATE "fr_FR"; - In case a collation is needed that has different values for - LC_COLLATE and LC_CTYPE, a new - collation may be created using - the command. That command - can also be used to create a new collation from an existing - collation, which can be useful to be able to use - operating-system-independent collation names in applications. + The default set of collations provided by libc map + directly to the locales installed in the operating system, which can be + listed using the command locale -a. In case + a libc collation is needed that has different values + for LC_COLLATE and LC_CTYPE, or new + locales are installed in the operating system after the database system + was initialized, then a new collation may be created using + the command. @@ -566,8 +617,8 @@ SELECT * FROM test1 ORDER BY a || b COLLATE "fr_FR"; Use of the stripped collation names is recommended, since it will make one less thing you need to change if you decide to change to another database encoding. Note however that the default, - C, and POSIX collations can be used - regardless of the database encoding. + C, and POSIX collations, as well as all collations + provided by ICU can be used regardless of the database encoding. @@ -581,6 +632,104 @@ SELECT a COLLATE "C" < b COLLATE "POSIX" FROM test1; collations have identical behaviors. Mixing stripped and non-stripped collation names is therefore not recommended. + + + + ICU collations + + + Collations provided by ICU are created with names in BCP 47 language tag + format, with a private use + extension -x-icu appended, to distinguish them from + libc locales. So de-x-icu would be an example. + + + + With ICU, it is not sensible to enumerate all possible locale names. ICU + uses a particular naming system for locales, but there are many more ways + to name a locale than there are actually distinct locales. (In fact, any + string will be accepted as a locale name.) + See for + information on ICU locale naming. initdb uses the ICU + APIs to extract a set of locales with distinct collation rules to populate + the initial set of collations. Here are some examples collations that + might be created: + + + + de-x-icu + + German collation, default variant + + + + + de-u-co-phonebk-x-icu + + German collation, phone book variant + + + + + de-AT-x-icu + + German collation for Austria, default variant + + (Note that as of this writing, there is no, + say, de-DE-x-icu or de-CH-x-icu, + because those are equivalent to de-x-icu.) + + + + + + de-AT-u-co-phonebk-x-icu + + German collation for Austria, phone book variant + + + + und-x-icu (for undefined) + + + ICU root collation. Use this to get a reasonable + language-agnostic sort order. + + + + + + + + Some (less frequently used) encodings are not supported by ICU. If the + database cluster was initialized with such an encoding, no ICU collations + will be predefined. + + + + + + Copying Collations + + + The command can also be used to + create a new collation from an existing collation, which can be useful to + be able to use operating-system-independent collation names in + applications, create compatibility names, or use an ICU-provided collation + under a more readable name. For example: + +CREATE COLLATION german FROM "de_DE"; +CREATE COLLATION french FROM "fr-x-icu"; +CREATE COLLATION "de-DE-x-icu" FROM "de-x-icu"; + + + + + The standard and predefined collations are in the + schema pg_catalog, like all predefined objects. + User-defined collations should be created in user schemas. This also + ensures that they are saved by pg_dump. + diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 4dc30caccb..64f86ce680 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -19545,6 +19545,14 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); + + + pg_collation_actual_version + pg_collation_actual_version(oid) + + text + Return actual version of collation from operating system + pg_import_system_collations @@ -19557,6 +19565,15 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); + + pg_collation_actual_version returns the actual + version of the collation object as it is currently installed in the + operating system. If this is different from the value + in pg_collation.collversion, then objects depending on + the collation might need to be rebuilt. See also + . + + pg_import_system_collations populates the system catalog pg_collation with collations based on all the diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml index f8a222e637..39774a058a 100644 --- a/doc/src/sgml/installation.sgml +++ b/doc/src/sgml/installation.sgml @@ -766,6 +766,21 @@ su - postgres + + + + + Build with support for + the ICUICU + library. This requires the ICU4C package + as well + as pkg-configpkg-config + to be installed. The minimum required version + of ICU4C is currently 4.6. + + + + diff --git a/doc/src/sgml/mvcc.sgml b/doc/src/sgml/mvcc.sgml index 306def4a15..82e69fe2d2 100644 --- a/doc/src/sgml/mvcc.sgml +++ b/doc/src/sgml/mvcc.sgml @@ -967,7 +967,8 @@ ERROR: could not serialize access due to read/write dependencies among transact - Acquired by CREATE TRIGGER and many forms of + Acquired by CREATE COLLATION, + CREATE TRIGGER, and many forms of ALTER TABLE (see ). diff --git a/doc/src/sgml/ref/alter_collation.sgml b/doc/src/sgml/ref/alter_collation.sgml index 6708c7e10e..bf934ce75f 100644 --- a/doc/src/sgml/ref/alter_collation.sgml +++ b/doc/src/sgml/ref/alter_collation.sgml @@ -21,6 +21,8 @@ PostgreSQL documentation +ALTER COLLATION name REFRESH VERSION + ALTER COLLATION name RENAME TO new_name ALTER COLLATION name OWNER TO { new_owner | CURRENT_USER | SESSION_USER } ALTER COLLATION name SET SCHEMA new_schema @@ -85,9 +87,62 @@ ALTER COLLATION name SET SCHEMA new_sche     + + + REFRESH VERSION + + + Updated the collation version. + See below. + + + + + Notes + + + When using collations provided by the ICU library, the ICU-specific version + of the collator is recorded in the system catalog when the collation object + is created. When the collation is then used, the current version is + checked against the recorded version, and a warning is issued when there is + a mismatch, for example: + +WARNING: ICU collator version mismatch +DETAIL: The database was created using version 1.2.3.4, the library provides version 2.3.4.5. +HINT: Rebuild all objects affected by this collation and run ALTER COLLATION pg_catalog."xx-x-icu" REFRESH VERSION, or build PostgreSQL with the right version of ICU. + + A change in collation definitions can lead to corrupt indexes and other + problems where the database system relies on stored objects having a + certain sort order. Generally, this should be avoided, but it can happen + in legitimate circumstances, such as when + using pg_upgrade to upgrade to server binaries linked + with a newer version of ICU. When this happens, all objects depending on + the collation should be rebuilt, for example, + using REINDEX. When that is done, the collation version + can be refreshed using the command ALTER COLLATION ... REFRESH + VERSION. This will update the system catalog to record the + current collator version and will make the warning go away. Note that this + does not actually check whether all affected objects have been rebuilt + correctly. + + + + The following query can be used to identify all collations in the current + database that need to be refreshed and the objects that depend on them: + pg_collation_actual_version(c.oid) + ORDER BY 1, 2; +]]> + + + Examples diff --git a/doc/src/sgml/ref/create_collation.sgml b/doc/src/sgml/ref/create_collation.sgml index c09e5bd6d4..47de9a09b6 100644 --- a/doc/src/sgml/ref/create_collation.sgml +++ b/doc/src/sgml/ref/create_collation.sgml @@ -21,7 +21,9 @@ CREATE COLLATION [ IF NOT EXISTS ] name ( [ LOCALE = locale, ] [ LC_COLLATE = lc_collate, ] - [ LC_CTYPE = lc_ctype ] + [ LC_CTYPE = lc_ctype, ] + [ PROVIDER = provider, ] + [ VERSION = version ] ) CREATE COLLATION [ IF NOT EXISTS ] name FROM existing_collation @@ -113,6 +115,39 @@ CREATE COLLATION [ IF NOT EXISTS ] name FROM + + provider + + + + Specifies the provider to use for locale services associated with this + collation. Possible values + are: icu,ICU libc. + The available choices depend on the operating system and build options. + + + + + + version + + + + Specifies the version string to store with the collation. Normally, + this should be omitted, which will cause the version to be computed + from the actual version of the collation as provided by the operating + system. This option is intended to be used + by pg_upgrade for copying the version from an + existing installation. + + + + See also for how to handle + collation version mismatches. + + + + existing_collation diff --git a/src/Makefile.global.in b/src/Makefile.global.in index 8e1d6e3bd4..4acf7d2f06 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -179,6 +179,7 @@ pgxsdir = $(pkglibdir)/pgxs # # Records the choice of the various --enable-xxx and --with-xxx options. +with_icu = @with_icu@ with_perl = @with_perl@ with_python = @with_python@ with_tcl = @with_tcl@ @@ -208,6 +209,9 @@ python_version = @python_version@ krb_srvtab = @krb_srvtab@ +ICU_CFLAGS = @ICU_CFLAGS@ +ICU_LIBS = @ICU_LIBS@ + TCLSH = @TCLSH@ TCL_LIBS = @TCL_LIBS@ TCL_LIB_SPEC = @TCL_LIB_SPEC@ diff --git a/src/backend/Makefile b/src/backend/Makefile index 7a0bbb2942..fffb0d95ba 100644 --- a/src/backend/Makefile +++ b/src/backend/Makefile @@ -58,7 +58,7 @@ ifneq ($(PORTNAME), win32) ifneq ($(PORTNAME), aix) postgres: $(OBJS) - $(CC) $(CFLAGS) $(LDFLAGS) $(LDFLAGS_EX) $(export_dynamic) $(call expand_subsys,$^) $(LIBS) -o $@ + $(CC) $(CFLAGS) $(LDFLAGS) $(LDFLAGS_EX) $(export_dynamic) $(call expand_subsys,$^) $(LIBS) $(ICU_LIBS) -o $@ endif endif diff --git a/src/backend/catalog/pg_collation.c b/src/backend/catalog/pg_collation.c index 65b6051c0d..ede920955d 100644 --- a/src/backend/catalog/pg_collation.c +++ b/src/backend/catalog/pg_collation.c @@ -27,6 +27,7 @@ #include "mb/pg_wchar.h" #include "utils/builtins.h" #include "utils/fmgroids.h" +#include "utils/pg_locale.h" #include "utils/rel.h" #include "utils/syscache.h" #include "utils/tqual.h" @@ -40,8 +41,10 @@ Oid CollationCreate(const char *collname, Oid collnamespace, Oid collowner, + char collprovider, int32 collencoding, const char *collcollate, const char *collctype, + const char *collversion, bool if_not_exists) { Relation rel; @@ -78,29 +81,47 @@ CollationCreate(const char *collname, Oid collnamespace, { ereport(NOTICE, (errcode(ERRCODE_DUPLICATE_OBJECT), - errmsg("collation \"%s\" for encoding \"%s\" already exists, skipping", - collname, pg_encoding_to_char(collencoding)))); + collencoding == -1 + ? errmsg("collation \"%s\" already exists, skipping", + collname) + : errmsg("collation \"%s\" for encoding \"%s\" already exists, skipping", + collname, pg_encoding_to_char(collencoding)))); return InvalidOid; } else ereport(ERROR, (errcode(ERRCODE_DUPLICATE_OBJECT), - errmsg("collation \"%s\" for encoding \"%s\" already exists", - collname, pg_encoding_to_char(collencoding)))); + collencoding == -1 + ? errmsg("collation \"%s\" already exists", + collname) + : errmsg("collation \"%s\" for encoding \"%s\" already exists", + collname, pg_encoding_to_char(collencoding)))); } + /* open pg_collation; see below about the lock level */ + rel = heap_open(CollationRelationId, ShareRowExclusiveLock); + /* - * Also forbid matching an any-encoding entry. This test of course is not - * backed up by the unique index, but it's not a problem since we don't - * support adding any-encoding entries after initdb. + * Also forbid a specific-encoding collation shadowing an any-encoding + * collation, or an any-encoding collation being shadowed (see + * get_collation_name()). This test is not backed up by the unique index, + * so we take a ShareRowExclusiveLock earlier, to protect against + * concurrent changes fooling this check. */ - if (SearchSysCacheExists3(COLLNAMEENCNSP, - PointerGetDatum(collname), - Int32GetDatum(-1), - ObjectIdGetDatum(collnamespace))) + if ((collencoding == -1 && + SearchSysCacheExists3(COLLNAMEENCNSP, + PointerGetDatum(collname), + Int32GetDatum(GetDatabaseEncoding()), + ObjectIdGetDatum(collnamespace))) || + (collencoding != -1 && + SearchSysCacheExists3(COLLNAMEENCNSP, + PointerGetDatum(collname), + Int32GetDatum(-1), + ObjectIdGetDatum(collnamespace)))) { if (if_not_exists) { + heap_close(rel, NoLock); ereport(NOTICE, (errcode(ERRCODE_DUPLICATE_OBJECT), errmsg("collation \"%s\" already exists, skipping", @@ -114,8 +135,6 @@ CollationCreate(const char *collname, Oid collnamespace, collname))); } - /* open pg_collation */ - rel = heap_open(CollationRelationId, RowExclusiveLock); tupDesc = RelationGetDescr(rel); /* form a tuple */ @@ -125,11 +144,16 @@ CollationCreate(const char *collname, Oid collnamespace, values[Anum_pg_collation_collname - 1] = NameGetDatum(&name_name); values[Anum_pg_collation_collnamespace - 1] = ObjectIdGetDatum(collnamespace); values[Anum_pg_collation_collowner - 1] = ObjectIdGetDatum(collowner); + values[Anum_pg_collation_collprovider - 1] = CharGetDatum(collprovider); values[Anum_pg_collation_collencoding - 1] = Int32GetDatum(collencoding); namestrcpy(&name_collate, collcollate); values[Anum_pg_collation_collcollate - 1] = NameGetDatum(&name_collate); namestrcpy(&name_ctype, collctype); values[Anum_pg_collation_collctype - 1] = NameGetDatum(&name_ctype); + if (collversion) + values[Anum_pg_collation_collversion - 1] = CStringGetTextDatum(collversion); + else + nulls[Anum_pg_collation_collversion - 1] = true; tup = heap_form_tuple(tupDesc, values, nulls); @@ -159,7 +183,7 @@ CollationCreate(const char *collname, Oid collnamespace, InvokeObjectPostCreateHook(CollationRelationId, oid, 0); heap_freetuple(tup); - heap_close(rel, RowExclusiveLock); + heap_close(rel, NoLock); return oid; } diff --git a/src/backend/commands/collationcmds.c b/src/backend/commands/collationcmds.c index 919cfc6a06..835cb263db 100644 --- a/src/backend/commands/collationcmds.c +++ b/src/backend/commands/collationcmds.c @@ -14,15 +14,18 @@ */ #include "postgres.h" +#include "access/heapam.h" #include "access/htup_details.h" #include "access/xact.h" #include "catalog/dependency.h" #include "catalog/indexing.h" #include "catalog/namespace.h" +#include "catalog/objectaccess.h" #include "catalog/pg_collation.h" #include "catalog/pg_collation_fn.h" #include "commands/alter.h" #include "commands/collationcmds.h" +#include "commands/comment.h" #include "commands/dbcommands.h" #include "commands/defrem.h" #include "mb/pg_wchar.h" @@ -33,6 +36,7 @@ #include "utils/rel.h" #include "utils/syscache.h" + /* * CREATE COLLATION */ @@ -47,8 +51,14 @@ DefineCollation(ParseState *pstate, List *names, List *parameters, bool if_not_e DefElem *localeEl = NULL; DefElem *lccollateEl = NULL; DefElem *lcctypeEl = NULL; + DefElem *providerEl = NULL; + DefElem *versionEl = NULL; char *collcollate = NULL; char *collctype = NULL; + char *collproviderstr = NULL; + int collencoding; + char collprovider = 0; + char *collversion = NULL; Oid newoid; ObjectAddress address; @@ -72,6 +82,10 @@ DefineCollation(ParseState *pstate, List *names, List *parameters, bool if_not_e defelp = &lccollateEl; else if (pg_strcasecmp(defel->defname, "lc_ctype") == 0) defelp = &lcctypeEl; + else if (pg_strcasecmp(defel->defname, "provider") == 0) + defelp = &providerEl; + else if (pg_strcasecmp(defel->defname, "version") == 0) + defelp = &versionEl; else { ereport(ERROR, @@ -103,6 +117,7 @@ DefineCollation(ParseState *pstate, List *names, List *parameters, bool if_not_e collcollate = pstrdup(NameStr(((Form_pg_collation) GETSTRUCT(tp))->collcollate)); collctype = pstrdup(NameStr(((Form_pg_collation) GETSTRUCT(tp))->collctype)); + collprovider = ((Form_pg_collation) GETSTRUCT(tp))->collprovider; ReleaseSysCache(tp); } @@ -119,6 +134,27 @@ DefineCollation(ParseState *pstate, List *names, List *parameters, bool if_not_e if (lcctypeEl) collctype = defGetString(lcctypeEl); + if (providerEl) + collproviderstr = defGetString(providerEl); + + if (versionEl) + collversion = defGetString(versionEl); + + if (collproviderstr) + { + if (pg_strcasecmp(collproviderstr, "icu") == 0) + collprovider = COLLPROVIDER_ICU; + else if (pg_strcasecmp(collproviderstr, "libc") == 0) + collprovider = COLLPROVIDER_LIBC; + else + ereport(ERROR, + (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), + errmsg("unrecognized collation provider: %s", + collproviderstr))); + } + else if (!fromEl) + collprovider = COLLPROVIDER_LIBC; + if (!collcollate) ereport(ERROR, (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), @@ -129,14 +165,25 @@ DefineCollation(ParseState *pstate, List *names, List *parameters, bool if_not_e (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), errmsg("parameter \"lc_ctype\" must be specified"))); - check_encoding_locale_matches(GetDatabaseEncoding(), collcollate, collctype); + if (collprovider == COLLPROVIDER_ICU) + collencoding = -1; + else + { + collencoding = GetDatabaseEncoding(); + check_encoding_locale_matches(collencoding, collcollate, collctype); + } + + if (!collversion) + collversion = get_collation_actual_version(collprovider, collcollate); newoid = CollationCreate(collName, collNamespace, GetUserId(), - GetDatabaseEncoding(), + collprovider, + collencoding, collcollate, collctype, + collversion, if_not_exists); if (!OidIsValid(newoid)) @@ -182,16 +229,118 @@ IsThereCollationInNamespace(const char *collname, Oid nspOid) collname, get_namespace_name(nspOid)))); } +/* + * ALTER COLLATION + */ +ObjectAddress +AlterCollation(AlterCollationStmt *stmt) +{ + Relation rel; + Oid collOid; + HeapTuple tup; + Form_pg_collation collForm; + Datum collversion; + bool isnull; + char *oldversion; + char *newversion; + ObjectAddress address; + + rel = heap_open(CollationRelationId, RowExclusiveLock); + collOid = get_collation_oid(stmt->collname, false); + + if (!pg_collation_ownercheck(collOid, GetUserId())) + aclcheck_error(ACLCHECK_NOT_OWNER, ACL_KIND_COLLATION, + NameListToString(stmt->collname)); + + tup = SearchSysCacheCopy1(COLLOID, ObjectIdGetDatum(collOid)); + if (!HeapTupleIsValid(tup)) + elog(ERROR, "cache lookup failed for collation %u", collOid); + + collForm = (Form_pg_collation) GETSTRUCT(tup); + collversion = SysCacheGetAttr(COLLOID, tup, Anum_pg_collation_collversion, + &isnull); + oldversion = isnull ? NULL : TextDatumGetCString(collversion); + + newversion = get_collation_actual_version(collForm->collprovider, NameStr(collForm->collcollate)); + + /* cannot change from NULL to non-NULL or vice versa */ + if ((!oldversion && newversion) || (oldversion && !newversion)) + elog(ERROR, "invalid collation version change"); + else if (oldversion && newversion && strcmp(newversion, oldversion) != 0) + { + bool nulls[Natts_pg_collation]; + bool replaces[Natts_pg_collation]; + Datum values[Natts_pg_collation]; + + ereport(NOTICE, + (errmsg("changing version from %s to %s", + oldversion, newversion))); + + memset(values, 0, sizeof(values)); + memset(nulls, false, sizeof(nulls)); + memset(replaces, false, sizeof(replaces)); + + values[Anum_pg_collation_collversion - 1] = CStringGetTextDatum(newversion); + replaces[Anum_pg_collation_collversion - 1] = true; + + tup = heap_modify_tuple(tup, RelationGetDescr(rel), + values, nulls, replaces); + } + else + ereport(NOTICE, + (errmsg("version has not changed"))); + + CatalogTupleUpdate(rel, &tup->t_self, tup); + + InvokeObjectPostAlterHook(CollationRelationId, collOid, 0); + + ObjectAddressSet(address, CollationRelationId, collOid); + + heap_freetuple(tup); + heap_close(rel, NoLock); + + return address; +} + + +Datum +pg_collation_actual_version(PG_FUNCTION_ARGS) +{ + Oid collid = PG_GETARG_OID(0); + HeapTuple tp; + char *collcollate; + char collprovider; + char *version; + + tp = SearchSysCache1(COLLOID, ObjectIdGetDatum(collid)); + if (!HeapTupleIsValid(tp)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("collation with OID %u does not exist", collid))); + + collcollate = pstrdup(NameStr(((Form_pg_collation) GETSTRUCT(tp))->collcollate)); + collprovider = ((Form_pg_collation) GETSTRUCT(tp))->collprovider; + + ReleaseSysCache(tp); + + version = get_collation_actual_version(collprovider, collcollate); + + if (version) + PG_RETURN_TEXT_P(cstring_to_text(version)); + else + PG_RETURN_NULL(); +} + /* - * "Normalize" a locale name, stripping off encoding tags such as + * "Normalize" a libc locale name, stripping off encoding tags such as * ".utf8" (e.g., "en_US.utf8" -> "en_US", but "br_FR.iso885915@euro" * -> "br_FR@euro"). Return true if a new, different name was * generated. */ pg_attribute_unused() static bool -normalize_locale_name(char *new, const char *old) +normalize_libc_locale_name(char *new, const char *old) { char *n = new; const char *o = old; @@ -219,6 +368,46 @@ normalize_locale_name(char *new, const char *old) } +#ifdef USE_ICU +static char * +get_icu_language_tag(const char *localename) +{ + char buf[ULOC_FULLNAME_CAPACITY]; + UErrorCode status; + + status = U_ZERO_ERROR; + uloc_toLanguageTag(localename, buf, sizeof(buf), TRUE, &status); + if (U_FAILURE(status)) + ereport(ERROR, + (errmsg("could not convert locale name \"%s\" to language tag: %s", + localename, u_errorName(status)))); + + return pstrdup(buf); +} + + +static char * +get_icu_locale_comment(const char *localename) +{ + UErrorCode status; + UChar displayname[128]; + int32 len_uchar; + char *result; + + status = U_ZERO_ERROR; + len_uchar = uloc_getDisplayName(localename, "en", &displayname[0], sizeof(displayname), &status); + if (U_FAILURE(status)) + ereport(ERROR, + (errmsg("could get display name for locale \"%s\": %s", + localename, u_errorName(status)))); + + icu_from_uchar(&result, displayname, len_uchar); + + return result; +} +#endif /* USE_ICU */ + + Datum pg_import_system_collations(PG_FUNCTION_ARGS) { @@ -302,8 +491,10 @@ pg_import_system_collations(PG_FUNCTION_ARGS) count++; - CollationCreate(localebuf, nspid, GetUserId(), enc, - localebuf, localebuf, if_not_exists); + CollationCreate(localebuf, nspid, GetUserId(), COLLPROVIDER_LIBC, enc, + localebuf, localebuf, + get_collation_actual_version(COLLPROVIDER_LIBC, localebuf), + if_not_exists); CommandCounterIncrement(); @@ -316,7 +507,7 @@ pg_import_system_collations(PG_FUNCTION_ARGS) * "locale -a" output. So save up the aliases and try to add them * after we've read all the output. */ - if (normalize_locale_name(alias, localebuf)) + if (normalize_libc_locale_name(alias, localebuf)) { aliaslist = lappend(aliaslist, pstrdup(alias)); localelist = lappend(localelist, pstrdup(localebuf)); @@ -333,8 +524,10 @@ pg_import_system_collations(PG_FUNCTION_ARGS) char *locale = (char *) lfirst(lcl); int enc = lfirst_int(lce); - CollationCreate(alias, nspid, GetUserId(), enc, - locale, locale, true); + CollationCreate(alias, nspid, GetUserId(), COLLPROVIDER_LIBC, enc, + locale, locale, + get_collation_actual_version(COLLPROVIDER_LIBC, locale), + true); CommandCounterIncrement(); } @@ -343,5 +536,82 @@ pg_import_system_collations(PG_FUNCTION_ARGS) (errmsg("no usable system locales were found"))); #endif /* not HAVE_LOCALE_T && not WIN32 */ +#ifdef USE_ICU + if (!is_encoding_supported_by_icu(GetDatabaseEncoding())) + { + ereport(NOTICE, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("encoding \"%s\" not supported by ICU", + pg_encoding_to_char(GetDatabaseEncoding())))); + } + else + { + int i; + + /* + * Start the loop at -1 to sneak in the root locale without too much + * code duplication. + */ + for (i = -1; i < ucol_countAvailable(); i++) + { + const char *name; + char *langtag; + const char *collcollate; + UEnumeration *en; + UErrorCode status; + const char *val; + Oid collid; + + if (i == -1) + name = ""; /* ICU root locale */ + else + name = ucol_getAvailable(i); + + langtag = get_icu_language_tag(name); + collcollate = U_ICU_VERSION_MAJOR_NUM >= 54 ? langtag : name; + collid = CollationCreate(psprintf("%s-x-icu", langtag), + nspid, GetUserId(), COLLPROVIDER_ICU, -1, + collcollate, collcollate, + get_collation_actual_version(COLLPROVIDER_ICU, collcollate), + if_not_exists); + + CreateComments(collid, CollationRelationId, 0, + get_icu_locale_comment(name)); + + /* + * Add keyword variants + */ + status = U_ZERO_ERROR; + en = ucol_getKeywordValuesForLocale("collation", name, TRUE, &status); + if (U_FAILURE(status)) + ereport(ERROR, + (errmsg("could not get keyword values for locale \"%s\": %s", + name, u_errorName(status)))); + + status = U_ZERO_ERROR; + uenum_reset(en, &status); + while ((val = uenum_next(en, NULL, &status))) + { + char *localeid = psprintf("%s@collation=%s", name, val); + + langtag = get_icu_language_tag(localeid); + collcollate = U_ICU_VERSION_MAJOR_NUM >= 54 ? langtag : localeid; + collid = CollationCreate(psprintf("%s-x-icu", langtag), + nspid, GetUserId(), COLLPROVIDER_ICU, -1, + collcollate, collcollate, + get_collation_actual_version(COLLPROVIDER_ICU, collcollate), + if_not_exists); + CreateComments(collid, CollationRelationId, 0, + get_icu_locale_comment(localeid)); + } + if (U_FAILURE(status)) + ereport(ERROR, + (errmsg("could not get keyword values for locale \"%s\": %s", + name, u_errorName(status)))); + uenum_close(en); + } + } +#endif + PG_RETURN_VOID(); } diff --git a/src/backend/common.mk b/src/backend/common.mk index 5d599dbd0c..0b57543bc4 100644 --- a/src/backend/common.mk +++ b/src/backend/common.mk @@ -8,6 +8,8 @@ # this directory and SUBDIRS to subdirectories containing more things # to build. +override CPPFLAGS := $(CPPFLAGS) $(ICU_CFLAGS) + ifdef PARTIAL_LINKING # old style: linking using SUBSYS.o subsysfilename = SUBSYS.o diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index 93d4eb207f..93bda42715 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -3046,6 +3046,16 @@ _copyAlterTableCmd(const AlterTableCmd *from) return newnode; } +static AlterCollationStmt * +_copyAlterCollationStmt(const AlterCollationStmt *from) +{ + AlterCollationStmt *newnode = makeNode(AlterCollationStmt); + + COPY_NODE_FIELD(collname); + + return newnode; +} + static AlterDomainStmt * _copyAlterDomainStmt(const AlterDomainStmt *from) { @@ -4986,6 +4996,9 @@ copyObject(const void *from) case T_AlterTableCmd: retval = _copyAlterTableCmd(from); break; + case T_AlterCollationStmt: + retval = _copyAlterCollationStmt(from); + break; case T_AlterDomainStmt: retval = _copyAlterDomainStmt(from); break; diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index 6b40b56f71..0d12636d92 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -1095,6 +1095,14 @@ _equalAlterTableCmd(const AlterTableCmd *a, const AlterTableCmd *b) return true; } +static bool +_equalAlterCollationStmt(const AlterCollationStmt *a, const AlterCollationStmt *b) +{ + COMPARE_NODE_FIELD(collname); + + return true; +} + static bool _equalAlterDomainStmt(const AlterDomainStmt *a, const AlterDomainStmt *b) { @@ -3174,6 +3182,9 @@ equal(const void *a, const void *b) case T_AlterTableCmd: retval = _equalAlterTableCmd(a, b); break; + case T_AlterCollationStmt: + retval = _equalAlterCollationStmt(a, b); + break; case T_AlterDomainStmt: retval = _equalAlterDomainStmt(a, b); break; diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 50126baacf..82844a0399 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -244,7 +244,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); } %type stmt schema_stmt - AlterEventTrigStmt + AlterEventTrigStmt AlterCollationStmt AlterDatabaseStmt AlterDatabaseSetStmt AlterDomainStmt AlterEnumStmt AlterFdwStmt AlterForeignServerStmt AlterGroupStmt AlterObjectDependsStmt AlterObjectSchemaStmt AlterOwnerStmt @@ -812,6 +812,7 @@ stmtmulti: stmtmulti ';' stmt stmt : AlterEventTrigStmt + | AlterCollationStmt | AlterDatabaseStmt | AlterDatabaseSetStmt | AlterDefaultPrivilegesStmt @@ -9705,6 +9706,21 @@ DropdbStmt: DROP DATABASE database_name ; +/***************************************************************************** + * + * ALTER COLLATION + * + *****************************************************************************/ + +AlterCollationStmt: ALTER COLLATION any_name REFRESH VERSION_P + { + AlterCollationStmt *n = makeNode(AlterCollationStmt); + n->collname = $3; + $$ = (Node *)n; + } + ; + + /***************************************************************************** * * ALTER SYSTEM diff --git a/src/backend/regex/regc_pg_locale.c b/src/backend/regex/regc_pg_locale.c index 0121cbb2ad..4bdcb4fd6a 100644 --- a/src/backend/regex/regc_pg_locale.c +++ b/src/backend/regex/regc_pg_locale.c @@ -68,7 +68,8 @@ typedef enum PG_REGEX_LOCALE_WIDE, /* Use functions */ PG_REGEX_LOCALE_1BYTE, /* Use functions */ PG_REGEX_LOCALE_WIDE_L, /* Use locale_t functions */ - PG_REGEX_LOCALE_1BYTE_L /* Use locale_t functions */ + PG_REGEX_LOCALE_1BYTE_L, /* Use locale_t functions */ + PG_REGEX_LOCALE_ICU /* Use ICU uchar.h functions */ } PG_Locale_Strategy; static PG_Locale_Strategy pg_regex_strategy; @@ -262,6 +263,11 @@ pg_set_regex_collation(Oid collation) errhint("Use the COLLATE clause to set the collation explicitly."))); } +#ifdef USE_ICU + if (pg_regex_locale && pg_regex_locale->provider == COLLPROVIDER_ICU) + pg_regex_strategy = PG_REGEX_LOCALE_ICU; + else +#endif #ifdef USE_WIDE_UPPER_LOWER if (GetDatabaseEncoding() == PG_UTF8) { @@ -303,13 +309,18 @@ pg_wc_isdigit(pg_wchar c) case PG_REGEX_LOCALE_WIDE_L: #if defined(HAVE_LOCALE_T) && defined(USE_WIDE_UPPER_LOWER) if (sizeof(wchar_t) >= 4 || c <= (pg_wchar) 0xFFFF) - return iswdigit_l((wint_t) c, pg_regex_locale); + return iswdigit_l((wint_t) c, pg_regex_locale->info.lt); #endif /* FALL THRU */ case PG_REGEX_LOCALE_1BYTE_L: #ifdef HAVE_LOCALE_T return (c <= (pg_wchar) UCHAR_MAX && - isdigit_l((unsigned char) c, pg_regex_locale)); + isdigit_l((unsigned char) c, pg_regex_locale->info.lt)); +#endif + break; + case PG_REGEX_LOCALE_ICU: +#ifdef USE_ICU + return u_isdigit(c); #endif break; } @@ -336,13 +347,18 @@ pg_wc_isalpha(pg_wchar c) case PG_REGEX_LOCALE_WIDE_L: #if defined(HAVE_LOCALE_T) && defined(USE_WIDE_UPPER_LOWER) if (sizeof(wchar_t) >= 4 || c <= (pg_wchar) 0xFFFF) - return iswalpha_l((wint_t) c, pg_regex_locale); + return iswalpha_l((wint_t) c, pg_regex_locale->info.lt); #endif /* FALL THRU */ case PG_REGEX_LOCALE_1BYTE_L: #ifdef HAVE_LOCALE_T return (c <= (pg_wchar) UCHAR_MAX && - isalpha_l((unsigned char) c, pg_regex_locale)); + isalpha_l((unsigned char) c, pg_regex_locale->info.lt)); +#endif + break; + case PG_REGEX_LOCALE_ICU: +#ifdef USE_ICU + return u_isalpha(c); #endif break; } @@ -369,13 +385,18 @@ pg_wc_isalnum(pg_wchar c) case PG_REGEX_LOCALE_WIDE_L: #if defined(HAVE_LOCALE_T) && defined(USE_WIDE_UPPER_LOWER) if (sizeof(wchar_t) >= 4 || c <= (pg_wchar) 0xFFFF) - return iswalnum_l((wint_t) c, pg_regex_locale); + return iswalnum_l((wint_t) c, pg_regex_locale->info.lt); #endif /* FALL THRU */ case PG_REGEX_LOCALE_1BYTE_L: #ifdef HAVE_LOCALE_T return (c <= (pg_wchar) UCHAR_MAX && - isalnum_l((unsigned char) c, pg_regex_locale)); + isalnum_l((unsigned char) c, pg_regex_locale->info.lt)); +#endif + break; + case PG_REGEX_LOCALE_ICU: +#ifdef USE_ICU + return u_isalnum(c); #endif break; } @@ -402,13 +423,18 @@ pg_wc_isupper(pg_wchar c) case PG_REGEX_LOCALE_WIDE_L: #if defined(HAVE_LOCALE_T) && defined(USE_WIDE_UPPER_LOWER) if (sizeof(wchar_t) >= 4 || c <= (pg_wchar) 0xFFFF) - return iswupper_l((wint_t) c, pg_regex_locale); + return iswupper_l((wint_t) c, pg_regex_locale->info.lt); #endif /* FALL THRU */ case PG_REGEX_LOCALE_1BYTE_L: #ifdef HAVE_LOCALE_T return (c <= (pg_wchar) UCHAR_MAX && - isupper_l((unsigned char) c, pg_regex_locale)); + isupper_l((unsigned char) c, pg_regex_locale->info.lt)); +#endif + break; + case PG_REGEX_LOCALE_ICU: +#ifdef USE_ICU + return u_isupper(c); #endif break; } @@ -435,13 +461,18 @@ pg_wc_islower(pg_wchar c) case PG_REGEX_LOCALE_WIDE_L: #if defined(HAVE_LOCALE_T) && defined(USE_WIDE_UPPER_LOWER) if (sizeof(wchar_t) >= 4 || c <= (pg_wchar) 0xFFFF) - return iswlower_l((wint_t) c, pg_regex_locale); + return iswlower_l((wint_t) c, pg_regex_locale->info.lt); #endif /* FALL THRU */ case PG_REGEX_LOCALE_1BYTE_L: #ifdef HAVE_LOCALE_T return (c <= (pg_wchar) UCHAR_MAX && - islower_l((unsigned char) c, pg_regex_locale)); + islower_l((unsigned char) c, pg_regex_locale->info.lt)); +#endif + break; + case PG_REGEX_LOCALE_ICU: +#ifdef USE_ICU + return u_islower(c); #endif break; } @@ -468,13 +499,18 @@ pg_wc_isgraph(pg_wchar c) case PG_REGEX_LOCALE_WIDE_L: #if defined(HAVE_LOCALE_T) && defined(USE_WIDE_UPPER_LOWER) if (sizeof(wchar_t) >= 4 || c <= (pg_wchar) 0xFFFF) - return iswgraph_l((wint_t) c, pg_regex_locale); + return iswgraph_l((wint_t) c, pg_regex_locale->info.lt); #endif /* FALL THRU */ case PG_REGEX_LOCALE_1BYTE_L: #ifdef HAVE_LOCALE_T return (c <= (pg_wchar) UCHAR_MAX && - isgraph_l((unsigned char) c, pg_regex_locale)); + isgraph_l((unsigned char) c, pg_regex_locale->info.lt)); +#endif + break; + case PG_REGEX_LOCALE_ICU: +#ifdef USE_ICU + return u_isgraph(c); #endif break; } @@ -501,13 +537,18 @@ pg_wc_isprint(pg_wchar c) case PG_REGEX_LOCALE_WIDE_L: #if defined(HAVE_LOCALE_T) && defined(USE_WIDE_UPPER_LOWER) if (sizeof(wchar_t) >= 4 || c <= (pg_wchar) 0xFFFF) - return iswprint_l((wint_t) c, pg_regex_locale); + return iswprint_l((wint_t) c, pg_regex_locale->info.lt); #endif /* FALL THRU */ case PG_REGEX_LOCALE_1BYTE_L: #ifdef HAVE_LOCALE_T return (c <= (pg_wchar) UCHAR_MAX && - isprint_l((unsigned char) c, pg_regex_locale)); + isprint_l((unsigned char) c, pg_regex_locale->info.lt)); +#endif + break; + case PG_REGEX_LOCALE_ICU: +#ifdef USE_ICU + return u_isprint(c); #endif break; } @@ -534,13 +575,18 @@ pg_wc_ispunct(pg_wchar c) case PG_REGEX_LOCALE_WIDE_L: #if defined(HAVE_LOCALE_T) && defined(USE_WIDE_UPPER_LOWER) if (sizeof(wchar_t) >= 4 || c <= (pg_wchar) 0xFFFF) - return iswpunct_l((wint_t) c, pg_regex_locale); + return iswpunct_l((wint_t) c, pg_regex_locale->info.lt); #endif /* FALL THRU */ case PG_REGEX_LOCALE_1BYTE_L: #ifdef HAVE_LOCALE_T return (c <= (pg_wchar) UCHAR_MAX && - ispunct_l((unsigned char) c, pg_regex_locale)); + ispunct_l((unsigned char) c, pg_regex_locale->info.lt)); +#endif + break; + case PG_REGEX_LOCALE_ICU: +#ifdef USE_ICU + return u_ispunct(c); #endif break; } @@ -567,13 +613,18 @@ pg_wc_isspace(pg_wchar c) case PG_REGEX_LOCALE_WIDE_L: #if defined(HAVE_LOCALE_T) && defined(USE_WIDE_UPPER_LOWER) if (sizeof(wchar_t) >= 4 || c <= (pg_wchar) 0xFFFF) - return iswspace_l((wint_t) c, pg_regex_locale); + return iswspace_l((wint_t) c, pg_regex_locale->info.lt); #endif /* FALL THRU */ case PG_REGEX_LOCALE_1BYTE_L: #ifdef HAVE_LOCALE_T return (c <= (pg_wchar) UCHAR_MAX && - isspace_l((unsigned char) c, pg_regex_locale)); + isspace_l((unsigned char) c, pg_regex_locale->info.lt)); +#endif + break; + case PG_REGEX_LOCALE_ICU: +#ifdef USE_ICU + return u_isspace(c); #endif break; } @@ -608,15 +659,20 @@ pg_wc_toupper(pg_wchar c) case PG_REGEX_LOCALE_WIDE_L: #if defined(HAVE_LOCALE_T) && defined(USE_WIDE_UPPER_LOWER) if (sizeof(wchar_t) >= 4 || c <= (pg_wchar) 0xFFFF) - return towupper_l((wint_t) c, pg_regex_locale); + return towupper_l((wint_t) c, pg_regex_locale->info.lt); #endif /* FALL THRU */ case PG_REGEX_LOCALE_1BYTE_L: #ifdef HAVE_LOCALE_T if (c <= (pg_wchar) UCHAR_MAX) - return toupper_l((unsigned char) c, pg_regex_locale); + return toupper_l((unsigned char) c, pg_regex_locale->info.lt); #endif return c; + case PG_REGEX_LOCALE_ICU: +#ifdef USE_ICU + return u_toupper(c); +#endif + break; } return 0; /* can't get here, but keep compiler quiet */ } @@ -649,15 +705,20 @@ pg_wc_tolower(pg_wchar c) case PG_REGEX_LOCALE_WIDE_L: #if defined(HAVE_LOCALE_T) && defined(USE_WIDE_UPPER_LOWER) if (sizeof(wchar_t) >= 4 || c <= (pg_wchar) 0xFFFF) - return towlower_l((wint_t) c, pg_regex_locale); + return towlower_l((wint_t) c, pg_regex_locale->info.lt); #endif /* FALL THRU */ case PG_REGEX_LOCALE_1BYTE_L: #ifdef HAVE_LOCALE_T if (c <= (pg_wchar) UCHAR_MAX) - return tolower_l((unsigned char) c, pg_regex_locale); + return tolower_l((unsigned char) c, pg_regex_locale->info.lt); #endif return c; + case PG_REGEX_LOCALE_ICU: +#ifdef USE_ICU + return u_tolower(c); +#endif + break; } return 0; /* can't get here, but keep compiler quiet */ } @@ -808,6 +869,9 @@ pg_ctype_get_cache(pg_wc_probefunc probefunc, int cclasscode) max_chr = (pg_wchar) MAX_SIMPLE_CHR; #endif break; + case PG_REGEX_LOCALE_ICU: + max_chr = (pg_wchar) MAX_SIMPLE_CHR; + break; default: max_chr = 0; /* can't get here, but keep compiler quiet */ break; diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index 20b5273405..c8d20fffea 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -1623,6 +1623,10 @@ ProcessUtilitySlow(ParseState *pstate, commandCollected = true; break; + case T_AlterCollationStmt: + address = AlterCollation((AlterCollationStmt *) parsetree); + break; + default: elog(ERROR, "unrecognized node type: %d", (int) nodeTag(parsetree)); @@ -2673,6 +2677,10 @@ CreateCommandTag(Node *parsetree) tag = "DROP SUBSCRIPTION"; break; + case T_AlterCollationStmt: + tag = "ALTER COLLATION"; + break; + case T_PrepareStmt: tag = "PREPARE"; break; diff --git a/src/backend/utils/adt/formatting.c b/src/backend/utils/adt/formatting.c index c16bfbca93..0566abd314 100644 --- a/src/backend/utils/adt/formatting.c +++ b/src/backend/utils/adt/formatting.c @@ -82,6 +82,10 @@ #include #endif +#ifdef USE_ICU +#include +#endif + #include "catalog/pg_collation.h" #include "mb/pg_wchar.h" #include "utils/builtins.h" @@ -1443,6 +1447,42 @@ str_numth(char *dest, char *num, int type) * upper/lower/initcap functions *****************************************************************************/ +#ifdef USE_ICU +static int32_t +icu_convert_case(int32_t (*func)(UChar *, int32_t, const UChar *, int32_t, const char *, UErrorCode *), + pg_locale_t mylocale, UChar **buff_dest, UChar *buff_source, int32_t len_source) +{ + UErrorCode status; + int32_t len_dest; + + len_dest = len_source; /* try first with same length */ + *buff_dest = palloc(len_dest * sizeof(**buff_dest)); + status = U_ZERO_ERROR; + len_dest = func(*buff_dest, len_dest, buff_source, len_source, mylocale->info.icu.locale, &status); + if (status == U_BUFFER_OVERFLOW_ERROR) + { + /* try again with adjusted length */ + pfree(buff_dest); + buff_dest = palloc(len_dest * sizeof(**buff_dest)); + status = U_ZERO_ERROR; + len_dest = func(*buff_dest, len_dest, buff_source, len_source, mylocale->info.icu.locale, &status); + } + if (U_FAILURE(status)) + ereport(ERROR, + (errmsg("case conversion failed: %s", u_errorName(status)))); + return len_dest; +} + +static int32_t +u_strToTitle_default_BI(UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + const char *locale, + UErrorCode *pErrorCode) +{ + return u_strToTitle(dest, destCapacity, src, srcLength, NULL, locale, pErrorCode); +} +#endif + /* * If the system provides the needed functions for wide-character manipulation * (which are all standardized by C99), then we implement upper/lower/initcap @@ -1479,12 +1519,9 @@ str_tolower(const char *buff, size_t nbytes, Oid collid) result = asc_tolower(buff, nbytes); } #ifdef USE_WIDE_UPPER_LOWER - else if (pg_database_encoding_max_length() > 1) + else { pg_locale_t mylocale = 0; - wchar_t *workspace; - size_t curr_char; - size_t result_size; if (collid != DEFAULT_COLLATION_OID) { @@ -1502,77 +1539,79 @@ str_tolower(const char *buff, size_t nbytes, Oid collid) mylocale = pg_newlocale_from_collation(collid); } - /* Overflow paranoia */ - if ((nbytes + 1) > (INT_MAX / sizeof(wchar_t))) - ereport(ERROR, - (errcode(ERRCODE_OUT_OF_MEMORY), - errmsg("out of memory"))); +#ifdef USE_ICU + if (mylocale && mylocale->provider == COLLPROVIDER_ICU) + { + int32_t len_uchar; + int32_t len_conv; + UChar *buff_uchar; + UChar *buff_conv; + + len_uchar = icu_to_uchar(&buff_uchar, buff, nbytes); + len_conv = icu_convert_case(u_strToLower, mylocale, &buff_conv, buff_uchar, len_uchar); + icu_from_uchar(&result, buff_conv, len_conv); + } + else +#endif + { + if (pg_database_encoding_max_length() > 1) + { + wchar_t *workspace; + size_t curr_char; + size_t result_size; - /* Output workspace cannot have more codes than input bytes */ - workspace = (wchar_t *) palloc((nbytes + 1) * sizeof(wchar_t)); + /* Overflow paranoia */ + if ((nbytes + 1) > (INT_MAX / sizeof(wchar_t))) + ereport(ERROR, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("out of memory"))); - char2wchar(workspace, nbytes + 1, buff, nbytes, mylocale); + /* Output workspace cannot have more codes than input bytes */ + workspace = (wchar_t *) palloc((nbytes + 1) * sizeof(wchar_t)); - for (curr_char = 0; workspace[curr_char] != 0; curr_char++) - { + char2wchar(workspace, nbytes + 1, buff, nbytes, mylocale); + + for (curr_char = 0; workspace[curr_char] != 0; curr_char++) + { #ifdef HAVE_LOCALE_T - if (mylocale) - workspace[curr_char] = towlower_l(workspace[curr_char], mylocale); - else + if (mylocale) + workspace[curr_char] = towlower_l(workspace[curr_char], mylocale->info.lt); + else #endif - workspace[curr_char] = towlower(workspace[curr_char]); - } + workspace[curr_char] = towlower(workspace[curr_char]); + } - /* Make result large enough; case change might change number of bytes */ - result_size = curr_char * pg_database_encoding_max_length() + 1; - result = palloc(result_size); + /* Make result large enough; case change might change number of bytes */ + result_size = curr_char * pg_database_encoding_max_length() + 1; + result = palloc(result_size); - wchar2char(result, workspace, result_size, mylocale); - pfree(workspace); - } + wchar2char(result, workspace, result_size, mylocale); + pfree(workspace); + } #endif /* USE_WIDE_UPPER_LOWER */ - else - { -#ifdef HAVE_LOCALE_T - pg_locale_t mylocale = 0; -#endif - char *p; - - if (collid != DEFAULT_COLLATION_OID) - { - if (!OidIsValid(collid)) + else { - /* - * This typically means that the parser could not resolve a - * conflict of implicit collations, so report it that way. - */ - ereport(ERROR, - (errcode(ERRCODE_INDETERMINATE_COLLATION), - errmsg("could not determine which collation to use for lower() function"), - errhint("Use the COLLATE clause to set the collation explicitly."))); - } -#ifdef HAVE_LOCALE_T - mylocale = pg_newlocale_from_collation(collid); -#endif - } + char *p; - result = pnstrdup(buff, nbytes); + result = pnstrdup(buff, nbytes); - /* - * Note: we assume that tolower_l() will not be so broken as to need - * an isupper_l() guard test. When using the default collation, we - * apply the traditional Postgres behavior that forces ASCII-style - * treatment of I/i, but in non-default collations you get exactly - * what the collation says. - */ - for (p = result; *p; p++) - { + /* + * Note: we assume that tolower_l() will not be so broken as to need + * an isupper_l() guard test. When using the default collation, we + * apply the traditional Postgres behavior that forces ASCII-style + * treatment of I/i, but in non-default collations you get exactly + * what the collation says. + */ + for (p = result; *p; p++) + { #ifdef HAVE_LOCALE_T - if (mylocale) - *p = tolower_l((unsigned char) *p, mylocale); - else + if (mylocale) + *p = tolower_l((unsigned char) *p, mylocale->info.lt); + else #endif - *p = pg_tolower((unsigned char) *p); + *p = pg_tolower((unsigned char) *p); + } + } } } @@ -1599,12 +1638,9 @@ str_toupper(const char *buff, size_t nbytes, Oid collid) result = asc_toupper(buff, nbytes); } #ifdef USE_WIDE_UPPER_LOWER - else if (pg_database_encoding_max_length() > 1) + else { pg_locale_t mylocale = 0; - wchar_t *workspace; - size_t curr_char; - size_t result_size; if (collid != DEFAULT_COLLATION_OID) { @@ -1622,77 +1658,78 @@ str_toupper(const char *buff, size_t nbytes, Oid collid) mylocale = pg_newlocale_from_collation(collid); } - /* Overflow paranoia */ - if ((nbytes + 1) > (INT_MAX / sizeof(wchar_t))) - ereport(ERROR, - (errcode(ERRCODE_OUT_OF_MEMORY), - errmsg("out of memory"))); +#ifdef USE_ICU + if (mylocale && mylocale->provider == COLLPROVIDER_ICU) + { + int32_t len_uchar, len_conv; + UChar *buff_uchar; + UChar *buff_conv; - /* Output workspace cannot have more codes than input bytes */ - workspace = (wchar_t *) palloc((nbytes + 1) * sizeof(wchar_t)); + len_uchar = icu_to_uchar(&buff_uchar, buff, nbytes); + len_conv = icu_convert_case(u_strToUpper, mylocale, &buff_conv, buff_uchar, len_uchar); + icu_from_uchar(&result, buff_conv, len_conv); + } + else +#endif + { + if (pg_database_encoding_max_length() > 1) + { + wchar_t *workspace; + size_t curr_char; + size_t result_size; - char2wchar(workspace, nbytes + 1, buff, nbytes, mylocale); + /* Overflow paranoia */ + if ((nbytes + 1) > (INT_MAX / sizeof(wchar_t))) + ereport(ERROR, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("out of memory"))); - for (curr_char = 0; workspace[curr_char] != 0; curr_char++) - { -#ifdef HAVE_LOCALE_T - if (mylocale) - workspace[curr_char] = towupper_l(workspace[curr_char], mylocale); - else -#endif - workspace[curr_char] = towupper(workspace[curr_char]); - } + /* Output workspace cannot have more codes than input bytes */ + workspace = (wchar_t *) palloc((nbytes + 1) * sizeof(wchar_t)); - /* Make result large enough; case change might change number of bytes */ - result_size = curr_char * pg_database_encoding_max_length() + 1; - result = palloc(result_size); + char2wchar(workspace, nbytes + 1, buff, nbytes, mylocale); - wchar2char(result, workspace, result_size, mylocale); - pfree(workspace); - } -#endif /* USE_WIDE_UPPER_LOWER */ - else - { + for (curr_char = 0; workspace[curr_char] != 0; curr_char++) + { #ifdef HAVE_LOCALE_T - pg_locale_t mylocale = 0; + if (mylocale) + workspace[curr_char] = towupper_l(workspace[curr_char], mylocale->info.lt); + else #endif - char *p; + workspace[curr_char] = towupper(workspace[curr_char]); + } - if (collid != DEFAULT_COLLATION_OID) - { - if (!OidIsValid(collid)) - { - /* - * This typically means that the parser could not resolve a - * conflict of implicit collations, so report it that way. - */ - ereport(ERROR, - (errcode(ERRCODE_INDETERMINATE_COLLATION), - errmsg("could not determine which collation to use for upper() function"), - errhint("Use the COLLATE clause to set the collation explicitly."))); + /* Make result large enough; case change might change number of bytes */ + result_size = curr_char * pg_database_encoding_max_length() + 1; + result = palloc(result_size); + + wchar2char(result, workspace, result_size, mylocale); + pfree(workspace); } -#ifdef HAVE_LOCALE_T - mylocale = pg_newlocale_from_collation(collid); -#endif - } +#endif /* USE_WIDE_UPPER_LOWER */ + else + { + char *p; - result = pnstrdup(buff, nbytes); + result = pnstrdup(buff, nbytes); - /* - * Note: we assume that toupper_l() will not be so broken as to need - * an islower_l() guard test. When using the default collation, we - * apply the traditional Postgres behavior that forces ASCII-style - * treatment of I/i, but in non-default collations you get exactly - * what the collation says. - */ - for (p = result; *p; p++) - { + /* + * Note: we assume that toupper_l() will not be so broken as to need + * an islower_l() guard test. When using the default collation, we + * apply the traditional Postgres behavior that forces ASCII-style + * treatment of I/i, but in non-default collations you get exactly + * what the collation says. + */ + for (p = result; *p; p++) + { #ifdef HAVE_LOCALE_T - if (mylocale) - *p = toupper_l((unsigned char) *p, mylocale); - else + if (mylocale) + *p = toupper_l((unsigned char) *p, mylocale->info.lt); + else #endif - *p = pg_toupper((unsigned char) *p); + *p = pg_toupper((unsigned char) *p); + } + } } } @@ -1720,12 +1757,9 @@ str_initcap(const char *buff, size_t nbytes, Oid collid) result = asc_initcap(buff, nbytes); } #ifdef USE_WIDE_UPPER_LOWER - else if (pg_database_encoding_max_length() > 1) + else { pg_locale_t mylocale = 0; - wchar_t *workspace; - size_t curr_char; - size_t result_size; if (collid != DEFAULT_COLLATION_OID) { @@ -1743,100 +1777,101 @@ str_initcap(const char *buff, size_t nbytes, Oid collid) mylocale = pg_newlocale_from_collation(collid); } - /* Overflow paranoia */ - if ((nbytes + 1) > (INT_MAX / sizeof(wchar_t))) - ereport(ERROR, - (errcode(ERRCODE_OUT_OF_MEMORY), - errmsg("out of memory"))); - - /* Output workspace cannot have more codes than input bytes */ - workspace = (wchar_t *) palloc((nbytes + 1) * sizeof(wchar_t)); - - char2wchar(workspace, nbytes + 1, buff, nbytes, mylocale); - - for (curr_char = 0; workspace[curr_char] != 0; curr_char++) +#ifdef USE_ICU + if (mylocale && mylocale->provider == COLLPROVIDER_ICU) { -#ifdef HAVE_LOCALE_T - if (mylocale) - { - if (wasalnum) - workspace[curr_char] = towlower_l(workspace[curr_char], mylocale); - else - workspace[curr_char] = towupper_l(workspace[curr_char], mylocale); - wasalnum = iswalnum_l(workspace[curr_char], mylocale); - } - else + int32_t len_uchar, len_conv; + UChar *buff_uchar; + UChar *buff_conv; + + len_uchar = icu_to_uchar(&buff_uchar, buff, nbytes); + len_conv = icu_convert_case(u_strToTitle_default_BI, mylocale, &buff_conv, buff_uchar, len_uchar); + icu_from_uchar(&result, buff_conv, len_conv); + } + else #endif + { + if (pg_database_encoding_max_length() > 1) { - if (wasalnum) - workspace[curr_char] = towlower(workspace[curr_char]); - else - workspace[curr_char] = towupper(workspace[curr_char]); - wasalnum = iswalnum(workspace[curr_char]); - } - } + wchar_t *workspace; + size_t curr_char; + size_t result_size; - /* Make result large enough; case change might change number of bytes */ - result_size = curr_char * pg_database_encoding_max_length() + 1; - result = palloc(result_size); + /* Overflow paranoia */ + if ((nbytes + 1) > (INT_MAX / sizeof(wchar_t))) + ereport(ERROR, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("out of memory"))); - wchar2char(result, workspace, result_size, mylocale); - pfree(workspace); - } -#endif /* USE_WIDE_UPPER_LOWER */ - else - { -#ifdef HAVE_LOCALE_T - pg_locale_t mylocale = 0; -#endif - char *p; + /* Output workspace cannot have more codes than input bytes */ + workspace = (wchar_t *) palloc((nbytes + 1) * sizeof(wchar_t)); - if (collid != DEFAULT_COLLATION_OID) - { - if (!OidIsValid(collid)) - { - /* - * This typically means that the parser could not resolve a - * conflict of implicit collations, so report it that way. - */ - ereport(ERROR, - (errcode(ERRCODE_INDETERMINATE_COLLATION), - errmsg("could not determine which collation to use for initcap() function"), - errhint("Use the COLLATE clause to set the collation explicitly."))); - } + char2wchar(workspace, nbytes + 1, buff, nbytes, mylocale); + + for (curr_char = 0; workspace[curr_char] != 0; curr_char++) + { #ifdef HAVE_LOCALE_T - mylocale = pg_newlocale_from_collation(collid); + if (mylocale) + { + if (wasalnum) + workspace[curr_char] = towlower_l(workspace[curr_char], mylocale->info.lt); + else + workspace[curr_char] = towupper_l(workspace[curr_char], mylocale->info.lt); + wasalnum = iswalnum_l(workspace[curr_char], mylocale->info.lt); + } + else #endif - } + { + if (wasalnum) + workspace[curr_char] = towlower(workspace[curr_char]); + else + workspace[curr_char] = towupper(workspace[curr_char]); + wasalnum = iswalnum(workspace[curr_char]); + } + } - result = pnstrdup(buff, nbytes); + /* Make result large enough; case change might change number of bytes */ + result_size = curr_char * pg_database_encoding_max_length() + 1; + result = palloc(result_size); - /* - * Note: we assume that toupper_l()/tolower_l() will not be so broken - * as to need guard tests. When using the default collation, we apply - * the traditional Postgres behavior that forces ASCII-style treatment - * of I/i, but in non-default collations you get exactly what the - * collation says. - */ - for (p = result; *p; p++) - { -#ifdef HAVE_LOCALE_T - if (mylocale) - { - if (wasalnum) - *p = tolower_l((unsigned char) *p, mylocale); - else - *p = toupper_l((unsigned char) *p, mylocale); - wasalnum = isalnum_l((unsigned char) *p, mylocale); + wchar2char(result, workspace, result_size, mylocale); + pfree(workspace); } +#endif /* USE_WIDE_UPPER_LOWER */ else -#endif { - if (wasalnum) - *p = pg_tolower((unsigned char) *p); - else - *p = pg_toupper((unsigned char) *p); - wasalnum = isalnum((unsigned char) *p); + char *p; + + result = pnstrdup(buff, nbytes); + + /* + * Note: we assume that toupper_l()/tolower_l() will not be so broken + * as to need guard tests. When using the default collation, we apply + * the traditional Postgres behavior that forces ASCII-style treatment + * of I/i, but in non-default collations you get exactly what the + * collation says. + */ + for (p = result; *p; p++) + { +#ifdef HAVE_LOCALE_T + if (mylocale) + { + if (wasalnum) + *p = tolower_l((unsigned char) *p, mylocale->info.lt); + else + *p = toupper_l((unsigned char) *p, mylocale->info.lt); + wasalnum = isalnum_l((unsigned char) *p, mylocale->info.lt); + } + else +#endif + { + if (wasalnum) + *p = pg_tolower((unsigned char) *p); + else + *p = pg_toupper((unsigned char) *p); + wasalnum = isalnum((unsigned char) *p); + } + } } } } diff --git a/src/backend/utils/adt/like.c b/src/backend/utils/adt/like.c index 8d9d285fb5..1f683ccd0f 100644 --- a/src/backend/utils/adt/like.c +++ b/src/backend/utils/adt/like.c @@ -96,7 +96,7 @@ SB_lower_char(unsigned char c, pg_locale_t locale, bool locale_is_c) return pg_ascii_tolower(c); #ifdef HAVE_LOCALE_T else if (locale) - return tolower_l(c, locale); + return tolower_l(c, locale->info.lt); #endif else return pg_tolower(c); @@ -165,14 +165,36 @@ Generic_Text_IC_like(text *str, text *pat, Oid collation) *p; int slen, plen; + pg_locale_t locale = 0; + bool locale_is_c = false; + + if (lc_ctype_is_c(collation)) + locale_is_c = true; + else if (collation != DEFAULT_COLLATION_OID) + { + if (!OidIsValid(collation)) + { + /* + * This typically means that the parser could not resolve a + * conflict of implicit collations, so report it that way. + */ + ereport(ERROR, + (errcode(ERRCODE_INDETERMINATE_COLLATION), + errmsg("could not determine which collation to use for ILIKE"), + errhint("Use the COLLATE clause to set the collation explicitly."))); + } + locale = pg_newlocale_from_collation(collation); + } /* * For efficiency reasons, in the single byte case we don't call lower() * on the pattern and text, but instead call SB_lower_char on each - * character. In the multi-byte case we don't have much choice :-( + * character. In the multi-byte case we don't have much choice :-(. + * Also, ICU does not support single-character case folding, so we go the + * long way. */ - if (pg_database_encoding_max_length() > 1) + if (pg_database_encoding_max_length() > 1 || locale->provider == COLLPROVIDER_ICU) { /* lower's result is never packed, so OK to use old macros here */ pat = DatumGetTextPP(DirectFunctionCall1Coll(lower, collation, @@ -190,31 +212,6 @@ Generic_Text_IC_like(text *str, text *pat, Oid collation) } else { - /* - * Here we need to prepare locale information for SB_lower_char. This - * should match the methods used in str_tolower(). - */ - pg_locale_t locale = 0; - bool locale_is_c = false; - - if (lc_ctype_is_c(collation)) - locale_is_c = true; - else if (collation != DEFAULT_COLLATION_OID) - { - if (!OidIsValid(collation)) - { - /* - * This typically means that the parser could not resolve a - * conflict of implicit collations, so report it that way. - */ - ereport(ERROR, - (errcode(ERRCODE_INDETERMINATE_COLLATION), - errmsg("could not determine which collation to use for ILIKE"), - errhint("Use the COLLATE clause to set the collation explicitly."))); - } - locale = pg_newlocale_from_collation(collation); - } - p = VARDATA_ANY(pat); plen = VARSIZE_ANY_EXHDR(pat); s = VARDATA_ANY(str); diff --git a/src/backend/utils/adt/pg_locale.c b/src/backend/utils/adt/pg_locale.c index ab197025f8..2a2c9bc504 100644 --- a/src/backend/utils/adt/pg_locale.c +++ b/src/backend/utils/adt/pg_locale.c @@ -57,11 +57,17 @@ #include "catalog/pg_collation.h" #include "catalog/pg_control.h" #include "mb/pg_wchar.h" +#include "utils/builtins.h" #include "utils/hsearch.h" +#include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/pg_locale.h" #include "utils/syscache.h" +#ifdef USE_ICU +#include +#endif + #ifdef WIN32 /* * This Windows file defines StrNCpy. We don't need it here, so we undefine @@ -1272,12 +1278,13 @@ pg_newlocale_from_collation(Oid collid) if (cache_entry->locale == 0) { /* We haven't computed this yet in this session, so do it */ -#ifdef HAVE_LOCALE_T HeapTuple tp; Form_pg_collation collform; const char *collcollate; - const char *collctype; - locale_t result; + const char *collctype pg_attribute_unused(); + pg_locale_t result; + Datum collversion; + bool isnull; tp = SearchSysCache1(COLLOID, ObjectIdGetDatum(collid)); if (!HeapTupleIsValid(tp)) @@ -1287,61 +1294,230 @@ pg_newlocale_from_collation(Oid collid) collcollate = NameStr(collform->collcollate); collctype = NameStr(collform->collctype); - if (strcmp(collcollate, collctype) == 0) + result = malloc(sizeof(* result)); + memset(result, 0, sizeof(* result)); + result->provider = collform->collprovider; + + if (collform->collprovider == COLLPROVIDER_LIBC) { - /* Normal case where they're the same */ +#ifdef HAVE_LOCALE_T + locale_t loc; + + if (strcmp(collcollate, collctype) == 0) + { + /* Normal case where they're the same */ #ifndef WIN32 - result = newlocale(LC_COLLATE_MASK | LC_CTYPE_MASK, collcollate, - NULL); + loc = newlocale(LC_COLLATE_MASK | LC_CTYPE_MASK, collcollate, + NULL); #else - result = _create_locale(LC_ALL, collcollate); + loc = _create_locale(LC_ALL, collcollate); #endif - if (!result) - report_newlocale_failure(collcollate); - } - else - { + if (!loc) + report_newlocale_failure(collcollate); + } + else + { #ifndef WIN32 - /* We need two newlocale() steps */ - locale_t loc1; - - loc1 = newlocale(LC_COLLATE_MASK, collcollate, NULL); - if (!loc1) - report_newlocale_failure(collcollate); - result = newlocale(LC_CTYPE_MASK, collctype, loc1); - if (!result) - report_newlocale_failure(collctype); + /* We need two newlocale() steps */ + locale_t loc1; + + loc1 = newlocale(LC_COLLATE_MASK, collcollate, NULL); + if (!loc1) + report_newlocale_failure(collcollate); + loc = newlocale(LC_CTYPE_MASK, collctype, loc1); + if (!loc) + report_newlocale_failure(collctype); #else - /* - * XXX The _create_locale() API doesn't appear to support this. - * Could perhaps be worked around by changing pg_locale_t to - * contain two separate fields. - */ + /* + * XXX The _create_locale() API doesn't appear to support this. + * Could perhaps be worked around by changing pg_locale_t to + * contain two separate fields. + */ + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("collations with different collate and ctype values are not supported on this platform"))); +#endif + } + + result->info.lt = loc; +#else /* not HAVE_LOCALE_T */ + /* platform that doesn't support locale_t */ ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("collations with different collate and ctype values are not supported on this platform"))); -#endif + errmsg("collation provider LIBC is not supported on this platform"))); +#endif /* not HAVE_LOCALE_T */ + } + else if (collform->collprovider == COLLPROVIDER_ICU) + { +#ifdef USE_ICU + UCollator *collator; + UErrorCode status; + + status = U_ZERO_ERROR; + collator = ucol_open(collcollate, &status); + if (U_FAILURE(status)) + ereport(ERROR, + (errmsg("could not open collator for locale \"%s\": %s", + collcollate, u_errorName(status)))); + + result->info.icu.locale = strdup(collcollate); + result->info.icu.ucol = collator; +#else /* not USE_ICU */ + /* could get here if a collation was created by a build with ICU */ + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("ICU is not supported in this build"), \ + errhint("You need to rebuild PostgreSQL using --with-icu."))); +#endif /* not USE_ICU */ } - cache_entry->locale = result; + collversion = SysCacheGetAttr(COLLOID, tp, Anum_pg_collation_collversion, + &isnull); + if (!isnull) + { + char *actual_versionstr; + char *collversionstr; + + actual_versionstr = get_collation_actual_version(collform->collprovider, collcollate); + if (!actual_versionstr) + /* This could happen when specifying a version in CREATE + * COLLATION for a libc locale, or manually creating a mess + * in the catalogs. */ + ereport(ERROR, + (errmsg("collation \"%s\" has no actual version, but a version was specified", + NameStr(collform->collname)))); + collversionstr = TextDatumGetCString(collversion); + + if (strcmp(actual_versionstr, collversionstr) != 0) + ereport(WARNING, + (errmsg("collation \"%s\" has version mismatch", + NameStr(collform->collname)), + errdetail("The collation in the database was created using version %s, " + "but the operating system provides version %s.", + collversionstr, actual_versionstr), + errhint("Rebuild all objects affected by this collation and run " + "ALTER COLLATION %s REFRESH VERSION, " + "or build PostgreSQL with the right library version.", + quote_qualified_identifier(get_namespace_name(collform->collnamespace), + NameStr(collform->collname))))); + } ReleaseSysCache(tp); -#else /* not HAVE_LOCALE_T */ - /* - * For platforms that don't support locale_t, we can't do anything - * with non-default collations. - */ - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("nondefault collations are not supported on this platform"))); -#endif /* not HAVE_LOCALE_T */ + cache_entry->locale = result; } return cache_entry->locale; } +/* + * Get provider-specific collation version string for the given collation from + * the operating system/library. + * + * A particular provider must always either return a non-NULL string or return + * NULL (if it doesn't support versions). It must not return NULL for some + * collcollate and not NULL for others. + */ +char * +get_collation_actual_version(char collprovider, const char *collcollate) +{ + char *collversion; + +#ifdef USE_ICU + if (collprovider == COLLPROVIDER_ICU) + { + UCollator *collator; + UErrorCode status; + UVersionInfo versioninfo; + char buf[U_MAX_VERSION_STRING_LENGTH]; + + status = U_ZERO_ERROR; + collator = ucol_open(collcollate, &status); + if (U_FAILURE(status)) + ereport(ERROR, + (errmsg("could not open collator for locale \"%s\": %s", + collcollate, u_errorName(status)))); + ucol_getVersion(collator, versioninfo); + ucol_close(collator); + + u_versionToString(versioninfo, buf); + collversion = pstrdup(buf); + } + else +#endif + collversion = NULL; + + return collversion; +} + + +#ifdef USE_ICU +/* + * Converter object for converting between ICU's UChar strings and C strings + * in database encoding. Since the database encoding doesn't change, we only + * need one of these per session. + */ +static UConverter *icu_converter = NULL; + +static void +init_icu_converter(void) +{ + const char *icu_encoding_name; + UErrorCode status; + UConverter *conv; + + if (icu_converter) + return; + + icu_encoding_name = get_encoding_name_for_icu(GetDatabaseEncoding()); + + status = U_ZERO_ERROR; + conv = ucnv_open(icu_encoding_name, &status); + if (U_FAILURE(status)) + ereport(ERROR, + (errmsg("could not open ICU converter for encoding \"%s\": %s", + icu_encoding_name, u_errorName(status)))); + + icu_converter = conv; +} + +int32_t +icu_to_uchar(UChar **buff_uchar, const char *buff, size_t nbytes) +{ + UErrorCode status; + int32_t len_uchar; + + init_icu_converter(); + + len_uchar = 2 * nbytes; /* max length per docs */ + *buff_uchar = palloc(len_uchar * sizeof(**buff_uchar)); + status = U_ZERO_ERROR; + len_uchar = ucnv_toUChars(icu_converter, *buff_uchar, len_uchar, buff, nbytes, &status); + if (U_FAILURE(status)) + ereport(ERROR, + (errmsg("ucnv_toUChars failed: %s", u_errorName(status)))); + return len_uchar; +} + +int32_t +icu_from_uchar(char **result, UChar *buff_uchar, int32_t len_uchar) +{ + UErrorCode status; + int32_t len_result; + + init_icu_converter(); + + len_result = UCNV_GET_MAX_BYTES_FOR_STRING(len_uchar, ucnv_getMaxCharSize(icu_converter)); + *result = palloc(len_result + 1); + status = U_ZERO_ERROR; + ucnv_fromUChars(icu_converter, *result, len_result, buff_uchar, len_uchar, &status); + if (U_FAILURE(status)) + ereport(ERROR, + (errmsg("ucnv_fromUChars failed: %s", u_errorName(status)))); + return len_result; +} +#endif /* * These functions convert from/to libc's wchar_t, *not* pg_wchar_t. @@ -1362,6 +1538,8 @@ wchar2char(char *to, const wchar_t *from, size_t tolen, pg_locale_t locale) { size_t result; + Assert(!locale || locale->provider == COLLPROVIDER_LIBC); + if (tolen == 0) return 0; @@ -1398,10 +1576,10 @@ wchar2char(char *to, const wchar_t *from, size_t tolen, pg_locale_t locale) #ifdef HAVE_LOCALE_T #ifdef HAVE_WCSTOMBS_L /* Use wcstombs_l for nondefault locales */ - result = wcstombs_l(to, from, tolen, locale); + result = wcstombs_l(to, from, tolen, locale->info.lt); #else /* !HAVE_WCSTOMBS_L */ /* We have to temporarily set the locale as current ... ugh */ - locale_t save_locale = uselocale(locale); + locale_t save_locale = uselocale(locale->info.lt); result = wcstombs(to, from, tolen); @@ -1432,6 +1610,8 @@ char2wchar(wchar_t *to, size_t tolen, const char *from, size_t fromlen, { size_t result; + Assert(!locale || locale->provider == COLLPROVIDER_LIBC); + if (tolen == 0) return 0; @@ -1473,10 +1653,10 @@ char2wchar(wchar_t *to, size_t tolen, const char *from, size_t fromlen, #ifdef HAVE_LOCALE_T #ifdef HAVE_MBSTOWCS_L /* Use mbstowcs_l for nondefault locales */ - result = mbstowcs_l(to, str, tolen, locale); + result = mbstowcs_l(to, str, tolen, locale->info.lt); #else /* !HAVE_MBSTOWCS_L */ /* We have to temporarily set the locale as current ... ugh */ - locale_t save_locale = uselocale(locale); + locale_t save_locale = uselocale(locale->info.lt); result = mbstowcs(to, str, tolen); diff --git a/src/backend/utils/adt/selfuncs.c b/src/backend/utils/adt/selfuncs.c index bb9a544686..f8b28fe0e6 100644 --- a/src/backend/utils/adt/selfuncs.c +++ b/src/backend/utils/adt/selfuncs.c @@ -5259,7 +5259,7 @@ find_join_input_rel(PlannerInfo *root, Relids relids) /* * Check whether char is a letter (and, hence, subject to case-folding) * - * In multibyte character sets, we can't use isalpha, and it does not seem + * In multibyte character sets or with ICU, we can't use isalpha, and it does not seem * worth trying to convert to wchar_t to use iswalpha. Instead, just assume * any multibyte char is potentially case-varying. */ @@ -5271,9 +5271,11 @@ pattern_char_isalpha(char c, bool is_multibyte, return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'); else if (is_multibyte && IS_HIGHBIT_SET(c)) return true; + else if (locale && locale->provider == COLLPROVIDER_ICU) + return IS_HIGHBIT_SET(c) ? true : false; #ifdef HAVE_LOCALE_T - else if (locale) - return isalpha_l((unsigned char) c, locale); + else if (locale && locale->provider == COLLPROVIDER_LIBC) + return isalpha_l((unsigned char) c, locale->info.lt); #endif else return isalpha((unsigned char) c); diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c index cd036afc00..aa556aa5de 100644 --- a/src/backend/utils/adt/varlena.c +++ b/src/backend/utils/adt/varlena.c @@ -73,9 +73,7 @@ typedef struct hyperLogLogState abbr_card; /* Abbreviated key cardinality state */ hyperLogLogState full_card; /* Full key cardinality state */ double prop_card; /* Required cardinality proportion */ -#ifdef HAVE_LOCALE_T pg_locale_t locale; -#endif } VarStringSortSupport; /* @@ -1403,10 +1401,7 @@ varstr_cmp(char *arg1, int len1, char *arg2, int len2, Oid collid) char a2buf[TEXTBUFLEN]; char *a1p, *a2p; - -#ifdef HAVE_LOCALE_T pg_locale_t mylocale = 0; -#endif if (collid != DEFAULT_COLLATION_OID) { @@ -1421,9 +1416,7 @@ varstr_cmp(char *arg1, int len1, char *arg2, int len2, Oid collid) errmsg("could not determine which collation to use for string comparison"), errhint("Use the COLLATE clause to set the collation explicitly."))); } -#ifdef HAVE_LOCALE_T mylocale = pg_newlocale_from_collation(collid); -#endif } /* @@ -1542,11 +1535,54 @@ varstr_cmp(char *arg1, int len1, char *arg2, int len2, Oid collid) memcpy(a2p, arg2, len2); a2p[len2] = '\0'; -#ifdef HAVE_LOCALE_T if (mylocale) - result = strcoll_l(a1p, a2p, mylocale); - else + { + if (mylocale->provider == COLLPROVIDER_ICU) + { +#ifdef USE_ICU +#ifdef HAVE_UCOL_STRCOLLUTF8 + if (GetDatabaseEncoding() == PG_UTF8) + { + UErrorCode status; + + status = U_ZERO_ERROR; + result = ucol_strcollUTF8(mylocale->info.icu.ucol, + arg1, len1, + arg2, len2, + &status); + if (U_FAILURE(status)) + ereport(ERROR, + (errmsg("collation failed: %s", u_errorName(status)))); + } + else +#endif + { + int32_t ulen1, ulen2; + UChar *uchar1, *uchar2; + + ulen1 = icu_to_uchar(&uchar1, arg1, len1); + ulen2 = icu_to_uchar(&uchar2, arg2, len2); + + result = ucol_strcoll(mylocale->info.icu.ucol, + uchar1, ulen1, + uchar2, ulen2); + } +#else /* not USE_ICU */ + /* shouldn't happen */ + elog(ERROR, "unsupported collprovider: %c", mylocale->provider); +#endif /* not USE_ICU */ + } + else + { +#ifdef HAVE_LOCALE_T + result = strcoll_l(a1p, a2p, mylocale->info.lt); +#else + /* shouldn't happen */ + elog(ERROR, "unsupported collprovider: %c", mylocale->provider); #endif + } + } + else result = strcoll(a1p, a2p); /* @@ -1768,10 +1804,7 @@ varstr_sortsupport(SortSupport ssup, Oid collid, bool bpchar) bool abbreviate = ssup->abbreviate; bool collate_c = false; VarStringSortSupport *sss; - -#ifdef HAVE_LOCALE_T pg_locale_t locale = 0; -#endif /* * If possible, set ssup->comparator to a function which can be used to @@ -1826,9 +1859,7 @@ varstr_sortsupport(SortSupport ssup, Oid collid, bool bpchar) errmsg("could not determine which collation to use for string comparison"), errhint("Use the COLLATE clause to set the collation explicitly."))); } -#ifdef HAVE_LOCALE_T locale = pg_newlocale_from_collation(collid); -#endif } } @@ -1854,7 +1885,7 @@ varstr_sortsupport(SortSupport ssup, Oid collid, bool bpchar) * platforms. */ #ifndef TRUST_STRXFRM - if (!collate_c) + if (!collate_c && !(locale && locale->provider == COLLPROVIDER_ICU)) abbreviate = false; #endif @@ -1877,9 +1908,7 @@ varstr_sortsupport(SortSupport ssup, Oid collid, bool bpchar) sss->last_len2 = -1; /* Initialize */ sss->last_returned = 0; -#ifdef HAVE_LOCALE_T sss->locale = locale; -#endif /* * To avoid somehow confusing a strxfrm() blob and an original string, @@ -2090,11 +2119,54 @@ varstrfastcmp_locale(Datum x, Datum y, SortSupport ssup) goto done; } -#ifdef HAVE_LOCALE_T if (sss->locale) - result = strcoll_l(sss->buf1, sss->buf2, sss->locale); - else + { + if (sss->locale->provider == COLLPROVIDER_ICU) + { +#ifdef USE_ICU +#ifdef HAVE_UCOL_STRCOLLUTF8 + if (GetDatabaseEncoding() == PG_UTF8) + { + UErrorCode status; + + status = U_ZERO_ERROR; + result = ucol_strcollUTF8(sss->locale->info.icu.ucol, + a1p, len1, + a2p, len2, + &status); + if (U_FAILURE(status)) + ereport(ERROR, + (errmsg("collation failed: %s", u_errorName(status)))); + } + else #endif + { + int32_t ulen1, ulen2; + UChar *uchar1, *uchar2; + + ulen1 = icu_to_uchar(&uchar1, a1p, len1); + ulen2 = icu_to_uchar(&uchar2, a2p, len2); + + result = ucol_strcoll(sss->locale->info.icu.ucol, + uchar1, ulen1, + uchar2, ulen2); + } +#else /* not USE_ICU */ + /* shouldn't happen */ + elog(ERROR, "unsupported collprovider: %c", sss->locale->provider); +#endif /* not USE_ICU */ + } + else + { +#ifdef HAVE_LOCALE_T + result = strcoll_l(sss->buf1, sss->buf2, sss->locale->info.lt); +#else + /* shouldn't happen */ + elog(ERROR, "unsupported collprovider: %c", sss->locale->provider); +#endif + } + } + else result = strcoll(sss->buf1, sss->buf2); /* @@ -2200,9 +2272,14 @@ varstr_abbrev_convert(Datum original, SortSupport ssup) else { Size bsize; +#ifdef USE_ICU + int32_t ulen = -1; + UChar *uchar; +#endif /* - * We're not using the C collation, so fall back on strxfrm. + * We're not using the C collation, so fall back on strxfrm or ICU + * analogs. */ /* By convention, we use buffer 1 to store and NUL-terminate */ @@ -2222,17 +2299,66 @@ varstr_abbrev_convert(Datum original, SortSupport ssup) goto done; } - /* Just like strcoll(), strxfrm() expects a NUL-terminated string */ memcpy(sss->buf1, authoritative_data, len); + /* Just like strcoll(), strxfrm() expects a NUL-terminated string. + * Not necessary for ICU, but doesn't hurt. */ sss->buf1[len] = '\0'; sss->last_len1 = len; +#ifdef USE_ICU + /* When using ICU and not UTF8, convert string to UChar. */ + if (sss->locale && sss->locale->provider == COLLPROVIDER_ICU && + GetDatabaseEncoding() != PG_UTF8) + ulen = icu_to_uchar(&uchar, sss->buf1, len); +#endif + + /* + * Loop: Call strxfrm() or ucol_getSortKey(), possibly enlarge buffer, + * and try again. Both of these functions have the result buffer + * content undefined if the result did not fit, so we need to retry + * until everything fits, even though we only need the first few bytes + * in the end. When using ucol_nextSortKeyPart(), however, we only + * ask for as many bytes as we actually need. + */ for (;;) { +#ifdef USE_ICU + if (sss->locale && sss->locale->provider == COLLPROVIDER_ICU) + { + /* + * When using UTF8, use the iteration interface so we only + * need to produce as many bytes as we actually need. + */ + if (GetDatabaseEncoding() == PG_UTF8) + { + UCharIterator iter; + uint32_t state[2]; + UErrorCode status; + + uiter_setUTF8(&iter, sss->buf1, len); + state[0] = state[1] = 0; /* won't need that again */ + status = U_ZERO_ERROR; + bsize = ucol_nextSortKeyPart(sss->locale->info.icu.ucol, + &iter, + state, + (uint8_t *) sss->buf2, + Min(sizeof(Datum), sss->buflen2), + &status); + if (U_FAILURE(status)) + ereport(ERROR, + (errmsg("sort key generation failed: %s", u_errorName(status)))); + } + else + bsize = ucol_getSortKey(sss->locale->info.icu.ucol, + uchar, ulen, + (uint8_t *) sss->buf2, sss->buflen2); + } + else +#endif #ifdef HAVE_LOCALE_T - if (sss->locale) + if (sss->locale && sss->locale->provider == COLLPROVIDER_LIBC) bsize = strxfrm_l(sss->buf2, sss->buf1, - sss->buflen2, sss->locale); + sss->buflen2, sss->locale->info.lt); else #endif bsize = strxfrm(sss->buf2, sss->buf1, sss->buflen2); @@ -2242,8 +2368,7 @@ varstr_abbrev_convert(Datum original, SortSupport ssup) break; /* - * The C standard states that the contents of the buffer is now - * unspecified. Grow buffer, and retry. + * Grow buffer and retry. */ pfree(sss->buf2); sss->buflen2 = Max(bsize + 1, diff --git a/src/backend/utils/mb/encnames.c b/src/backend/utils/mb/encnames.c index 11099b844f..444eec25b5 100644 --- a/src/backend/utils/mb/encnames.c +++ b/src/backend/utils/mb/encnames.c @@ -403,6 +403,82 @@ const pg_enc2gettext pg_enc2gettext_tbl[] = }; +#ifndef FRONTEND + +/* + * Table of encoding names for ICU + * + * Reference: + * + * NULL entries are not supported by ICU, or their mapping is unclear. + */ +static const char * const pg_enc2icu_tbl[] = +{ + NULL, /* PG_SQL_ASCII */ + "EUC-JP", /* PG_EUC_JP */ + "EUC-CN", /* PG_EUC_CN */ + "EUC-KR", /* PG_EUC_KR */ + "EUC-TW", /* PG_EUC_TW */ + NULL, /* PG_EUC_JIS_2004 */ + "UTF-8", /* PG_UTF8 */ + NULL, /* PG_MULE_INTERNAL */ + "ISO-8859-1", /* PG_LATIN1 */ + "ISO-8859-2", /* PG_LATIN2 */ + "ISO-8859-3", /* PG_LATIN3 */ + "ISO-8859-4", /* PG_LATIN4 */ + "ISO-8859-9", /* PG_LATIN5 */ + "ISO-8859-10", /* PG_LATIN6 */ + "ISO-8859-13", /* PG_LATIN7 */ + "ISO-8859-14", /* PG_LATIN8 */ + "ISO-8859-15", /* PG_LATIN9 */ + NULL, /* PG_LATIN10 */ + "CP1256", /* PG_WIN1256 */ + "CP1258", /* PG_WIN1258 */ + "CP866", /* PG_WIN866 */ + NULL, /* PG_WIN874 */ + "KOI8-R", /* PG_KOI8R */ + "CP1251", /* PG_WIN1251 */ + "CP1252", /* PG_WIN1252 */ + "ISO-8859-5", /* PG_ISO_8859_5 */ + "ISO-8859-6", /* PG_ISO_8859_6 */ + "ISO-8859-7", /* PG_ISO_8859_7 */ + "ISO-8859-8", /* PG_ISO_8859_8 */ + "CP1250", /* PG_WIN1250 */ + "CP1253", /* PG_WIN1253 */ + "CP1254", /* PG_WIN1254 */ + "CP1255", /* PG_WIN1255 */ + "CP1257", /* PG_WIN1257 */ + "KOI8-U", /* PG_KOI8U */ +}; + +bool +is_encoding_supported_by_icu(int encoding) +{ + return (pg_enc2icu_tbl[encoding] != NULL); +} + +const char * +get_encoding_name_for_icu(int encoding) +{ + const char *icu_encoding_name; + + StaticAssertStmt(lengthof(pg_enc2icu_tbl) == PG_ENCODING_BE_LAST + 1, + "pg_enc2icu_tbl incomplete"); + + icu_encoding_name = pg_enc2icu_tbl[encoding]; + + if (!icu_encoding_name) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("encoding \"%s\" not supported by ICU", + pg_encoding_to_char(encoding)))); + + return icu_encoding_name; +} + +#endif /* not FRONTEND */ + + /* ---------- * Encoding checks, for error returns -1 else encoding id * ---------- diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index e0c72fbb80..8dde1e8f9d 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -62,6 +62,7 @@ #include "catalog/catalog.h" #include "catalog/pg_authid.h" #include "catalog/pg_class.h" +#include "catalog/pg_collation.h" #include "common/file_utils.h" #include "common/restricted_token.h" #include "common/username.h" @@ -1629,7 +1630,7 @@ setup_collation(FILE *cmdfd) PG_CMD_PUTS("SELECT pg_import_system_collations(if_not_exists => false, schema => 'pg_catalog');\n\n"); /* Add an SQL-standard name */ - PG_CMD_PRINTF2("INSERT INTO pg_collation (collname, collnamespace, collowner, collencoding, collcollate, collctype) VALUES ('ucs_basic', 'pg_catalog'::regnamespace, %u, %d, 'C', 'C');\n\n", BOOTSTRAP_SUPERUSERID, PG_UTF8); + PG_CMD_PRINTF3("INSERT INTO pg_collation (collname, collnamespace, collowner, collprovider, collencoding, collcollate, collctype) VALUES ('ucs_basic', 'pg_catalog'::regnamespace, %u, '%c', %d, 'C', 'C');\n\n", BOOTSTRAP_SUPERUSERID, COLLPROVIDER_LIBC, PG_UTF8); } /* diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index a98747d89a..b3d95d7f6e 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -12834,8 +12834,10 @@ dumpCollation(Archive *fout, CollInfo *collinfo) PQExpBuffer delq; PQExpBuffer labelq; PGresult *res; + int i_collprovider; int i_collcollate; int i_collctype; + const char *collprovider; const char *collcollate; const char *collctype; @@ -12852,18 +12854,32 @@ dumpCollation(Archive *fout, CollInfo *collinfo) selectSourceSchema(fout, collinfo->dobj.namespace->dobj.name); /* Get collation-specific details */ - appendPQExpBuffer(query, "SELECT " - "collcollate, " - "collctype " - "FROM pg_catalog.pg_collation c " - "WHERE c.oid = '%u'::pg_catalog.oid", - collinfo->dobj.catId.oid); + if (fout->remoteVersion >= 100000) + appendPQExpBuffer(query, "SELECT " + "collprovider, " + "collcollate, " + "collctype, " + "collversion " + "FROM pg_catalog.pg_collation c " + "WHERE c.oid = '%u'::pg_catalog.oid", + collinfo->dobj.catId.oid); + else + appendPQExpBuffer(query, "SELECT " + "'p'::char AS collprovider, " + "collcollate, " + "collctype, " + "NULL AS collversion " + "FROM pg_catalog.pg_collation c " + "WHERE c.oid = '%u'::pg_catalog.oid", + collinfo->dobj.catId.oid); res = ExecuteSqlQueryForSingleRow(fout, query->data); + i_collprovider = PQfnumber(res, "collprovider"); i_collcollate = PQfnumber(res, "collcollate"); i_collctype = PQfnumber(res, "collctype"); + collprovider = PQgetvalue(res, 0, i_collprovider); collcollate = PQgetvalue(res, 0, i_collcollate); collctype = PQgetvalue(res, 0, i_collctype); @@ -12875,11 +12891,50 @@ dumpCollation(Archive *fout, CollInfo *collinfo) appendPQExpBuffer(delq, ".%s;\n", fmtId(collinfo->dobj.name)); - appendPQExpBuffer(q, "CREATE COLLATION %s (lc_collate = ", + appendPQExpBuffer(q, "CREATE COLLATION %s (", fmtId(collinfo->dobj.name)); - appendStringLiteralAH(q, collcollate, fout); - appendPQExpBufferStr(q, ", lc_ctype = "); - appendStringLiteralAH(q, collctype, fout); + + appendPQExpBufferStr(q, "provider = "); + if (collprovider[0] == 'c') + appendPQExpBufferStr(q, "libc"); + else if (collprovider[0] == 'i') + appendPQExpBufferStr(q, "icu"); + else + exit_horribly(NULL, + "unrecognized collation provider: %s\n", + collprovider); + + if (strcmp(collcollate, collctype) == 0) + { + appendPQExpBufferStr(q, ", locale = "); + appendStringLiteralAH(q, collcollate, fout); + } + else + { + appendPQExpBufferStr(q, ", lc_collate = "); + appendStringLiteralAH(q, collcollate, fout); + appendPQExpBufferStr(q, ", lc_ctype = "); + appendStringLiteralAH(q, collctype, fout); + } + + /* + * For binary upgrade, carry over the collation version. For normal + * dump/restore, omit the version, so that it is computed upon restore. + */ + if (dopt->binary_upgrade) + { + int i_collversion; + + i_collversion = PQfnumber(res, "collversion"); + if (!PQgetisnull(res, 0, i_collversion)) + { + appendPQExpBufferStr(q, ", version = "); + appendStringLiteralAH(q, + PQgetvalue(res, 0, i_collversion), + fout); + } + } + appendPQExpBufferStr(q, ");\n"); appendPQExpBuffer(labelq, "COLLATION %s", fmtId(collinfo->dobj.name)); diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl index 021f4bf081..366737440c 100644 --- a/src/bin/pg_dump/t/002_pg_dump.pl +++ b/src/bin/pg_dump/t/002_pg_dump.pl @@ -2424,7 +2424,7 @@ qr/^\QINSERT INTO test_fifth_table (col1, col2, col3, col4, col5) VALUES (NULL, 'CREATE COLLATION test0 FROM "C";', regexp => qr/^ - \QCREATE COLLATION test0 (lc_collate = 'C', lc_ctype = 'C');\E/xm, + \QCREATE COLLATION test0 (provider = libc, locale = 'C');\E/xm, collation => 1, like => { binary_upgrade => 1, diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index 61a3e2a848..8c583127fd 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -3738,7 +3738,7 @@ listCollations(const char *pattern, bool verbose, bool showSystem) PQExpBufferData buf; PGresult *res; printQueryOpt myopt = pset.popt; - static const bool translate_columns[] = {false, false, false, false, false}; + static const bool translate_columns[] = {false, false, false, false, false, false}; if (pset.sversion < 90100) { @@ -3762,6 +3762,11 @@ listCollations(const char *pattern, bool verbose, bool showSystem) gettext_noop("Collate"), gettext_noop("Ctype")); + if (pset.sversion >= 100000) + appendPQExpBuffer(&buf, + ",\n CASE c.collprovider WHEN 'd' THEN 'default' WHEN 'c' THEN 'libc' WHEN 'i' THEN 'icu' END AS \"%s\"", + gettext_noop("Provider")); + if (verbose) appendPQExpBuffer(&buf, ",\n pg_catalog.obj_description(c.oid, 'pg_collation') AS \"%s\"", diff --git a/src/include/catalog/pg_collation.h b/src/include/catalog/pg_collation.h index 30c87e004e..8edd8aa066 100644 --- a/src/include/catalog/pg_collation.h +++ b/src/include/catalog/pg_collation.h @@ -34,9 +34,13 @@ CATALOG(pg_collation,3456) NameData collname; /* collation name */ Oid collnamespace; /* OID of namespace containing collation */ Oid collowner; /* owner of collation */ + char collprovider; /* see constants below */ int32 collencoding; /* encoding for this collation; -1 = "all" */ NameData collcollate; /* LC_COLLATE setting */ NameData collctype; /* LC_CTYPE setting */ +#ifdef CATALOG_VARLEN /* variable-length fields start here */ + text collversion; /* provider-dependent version of collation data */ +#endif } FormData_pg_collation; /* ---------------- @@ -50,27 +54,34 @@ typedef FormData_pg_collation *Form_pg_collation; * compiler constants for pg_collation * ---------------- */ -#define Natts_pg_collation 6 +#define Natts_pg_collation 8 #define Anum_pg_collation_collname 1 #define Anum_pg_collation_collnamespace 2 #define Anum_pg_collation_collowner 3 -#define Anum_pg_collation_collencoding 4 -#define Anum_pg_collation_collcollate 5 -#define Anum_pg_collation_collctype 6 +#define Anum_pg_collation_collprovider 4 +#define Anum_pg_collation_collencoding 5 +#define Anum_pg_collation_collcollate 6 +#define Anum_pg_collation_collctype 7 +#define Anum_pg_collation_collversion 8 /* ---------------- * initial contents of pg_collation * ---------------- */ -DATA(insert OID = 100 ( default PGNSP PGUID -1 "" "" )); +DATA(insert OID = 100 ( default PGNSP PGUID d -1 "" "" 0 )); DESCR("database's default collation"); #define DEFAULT_COLLATION_OID 100 -DATA(insert OID = 950 ( C PGNSP PGUID -1 "C" "C" )); +DATA(insert OID = 950 ( C PGNSP PGUID c -1 "C" "C" 0 )); DESCR("standard C collation"); #define C_COLLATION_OID 950 -DATA(insert OID = 951 ( POSIX PGNSP PGUID -1 "POSIX" "POSIX" )); +DATA(insert OID = 951 ( POSIX PGNSP PGUID c -1 "POSIX" "POSIX" 0 )); DESCR("standard POSIX collation"); #define POSIX_COLLATION_OID 951 + +#define COLLPROVIDER_DEFAULT 'd' +#define COLLPROVIDER_ICU 'i' +#define COLLPROVIDER_LIBC 'c' + #endif /* PG_COLLATION_H */ diff --git a/src/include/catalog/pg_collation_fn.h b/src/include/catalog/pg_collation_fn.h index 482ba7920e..dfebdbaa0b 100644 --- a/src/include/catalog/pg_collation_fn.h +++ b/src/include/catalog/pg_collation_fn.h @@ -16,8 +16,10 @@ extern Oid CollationCreate(const char *collname, Oid collnamespace, Oid collowner, + char collprovider, int32 collencoding, const char *collcollate, const char *collctype, + const char *collversion, bool if_not_exists); extern void RemoveCollationById(Oid collationOid); diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index a5b415346b..0d18ab8c0d 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -5401,6 +5401,9 @@ DESCR("pg_controldata init state information as a function"); DATA(insert OID = 3445 ( pg_import_system_collations PGNSP PGUID 12 100 0 0 0 f f f f t f v r 2 0 2278 "16 4089" _null_ _null_ "{if_not_exists,schema}" _null_ _null_ pg_import_system_collations _null_ _null_ _null_ )); DESCR("import collations from operating system"); +DATA(insert OID = 3448 ( pg_collation_actual_version PGNSP PGUID 12 100 0 0 0 f f f f t f v s 1 0 25 "26" _null_ _null_ _null_ _null_ _null_ pg_collation_actual_version _null_ _null_ _null_ )); +DESCR("import collations from operating system"); + /* system management/monitoring related functions */ DATA(insert OID = 3353 ( pg_ls_logdir PGNSP PGUID 12 10 20 0 0 f f f f t t v s 0 0 2249 "" "{25,20,1184}" "{o,o,o}" "{name,size,modification}" _null_ _null_ pg_ls_logdir _null_ _null_ _null_ )); DESCR("list files in the log directory"); diff --git a/src/include/commands/collationcmds.h b/src/include/commands/collationcmds.h index 3b2fcb8271..df5623ccb6 100644 --- a/src/include/commands/collationcmds.h +++ b/src/include/commands/collationcmds.h @@ -20,5 +20,6 @@ extern ObjectAddress DefineCollation(ParseState *pstate, List *names, List *parameters, bool if_not_exists); extern void IsThereCollationInNamespace(const char *collname, Oid nspOid); +extern ObjectAddress AlterCollation(AlterCollationStmt *stmt); #endif /* COLLATIONCMDS_H */ diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h index 5f54697302..9c5e749c9e 100644 --- a/src/include/mb/pg_wchar.h +++ b/src/include/mb/pg_wchar.h @@ -332,6 +332,12 @@ typedef struct pg_enc2gettext extern const pg_enc2gettext pg_enc2gettext_tbl[]; +/* + * Encoding names for ICU + */ +extern bool is_encoding_supported_by_icu(int encoding); +extern const char *get_encoding_name_for_icu(int encoding); + /* * pg_wchar stuff */ diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index 9a4221a9e7..b2d8514f89 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -424,6 +424,7 @@ typedef enum NodeTag T_CreateSubscriptionStmt, T_AlterSubscriptionStmt, T_DropSubscriptionStmt, + T_AlterCollationStmt, /* * TAGS FOR PARSE TREE NODES (parsenodes.h) diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 582e0e0ebe..f3773ca929 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -1732,6 +1732,17 @@ typedef struct AlterTableCmd /* one subcommand of an ALTER TABLE */ } AlterTableCmd; +/* ---------------------- + * Alter Collation + * ---------------------- + */ +typedef struct AlterCollationStmt +{ + NodeTag type; + List *collname; +} AlterCollationStmt; + + /* ---------------------- * Alter Domain * diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 6a8176b323..e1c1c9e9b4 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -606,6 +606,9 @@ /* Define to 1 if you have the external array `tzname'. */ #undef HAVE_TZNAME +/* Define to 1 if you have the `ucol_strcollUTF8' function. */ +#undef HAVE_UCOL_STRCOLLUTF8 + /* Define to 1 if you have the header file. */ #undef HAVE_UCRED_H @@ -819,6 +822,9 @@ (--enable-float8-byval) */ #undef USE_FLOAT8_BYVAL +/* Define to build with ICU support. (--with-icu) */ +#undef USE_ICU + /* Define to 1 to build with LDAP support. (--with-ldap) */ #undef USE_LDAP diff --git a/src/include/utils/pg_locale.h b/src/include/utils/pg_locale.h index cb509e2b6b..12d7547413 100644 --- a/src/include/utils/pg_locale.h +++ b/src/include/utils/pg_locale.h @@ -15,6 +15,9 @@ #if defined(LOCALE_T_IN_XLOCALE) || defined(WCSTOMBS_L_IN_XLOCALE) #include #endif +#ifdef USE_ICU +#include +#endif #include "utils/guc.h" @@ -61,17 +64,36 @@ extern void cache_locale_time(void); * We define our own wrapper around locale_t so we can keep the same * function signatures for all builds, while not having to create a * fake version of the standard type locale_t in the global namespace. - * The fake version of pg_locale_t can be checked for truth; that's - * about all it will be needed for. + * pg_locale_t is occasionally checked for truth, so make it a pointer. */ +struct pg_locale_t +{ + char provider; + union + { #ifdef HAVE_LOCALE_T -typedef locale_t pg_locale_t; -#else -typedef int pg_locale_t; + locale_t lt; +#endif +#ifdef USE_ICU + struct { + const char *locale; + UCollator *ucol; + } icu; #endif + } info; +}; + +typedef struct pg_locale_t *pg_locale_t; extern pg_locale_t pg_newlocale_from_collation(Oid collid); +extern char *get_collation_actual_version(char collprovider, const char *collcollate); + +#ifdef USE_ICU +extern int32_t icu_to_uchar(UChar **buff_uchar, const char *buff, size_t nbytes); +extern int32_t icu_from_uchar(char **result, UChar *buff_uchar, int32_t len_uchar); +#endif + /* These functions convert from/to libc's wchar_t, *not* pg_wchar_t */ #ifdef USE_WIDE_UPPER_LOWER extern size_t wchar2char(char *to, const wchar_t *from, size_t tolen, diff --git a/src/test/regress/GNUmakefile b/src/test/regress/GNUmakefile index b923ea1420..a747facb9a 100644 --- a/src/test/regress/GNUmakefile +++ b/src/test/regress/GNUmakefile @@ -125,6 +125,9 @@ tablespace-setup: ## REGRESS_OPTS = --dlpath=. $(EXTRA_REGRESS_OPTS) +ifeq ($(with_icu),yes) +override EXTRA_TESTS := collate.icu $(EXTRA_TESTS) +endif check: all tablespace-setup $(pg_regress_check) $(REGRESS_OPTS) --schedule=$(srcdir)/parallel_schedule $(MAXCONNOPT) $(EXTRA_TESTS) diff --git a/src/test/regress/expected/collate.icu.out b/src/test/regress/expected/collate.icu.out new file mode 100644 index 0000000000..e1fc9984f2 --- /dev/null +++ b/src/test/regress/expected/collate.icu.out @@ -0,0 +1,1126 @@ +/* + * This test is for ICU collations. + */ +SET client_encoding TO UTF8; +CREATE SCHEMA collate_tests; +SET search_path = collate_tests; +CREATE TABLE collate_test1 ( + a int, + b text COLLATE "en-x-icu" NOT NULL +); +\d collate_test1 + Table "collate_tests.collate_test1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | text | en-x-icu | not null | + +CREATE TABLE collate_test_fail ( + a int, + b text COLLATE "ja_JP.eucjp-x-icu" +); +ERROR: collation "ja_JP.eucjp-x-icu" for encoding "UTF8" does not exist +LINE 3: b text COLLATE "ja_JP.eucjp-x-icu" + ^ +CREATE TABLE collate_test_fail ( + a int, + b text COLLATE "foo-x-icu" +); +ERROR: collation "foo-x-icu" for encoding "UTF8" does not exist +LINE 3: b text COLLATE "foo-x-icu" + ^ +CREATE TABLE collate_test_fail ( + a int COLLATE "en-x-icu", + b text +); +ERROR: collations are not supported by type integer +LINE 2: a int COLLATE "en-x-icu", + ^ +CREATE TABLE collate_test_like ( + LIKE collate_test1 +); +\d collate_test_like + Table "collate_tests.collate_test_like" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | text | en-x-icu | not null | + +CREATE TABLE collate_test2 ( + a int, + b text COLLATE "sv-x-icu" +); +CREATE TABLE collate_test3 ( + a int, + b text COLLATE "C" +); +INSERT INTO collate_test1 VALUES (1, 'abc'), (2, 'äbc'), (3, 'bbc'), (4, 'ABC'); +INSERT INTO collate_test2 SELECT * FROM collate_test1; +INSERT INTO collate_test3 SELECT * FROM collate_test1; +SELECT * FROM collate_test1 WHERE b >= 'bbc'; + a | b +---+----- + 3 | bbc +(1 row) + +SELECT * FROM collate_test2 WHERE b >= 'bbc'; + a | b +---+----- + 2 | äbc + 3 | bbc +(2 rows) + +SELECT * FROM collate_test3 WHERE b >= 'bbc'; + a | b +---+----- + 2 | äbc + 3 | bbc +(2 rows) + +SELECT * FROM collate_test3 WHERE b >= 'BBC'; + a | b +---+----- + 1 | abc + 2 | äbc + 3 | bbc +(3 rows) + +SELECT * FROM collate_test1 WHERE b COLLATE "C" >= 'bbc'; + a | b +---+----- + 2 | äbc + 3 | bbc +(2 rows) + +SELECT * FROM collate_test1 WHERE b >= 'bbc' COLLATE "C"; + a | b +---+----- + 2 | äbc + 3 | bbc +(2 rows) + +SELECT * FROM collate_test1 WHERE b COLLATE "C" >= 'bbc' COLLATE "C"; + a | b +---+----- + 2 | äbc + 3 | bbc +(2 rows) + +SELECT * FROM collate_test1 WHERE b COLLATE "C" >= 'bbc' COLLATE "en-x-icu"; +ERROR: collation mismatch between explicit collations "C" and "en-x-icu" +LINE 1: ...* FROM collate_test1 WHERE b COLLATE "C" >= 'bbc' COLLATE "e... + ^ +CREATE DOMAIN testdomain_sv AS text COLLATE "sv-x-icu"; +CREATE DOMAIN testdomain_i AS int COLLATE "sv-x-icu"; -- fails +ERROR: collations are not supported by type integer +CREATE TABLE collate_test4 ( + a int, + b testdomain_sv +); +INSERT INTO collate_test4 SELECT * FROM collate_test1; +SELECT a, b FROM collate_test4 ORDER BY b; + a | b +---+----- + 1 | abc + 4 | ABC + 3 | bbc + 2 | äbc +(4 rows) + +CREATE TABLE collate_test5 ( + a int, + b testdomain_sv COLLATE "en-x-icu" +); +INSERT INTO collate_test5 SELECT * FROM collate_test1; +SELECT a, b FROM collate_test5 ORDER BY b; + a | b +---+----- + 1 | abc + 4 | ABC + 2 | äbc + 3 | bbc +(4 rows) + +SELECT a, b FROM collate_test1 ORDER BY b; + a | b +---+----- + 1 | abc + 4 | ABC + 2 | äbc + 3 | bbc +(4 rows) + +SELECT a, b FROM collate_test2 ORDER BY b; + a | b +---+----- + 1 | abc + 4 | ABC + 3 | bbc + 2 | äbc +(4 rows) + +SELECT a, b FROM collate_test3 ORDER BY b; + a | b +---+----- + 4 | ABC + 1 | abc + 3 | bbc + 2 | äbc +(4 rows) + +SELECT a, b FROM collate_test1 ORDER BY b COLLATE "C"; + a | b +---+----- + 4 | ABC + 1 | abc + 3 | bbc + 2 | äbc +(4 rows) + +-- star expansion +SELECT * FROM collate_test1 ORDER BY b; + a | b +---+----- + 1 | abc + 4 | ABC + 2 | äbc + 3 | bbc +(4 rows) + +SELECT * FROM collate_test2 ORDER BY b; + a | b +---+----- + 1 | abc + 4 | ABC + 3 | bbc + 2 | äbc +(4 rows) + +SELECT * FROM collate_test3 ORDER BY b; + a | b +---+----- + 4 | ABC + 1 | abc + 3 | bbc + 2 | äbc +(4 rows) + +-- constant expression folding +SELECT 'bbc' COLLATE "en-x-icu" > 'äbc' COLLATE "en-x-icu" AS "true"; + true +------ + t +(1 row) + +SELECT 'bbc' COLLATE "sv-x-icu" > 'äbc' COLLATE "sv-x-icu" AS "false"; + false +------- + f +(1 row) + +-- upper/lower +CREATE TABLE collate_test10 ( + a int, + x text COLLATE "en-x-icu", + y text COLLATE "tr-x-icu" +); +INSERT INTO collate_test10 VALUES (1, 'hij', 'hij'), (2, 'HIJ', 'HIJ'); +SELECT a, lower(x), lower(y), upper(x), upper(y), initcap(x), initcap(y) FROM collate_test10; + a | lower | lower | upper | upper | initcap | initcap +---+-------+-------+-------+-------+---------+--------- + 1 | hij | hij | HIJ | HİJ | Hij | Hij + 2 | hij | hıj | HIJ | HIJ | Hij | Hıj +(2 rows) + +SELECT a, lower(x COLLATE "C"), lower(y COLLATE "C") FROM collate_test10; + a | lower | lower +---+-------+------- + 1 | hij | hij + 2 | hij | hij +(2 rows) + +SELECT a, x, y FROM collate_test10 ORDER BY lower(y), a; + a | x | y +---+-----+----- + 2 | HIJ | HIJ + 1 | hij | hij +(2 rows) + +-- LIKE/ILIKE +SELECT * FROM collate_test1 WHERE b LIKE 'abc'; + a | b +---+----- + 1 | abc +(1 row) + +SELECT * FROM collate_test1 WHERE b LIKE 'abc%'; + a | b +---+----- + 1 | abc +(1 row) + +SELECT * FROM collate_test1 WHERE b LIKE '%bc%'; + a | b +---+----- + 1 | abc + 2 | äbc + 3 | bbc +(3 rows) + +SELECT * FROM collate_test1 WHERE b ILIKE 'abc'; + a | b +---+----- + 1 | abc + 4 | ABC +(2 rows) + +SELECT * FROM collate_test1 WHERE b ILIKE 'abc%'; + a | b +---+----- + 1 | abc + 4 | ABC +(2 rows) + +SELECT * FROM collate_test1 WHERE b ILIKE '%bc%'; + a | b +---+----- + 1 | abc + 2 | äbc + 3 | bbc + 4 | ABC +(4 rows) + +SELECT 'Türkiye' COLLATE "en-x-icu" ILIKE '%KI%' AS "true"; + true +------ + t +(1 row) + +SELECT 'Türkiye' COLLATE "tr-x-icu" ILIKE '%KI%' AS "false"; + false +------- + f +(1 row) + +SELECT 'bıt' ILIKE 'BIT' COLLATE "en-x-icu" AS "false"; + false +------- + f +(1 row) + +SELECT 'bıt' ILIKE 'BIT' COLLATE "tr-x-icu" AS "true"; + true +------ + t +(1 row) + +-- The following actually exercises the selectivity estimation for ILIKE. +SELECT relname FROM pg_class WHERE relname ILIKE 'abc%'; + relname +--------- +(0 rows) + +-- regular expressions +SELECT * FROM collate_test1 WHERE b ~ '^abc$'; + a | b +---+----- + 1 | abc +(1 row) + +SELECT * FROM collate_test1 WHERE b ~ '^abc'; + a | b +---+----- + 1 | abc +(1 row) + +SELECT * FROM collate_test1 WHERE b ~ 'bc'; + a | b +---+----- + 1 | abc + 2 | äbc + 3 | bbc +(3 rows) + +SELECT * FROM collate_test1 WHERE b ~* '^abc$'; + a | b +---+----- + 1 | abc + 4 | ABC +(2 rows) + +SELECT * FROM collate_test1 WHERE b ~* '^abc'; + a | b +---+----- + 1 | abc + 4 | ABC +(2 rows) + +SELECT * FROM collate_test1 WHERE b ~* 'bc'; + a | b +---+----- + 1 | abc + 2 | äbc + 3 | bbc + 4 | ABC +(4 rows) + +CREATE TABLE collate_test6 ( + a int, + b text COLLATE "en-x-icu" +); +INSERT INTO collate_test6 VALUES (1, 'abc'), (2, 'ABC'), (3, '123'), (4, 'ab1'), + (5, 'a1!'), (6, 'a c'), (7, '!.;'), (8, ' '), + (9, 'äbç'), (10, 'ÄBÇ'); +SELECT b, + b ~ '^[[:alpha:]]+$' AS is_alpha, + b ~ '^[[:upper:]]+$' AS is_upper, + b ~ '^[[:lower:]]+$' AS is_lower, + b ~ '^[[:digit:]]+$' AS is_digit, + b ~ '^[[:alnum:]]+$' AS is_alnum, + b ~ '^[[:graph:]]+$' AS is_graph, + b ~ '^[[:print:]]+$' AS is_print, + b ~ '^[[:punct:]]+$' AS is_punct, + b ~ '^[[:space:]]+$' AS is_space +FROM collate_test6; + b | is_alpha | is_upper | is_lower | is_digit | is_alnum | is_graph | is_print | is_punct | is_space +-----+----------+----------+----------+----------+----------+----------+----------+----------+---------- + abc | t | f | t | f | t | t | t | f | f + ABC | t | t | f | f | t | t | t | f | f + 123 | f | f | f | t | t | t | t | f | f + ab1 | f | f | f | f | t | t | t | f | f + a1! | f | f | f | f | f | t | t | f | f + a c | f | f | f | f | f | f | t | f | f + !.; | f | f | f | f | f | t | t | t | f + | f | f | f | f | f | f | t | f | t + äbç | t | f | t | f | t | t | t | f | f + ÄBÇ | t | t | f | f | t | t | t | f | f +(10 rows) + +SELECT 'Türkiye' COLLATE "en-x-icu" ~* 'KI' AS "true"; + true +------ + t +(1 row) + +SELECT 'Türkiye' COLLATE "tr-x-icu" ~* 'KI' AS "true"; -- true with ICU + true +------ + t +(1 row) + +SELECT 'bıt' ~* 'BIT' COLLATE "en-x-icu" AS "false"; + false +------- + f +(1 row) + +SELECT 'bıt' ~* 'BIT' COLLATE "tr-x-icu" AS "false"; -- false with ICU + false +------- + f +(1 row) + +-- The following actually exercises the selectivity estimation for ~*. +SELECT relname FROM pg_class WHERE relname ~* '^abc'; + relname +--------- +(0 rows) + +/* not run by default because it requires tr_TR system locale +-- to_char + +SET lc_time TO 'tr_TR'; +SELECT to_char(date '2010-04-01', 'DD TMMON YYYY'); +SELECT to_char(date '2010-04-01', 'DD TMMON YYYY' COLLATE "tr-x-icu"); +*/ +-- backwards parsing +CREATE VIEW collview1 AS SELECT * FROM collate_test1 WHERE b COLLATE "C" >= 'bbc'; +CREATE VIEW collview2 AS SELECT a, b FROM collate_test1 ORDER BY b COLLATE "C"; +CREATE VIEW collview3 AS SELECT a, lower((x || x) COLLATE "C") FROM collate_test10; +SELECT table_name, view_definition FROM information_schema.views + WHERE table_name LIKE 'collview%' ORDER BY 1; + table_name | view_definition +------------+-------------------------------------------------------------------------- + collview1 | SELECT collate_test1.a, + + | collate_test1.b + + | FROM collate_test1 + + | WHERE ((collate_test1.b COLLATE "C") >= 'bbc'::text); + collview2 | SELECT collate_test1.a, + + | collate_test1.b + + | FROM collate_test1 + + | ORDER BY (collate_test1.b COLLATE "C"); + collview3 | SELECT collate_test10.a, + + | lower(((collate_test10.x || collate_test10.x) COLLATE "C")) AS lower+ + | FROM collate_test10; +(3 rows) + +-- collation propagation in various expression types +SELECT a, coalesce(b, 'foo') FROM collate_test1 ORDER BY 2; + a | coalesce +---+---------- + 1 | abc + 4 | ABC + 2 | äbc + 3 | bbc +(4 rows) + +SELECT a, coalesce(b, 'foo') FROM collate_test2 ORDER BY 2; + a | coalesce +---+---------- + 1 | abc + 4 | ABC + 3 | bbc + 2 | äbc +(4 rows) + +SELECT a, coalesce(b, 'foo') FROM collate_test3 ORDER BY 2; + a | coalesce +---+---------- + 4 | ABC + 1 | abc + 3 | bbc + 2 | äbc +(4 rows) + +SELECT a, lower(coalesce(x, 'foo')), lower(coalesce(y, 'foo')) FROM collate_test10; + a | lower | lower +---+-------+------- + 1 | hij | hij + 2 | hij | hıj +(2 rows) + +SELECT a, b, greatest(b, 'CCC') FROM collate_test1 ORDER BY 3; + a | b | greatest +---+-----+---------- + 1 | abc | CCC + 2 | äbc | CCC + 3 | bbc | CCC + 4 | ABC | CCC +(4 rows) + +SELECT a, b, greatest(b, 'CCC') FROM collate_test2 ORDER BY 3; + a | b | greatest +---+-----+---------- + 1 | abc | CCC + 3 | bbc | CCC + 4 | ABC | CCC + 2 | äbc | äbc +(4 rows) + +SELECT a, b, greatest(b, 'CCC') FROM collate_test3 ORDER BY 3; + a | b | greatest +---+-----+---------- + 4 | ABC | CCC + 1 | abc | abc + 3 | bbc | bbc + 2 | äbc | äbc +(4 rows) + +SELECT a, x, y, lower(greatest(x, 'foo')), lower(greatest(y, 'foo')) FROM collate_test10; + a | x | y | lower | lower +---+-----+-----+-------+------- + 1 | hij | hij | hij | hij + 2 | HIJ | HIJ | hij | hıj +(2 rows) + +SELECT a, nullif(b, 'abc') FROM collate_test1 ORDER BY 2; + a | nullif +---+-------- + 4 | ABC + 2 | äbc + 3 | bbc + 1 | +(4 rows) + +SELECT a, nullif(b, 'abc') FROM collate_test2 ORDER BY 2; + a | nullif +---+-------- + 4 | ABC + 3 | bbc + 2 | äbc + 1 | +(4 rows) + +SELECT a, nullif(b, 'abc') FROM collate_test3 ORDER BY 2; + a | nullif +---+-------- + 4 | ABC + 3 | bbc + 2 | äbc + 1 | +(4 rows) + +SELECT a, lower(nullif(x, 'foo')), lower(nullif(y, 'foo')) FROM collate_test10; + a | lower | lower +---+-------+------- + 1 | hij | hij + 2 | hij | hıj +(2 rows) + +SELECT a, CASE b WHEN 'abc' THEN 'abcd' ELSE b END FROM collate_test1 ORDER BY 2; + a | b +---+------ + 4 | ABC + 2 | äbc + 1 | abcd + 3 | bbc +(4 rows) + +SELECT a, CASE b WHEN 'abc' THEN 'abcd' ELSE b END FROM collate_test2 ORDER BY 2; + a | b +---+------ + 4 | ABC + 1 | abcd + 3 | bbc + 2 | äbc +(4 rows) + +SELECT a, CASE b WHEN 'abc' THEN 'abcd' ELSE b END FROM collate_test3 ORDER BY 2; + a | b +---+------ + 4 | ABC + 1 | abcd + 3 | bbc + 2 | äbc +(4 rows) + +CREATE DOMAIN testdomain AS text; +SELECT a, b::testdomain FROM collate_test1 ORDER BY 2; + a | b +---+----- + 1 | abc + 4 | ABC + 2 | äbc + 3 | bbc +(4 rows) + +SELECT a, b::testdomain FROM collate_test2 ORDER BY 2; + a | b +---+----- + 1 | abc + 4 | ABC + 3 | bbc + 2 | äbc +(4 rows) + +SELECT a, b::testdomain FROM collate_test3 ORDER BY 2; + a | b +---+----- + 4 | ABC + 1 | abc + 3 | bbc + 2 | äbc +(4 rows) + +SELECT a, b::testdomain_sv FROM collate_test3 ORDER BY 2; + a | b +---+----- + 1 | abc + 4 | ABC + 3 | bbc + 2 | äbc +(4 rows) + +SELECT a, lower(x::testdomain), lower(y::testdomain) FROM collate_test10; + a | lower | lower +---+-------+------- + 1 | hij | hij + 2 | hij | hıj +(2 rows) + +SELECT min(b), max(b) FROM collate_test1; + min | max +-----+----- + abc | bbc +(1 row) + +SELECT min(b), max(b) FROM collate_test2; + min | max +-----+----- + abc | äbc +(1 row) + +SELECT min(b), max(b) FROM collate_test3; + min | max +-----+----- + ABC | äbc +(1 row) + +SELECT array_agg(b ORDER BY b) FROM collate_test1; + array_agg +------------------- + {abc,ABC,äbc,bbc} +(1 row) + +SELECT array_agg(b ORDER BY b) FROM collate_test2; + array_agg +------------------- + {abc,ABC,bbc,äbc} +(1 row) + +SELECT array_agg(b ORDER BY b) FROM collate_test3; + array_agg +------------------- + {ABC,abc,bbc,äbc} +(1 row) + +SELECT a, b FROM collate_test1 UNION ALL SELECT a, b FROM collate_test1 ORDER BY 2; + a | b +---+----- + 1 | abc + 1 | abc + 4 | ABC + 4 | ABC + 2 | äbc + 2 | äbc + 3 | bbc + 3 | bbc +(8 rows) + +SELECT a, b FROM collate_test2 UNION SELECT a, b FROM collate_test2 ORDER BY 2; + a | b +---+----- + 1 | abc + 4 | ABC + 3 | bbc + 2 | äbc +(4 rows) + +SELECT a, b FROM collate_test3 WHERE a < 4 INTERSECT SELECT a, b FROM collate_test3 WHERE a > 1 ORDER BY 2; + a | b +---+----- + 3 | bbc + 2 | äbc +(2 rows) + +SELECT a, b FROM collate_test3 EXCEPT SELECT a, b FROM collate_test3 WHERE a < 2 ORDER BY 2; + a | b +---+----- + 4 | ABC + 3 | bbc + 2 | äbc +(3 rows) + +SELECT a, b FROM collate_test1 UNION ALL SELECT a, b FROM collate_test3 ORDER BY 2; -- fail +ERROR: could not determine which collation to use for string comparison +HINT: Use the COLLATE clause to set the collation explicitly. +SELECT a, b FROM collate_test1 UNION ALL SELECT a, b FROM collate_test3; -- ok + a | b +---+----- + 1 | abc + 2 | äbc + 3 | bbc + 4 | ABC + 1 | abc + 2 | äbc + 3 | bbc + 4 | ABC +(8 rows) + +SELECT a, b FROM collate_test1 UNION SELECT a, b FROM collate_test3 ORDER BY 2; -- fail +ERROR: collation mismatch between implicit collations "en-x-icu" and "C" +LINE 1: SELECT a, b FROM collate_test1 UNION SELECT a, b FROM collat... + ^ +HINT: You can choose the collation by applying the COLLATE clause to one or both expressions. +SELECT a, b COLLATE "C" FROM collate_test1 UNION SELECT a, b FROM collate_test3 ORDER BY 2; -- ok + a | b +---+----- + 4 | ABC + 1 | abc + 3 | bbc + 2 | äbc +(4 rows) + +SELECT a, b FROM collate_test1 INTERSECT SELECT a, b FROM collate_test3 ORDER BY 2; -- fail +ERROR: collation mismatch between implicit collations "en-x-icu" and "C" +LINE 1: ...ELECT a, b FROM collate_test1 INTERSECT SELECT a, b FROM col... + ^ +HINT: You can choose the collation by applying the COLLATE clause to one or both expressions. +SELECT a, b FROM collate_test1 EXCEPT SELECT a, b FROM collate_test3 ORDER BY 2; -- fail +ERROR: collation mismatch between implicit collations "en-x-icu" and "C" +LINE 1: SELECT a, b FROM collate_test1 EXCEPT SELECT a, b FROM colla... + ^ +HINT: You can choose the collation by applying the COLLATE clause to one or both expressions. +CREATE TABLE test_u AS SELECT a, b FROM collate_test1 UNION ALL SELECT a, b FROM collate_test3; -- fail +ERROR: no collation was derived for column "b" with collatable type text +HINT: Use the COLLATE clause to set the collation explicitly. +-- ideally this would be a parse-time error, but for now it must be run-time: +select x < y from collate_test10; -- fail +ERROR: could not determine which collation to use for string comparison +HINT: Use the COLLATE clause to set the collation explicitly. +select x || y from collate_test10; -- ok, because || is not collation aware + ?column? +---------- + hijhij + HIJHIJ +(2 rows) + +select x, y from collate_test10 order by x || y; -- not so ok +ERROR: collation mismatch between implicit collations "en-x-icu" and "tr-x-icu" +LINE 1: select x, y from collate_test10 order by x || y; + ^ +HINT: You can choose the collation by applying the COLLATE clause to one or both expressions. +-- collation mismatch between recursive and non-recursive term +WITH RECURSIVE foo(x) AS + (SELECT x FROM (VALUES('a' COLLATE "en-x-icu"),('b')) t(x) + UNION ALL + SELECT (x || 'c') COLLATE "de-x-icu" FROM foo WHERE length(x) < 10) +SELECT * FROM foo; +ERROR: recursive query "foo" column 1 has collation "en-x-icu" in non-recursive term but collation "de-x-icu" overall +LINE 2: (SELECT x FROM (VALUES('a' COLLATE "en-x-icu"),('b')) t(x... + ^ +HINT: Use the COLLATE clause to set the collation of the non-recursive term. +-- casting +SELECT CAST('42' AS text COLLATE "C"); +ERROR: syntax error at or near "COLLATE" +LINE 1: SELECT CAST('42' AS text COLLATE "C"); + ^ +SELECT a, CAST(b AS varchar) FROM collate_test1 ORDER BY 2; + a | b +---+----- + 1 | abc + 4 | ABC + 2 | äbc + 3 | bbc +(4 rows) + +SELECT a, CAST(b AS varchar) FROM collate_test2 ORDER BY 2; + a | b +---+----- + 1 | abc + 4 | ABC + 3 | bbc + 2 | äbc +(4 rows) + +SELECT a, CAST(b AS varchar) FROM collate_test3 ORDER BY 2; + a | b +---+----- + 4 | ABC + 1 | abc + 3 | bbc + 2 | äbc +(4 rows) + +-- propagation of collation in SQL functions (inlined and non-inlined cases) +-- and plpgsql functions too +CREATE FUNCTION mylt (text, text) RETURNS boolean LANGUAGE sql + AS $$ select $1 < $2 $$; +CREATE FUNCTION mylt_noninline (text, text) RETURNS boolean LANGUAGE sql + AS $$ select $1 < $2 limit 1 $$; +CREATE FUNCTION mylt_plpgsql (text, text) RETURNS boolean LANGUAGE plpgsql + AS $$ begin return $1 < $2; end $$; +SELECT a.b AS a, b.b AS b, a.b < b.b AS lt, + mylt(a.b, b.b), mylt_noninline(a.b, b.b), mylt_plpgsql(a.b, b.b) +FROM collate_test1 a, collate_test1 b +ORDER BY a.b, b.b; + a | b | lt | mylt | mylt_noninline | mylt_plpgsql +-----+-----+----+------+----------------+-------------- + abc | abc | f | f | f | f + abc | ABC | t | t | t | t + abc | äbc | t | t | t | t + abc | bbc | t | t | t | t + ABC | abc | f | f | f | f + ABC | ABC | f | f | f | f + ABC | äbc | t | t | t | t + ABC | bbc | t | t | t | t + äbc | abc | f | f | f | f + äbc | ABC | f | f | f | f + äbc | äbc | f | f | f | f + äbc | bbc | t | t | t | t + bbc | abc | f | f | f | f + bbc | ABC | f | f | f | f + bbc | äbc | f | f | f | f + bbc | bbc | f | f | f | f +(16 rows) + +SELECT a.b AS a, b.b AS b, a.b < b.b COLLATE "C" AS lt, + mylt(a.b, b.b COLLATE "C"), mylt_noninline(a.b, b.b COLLATE "C"), + mylt_plpgsql(a.b, b.b COLLATE "C") +FROM collate_test1 a, collate_test1 b +ORDER BY a.b, b.b; + a | b | lt | mylt | mylt_noninline | mylt_plpgsql +-----+-----+----+------+----------------+-------------- + abc | abc | f | f | f | f + abc | ABC | f | f | f | f + abc | äbc | t | t | t | t + abc | bbc | t | t | t | t + ABC | abc | t | t | t | t + ABC | ABC | f | f | f | f + ABC | äbc | t | t | t | t + ABC | bbc | t | t | t | t + äbc | abc | f | f | f | f + äbc | ABC | f | f | f | f + äbc | äbc | f | f | f | f + äbc | bbc | f | f | f | f + bbc | abc | f | f | f | f + bbc | ABC | f | f | f | f + bbc | äbc | t | t | t | t + bbc | bbc | f | f | f | f +(16 rows) + +-- collation override in plpgsql +CREATE FUNCTION mylt2 (x text, y text) RETURNS boolean LANGUAGE plpgsql AS $$ +declare + xx text := x; + yy text := y; +begin + return xx < yy; +end +$$; +SELECT mylt2('a', 'B' collate "en-x-icu") as t, mylt2('a', 'B' collate "C") as f; + t | f +---+--- + t | f +(1 row) + +CREATE OR REPLACE FUNCTION + mylt2 (x text, y text) RETURNS boolean LANGUAGE plpgsql AS $$ +declare + xx text COLLATE "POSIX" := x; + yy text := y; +begin + return xx < yy; +end +$$; +SELECT mylt2('a', 'B') as f; + f +--- + f +(1 row) + +SELECT mylt2('a', 'B' collate "C") as fail; -- conflicting collations +ERROR: could not determine which collation to use for string comparison +HINT: Use the COLLATE clause to set the collation explicitly. +CONTEXT: PL/pgSQL function mylt2(text,text) line 6 at RETURN +SELECT mylt2('a', 'B' collate "POSIX") as f; + f +--- + f +(1 row) + +-- polymorphism +SELECT * FROM unnest((SELECT array_agg(b ORDER BY b) FROM collate_test1)) ORDER BY 1; + unnest +-------- + abc + ABC + äbc + bbc +(4 rows) + +SELECT * FROM unnest((SELECT array_agg(b ORDER BY b) FROM collate_test2)) ORDER BY 1; + unnest +-------- + abc + ABC + bbc + äbc +(4 rows) + +SELECT * FROM unnest((SELECT array_agg(b ORDER BY b) FROM collate_test3)) ORDER BY 1; + unnest +-------- + ABC + abc + bbc + äbc +(4 rows) + +CREATE FUNCTION dup (anyelement) RETURNS anyelement + AS 'select $1' LANGUAGE sql; +SELECT a, dup(b) FROM collate_test1 ORDER BY 2; + a | dup +---+----- + 1 | abc + 4 | ABC + 2 | äbc + 3 | bbc +(4 rows) + +SELECT a, dup(b) FROM collate_test2 ORDER BY 2; + a | dup +---+----- + 1 | abc + 4 | ABC + 3 | bbc + 2 | äbc +(4 rows) + +SELECT a, dup(b) FROM collate_test3 ORDER BY 2; + a | dup +---+----- + 4 | ABC + 1 | abc + 3 | bbc + 2 | äbc +(4 rows) + +-- indexes +CREATE INDEX collate_test1_idx1 ON collate_test1 (b); +CREATE INDEX collate_test1_idx2 ON collate_test1 (b COLLATE "C"); +CREATE INDEX collate_test1_idx3 ON collate_test1 ((b COLLATE "C")); -- this is different grammatically +CREATE INDEX collate_test1_idx4 ON collate_test1 (((b||'foo') COLLATE "POSIX")); +CREATE INDEX collate_test1_idx5 ON collate_test1 (a COLLATE "C"); -- fail +ERROR: collations are not supported by type integer +CREATE INDEX collate_test1_idx6 ON collate_test1 ((a COLLATE "C")); -- fail +ERROR: collations are not supported by type integer +LINE 1: ...ATE INDEX collate_test1_idx6 ON collate_test1 ((a COLLATE "C... + ^ +SELECT relname, pg_get_indexdef(oid) FROM pg_class WHERE relname LIKE 'collate_test%_idx%' ORDER BY 1; + relname | pg_get_indexdef +--------------------+----------------------------------------------------------------------------------------------------- + collate_test1_idx1 | CREATE INDEX collate_test1_idx1 ON collate_test1 USING btree (b) + collate_test1_idx2 | CREATE INDEX collate_test1_idx2 ON collate_test1 USING btree (b COLLATE "C") + collate_test1_idx3 | CREATE INDEX collate_test1_idx3 ON collate_test1 USING btree (b COLLATE "C") + collate_test1_idx4 | CREATE INDEX collate_test1_idx4 ON collate_test1 USING btree (((b || 'foo'::text)) COLLATE "POSIX") +(4 rows) + +-- schema manipulation commands +CREATE ROLE regress_test_role; +CREATE SCHEMA test_schema; +-- We need to do this this way to cope with varying names for encodings: +do $$ +BEGIN + EXECUTE 'CREATE COLLATION test0 (provider = icu, locale = ' || + quote_literal(current_setting('lc_collate')) || ');'; +END +$$; +CREATE COLLATION test0 FROM "C"; -- fail, duplicate name +ERROR: collation "test0" already exists +do $$ +BEGIN + EXECUTE 'CREATE COLLATION test1 (provider = icu, lc_collate = ' || + quote_literal(current_setting('lc_collate')) || + ', lc_ctype = ' || + quote_literal(current_setting('lc_ctype')) || ');'; +END +$$; +CREATE COLLATION test3 (provider = icu, lc_collate = 'en_US.utf8'); -- fail, need lc_ctype +ERROR: parameter "lc_ctype" must be specified +CREATE COLLATION testx (provider = icu, locale = 'nonsense'); /* never fails with ICU */ DROP COLLATION testx; +CREATE COLLATION test4 FROM nonsense; +ERROR: collation "nonsense" for encoding "UTF8" does not exist +CREATE COLLATION test5 FROM test0; +SELECT collname FROM pg_collation WHERE collname LIKE 'test%' ORDER BY 1; + collname +---------- + test0 + test1 + test5 +(3 rows) + +ALTER COLLATION test1 RENAME TO test11; +ALTER COLLATION test0 RENAME TO test11; -- fail +ERROR: collation "test11" already exists in schema "collate_tests" +ALTER COLLATION test1 RENAME TO test22; -- fail +ERROR: collation "test1" for encoding "UTF8" does not exist +ALTER COLLATION test11 OWNER TO regress_test_role; +ALTER COLLATION test11 OWNER TO nonsense; +ERROR: role "nonsense" does not exist +ALTER COLLATION test11 SET SCHEMA test_schema; +COMMENT ON COLLATION test0 IS 'US English'; +SELECT collname, nspname, obj_description(pg_collation.oid, 'pg_collation') + FROM pg_collation JOIN pg_namespace ON (collnamespace = pg_namespace.oid) + WHERE collname LIKE 'test%' + ORDER BY 1; + collname | nspname | obj_description +----------+---------------+----------------- + test0 | collate_tests | US English + test11 | test_schema | + test5 | collate_tests | +(3 rows) + +DROP COLLATION test0, test_schema.test11, test5; +DROP COLLATION test0; -- fail +ERROR: collation "test0" for encoding "UTF8" does not exist +DROP COLLATION IF EXISTS test0; +NOTICE: collation "test0" does not exist, skipping +SELECT collname FROM pg_collation WHERE collname LIKE 'test%'; + collname +---------- +(0 rows) + +DROP SCHEMA test_schema; +DROP ROLE regress_test_role; +-- ALTER +ALTER COLLATION "en-x-icu" REFRESH VERSION; +NOTICE: version has not changed +-- dependencies +CREATE COLLATION test0 FROM "C"; +CREATE TABLE collate_dep_test1 (a int, b text COLLATE test0); +CREATE DOMAIN collate_dep_dom1 AS text COLLATE test0; +CREATE TYPE collate_dep_test2 AS (x int, y text COLLATE test0); +CREATE VIEW collate_dep_test3 AS SELECT text 'foo' COLLATE test0 AS foo; +CREATE TABLE collate_dep_test4t (a int, b text); +CREATE INDEX collate_dep_test4i ON collate_dep_test4t (b COLLATE test0); +DROP COLLATION test0 RESTRICT; -- fail +ERROR: cannot drop collation test0 because other objects depend on it +DETAIL: table collate_dep_test1 column b depends on collation test0 +type collate_dep_dom1 depends on collation test0 +composite type collate_dep_test2 column y depends on collation test0 +view collate_dep_test3 depends on collation test0 +index collate_dep_test4i depends on collation test0 +HINT: Use DROP ... CASCADE to drop the dependent objects too. +DROP COLLATION test0 CASCADE; +NOTICE: drop cascades to 5 other objects +DETAIL: drop cascades to table collate_dep_test1 column b +drop cascades to type collate_dep_dom1 +drop cascades to composite type collate_dep_test2 column y +drop cascades to view collate_dep_test3 +drop cascades to index collate_dep_test4i +\d collate_dep_test1 + Table "collate_tests.collate_dep_test1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + +\d collate_dep_test2 + Composite type "collate_tests.collate_dep_test2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + x | integer | | | + +DROP TABLE collate_dep_test1, collate_dep_test4t; +DROP TYPE collate_dep_test2; +-- test range types and collations +create type textrange_c as range(subtype=text, collation="C"); +create type textrange_en_us as range(subtype=text, collation="en-x-icu"); +select textrange_c('A','Z') @> 'b'::text; + ?column? +---------- + f +(1 row) + +select textrange_en_us('A','Z') @> 'b'::text; + ?column? +---------- + t +(1 row) + +drop type textrange_c; +drop type textrange_en_us; +-- cleanup +DROP SCHEMA collate_tests CASCADE; +NOTICE: drop cascades to 18 other objects +DETAIL: drop cascades to table collate_test1 +drop cascades to table collate_test_like +drop cascades to table collate_test2 +drop cascades to table collate_test3 +drop cascades to type testdomain_sv +drop cascades to table collate_test4 +drop cascades to table collate_test5 +drop cascades to table collate_test10 +drop cascades to table collate_test6 +drop cascades to view collview1 +drop cascades to view collview2 +drop cascades to view collview3 +drop cascades to type testdomain +drop cascades to function mylt(text,text) +drop cascades to function mylt_noninline(text,text) +drop cascades to function mylt_plpgsql(text,text) +drop cascades to function mylt2(text,text) +drop cascades to function dup(anyelement) +RESET search_path; +-- leave a collation for pg_upgrade test +CREATE COLLATION coll_icu_upgrade FROM "und-x-icu"; diff --git a/src/test/regress/expected/collate.linux.utf8.out b/src/test/regress/expected/collate.linux.utf8.out index 293e78641e..26275c3fb3 100644 --- a/src/test/regress/expected/collate.linux.utf8.out +++ b/src/test/regress/expected/collate.linux.utf8.out @@ -4,12 +4,14 @@ * because other encodings don't support all the characters used. */ SET client_encoding TO UTF8; +CREATE SCHEMA collate_tests; +SET search_path = collate_tests; CREATE TABLE collate_test1 ( a int, b text COLLATE "en_US" NOT NULL ); \d collate_test1 - Table "public.collate_test1" + Table "collate_tests.collate_test1" Column | Type | Collation | Nullable | Default --------+---------+-----------+----------+--------- a | integer | | | @@ -40,7 +42,7 @@ CREATE TABLE collate_test_like ( LIKE collate_test1 ); \d collate_test_like - Table "public.collate_test_like" + Table "collate_tests.collate_test_like" Column | Type | Collation | Nullable | Default --------+---------+-----------+----------+--------- a | integer | | | @@ -364,6 +366,38 @@ SELECT * FROM collate_test1 WHERE b ~* 'bc'; 4 | ABC (4 rows) +CREATE TABLE collate_test6 ( + a int, + b text COLLATE "en_US" +); +INSERT INTO collate_test6 VALUES (1, 'abc'), (2, 'ABC'), (3, '123'), (4, 'ab1'), + (5, 'a1!'), (6, 'a c'), (7, '!.;'), (8, ' '), + (9, 'äbç'), (10, 'ÄBÇ'); +SELECT b, + b ~ '^[[:alpha:]]+$' AS is_alpha, + b ~ '^[[:upper:]]+$' AS is_upper, + b ~ '^[[:lower:]]+$' AS is_lower, + b ~ '^[[:digit:]]+$' AS is_digit, + b ~ '^[[:alnum:]]+$' AS is_alnum, + b ~ '^[[:graph:]]+$' AS is_graph, + b ~ '^[[:print:]]+$' AS is_print, + b ~ '^[[:punct:]]+$' AS is_punct, + b ~ '^[[:space:]]+$' AS is_space +FROM collate_test6; + b | is_alpha | is_upper | is_lower | is_digit | is_alnum | is_graph | is_print | is_punct | is_space +-----+----------+----------+----------+----------+----------+----------+----------+----------+---------- + abc | t | f | t | f | t | t | t | f | f + ABC | t | t | f | f | t | t | t | f | f + 123 | f | f | f | t | t | t | t | f | f + ab1 | f | f | f | f | t | t | t | f | f + a1! | f | f | f | f | f | t | t | f | f + a c | f | f | f | f | f | f | t | f | f + !.; | f | f | f | f | f | t | t | t | f + | f | f | f | f | f | f | t | f | t + äbç | t | f | t | f | t | t | t | f | f + ÄBÇ | t | t | f | f | t | t | t | f | f +(10 rows) + SELECT 'Türkiye' COLLATE "en_US" ~* 'KI' AS "true"; true ------ @@ -980,6 +1014,8 @@ ERROR: parameter "lc_ctype" must be specified CREATE COLLATION testx (locale = 'nonsense'); -- fail ERROR: could not create locale "nonsense": No such file or directory DETAIL: The operating system could not find any locale data for the locale name "nonsense". +CREATE COLLATION testy (locale = 'en_US.utf8', version = 'foo'); -- fail, no versions for libc +ERROR: collation "testy" has no actual version, but a version was specified CREATE COLLATION test4 FROM nonsense; ERROR: collation "nonsense" for encoding "UTF8" does not exist CREATE COLLATION test5 FROM test0; @@ -993,7 +1029,7 @@ SELECT collname FROM pg_collation WHERE collname LIKE 'test%' ORDER BY 1; ALTER COLLATION test1 RENAME TO test11; ALTER COLLATION test0 RENAME TO test11; -- fail -ERROR: collation "test11" for encoding "UTF8" already exists in schema "public" +ERROR: collation "test11" for encoding "UTF8" already exists in schema "collate_tests" ALTER COLLATION test1 RENAME TO test22; -- fail ERROR: collation "test1" for encoding "UTF8" does not exist ALTER COLLATION test11 OWNER TO regress_test_role; @@ -1005,11 +1041,11 @@ SELECT collname, nspname, obj_description(pg_collation.oid, 'pg_collation') FROM pg_collation JOIN pg_namespace ON (collnamespace = pg_namespace.oid) WHERE collname LIKE 'test%' ORDER BY 1; - collname | nspname | obj_description -----------+-------------+----------------- - test0 | public | US English - test11 | test_schema | - test5 | public | + collname | nspname | obj_description +----------+---------------+----------------- + test0 | collate_tests | US English + test11 | test_schema | + test5 | collate_tests | (3 rows) DROP COLLATION test0, test_schema.test11, test5; @@ -1024,6 +1060,9 @@ SELECT collname FROM pg_collation WHERE collname LIKE 'test%'; DROP SCHEMA test_schema; DROP ROLE regress_test_role; +-- ALTER +ALTER COLLATION "en_US" REFRESH VERSION; +NOTICE: version has not changed -- dependencies CREATE COLLATION test0 FROM "C"; CREATE TABLE collate_dep_test1 (a int, b text COLLATE test0); @@ -1048,13 +1087,13 @@ drop cascades to composite type collate_dep_test2 column y drop cascades to view collate_dep_test3 drop cascades to index collate_dep_test4i \d collate_dep_test1 - Table "public.collate_dep_test1" + Table "collate_tests.collate_dep_test1" Column | Type | Collation | Nullable | Default --------+---------+-----------+----------+--------- a | integer | | | \d collate_dep_test2 - Composite type "public.collate_dep_test2" + Composite type "collate_tests.collate_dep_test2" Column | Type | Collation | Nullable | Default --------+---------+-----------+----------+--------- x | integer | | | @@ -1078,3 +1117,24 @@ select textrange_en_us('A','Z') @> 'b'::text; drop type textrange_c; drop type textrange_en_us; +-- cleanup +DROP SCHEMA collate_tests CASCADE; +NOTICE: drop cascades to 18 other objects +DETAIL: drop cascades to table collate_test1 +drop cascades to table collate_test_like +drop cascades to table collate_test2 +drop cascades to table collate_test3 +drop cascades to type testdomain_sv +drop cascades to table collate_test4 +drop cascades to table collate_test5 +drop cascades to table collate_test10 +drop cascades to table collate_test6 +drop cascades to view collview1 +drop cascades to view collview2 +drop cascades to view collview3 +drop cascades to type testdomain +drop cascades to function mylt(text,text) +drop cascades to function mylt_noninline(text,text) +drop cascades to function mylt_plpgsql(text,text) +drop cascades to function mylt2(text,text) +drop cascades to function dup(anyelement) diff --git a/src/test/regress/sql/collate.icu.sql b/src/test/regress/sql/collate.icu.sql new file mode 100644 index 0000000000..ef39445b30 --- /dev/null +++ b/src/test/regress/sql/collate.icu.sql @@ -0,0 +1,433 @@ +/* + * This test is for ICU collations. + */ + +SET client_encoding TO UTF8; + +CREATE SCHEMA collate_tests; +SET search_path = collate_tests; + + +CREATE TABLE collate_test1 ( + a int, + b text COLLATE "en-x-icu" NOT NULL +); + +\d collate_test1 + +CREATE TABLE collate_test_fail ( + a int, + b text COLLATE "ja_JP.eucjp-x-icu" +); + +CREATE TABLE collate_test_fail ( + a int, + b text COLLATE "foo-x-icu" +); + +CREATE TABLE collate_test_fail ( + a int COLLATE "en-x-icu", + b text +); + +CREATE TABLE collate_test_like ( + LIKE collate_test1 +); + +\d collate_test_like + +CREATE TABLE collate_test2 ( + a int, + b text COLLATE "sv-x-icu" +); + +CREATE TABLE collate_test3 ( + a int, + b text COLLATE "C" +); + +INSERT INTO collate_test1 VALUES (1, 'abc'), (2, 'äbc'), (3, 'bbc'), (4, 'ABC'); +INSERT INTO collate_test2 SELECT * FROM collate_test1; +INSERT INTO collate_test3 SELECT * FROM collate_test1; + +SELECT * FROM collate_test1 WHERE b >= 'bbc'; +SELECT * FROM collate_test2 WHERE b >= 'bbc'; +SELECT * FROM collate_test3 WHERE b >= 'bbc'; +SELECT * FROM collate_test3 WHERE b >= 'BBC'; + +SELECT * FROM collate_test1 WHERE b COLLATE "C" >= 'bbc'; +SELECT * FROM collate_test1 WHERE b >= 'bbc' COLLATE "C"; +SELECT * FROM collate_test1 WHERE b COLLATE "C" >= 'bbc' COLLATE "C"; +SELECT * FROM collate_test1 WHERE b COLLATE "C" >= 'bbc' COLLATE "en-x-icu"; + + +CREATE DOMAIN testdomain_sv AS text COLLATE "sv-x-icu"; +CREATE DOMAIN testdomain_i AS int COLLATE "sv-x-icu"; -- fails +CREATE TABLE collate_test4 ( + a int, + b testdomain_sv +); +INSERT INTO collate_test4 SELECT * FROM collate_test1; +SELECT a, b FROM collate_test4 ORDER BY b; + +CREATE TABLE collate_test5 ( + a int, + b testdomain_sv COLLATE "en-x-icu" +); +INSERT INTO collate_test5 SELECT * FROM collate_test1; +SELECT a, b FROM collate_test5 ORDER BY b; + + +SELECT a, b FROM collate_test1 ORDER BY b; +SELECT a, b FROM collate_test2 ORDER BY b; +SELECT a, b FROM collate_test3 ORDER BY b; + +SELECT a, b FROM collate_test1 ORDER BY b COLLATE "C"; + +-- star expansion +SELECT * FROM collate_test1 ORDER BY b; +SELECT * FROM collate_test2 ORDER BY b; +SELECT * FROM collate_test3 ORDER BY b; + +-- constant expression folding +SELECT 'bbc' COLLATE "en-x-icu" > 'äbc' COLLATE "en-x-icu" AS "true"; +SELECT 'bbc' COLLATE "sv-x-icu" > 'äbc' COLLATE "sv-x-icu" AS "false"; + +-- upper/lower + +CREATE TABLE collate_test10 ( + a int, + x text COLLATE "en-x-icu", + y text COLLATE "tr-x-icu" +); + +INSERT INTO collate_test10 VALUES (1, 'hij', 'hij'), (2, 'HIJ', 'HIJ'); + +SELECT a, lower(x), lower(y), upper(x), upper(y), initcap(x), initcap(y) FROM collate_test10; +SELECT a, lower(x COLLATE "C"), lower(y COLLATE "C") FROM collate_test10; + +SELECT a, x, y FROM collate_test10 ORDER BY lower(y), a; + +-- LIKE/ILIKE + +SELECT * FROM collate_test1 WHERE b LIKE 'abc'; +SELECT * FROM collate_test1 WHERE b LIKE 'abc%'; +SELECT * FROM collate_test1 WHERE b LIKE '%bc%'; +SELECT * FROM collate_test1 WHERE b ILIKE 'abc'; +SELECT * FROM collate_test1 WHERE b ILIKE 'abc%'; +SELECT * FROM collate_test1 WHERE b ILIKE '%bc%'; + +SELECT 'Türkiye' COLLATE "en-x-icu" ILIKE '%KI%' AS "true"; +SELECT 'Türkiye' COLLATE "tr-x-icu" ILIKE '%KI%' AS "false"; + +SELECT 'bıt' ILIKE 'BIT' COLLATE "en-x-icu" AS "false"; +SELECT 'bıt' ILIKE 'BIT' COLLATE "tr-x-icu" AS "true"; + +-- The following actually exercises the selectivity estimation for ILIKE. +SELECT relname FROM pg_class WHERE relname ILIKE 'abc%'; + +-- regular expressions + +SELECT * FROM collate_test1 WHERE b ~ '^abc$'; +SELECT * FROM collate_test1 WHERE b ~ '^abc'; +SELECT * FROM collate_test1 WHERE b ~ 'bc'; +SELECT * FROM collate_test1 WHERE b ~* '^abc$'; +SELECT * FROM collate_test1 WHERE b ~* '^abc'; +SELECT * FROM collate_test1 WHERE b ~* 'bc'; + +CREATE TABLE collate_test6 ( + a int, + b text COLLATE "en-x-icu" +); +INSERT INTO collate_test6 VALUES (1, 'abc'), (2, 'ABC'), (3, '123'), (4, 'ab1'), + (5, 'a1!'), (6, 'a c'), (7, '!.;'), (8, ' '), + (9, 'äbç'), (10, 'ÄBÇ'); +SELECT b, + b ~ '^[[:alpha:]]+$' AS is_alpha, + b ~ '^[[:upper:]]+$' AS is_upper, + b ~ '^[[:lower:]]+$' AS is_lower, + b ~ '^[[:digit:]]+$' AS is_digit, + b ~ '^[[:alnum:]]+$' AS is_alnum, + b ~ '^[[:graph:]]+$' AS is_graph, + b ~ '^[[:print:]]+$' AS is_print, + b ~ '^[[:punct:]]+$' AS is_punct, + b ~ '^[[:space:]]+$' AS is_space +FROM collate_test6; + +SELECT 'Türkiye' COLLATE "en-x-icu" ~* 'KI' AS "true"; +SELECT 'Türkiye' COLLATE "tr-x-icu" ~* 'KI' AS "true"; -- true with ICU + +SELECT 'bıt' ~* 'BIT' COLLATE "en-x-icu" AS "false"; +SELECT 'bıt' ~* 'BIT' COLLATE "tr-x-icu" AS "false"; -- false with ICU + +-- The following actually exercises the selectivity estimation for ~*. +SELECT relname FROM pg_class WHERE relname ~* '^abc'; + + +/* not run by default because it requires tr_TR system locale +-- to_char + +SET lc_time TO 'tr_TR'; +SELECT to_char(date '2010-04-01', 'DD TMMON YYYY'); +SELECT to_char(date '2010-04-01', 'DD TMMON YYYY' COLLATE "tr-x-icu"); +*/ + + +-- backwards parsing + +CREATE VIEW collview1 AS SELECT * FROM collate_test1 WHERE b COLLATE "C" >= 'bbc'; +CREATE VIEW collview2 AS SELECT a, b FROM collate_test1 ORDER BY b COLLATE "C"; +CREATE VIEW collview3 AS SELECT a, lower((x || x) COLLATE "C") FROM collate_test10; + +SELECT table_name, view_definition FROM information_schema.views + WHERE table_name LIKE 'collview%' ORDER BY 1; + + +-- collation propagation in various expression types + +SELECT a, coalesce(b, 'foo') FROM collate_test1 ORDER BY 2; +SELECT a, coalesce(b, 'foo') FROM collate_test2 ORDER BY 2; +SELECT a, coalesce(b, 'foo') FROM collate_test3 ORDER BY 2; +SELECT a, lower(coalesce(x, 'foo')), lower(coalesce(y, 'foo')) FROM collate_test10; + +SELECT a, b, greatest(b, 'CCC') FROM collate_test1 ORDER BY 3; +SELECT a, b, greatest(b, 'CCC') FROM collate_test2 ORDER BY 3; +SELECT a, b, greatest(b, 'CCC') FROM collate_test3 ORDER BY 3; +SELECT a, x, y, lower(greatest(x, 'foo')), lower(greatest(y, 'foo')) FROM collate_test10; + +SELECT a, nullif(b, 'abc') FROM collate_test1 ORDER BY 2; +SELECT a, nullif(b, 'abc') FROM collate_test2 ORDER BY 2; +SELECT a, nullif(b, 'abc') FROM collate_test3 ORDER BY 2; +SELECT a, lower(nullif(x, 'foo')), lower(nullif(y, 'foo')) FROM collate_test10; + +SELECT a, CASE b WHEN 'abc' THEN 'abcd' ELSE b END FROM collate_test1 ORDER BY 2; +SELECT a, CASE b WHEN 'abc' THEN 'abcd' ELSE b END FROM collate_test2 ORDER BY 2; +SELECT a, CASE b WHEN 'abc' THEN 'abcd' ELSE b END FROM collate_test3 ORDER BY 2; + +CREATE DOMAIN testdomain AS text; +SELECT a, b::testdomain FROM collate_test1 ORDER BY 2; +SELECT a, b::testdomain FROM collate_test2 ORDER BY 2; +SELECT a, b::testdomain FROM collate_test3 ORDER BY 2; +SELECT a, b::testdomain_sv FROM collate_test3 ORDER BY 2; +SELECT a, lower(x::testdomain), lower(y::testdomain) FROM collate_test10; + +SELECT min(b), max(b) FROM collate_test1; +SELECT min(b), max(b) FROM collate_test2; +SELECT min(b), max(b) FROM collate_test3; + +SELECT array_agg(b ORDER BY b) FROM collate_test1; +SELECT array_agg(b ORDER BY b) FROM collate_test2; +SELECT array_agg(b ORDER BY b) FROM collate_test3; + +SELECT a, b FROM collate_test1 UNION ALL SELECT a, b FROM collate_test1 ORDER BY 2; +SELECT a, b FROM collate_test2 UNION SELECT a, b FROM collate_test2 ORDER BY 2; +SELECT a, b FROM collate_test3 WHERE a < 4 INTERSECT SELECT a, b FROM collate_test3 WHERE a > 1 ORDER BY 2; +SELECT a, b FROM collate_test3 EXCEPT SELECT a, b FROM collate_test3 WHERE a < 2 ORDER BY 2; + +SELECT a, b FROM collate_test1 UNION ALL SELECT a, b FROM collate_test3 ORDER BY 2; -- fail +SELECT a, b FROM collate_test1 UNION ALL SELECT a, b FROM collate_test3; -- ok +SELECT a, b FROM collate_test1 UNION SELECT a, b FROM collate_test3 ORDER BY 2; -- fail +SELECT a, b COLLATE "C" FROM collate_test1 UNION SELECT a, b FROM collate_test3 ORDER BY 2; -- ok +SELECT a, b FROM collate_test1 INTERSECT SELECT a, b FROM collate_test3 ORDER BY 2; -- fail +SELECT a, b FROM collate_test1 EXCEPT SELECT a, b FROM collate_test3 ORDER BY 2; -- fail + +CREATE TABLE test_u AS SELECT a, b FROM collate_test1 UNION ALL SELECT a, b FROM collate_test3; -- fail + +-- ideally this would be a parse-time error, but for now it must be run-time: +select x < y from collate_test10; -- fail +select x || y from collate_test10; -- ok, because || is not collation aware +select x, y from collate_test10 order by x || y; -- not so ok + +-- collation mismatch between recursive and non-recursive term +WITH RECURSIVE foo(x) AS + (SELECT x FROM (VALUES('a' COLLATE "en-x-icu"),('b')) t(x) + UNION ALL + SELECT (x || 'c') COLLATE "de-x-icu" FROM foo WHERE length(x) < 10) +SELECT * FROM foo; + + +-- casting + +SELECT CAST('42' AS text COLLATE "C"); + +SELECT a, CAST(b AS varchar) FROM collate_test1 ORDER BY 2; +SELECT a, CAST(b AS varchar) FROM collate_test2 ORDER BY 2; +SELECT a, CAST(b AS varchar) FROM collate_test3 ORDER BY 2; + + +-- propagation of collation in SQL functions (inlined and non-inlined cases) +-- and plpgsql functions too + +CREATE FUNCTION mylt (text, text) RETURNS boolean LANGUAGE sql + AS $$ select $1 < $2 $$; + +CREATE FUNCTION mylt_noninline (text, text) RETURNS boolean LANGUAGE sql + AS $$ select $1 < $2 limit 1 $$; + +CREATE FUNCTION mylt_plpgsql (text, text) RETURNS boolean LANGUAGE plpgsql + AS $$ begin return $1 < $2; end $$; + +SELECT a.b AS a, b.b AS b, a.b < b.b AS lt, + mylt(a.b, b.b), mylt_noninline(a.b, b.b), mylt_plpgsql(a.b, b.b) +FROM collate_test1 a, collate_test1 b +ORDER BY a.b, b.b; + +SELECT a.b AS a, b.b AS b, a.b < b.b COLLATE "C" AS lt, + mylt(a.b, b.b COLLATE "C"), mylt_noninline(a.b, b.b COLLATE "C"), + mylt_plpgsql(a.b, b.b COLLATE "C") +FROM collate_test1 a, collate_test1 b +ORDER BY a.b, b.b; + + +-- collation override in plpgsql + +CREATE FUNCTION mylt2 (x text, y text) RETURNS boolean LANGUAGE plpgsql AS $$ +declare + xx text := x; + yy text := y; +begin + return xx < yy; +end +$$; + +SELECT mylt2('a', 'B' collate "en-x-icu") as t, mylt2('a', 'B' collate "C") as f; + +CREATE OR REPLACE FUNCTION + mylt2 (x text, y text) RETURNS boolean LANGUAGE plpgsql AS $$ +declare + xx text COLLATE "POSIX" := x; + yy text := y; +begin + return xx < yy; +end +$$; + +SELECT mylt2('a', 'B') as f; +SELECT mylt2('a', 'B' collate "C") as fail; -- conflicting collations +SELECT mylt2('a', 'B' collate "POSIX") as f; + + +-- polymorphism + +SELECT * FROM unnest((SELECT array_agg(b ORDER BY b) FROM collate_test1)) ORDER BY 1; +SELECT * FROM unnest((SELECT array_agg(b ORDER BY b) FROM collate_test2)) ORDER BY 1; +SELECT * FROM unnest((SELECT array_agg(b ORDER BY b) FROM collate_test3)) ORDER BY 1; + +CREATE FUNCTION dup (anyelement) RETURNS anyelement + AS 'select $1' LANGUAGE sql; + +SELECT a, dup(b) FROM collate_test1 ORDER BY 2; +SELECT a, dup(b) FROM collate_test2 ORDER BY 2; +SELECT a, dup(b) FROM collate_test3 ORDER BY 2; + + +-- indexes + +CREATE INDEX collate_test1_idx1 ON collate_test1 (b); +CREATE INDEX collate_test1_idx2 ON collate_test1 (b COLLATE "C"); +CREATE INDEX collate_test1_idx3 ON collate_test1 ((b COLLATE "C")); -- this is different grammatically +CREATE INDEX collate_test1_idx4 ON collate_test1 (((b||'foo') COLLATE "POSIX")); + +CREATE INDEX collate_test1_idx5 ON collate_test1 (a COLLATE "C"); -- fail +CREATE INDEX collate_test1_idx6 ON collate_test1 ((a COLLATE "C")); -- fail + +SELECT relname, pg_get_indexdef(oid) FROM pg_class WHERE relname LIKE 'collate_test%_idx%' ORDER BY 1; + + +-- schema manipulation commands + +CREATE ROLE regress_test_role; +CREATE SCHEMA test_schema; + +-- We need to do this this way to cope with varying names for encodings: +do $$ +BEGIN + EXECUTE 'CREATE COLLATION test0 (provider = icu, locale = ' || + quote_literal(current_setting('lc_collate')) || ');'; +END +$$; +CREATE COLLATION test0 FROM "C"; -- fail, duplicate name +do $$ +BEGIN + EXECUTE 'CREATE COLLATION test1 (provider = icu, lc_collate = ' || + quote_literal(current_setting('lc_collate')) || + ', lc_ctype = ' || + quote_literal(current_setting('lc_ctype')) || ');'; +END +$$; +CREATE COLLATION test3 (provider = icu, lc_collate = 'en_US.utf8'); -- fail, need lc_ctype +CREATE COLLATION testx (provider = icu, locale = 'nonsense'); /* never fails with ICU */ DROP COLLATION testx; + +CREATE COLLATION test4 FROM nonsense; +CREATE COLLATION test5 FROM test0; + +SELECT collname FROM pg_collation WHERE collname LIKE 'test%' ORDER BY 1; + +ALTER COLLATION test1 RENAME TO test11; +ALTER COLLATION test0 RENAME TO test11; -- fail +ALTER COLLATION test1 RENAME TO test22; -- fail + +ALTER COLLATION test11 OWNER TO regress_test_role; +ALTER COLLATION test11 OWNER TO nonsense; +ALTER COLLATION test11 SET SCHEMA test_schema; + +COMMENT ON COLLATION test0 IS 'US English'; + +SELECT collname, nspname, obj_description(pg_collation.oid, 'pg_collation') + FROM pg_collation JOIN pg_namespace ON (collnamespace = pg_namespace.oid) + WHERE collname LIKE 'test%' + ORDER BY 1; + +DROP COLLATION test0, test_schema.test11, test5; +DROP COLLATION test0; -- fail +DROP COLLATION IF EXISTS test0; + +SELECT collname FROM pg_collation WHERE collname LIKE 'test%'; + +DROP SCHEMA test_schema; +DROP ROLE regress_test_role; + + +-- ALTER + +ALTER COLLATION "en-x-icu" REFRESH VERSION; + + +-- dependencies + +CREATE COLLATION test0 FROM "C"; + +CREATE TABLE collate_dep_test1 (a int, b text COLLATE test0); +CREATE DOMAIN collate_dep_dom1 AS text COLLATE test0; +CREATE TYPE collate_dep_test2 AS (x int, y text COLLATE test0); +CREATE VIEW collate_dep_test3 AS SELECT text 'foo' COLLATE test0 AS foo; +CREATE TABLE collate_dep_test4t (a int, b text); +CREATE INDEX collate_dep_test4i ON collate_dep_test4t (b COLLATE test0); + +DROP COLLATION test0 RESTRICT; -- fail +DROP COLLATION test0 CASCADE; + +\d collate_dep_test1 +\d collate_dep_test2 + +DROP TABLE collate_dep_test1, collate_dep_test4t; +DROP TYPE collate_dep_test2; + +-- test range types and collations + +create type textrange_c as range(subtype=text, collation="C"); +create type textrange_en_us as range(subtype=text, collation="en-x-icu"); + +select textrange_c('A','Z') @> 'b'::text; +select textrange_en_us('A','Z') @> 'b'::text; + +drop type textrange_c; +drop type textrange_en_us; + + +-- cleanup +DROP SCHEMA collate_tests CASCADE; +RESET search_path; + +-- leave a collation for pg_upgrade test +CREATE COLLATION coll_icu_upgrade FROM "und-x-icu"; diff --git a/src/test/regress/sql/collate.linux.utf8.sql b/src/test/regress/sql/collate.linux.utf8.sql index c349cbde2b..b51162e3a1 100644 --- a/src/test/regress/sql/collate.linux.utf8.sql +++ b/src/test/regress/sql/collate.linux.utf8.sql @@ -6,6 +6,9 @@ SET client_encoding TO UTF8; +CREATE SCHEMA collate_tests; +SET search_path = collate_tests; + CREATE TABLE collate_test1 ( a int, @@ -134,6 +137,25 @@ SELECT * FROM collate_test1 WHERE b ~* '^abc$'; SELECT * FROM collate_test1 WHERE b ~* '^abc'; SELECT * FROM collate_test1 WHERE b ~* 'bc'; +CREATE TABLE collate_test6 ( + a int, + b text COLLATE "en_US" +); +INSERT INTO collate_test6 VALUES (1, 'abc'), (2, 'ABC'), (3, '123'), (4, 'ab1'), + (5, 'a1!'), (6, 'a c'), (7, '!.;'), (8, ' '), + (9, 'äbç'), (10, 'ÄBÇ'); +SELECT b, + b ~ '^[[:alpha:]]+$' AS is_alpha, + b ~ '^[[:upper:]]+$' AS is_upper, + b ~ '^[[:lower:]]+$' AS is_lower, + b ~ '^[[:digit:]]+$' AS is_digit, + b ~ '^[[:alnum:]]+$' AS is_alnum, + b ~ '^[[:graph:]]+$' AS is_graph, + b ~ '^[[:print:]]+$' AS is_print, + b ~ '^[[:punct:]]+$' AS is_punct, + b ~ '^[[:space:]]+$' AS is_space +FROM collate_test6; + SELECT 'Türkiye' COLLATE "en_US" ~* 'KI' AS "true"; SELECT 'Türkiye' COLLATE "tr_TR" ~* 'KI' AS "false"; @@ -337,6 +359,7 @@ END $$; CREATE COLLATION test3 (lc_collate = 'en_US.utf8'); -- fail, need lc_ctype CREATE COLLATION testx (locale = 'nonsense'); -- fail +CREATE COLLATION testy (locale = 'en_US.utf8', version = 'foo'); -- fail, no versions for libc CREATE COLLATION test4 FROM nonsense; CREATE COLLATION test5 FROM test0; @@ -368,6 +391,11 @@ DROP SCHEMA test_schema; DROP ROLE regress_test_role; +-- ALTER + +ALTER COLLATION "en_US" REFRESH VERSION; + + -- dependencies CREATE COLLATION test0 FROM "C"; @@ -398,3 +426,7 @@ select textrange_en_us('A','Z') @> 'b'::text; drop type textrange_c; drop type textrange_en_us; + + +-- cleanup +DROP SCHEMA collate_tests CASCADE; -- cgit v1.2.3 From 218747d2cf3cc1536ff77435e596280e0e739760 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Thu, 23 Mar 2017 16:59:24 -0400 Subject: doc: add missing closing 'sect3' tag for ICU patch --- doc/src/sgml/charset.sgml | 1 + 1 file changed, 1 insertion(+) (limited to 'doc/src') diff --git a/doc/src/sgml/charset.sgml b/doc/src/sgml/charset.sgml index 5c55f397f8..5e0a0bf7a7 100644 --- a/doc/src/sgml/charset.sgml +++ b/doc/src/sgml/charset.sgml @@ -730,6 +730,7 @@ CREATE COLLATION "de-DE-x-icu" FROM "de-x-icu"; User-defined collations should be created in user schemas. This also ensures that they are saved by pg_dump. + -- cgit v1.2.3 From 7ac955b34791500069179e1ea986f46d510126d9 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Fri, 24 Mar 2017 13:32:21 +0200 Subject: Allow SCRAM authentication, when pg_hba.conf says 'md5'. If a user has a SCRAM verifier in pg_authid.rolpassword, there's no reason we cannot attempt to perform SCRAM authentication instead of MD5. The worst that can happen is that the client doesn't support SCRAM, and the authentication will fail. But previously, it would fail for sure, because we would not even try. SCRAM is strictly more secure than MD5, so there's no harm in trying it. This allows for a more graceful transition from MD5 passwords to SCRAM, as user passwords can be changed to SCRAM verifiers incrementally, without changing pg_hba.conf. Refactor the code in auth.c to support that better. Notably, we now have to look up the user's pg_authid entry before sending the password challenge, also when performing MD5 authentication. Also simplify the concept of a "doomed" authentication. Previously, if a user had a password, but it had expired, we still performed SCRAM authentication (but always returned error at the end) using the salt and iteration count from the expired password. Now we construct a fake salt, like we do when the user doesn't have a password or doesn't exist at all. That simplifies get_role_password(), and we can don't need to distinguish the "user has expired password", and "user does not exist" cases in auth.c. On second thoughts, also rename uaSASL to uaSCRAM. It refers to the mechanism specified in pg_hba.conf, and while we use SASL for SCRAM authentication at the protocol level, the mechanism should be called SCRAM, not SASL. As a comparison, we have uaLDAP, even though it looks like the plain 'password' authentication at the protocol level. Discussion: https://www.postgresql.org/message-id/6425.1489506016@sss.pgh.pa.us Reviewed-by: Michael Paquier --- doc/src/sgml/client-auth.sgml | 37 +++--- src/backend/libpq/auth-scram.c | 104 +++++++++-------- src/backend/libpq/auth.c | 179 +++++++++++++++++++----------- src/backend/libpq/crypt.c | 44 +++----- src/backend/libpq/hba.c | 2 +- src/include/libpq/crypt.h | 3 +- src/include/libpq/hba.h | 2 +- src/include/libpq/scram.h | 2 +- src/test/authentication/t/001_password.pl | 6 +- 9 files changed, 214 insertions(+), 165 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/client-auth.sgml b/doc/src/sgml/client-auth.sgml index 28f5296b5a..c2fc6d3261 100644 --- a/doc/src/sgml/client-auth.sgml +++ b/doc/src/sgml/client-auth.sgml @@ -412,23 +412,22 @@ hostnossl database user - md5 + scram - Require the client to supply a double-MD5-hashed password for - authentication. - See for details. + Perform SCRAM-SHA-256 authentication to verify the user's + password. See for details. - scram + md5 - Perform SCRAM-SHA-256 authentication to verify the user's - password. - See for details. + Perform SCRAM-SHA-256 or MD5 authentication to verify the + user's password. See + for details. @@ -689,13 +688,12 @@ host postgres all 192.168.12.10/32 scram # Allow any user from hosts in the example.com domain to connect to # any database if the user's password is correctly supplied. # -# Most users use SCRAM authentication, but some users use older clients -# that don't support SCRAM authentication, and need to be able to log -# in using MD5 authentication. Such users are put in the @md5users -# group, everyone else must use SCRAM. +# Require SCRAM authentication for most users, but make an exception +# for user 'mike', who uses an older client that doesn't support SCRAM +# authentication. # # TYPE DATABASE USER ADDRESS METHOD -host all @md5users .example.com md5 +host all mike .example.com md5 host all all .example.com scram # In the absence of preceding "host" lines, these two lines will @@ -949,12 +947,13 @@ omicron bryanh guest1 - In md5, the client sends a hash of a random challenge, - generated by the server, and the password. It prevents password sniffing, - but is less secure than scram, and provides no protection - if an attacker manages to steal the password hash from the server. - md5 cannot be used with the feature. + md5 allows falling back to a less secure challenge-response + mechanism for those users with an MD5 hashed password. + The fallback mechanism also prevents password sniffing, but provides no + protection if an attacker manages to steal the password hash from the + server, and it cannot be used with the feature. For all other users, + md5 works the same as scram. diff --git a/src/backend/libpq/auth-scram.c b/src/backend/libpq/auth-scram.c index db15a2fac6..bcc8d03ef5 100644 --- a/src/backend/libpq/auth-scram.c +++ b/src/backend/libpq/auth-scram.c @@ -130,79 +130,91 @@ static char *scram_MockSalt(const char *username); * after the beginning of the exchange with verifier data. * * 'username' is the provided by the client. 'shadow_pass' is the role's - * password verifier, from pg_authid.rolpassword. If 'doomed' is true, the - * authentication must fail, as if an incorrect password was given. - * 'shadow_pass' may be NULL, when 'doomed' is set. + * password verifier, from pg_authid.rolpassword. If 'shadow_pass' is NULL, we + * still perform an authentication exchange, but it will fail, as if an + * incorrect password was given. */ void * -pg_be_scram_init(const char *username, const char *shadow_pass, bool doomed) +pg_be_scram_init(const char *username, const char *shadow_pass) { scram_state *state; - int password_type; + bool got_verifier; state = (scram_state *) palloc0(sizeof(scram_state)); state->state = SCRAM_AUTH_INIT; state->username = username; /* - * Perform sanity checks on the provided password after catalog lookup. - * The authentication is bound to fail if the lookup itself failed or if - * the password stored is MD5-encrypted. Authentication is possible for - * users with a valid plain password though. + * Parse the stored password verifier. */ + if (shadow_pass) + { + int password_type = get_password_type(shadow_pass); - if (shadow_pass == NULL || doomed) - password_type = -1; - else - password_type = get_password_type(shadow_pass); + if (password_type == PASSWORD_TYPE_SCRAM) + { + if (parse_scram_verifier(shadow_pass, &state->salt, &state->iterations, + state->StoredKey, state->ServerKey)) + got_verifier = true; + else + { + /* + * The password looked like a SCRAM verifier, but could not be + * parsed. + */ + elog(LOG, "invalid SCRAM verifier for user \"%s\"", username); + got_verifier = false; + } + } + else if (password_type == PASSWORD_TYPE_PLAINTEXT) + { + /* + * The stored password is in plain format. Generate a fresh SCRAM + * verifier from it, and proceed with that. + */ + char *verifier; - if (password_type == PASSWORD_TYPE_SCRAM) - { - if (!parse_scram_verifier(shadow_pass, &state->salt, &state->iterations, - state->StoredKey, state->ServerKey)) + verifier = scram_build_verifier(username, shadow_pass, 0); + + (void) parse_scram_verifier(verifier, &state->salt, &state->iterations, + state->StoredKey, state->ServerKey); + pfree(verifier); + + got_verifier = true; + } + else { /* - * The password looked like a SCRAM verifier, but could not be - * parsed. + * The user doesn't have SCRAM verifier, nor could we generate + * one. (You cannot do SCRAM authentication with an MD5 hash.) */ - elog(LOG, "invalid SCRAM verifier for user \"%s\"", username); - doomed = true; + state->logdetail = psprintf(_("User \"%s\" does not have a valid SCRAM verifier."), + state->username); + got_verifier = false; } } - else if (password_type == PASSWORD_TYPE_PLAINTEXT) + else { - char *verifier; - /* - * The password provided is in plain format, in which case a fresh - * SCRAM verifier can be generated and used for the rest of the - * processing. + * The caller requested us to perform a dummy authentication. This is + * considered normal, since the caller requested it, so don't set log + * detail. */ - verifier = scram_build_verifier(username, shadow_pass, 0); - - (void) parse_scram_verifier(verifier, &state->salt, &state->iterations, - state->StoredKey, state->ServerKey); - pfree(verifier); + got_verifier = false; } - else - doomed = true; - if (doomed) + /* + * If the user did not have a valid SCRAM verifier, we still go through + * the motions with a mock one, and fail as if the client supplied an + * incorrect password. This is to avoid revealing information to an + * attacker. + */ + if (!got_verifier) { - /* - * We don't have a valid SCRAM verifier, nor could we generate one, or - * the caller requested us to perform a dummy authentication. - * - * The authentication is bound to fail, but to avoid revealing - * information to the attacker, go through the motions with a fake - * SCRAM verifier, and fail as if the password was incorrect. - */ - state->logdetail = psprintf(_("User \"%s\" does not have a valid SCRAM verifier."), - state->username); mock_scram_verifier(username, &state->salt, &state->iterations, state->StoredKey, state->ServerKey); + state->doomed = true; } - state->doomed = doomed; return state; } diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c index a699a09e9a..5f4f55760c 100644 --- a/src/backend/libpq/auth.c +++ b/src/backend/libpq/auth.c @@ -24,6 +24,7 @@ #include #endif +#include "commands/user.h" #include "common/ip.h" #include "common/md5.h" #include "libpq/auth.h" @@ -49,17 +50,15 @@ static char *recv_password_packet(Port *port); /*---------------------------------------------------------------- - * MD5 authentication + * Password-based authentication methods (password, md5, and scram) *---------------------------------------------------------------- */ -static int CheckMD5Auth(Port *port, char **logdetail); +static int CheckPasswordAuth(Port *port, char **logdetail); +static int CheckPWChallengeAuth(Port *port, char **logdetail); -/*---------------------------------------------------------------- - * Plaintext password authentication - *---------------------------------------------------------------- - */ +static int CheckMD5Auth(Port *port, char *shadow_pass, char **logdetail); +static int CheckSCRAMAuth(Port *port, char *shadow_pass, char **logdetail); -static int CheckPasswordAuth(Port *port, char **logdetail); /*---------------------------------------------------------------- * Ident authentication @@ -200,12 +199,6 @@ static int CheckRADIUSAuth(Port *port); static int PerformRadiusTransaction(char *server, char *secret, char *portstr, char *identifier, char *user_name, char *passwd); -/*---------------------------------------------------------------- - * SASL authentication - *---------------------------------------------------------------- - */ -static int CheckSASLAuth(Port *port, char **logdetail); - /* * Maximum accepted size of GSS and SSPI authentication tokens. * @@ -291,7 +284,7 @@ auth_failed(Port *port, int status, char *logdetail) break; case uaPassword: case uaMD5: - case uaSASL: + case uaSCRAM: errstr = gettext_noop("password authentication failed for user \"%s\""); /* We use it to indicate if a .pgpass password failed. */ errcode_return = ERRCODE_INVALID_PASSWORD; @@ -552,17 +545,14 @@ ClientAuthentication(Port *port) break; case uaMD5: - status = CheckMD5Auth(port, &logdetail); + case uaSCRAM: + status = CheckPWChallengeAuth(port, &logdetail); break; case uaPassword: status = CheckPasswordAuth(port, &logdetail); break; - case uaSASL: - status = CheckSASLAuth(port, &logdetail); - break; - case uaPAM: #ifdef USE_PAM status = CheckPAMAuth(port, port->user_name, ""); @@ -710,41 +700,34 @@ recv_password_packet(Port *port) /*---------------------------------------------------------------- - * MD5 authentication + * Password-based authentication mechanisms *---------------------------------------------------------------- */ +/* + * Plaintext password authentication. + */ static int -CheckMD5Auth(Port *port, char **logdetail) +CheckPasswordAuth(Port *port, char **logdetail) { - char md5Salt[4]; /* Password salt */ char *passwd; - char *shadow_pass; int result; + char *shadow_pass; - if (Db_user_namespace) - ereport(FATAL, - (errcode(ERRCODE_INVALID_AUTHORIZATION_SPECIFICATION), - errmsg("MD5 authentication is not supported when \"db_user_namespace\" is enabled"))); - - /* include the salt to use for computing the response */ - if (!pg_backend_random(md5Salt, 4)) - { - ereport(LOG, - (errmsg("could not generate random MD5 salt"))); - return STATUS_ERROR; - } - - sendAuthRequest(port, AUTH_REQ_MD5, md5Salt, 4); + sendAuthRequest(port, AUTH_REQ_PASSWORD, NULL, 0); passwd = recv_password_packet(port); if (passwd == NULL) return STATUS_EOF; /* client wouldn't send password */ - result = get_role_password(port->user_name, &shadow_pass, logdetail); - if (result == STATUS_OK) - result = md5_crypt_verify(port->user_name, shadow_pass, passwd, - md5Salt, 4, logdetail); + shadow_pass = get_role_password(port->user_name, logdetail); + if (shadow_pass) + { + result = plain_crypt_verify(port->user_name, shadow_pass, passwd, + logdetail); + } + else + result = STATUS_ERROR; if (shadow_pass) pfree(shadow_pass); @@ -753,42 +736,114 @@ CheckMD5Auth(Port *port, char **logdetail) return result; } -/*---------------------------------------------------------------- - * Plaintext password authentication - *---------------------------------------------------------------- +/* + * MD5 and SCRAM authentication. */ +static int +CheckPWChallengeAuth(Port *port, char **logdetail) +{ + int auth_result; + char *shadow_pass; + PasswordType pwtype; + + Assert(port->hba->auth_method == uaSCRAM || + port->hba->auth_method == uaMD5); + + /* First look up the user's password. */ + shadow_pass = get_role_password(port->user_name, logdetail); + + /* + * If the user does not exist, or has no password, we still go through the + * motions of authentication, to avoid revealing to the client that the + * user didn't exist. If 'md5' is allowed, we choose whether to use 'md5' + * or 'scram' authentication based on current password_encryption setting. + * The idea is that most genuine users probably have a password of that + * type, if we pretend that this user had a password of that type, too, it + * "blends in" best. + * + * If the user had a password, but it was expired, we'll use the details + * of the expired password for the authentication, but report it as + * failure to the client even if correct password was given. + */ + if (!shadow_pass) + pwtype = Password_encryption; + else + pwtype = get_password_type(shadow_pass); + + /* + * If 'md5' authentication is allowed, decide whether to perform 'md5' or + * 'scram' authentication based on the type of password the user has. If + * it's an MD5 hash, we must do MD5 authentication, and if it's a SCRAM + * verifier, we must do SCRAM authentication. If it's stored in + * plaintext, we could do either one, so we opt for the more secure + * mechanism, SCRAM. + * + * If MD5 authentication is not allowed, always use SCRAM. If the user + * had an MD5 password, CheckSCRAMAuth() will fail. + */ + if (port->hba->auth_method == uaMD5 && pwtype == PASSWORD_TYPE_MD5) + { + auth_result = CheckMD5Auth(port, shadow_pass, logdetail); + } + else + { + auth_result = CheckSCRAMAuth(port, shadow_pass, logdetail); + } + + if (shadow_pass) + pfree(shadow_pass); + + /* + * If get_role_password() returned error, return error, even if the + * authentication succeeded. + */ + if (!shadow_pass) + { + Assert(auth_result != STATUS_OK); + return STATUS_ERROR; + } + return auth_result; +} static int -CheckPasswordAuth(Port *port, char **logdetail) +CheckMD5Auth(Port *port, char *shadow_pass, char **logdetail) { + char md5Salt[4]; /* Password salt */ char *passwd; int result; - char *shadow_pass; - sendAuthRequest(port, AUTH_REQ_PASSWORD, NULL, 0); + if (Db_user_namespace) + ereport(FATAL, + (errcode(ERRCODE_INVALID_AUTHORIZATION_SPECIFICATION), + errmsg("MD5 authentication is not supported when \"db_user_namespace\" is enabled"))); + + /* include the salt to use for computing the response */ + if (!pg_backend_random(md5Salt, 4)) + { + ereport(LOG, + (errmsg("could not generate random MD5 salt"))); + return STATUS_ERROR; + } + + sendAuthRequest(port, AUTH_REQ_MD5, md5Salt, 4); passwd = recv_password_packet(port); if (passwd == NULL) return STATUS_EOF; /* client wouldn't send password */ - result = get_role_password(port->user_name, &shadow_pass, logdetail); - if (result == STATUS_OK) - result = plain_crypt_verify(port->user_name, shadow_pass, passwd, - logdetail); - if (shadow_pass) - pfree(shadow_pass); + result = md5_crypt_verify(port->user_name, shadow_pass, passwd, + md5Salt, 4, logdetail); + else + result = STATUS_ERROR; + pfree(passwd); return result; } -/*---------------------------------------------------------------- - * SASL authentication system - *---------------------------------------------------------------- - */ static int -CheckSASLAuth(Port *port, char **logdetail) +CheckSCRAMAuth(Port *port, char *shadow_pass, char **logdetail) { int mtype; StringInfoData buf; @@ -796,8 +851,6 @@ CheckSASLAuth(Port *port, char **logdetail) char *output = NULL; int outputlen = 0; int result; - char *shadow_pass; - bool doomed = false; /* * SASL auth is not supported for protocol versions before 3, because it @@ -827,11 +880,9 @@ CheckSASLAuth(Port *port, char **logdetail) * This is because we don't want to reveal to an attacker what usernames * are valid, nor which users have a valid password. */ - if (get_role_password(port->user_name, &shadow_pass, logdetail) != STATUS_OK) - doomed = true; /* Initialize the status tracker for message exchanges */ - scram_opaq = pg_be_scram_init(port->user_name, shadow_pass, doomed); + scram_opaq = pg_be_scram_init(port->user_name, shadow_pass); /* * Loop through SASL message exchange. This exchange can consist of @@ -875,7 +926,7 @@ CheckSASLAuth(Port *port, char **logdetail) */ result = pg_be_scram_exchange(scram_opaq, buf.data, buf.len, &output, &outputlen, - doomed ? NULL : logdetail); + logdetail); /* input buffer no longer used */ pfree(buf.data); diff --git a/src/backend/libpq/crypt.c b/src/backend/libpq/crypt.c index ac10751ec2..34beab5334 100644 --- a/src/backend/libpq/crypt.c +++ b/src/backend/libpq/crypt.c @@ -31,25 +31,18 @@ /* * Fetch stored password for a user, for authentication. * - * Returns STATUS_OK on success. On error, returns STATUS_ERROR, and stores - * a palloc'd string describing the reason, for the postmaster log, in - * *logdetail. The error reason should *not* be sent to the client, to avoid - * giving away user information! - * - * If the password is expired, it is still returned in *shadow_pass, but the - * return code is STATUS_ERROR. On other errors, *shadow_pass is set to - * NULL. + * On error, returns NULL, and stores a palloc'd string describing the reason, + * for the postmaster log, in *logdetail. The error reason should *not* be + * sent to the client, to avoid giving away user information! */ -int -get_role_password(const char *role, char **shadow_pass, char **logdetail) +char * +get_role_password(const char *role, char **logdetail) { - int retval = STATUS_ERROR; TimestampTz vuntil = 0; HeapTuple roleTup; Datum datum; bool isnull; - - *shadow_pass = NULL; + char *shadow_pass; /* Get role info from pg_authid */ roleTup = SearchSysCache1(AUTHNAME, PointerGetDatum(role)); @@ -57,7 +50,7 @@ get_role_password(const char *role, char **shadow_pass, char **logdetail) { *logdetail = psprintf(_("Role \"%s\" does not exist."), role); - return STATUS_ERROR; /* no such user */ + return NULL; /* no such user */ } datum = SysCacheGetAttr(AUTHNAME, roleTup, @@ -67,9 +60,9 @@ get_role_password(const char *role, char **shadow_pass, char **logdetail) ReleaseSysCache(roleTup); *logdetail = psprintf(_("User \"%s\" has no password assigned."), role); - return STATUS_ERROR; /* user has no password */ + return NULL; /* user has no password */ } - *shadow_pass = TextDatumGetCString(datum); + shadow_pass = TextDatumGetCString(datum); datum = SysCacheGetAttr(AUTHNAME, roleTup, Anum_pg_authid_rolvaliduntil, &isnull); @@ -78,30 +71,25 @@ get_role_password(const char *role, char **shadow_pass, char **logdetail) ReleaseSysCache(roleTup); - if (**shadow_pass == '\0') + if (*shadow_pass == '\0') { *logdetail = psprintf(_("User \"%s\" has an empty password."), role); - pfree(*shadow_pass); - *shadow_pass = NULL; - return STATUS_ERROR; /* empty password */ + pfree(shadow_pass); + return NULL; /* empty password */ } /* - * Password OK, now check to be sure we are not past rolvaliduntil + * Password OK, but check to be sure we are not past rolvaliduntil */ - if (isnull) - retval = STATUS_OK; - else if (vuntil < GetCurrentTimestamp()) + if (!isnull && vuntil < GetCurrentTimestamp()) { *logdetail = psprintf(_("User \"%s\" has an expired password."), role); - retval = STATUS_ERROR; + return NULL; } - else - retval = STATUS_OK; - return retval; + return shadow_pass; } /* diff --git a/src/backend/libpq/hba.c b/src/backend/libpq/hba.c index 49be6638b8..af89fe898a 100644 --- a/src/backend/libpq/hba.c +++ b/src/backend/libpq/hba.c @@ -1328,7 +1328,7 @@ parse_hba_line(TokenizedLine *tok_line, int elevel) parsedline->auth_method = uaMD5; } else if (strcmp(token->string, "scram") == 0) - parsedline->auth_method = uaSASL; + parsedline->auth_method = uaSCRAM; else if (strcmp(token->string, "pam") == 0) #ifdef USE_PAM parsedline->auth_method = uaPAM; diff --git a/src/include/libpq/crypt.h b/src/include/libpq/crypt.h index 0502d6a0e5..3b5da69b08 100644 --- a/src/include/libpq/crypt.h +++ b/src/include/libpq/crypt.h @@ -32,8 +32,7 @@ extern PasswordType get_password_type(const char *shadow_pass); extern char *encrypt_password(PasswordType target_type, const char *role, const char *password); -extern int get_role_password(const char *role, char **shadow_pass, - char **logdetail); +extern char *get_role_password(const char *role, char **logdetail); extern int md5_crypt_verify(const char *role, const char *shadow_pass, const char *client_pass, const char *md5_salt, diff --git a/src/include/libpq/hba.h b/src/include/libpq/hba.h index 6c7382e67f..9a4f228d6a 100644 --- a/src/include/libpq/hba.h +++ b/src/include/libpq/hba.h @@ -30,7 +30,7 @@ typedef enum UserAuth uaIdent, uaPassword, uaMD5, - uaSASL, + uaSCRAM, uaGSS, uaSSPI, uaPAM, diff --git a/src/include/libpq/scram.h b/src/include/libpq/scram.h index fb21e056c8..e373f0c07e 100644 --- a/src/include/libpq/scram.h +++ b/src/include/libpq/scram.h @@ -22,7 +22,7 @@ #define SASL_EXCHANGE_FAILURE 2 /* Routines dedicated to authentication */ -extern void *pg_be_scram_init(const char *username, const char *shadow_pass, bool doomed); +extern void *pg_be_scram_init(const char *username, const char *shadow_pass); extern int pg_be_scram_exchange(void *opaq, char *input, int inputlen, char **output, int *outputlen, char **logdetail); diff --git a/src/test/authentication/t/001_password.pl b/src/test/authentication/t/001_password.pl index 8726a23e0d..d7bc13bd58 100644 --- a/src/test/authentication/t/001_password.pl +++ b/src/test/authentication/t/001_password.pl @@ -75,10 +75,10 @@ SKIP: test_role($node, 'md5_role', 'scram', 2); test_role($node, 'plain_role', 'scram', 0); - # For "md5" method, users "plain_role" and "md5_role" should be able to - # connect. + # For "md5" method, all users should be able to connect (SCRAM + # authentication will be performed for the user with a scram verifier.) reset_pg_hba($node, 'md5'); - test_role($node, 'scram_role', 'md5', 2); + test_role($node, 'scram_role', 'md5', 0); test_role($node, 'md5_role', 'md5', 0); test_role($node, 'plain_role', 'md5', 0); } -- cgit v1.2.3 From 857ee8e391ff6654ef9dcc5dd8b658d7709d0a3c Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Fri, 24 Mar 2017 12:00:53 -0400 Subject: Add a txid_status function. If your connection to the database server is lost while a COMMIT is in progress, it may be difficult to figure out whether the COMMIT was successful or not. This function will tell you, provided that you don't wait too long to ask. It may be useful in other situations, too. Craig Ringer, reviewed by Simon Riggs and by me Discussion: http://postgr.es/m/CAMsr+YHQiWNEi0daCTboS40T+V5s_+dst3PYv_8v2wNVH+Xx4g@mail.gmail.com --- doc/src/sgml/func.sgml | 27 ++++++ src/backend/utils/adt/txid.c | 132 ++++++++++++++++++++++++++++++ src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_proc.h | 2 + src/test/recovery/t/011_crash_recovery.pl | 46 +++++++++++ src/test/regress/expected/txid.out | 68 +++++++++++++++ src/test/regress/sql/txid.sql | 38 +++++++++ 7 files changed, 314 insertions(+), 1 deletion(-) create mode 100644 src/test/recovery/t/011_crash_recovery.pl (limited to 'doc/src') diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 64f86ce680..3f0f7363b9 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -17523,6 +17523,10 @@ SELECT collation for ('foo' COLLATE "de_DE"); txid_visible_in_snapshot + + txid_status + + The functions shown in provide server transaction information in an exportable form. The main @@ -17573,6 +17577,11 @@ SELECT collation for ('foo' COLLATE "de_DE"); boolean is transaction ID visible in snapshot? (do not use with subtransaction ids) + + txid_status(bigint) + txid_status + report the status of the given xact - committed, aborted, in progress, or NULL if the txid is too old + @@ -17642,6 +17651,24 @@ SELECT collation for ('foo' COLLATE "de_DE"); xmin=10, xmax=20, xip_list=10, 14, 15. + + txid_status(bigint) reports the commit status of a recent + transaction. Applications may use it to determine whether a transaction + committed or aborted when the application and database server become + disconnected while a COMMIT is in progress. + The status of a transaction will be reported as either + in progress, + committed, or aborted, provided that the + transaction is recent enough that the system retains the commit status + of that transaction. If is old enough that no references to that + transaction survive in the system and the commit status information has + been discarded, this function will return NULL. Note that prepared + transactions are reported as in progress; applications must + check pg_prepared_xacts if they + need to determine whether the txid is a prepared transaction. + + The functions shown in provide information about transactions that have been already committed. diff --git a/src/backend/utils/adt/txid.c b/src/backend/utils/adt/txid.c index 772d7c7203..5c64e32719 100644 --- a/src/backend/utils/adt/txid.c +++ b/src/backend/utils/adt/txid.c @@ -21,6 +21,7 @@ #include "postgres.h" +#include "access/clog.h" #include "access/transam.h" #include "access/xact.h" #include "access/xlog.h" @@ -28,6 +29,7 @@ #include "miscadmin.h" #include "libpq/pqformat.h" #include "postmaster/postmaster.h" +#include "storage/lwlock.h" #include "utils/builtins.h" #include "utils/memutils.h" #include "utils/snapmgr.h" @@ -92,6 +94,70 @@ load_xid_epoch(TxidEpoch *state) GetNextXidAndEpoch(&state->last_xid, &state->epoch); } +/* + * Helper to get a TransactionId from a 64-bit xid with wraparound detection. + * + * It is an ERROR if the xid is in the future. Otherwise, returns true if + * the transaction is still new enough that we can determine whether it + * committed and false otherwise. If *extracted_xid is not NULL, it is set + * to the low 32 bits of the transaction ID (i.e. the actual XID, without the + * epoch). + * + * The caller must hold CLogTruncationLock since it's dealing with arbitrary + * XIDs, and must continue to hold it until it's done with any clog lookups + * relating to those XIDs. + */ +static bool +TransactionIdInRecentPast(uint64 xid_with_epoch, TransactionId *extracted_xid) +{ + uint32 xid_epoch = (uint32) (xid_with_epoch >> 32); + TransactionId xid = (TransactionId) xid_with_epoch; + uint32 now_epoch; + TransactionId now_epoch_last_xid; + + GetNextXidAndEpoch(&now_epoch_last_xid, &now_epoch); + + if (extracted_xid != NULL) + *extracted_xid = xid; + + if (!TransactionIdIsValid(xid)) + return false; + + /* For non-normal transaction IDs, we can ignore the epoch. */ + if (!TransactionIdIsNormal(xid)) + return true; + + /* If the transaction ID is in the future, throw an error. */ + if (xid_epoch > now_epoch + || (xid_epoch == now_epoch && xid > now_epoch_last_xid)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("transaction ID " UINT64_FORMAT " is in the future", + xid_with_epoch))); + + /* + * ShmemVariableCache->oldestClogXid is protected by CLogTruncationLock, + * but we don't acquire that lock here. Instead, we require the caller to + * acquire it, because the caller is presumably going to look up the + * returned XID. If we took and released the lock within this function, a + * CLOG truncation could occur before the caller finished with the XID. + */ + Assert(LWLockHeldByMe(CLogTruncationLock)); + + /* + * If the transaction ID has wrapped around, it's definitely too old to + * determine the commit status. Otherwise, we can compare it to + * ShmemVariableCache->oldestClogXid to determine whether the relevant CLOG + * entry is guaranteed to still exist. + */ + if (xid_epoch + 1 < now_epoch + || (xid_epoch + 1 == now_epoch && xid < now_epoch_last_xid) + || TransactionIdPrecedes(xid, ShmemVariableCache->oldestClogXid)) + return false; + + return true; +} + /* * do a TransactionId -> txid conversion for an XID near the given epoch */ @@ -354,6 +420,9 @@ bad_format: * * Return the current toplevel transaction ID as TXID * If the current transaction does not have one, one is assigned. + * + * This value has the epoch as the high 32 bits and the 32-bit xid + * as the low 32 bits. */ Datum txid_current(PG_FUNCTION_ARGS) @@ -658,3 +727,66 @@ txid_snapshot_xip(PG_FUNCTION_ARGS) SRF_RETURN_DONE(fctx); } } + +/* + * Report the status of a recent transaction ID, or null for wrapped, + * truncated away or otherwise too old XIDs. + * + * The passed epoch-qualified xid is treated as a normal xid, not a + * multixact id. + * + * If it points to a committed subxact the result is the subxact status even + * though the parent xact may still be in progress or may have aborted. + */ +Datum +txid_status(PG_FUNCTION_ARGS) +{ + const char *status; + uint64 xid_with_epoch = PG_GETARG_INT64(0); + TransactionId xid; + + /* + * We must protect against concurrent truncation of clog entries to avoid + * an I/O error on SLRU lookup. + */ + LWLockAcquire(CLogTruncationLock, LW_SHARED); + if (TransactionIdInRecentPast(xid_with_epoch, &xid)) + { + Assert(TransactionIdIsValid(xid)); + + if (TransactionIdIsCurrentTransactionId(xid)) + status = gettext_noop("in progress"); + else if (TransactionIdDidCommit(xid)) + status = gettext_noop("committed"); + else if (TransactionIdDidAbort(xid)) + status = gettext_noop("aborted"); + else + { + /* + * The xact is not marked as either committed or aborted in clog. + * + * It could be a transaction that ended without updating clog or + * writing an abort record due to a crash. We can safely assume + * it's aborted if it isn't committed and is older than our + * snapshot xmin. + * + * Otherwise it must be in-progress (or have been at the time + * we checked commit/abort status). + */ + if (TransactionIdPrecedes(xid, GetActiveSnapshot()->xmin)) + status = gettext_noop("aborted"); + else + status = gettext_noop("in progress"); + } + } + else + { + status = NULL; + } + LWLockRelease(CLogTruncationLock); + + if (status == NULL) + PG_RETURN_NULL(); + else + PG_RETURN_TEXT_P(cstring_to_text(status)); +} diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index d8679f5f59..c9c9a18777 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201703231 +#define CATALOG_VERSION_NO 201703241 #endif diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index 0d18ab8c0d..a66d045100 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -4978,6 +4978,8 @@ DATA(insert OID = 2947 ( txid_snapshot_xip PGNSP PGUID 12 1 50 0 0 f f f f t DESCR("get set of in-progress txids in snapshot"); DATA(insert OID = 2948 ( txid_visible_in_snapshot PGNSP PGUID 12 1 0 0 0 f f f f t f i s 2 0 16 "20 2970" _null_ _null_ _null_ _null_ _null_ txid_visible_in_snapshot _null_ _null_ _null_ )); DESCR("is txid visible in snapshot?"); +DATA(insert OID = 3360 ( txid_status PGNSP PGUID 12 1 0 0 0 f f f f t f v s 1 0 25 "20" _null_ _null_ _null_ _null_ _null_ txid_status _null_ _null_ _null_ )); +DESCR("commit status of transaction"); /* record comparison using normal comparison rules */ DATA(insert OID = 2981 ( record_eq PGNSP PGUID 12 1 0 0 0 f f f f t f i s 2 0 16 "2249 2249" _null_ _null_ _null_ _null_ _null_ record_eq _null_ _null_ _null_ )); diff --git a/src/test/recovery/t/011_crash_recovery.pl b/src/test/recovery/t/011_crash_recovery.pl new file mode 100644 index 0000000000..e5d1a0f645 --- /dev/null +++ b/src/test/recovery/t/011_crash_recovery.pl @@ -0,0 +1,46 @@ +# +# Tests relating to PostgreSQL crash recovery and redo +# +use strict; +use warnings; +use PostgresNode; +use TestLib; +use Test::More tests => 3; + +my $node = get_new_node('master'); +$node->init(allows_streaming => 1); +$node->start; + +my ($stdin, $stdout, $stderr) = ('', '', ''); + +# Ensure that txid_status reports 'aborted' for xacts +# that were in-progress during crash. To do that, we need +# an xact to be in-progress when we crash and we need to know +# its xid. +my $tx = IPC::Run::start( + ['psql', '-qAt', '-v', 'ON_ERROR_STOP=1', '-f', '-', '-d', $node->connstr('postgres')], + '<', \$stdin, '>', \$stdout, '2>', \$stderr); +$stdin .= q[ +BEGIN; +CREATE TABLE mine(x integer); +SELECT txid_current(); +]; +$tx->pump until $stdout =~ /[[:digit:]]+[\r\n]$/; + +# Status should be in-progress +my $xid = $stdout; +chomp($xid); + +is($node->safe_psql('postgres', qq[SELECT txid_status('$xid');]), 'in progress', 'own xid is in-progres'); + +# Crash and restart the postmaster +$node->stop('immediate'); +$node->start; + +# Make sure we really got a new xid +cmp_ok($node->safe_psql('postgres', 'SELECT txid_current()'), '>', $xid, + 'new xid after restart is greater'); +# and make sure we show the in-progress xact as aborted +is($node->safe_psql('postgres', qq[SELECT txid_status('$xid');]), 'aborted', 'xid is aborted after crash'); + +$tx->kill_kill; diff --git a/src/test/regress/expected/txid.out b/src/test/regress/expected/txid.out index 802ccb949f..015dae3051 100644 --- a/src/test/regress/expected/txid.out +++ b/src/test/regress/expected/txid.out @@ -254,3 +254,71 @@ SELECT txid_current_if_assigned() IS NOT DISTINCT FROM BIGINT :'txid_current'; (1 row) COMMIT; +-- test xid status functions +BEGIN; +SELECT txid_current() AS committed \gset +COMMIT; +BEGIN; +SELECT txid_current() AS rolledback \gset +ROLLBACK; +BEGIN; +SELECT txid_current() AS inprogress \gset +SELECT txid_status(:committed) AS committed; + committed +----------- + committed +(1 row) + +SELECT txid_status(:rolledback) AS rolledback; + rolledback +------------ + aborted +(1 row) + +SELECT txid_status(:inprogress) AS inprogress; + inprogress +------------- + in progress +(1 row) + +SELECT txid_status(1); -- BootstrapTransactionId is always committed + txid_status +------------- + committed +(1 row) + +SELECT txid_status(2); -- FrozenTransactionId is always committed + txid_status +------------- + committed +(1 row) + +SELECT txid_status(3); -- in regress testing FirstNormalTransactionId will always be behind oldestXmin + txid_status +------------- + +(1 row) + +COMMIT; +BEGIN; +CREATE FUNCTION test_future_xid_status(bigint) +RETURNS void +LANGUAGE plpgsql +AS +$$ +BEGIN + PERFORM txid_status($1); + RAISE EXCEPTION 'didn''t ERROR at xid in the future as expected'; +EXCEPTION + WHEN invalid_parameter_value THEN + RAISE NOTICE 'Got expected error for xid in the future'; +END; +$$; +SELECT test_future_xid_status(:inprogress + 10000); +NOTICE: Got expected error for xid in the future + test_future_xid_status +------------------------ + +(1 row) + +ROLLBACK; diff --git a/src/test/regress/sql/txid.sql b/src/test/regress/sql/txid.sql index 4aefd9e64d..bd6decf0ef 100644 --- a/src/test/regress/sql/txid.sql +++ b/src/test/regress/sql/txid.sql @@ -59,3 +59,41 @@ SELECT txid_current_if_assigned() IS NULL; SELECT txid_current() \gset SELECT txid_current_if_assigned() IS NOT DISTINCT FROM BIGINT :'txid_current'; COMMIT; + +-- test xid status functions +BEGIN; +SELECT txid_current() AS committed \gset +COMMIT; + +BEGIN; +SELECT txid_current() AS rolledback \gset +ROLLBACK; + +BEGIN; +SELECT txid_current() AS inprogress \gset + +SELECT txid_status(:committed) AS committed; +SELECT txid_status(:rolledback) AS rolledback; +SELECT txid_status(:inprogress) AS inprogress; +SELECT txid_status(1); -- BootstrapTransactionId is always committed +SELECT txid_status(2); -- FrozenTransactionId is always committed +SELECT txid_status(3); -- in regress testing FirstNormalTransactionId will always be behind oldestXmin + +COMMIT; + +BEGIN; +CREATE FUNCTION test_future_xid_status(bigint) +RETURNS void +LANGUAGE plpgsql +AS +$$ +BEGIN + PERFORM txid_status($1); + RAISE EXCEPTION 'didn''t ERROR at xid in the future as expected'; +EXCEPTION + WHEN invalid_parameter_value THEN + RAISE NOTICE 'Got expected error for xid in the future'; +END; +$$; +SELECT test_future_xid_status(:inprogress + 10000); +ROLLBACK; -- cgit v1.2.3 From 7b504eb282ca2f5104b5c00b4f05a3ef6bb1385b Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Fri, 24 Mar 2017 14:06:10 -0300 Subject: Implement multivariate n-distinct coefficients Add support for explicitly declared statistic objects (CREATE STATISTICS), allowing collection of statistics on more complex combinations that individual table columns. Companion commands DROP STATISTICS and ALTER STATISTICS ... OWNER TO / SET SCHEMA / RENAME are added too. All this DDL has been designed so that more statistic types can be added later on, such as multivariate most-common-values and multivariate histograms between columns of a single table, leaving room for permitting columns on multiple tables, too, as well as expressions. This commit only adds support for collection of n-distinct coefficient on user-specified sets of columns in a single table. This is useful to estimate number of distinct groups in GROUP BY and DISTINCT clauses; estimation errors there can cause over-allocation of memory in hashed aggregates, for instance, so it's a worthwhile problem to solve. A new special pseudo-type pg_ndistinct is used. (num-distinct estimation was deemed sufficiently useful by itself that this is worthwhile even if no further statistic types are added immediately; so much so that another version of essentially the same functionality was submitted by Kyotaro Horiguchi: https://postgr.es/m/20150828.173334.114731693.horiguchi.kyotaro@lab.ntt.co.jp though this commit does not use that code.) Author: Tomas Vondra. Some code rework by Álvaro. Reviewed-by: Dean Rasheed, David Rowley, Kyotaro Horiguchi, Jeff Janes, Ideriha Takeshi Discussion: https://postgr.es/m/543AFA15.4080608@fuzzy.cz https://postgr.es/m/20170320190220.ixlaueanxegqd5gr@alvherre.pgsql --- doc/src/sgml/catalogs.sgml | 97 ++++ doc/src/sgml/func.sgml | 36 +- doc/src/sgml/ref/allfiles.sgml | 3 + doc/src/sgml/ref/alter_statistics.sgml | 115 ++++ doc/src/sgml/ref/alter_table.sgml | 9 +- doc/src/sgml/ref/comment.sgml | 6 +- doc/src/sgml/ref/create_statistics.sgml | 155 ++++++ doc/src/sgml/ref/drop_statistics.sgml | 98 ++++ doc/src/sgml/reference.sgml | 3 + src/backend/Makefile | 2 +- src/backend/catalog/Makefile | 1 + src/backend/catalog/aclchk.c | 35 ++ src/backend/catalog/dependency.c | 9 + src/backend/catalog/heap.c | 74 +++ src/backend/catalog/namespace.c | 56 ++ src/backend/catalog/objectaddress.c | 55 ++ src/backend/catalog/pg_shdepend.c | 2 + src/backend/catalog/system_views.sql | 10 + src/backend/commands/Makefile | 6 +- src/backend/commands/alter.c | 8 + src/backend/commands/analyze.c | 23 +- src/backend/commands/dropcmds.c | 7 + src/backend/commands/event_trigger.c | 3 + src/backend/commands/statscmds.c | 296 ++++++++++ src/backend/nodes/copyfuncs.c | 17 + src/backend/nodes/equalfuncs.c | 15 + src/backend/nodes/outfuncs.c | 31 ++ src/backend/optimizer/util/plancat.c | 67 ++- src/backend/parser/gram.y | 62 ++- src/backend/statistics/Makefile | 17 + src/backend/statistics/README | 34 ++ src/backend/statistics/extended_stats.c | 389 +++++++++++++ src/backend/statistics/mvdistinct.c | 671 +++++++++++++++++++++++ src/backend/tcop/utility.c | 12 + src/backend/utils/adt/ruleutils.c | 81 +++ src/backend/utils/adt/selfuncs.c | 181 +++++- src/backend/utils/cache/relcache.c | 79 +++ src/backend/utils/cache/syscache.c | 23 + src/bin/pg_dump/common.c | 4 + src/bin/pg_dump/pg_backup_archiver.c | 3 +- src/bin/pg_dump/pg_dump.c | 153 ++++++ src/bin/pg_dump/pg_dump.h | 9 + src/bin/pg_dump/pg_dump_sort.c | 12 +- src/bin/psql/describe.c | 51 ++ src/include/catalog/catversion.h | 2 +- src/include/catalog/dependency.h | 1 + src/include/catalog/heap.h | 1 + src/include/catalog/indexing.h | 7 + src/include/catalog/namespace.h | 2 + src/include/catalog/pg_cast.h | 4 + src/include/catalog/pg_proc.h | 11 + src/include/catalog/pg_statistic_ext.h | 75 +++ src/include/catalog/pg_type.h | 4 + src/include/catalog/toasting.h | 1 + src/include/commands/defrem.h | 4 + src/include/nodes/nodes.h | 2 + src/include/nodes/parsenodes.h | 15 + src/include/nodes/relation.h | 19 + src/include/statistics/extended_stats_internal.h | 64 +++ src/include/statistics/statistics.h | 47 ++ src/include/utils/acl.h | 2 + src/include/utils/rel.h | 4 + src/include/utils/relcache.h | 1 + src/include/utils/syscache.h | 2 + src/test/regress/expected/alter_generic.out | 45 +- src/test/regress/expected/object_address.out | 7 +- src/test/regress/expected/opr_sanity.out | 3 +- src/test/regress/expected/rules.out | 8 + src/test/regress/expected/sanity_check.out | 1 + src/test/regress/expected/stats_ext.out | 155 ++++++ src/test/regress/expected/type_sanity.out | 13 +- src/test/regress/parallel_schedule | 2 +- src/test/regress/serial_schedule | 1 + src/test/regress/sql/alter_generic.sql | 31 ++ src/test/regress/sql/object_address.sql | 4 +- src/test/regress/sql/stats_ext.sql | 102 ++++ src/test/regress/sql/type_sanity.sql | 2 +- 77 files changed, 3599 insertions(+), 63 deletions(-) create mode 100644 doc/src/sgml/ref/alter_statistics.sgml create mode 100644 doc/src/sgml/ref/create_statistics.sgml create mode 100644 doc/src/sgml/ref/drop_statistics.sgml create mode 100644 src/backend/commands/statscmds.c create mode 100644 src/backend/statistics/Makefile create mode 100644 src/backend/statistics/README create mode 100644 src/backend/statistics/extended_stats.c create mode 100644 src/backend/statistics/mvdistinct.c create mode 100644 src/include/catalog/pg_statistic_ext.h create mode 100644 src/include/statistics/extended_stats_internal.h create mode 100644 src/include/statistics/statistics.h create mode 100644 src/test/regress/expected/stats_ext.out create mode 100644 src/test/regress/sql/stats_ext.sql (limited to 'doc/src') diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index c531c73aac..ac39c639ed 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -295,6 +295,11 @@ planner statistics + + pg_statistic_ext + extended planner statistics + + pg_subscription logical replication subscriptions @@ -4247,6 +4252,98 @@ + + <structname>pg_statistic_ext</structname> + + + pg_statistic_ext + + + + The catalog pg_statistic_ext + holds extended planner statistics. + + + + <structname>pg_statistic_ext</> Columns + + + + + Name + Type + References + Description + + + + + + + starelid + oid + pg_class.oid + The table that the described columns belongs to + + + + staname + name + + Name of the statistic. + + + + stanamespace + oid + pg_namespace.oid + + The OID of the namespace that contains this statistic + + + + + staowner + oid + pg_authid.oid + Owner of the statistic + + + + staenabled + char[] + + + An array with the modes of the enabled statistic types, encoded as + d for ndistinct coefficients. + + + + + stakeys + int2vector + pg_attribute.attnum + + This is an array of values that indicate which table columns this + statistic covers. For example a value of 1 3 would + mean that the first and the third table columns make up the statistic key. + + + + + standistinct + pg_ndistinct + + + N-distinct coefficients, serialized as pg_ndistinct type. + + + + + +
    +     + <structname>pg_namespace</structname> diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 3f0f7363b9..ba6f8dd8d2 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -16720,6 +16720,10 @@ SELECT pg_type_is_visible('myschema.widget'::regtype); pg_get_serial_sequence + + pg_get_statisticsextdef + + pg_get_triggerdef @@ -16889,6 +16893,11 @@ SELECT pg_type_is_visible('myschema.widget'::regtype); get name of the sequence that a serial, smallserial or bigserial column uses     + + pg_get_statisticsextdef(statext_oid) + text + get CREATE STATISTICS command for extended statistics objects + pg_get_triggerdef(trigger_oid) text @@ -17034,19 +17043,20 @@ SELECT pg_type_is_visible('myschema.widget'::regtype); pg_get_constraintdef, pg_get_indexdef, pg_get_ruledef, - and pg_get_triggerdef, respectively reconstruct the - creating command for a constraint, index, rule, or trigger. (Note that this - is a decompiled reconstruction, not the original text of the command.) - pg_get_expr decompiles the internal form of an - individual expression, such as the default value for a column. It can be - useful when examining the contents of system catalogs. If the expression - might contain Vars, specify the OID of the relation they refer to as the - second parameter; if no Vars are expected, zero is sufficient. - pg_get_viewdef reconstructs the SELECT - query that defines a view. Most of these functions come in two variants, - one of which can optionally pretty-print the result. The - pretty-printed format is more readable, but the default format is more - likely to be interpreted the same way by future versions of + pg_get_statisticsextdef, and + pg_get_triggerdef, respectively reconstruct the + creating command for a constraint, index, rule, extended statistics object, + or trigger. (Note that this is a decompiled reconstruction, not the + original text of the command.) pg_get_expr decompiles + the internal form of an individual expression, such as the default value + for a column. It can be useful when examining the contents of system + catalogs. If the expression might contain Vars, specify the OID of the + relation they refer to as the second parameter; if no Vars are expected, + zero is sufficient. pg_get_viewdef reconstructs the + SELECT query that defines a view. Most of these functions come + in two variants, one of which can optionally pretty-print the + result. The pretty-printed format is more readable, but the default format + is more likely to be interpreted the same way by future versions of PostgreSQL; avoid using pretty-printed output for dump purposes. Passing false for the pretty-print parameter yields the same result as the variant that does not have the parameter at all. diff --git a/doc/src/sgml/ref/allfiles.sgml b/doc/src/sgml/ref/allfiles.sgml index 974e1b74e4..01acc2ef9d 100644 --- a/doc/src/sgml/ref/allfiles.sgml +++ b/doc/src/sgml/ref/allfiles.sgml @@ -34,6 +34,7 @@ Complete list of usable sgml source files in this directory. + @@ -80,6 +81,7 @@ Complete list of usable sgml source files in this directory. + @@ -126,6 +128,7 @@ Complete list of usable sgml source files in this directory. + diff --git a/doc/src/sgml/ref/alter_statistics.sgml b/doc/src/sgml/ref/alter_statistics.sgml new file mode 100644 index 0000000000..3e4d28614a --- /dev/null +++ b/doc/src/sgml/ref/alter_statistics.sgml @@ -0,0 +1,115 @@ + + + + + ALTER STATISTICS + + + + ALTER STATISTICS + 7 + SQL - Language Statements + + + + ALTER STATISTICS + + change the definition of a extended statistics + + + + + +ALTER STATISTICS name OWNER TO { new_owner | CURRENT_USER | SESSION_USER } +ALTER STATISTICS name RENAME TO new_name +ALTER STATISTICS name SET SCHEMA new_schema + + + + + Description + + + ALTER STATISTICS changes the parameters of an existing + extended statistics. Any parameters not specifically set in the + ALTER STATISTICS command retain their prior settings. + + + + You must own the statistics to use ALTER STATISTICS. + To change a statistics' schema, you must also have CREATE + privilege on the new schema. + To alter the owner, you must also be a direct or indirect member of the new + owning role, and that role must have CREATE privilege on + the statistics' schema. (These restrictions enforce that altering the owner + doesn't do anything you couldn't do by dropping and recreating the statistics. + However, a superuser can alter ownership of any statistics anyway.) + + + + + Parameters + + + + + name + + + The name (optionally schema-qualified) of the statistics to be altered. + + + + + + new_owner + + + The user name of the new owner of the statistics. + + + + + + new_name + + + The new name for the statistics. + + + + + + new_schema + + + The new schema for the statistics. + + + + + + + + + + Compatibility + + + There's no ALTER STATISTICS command in the SQL standard. + + + + + See Also + + + + + + + + diff --git a/doc/src/sgml/ref/alter_table.sgml b/doc/src/sgml/ref/alter_table.sgml index 767ea321da..75de226253 100644 --- a/doc/src/sgml/ref/alter_table.sgml +++ b/doc/src/sgml/ref/alter_table.sgml @@ -119,9 +119,12 @@ ALTER TABLE [ IF EXISTS ] name This form drops a column from a table. Indexes and table constraints involving the column will be automatically - dropped as well. You will need to say CASCADE if - anything outside the table depends on the column, for example, - foreign key references or views. + dropped as well. + Multivariate statistics referencing the dropped column will also be + removed if the removal of the column would cause the statistics to + contain data for only a single column. + You will need to say CASCADE if anything outside the table + depends on the column, for example, foreign key references or views. If IF EXISTS is specified and the column does not exist, no error is thrown. In this case a notice is issued instead. diff --git a/doc/src/sgml/ref/comment.sgml b/doc/src/sgml/ref/comment.sgml index 7483c8c03f..8fe17a5767 100644 --- a/doc/src/sgml/ref/comment.sgml +++ b/doc/src/sgml/ref/comment.sgml @@ -51,6 +51,7 @@ COMMENT ON SCHEMA object_name | SEQUENCE object_name | SERVER object_name | + STATISTICS object_name | TABLE object_name | TABLESPACE object_name | TEXT SEARCH CONFIGURATION object_name | @@ -125,8 +126,8 @@ COMMENT ON The name of the object to be commented. Names of tables, aggregates, collations, conversions, domains, foreign tables, functions, indexes, operators, operator classes, operator families, sequences, - text search objects, types, and views can be schema-qualified. - When commenting on a column, + statistics, text search objects, types, and views can be + schema-qualified. When commenting on a column, relation_name must refer to a table, view, composite type, or foreign table. @@ -327,6 +328,7 @@ COMMENT ON RULE my_rule ON my_table IS 'Logs updates of employee records'; COMMENT ON SCHEMA my_schema IS 'Departmental data'; COMMENT ON SEQUENCE my_sequence IS 'Used to generate primary keys'; COMMENT ON SERVER myserver IS 'my foreign server'; +COMMENT ON STATISTICS my_statistics IS 'Improves planner row estimations'; COMMENT ON TABLE my_schema.my_table IS 'Employee Information'; COMMENT ON TABLESPACE my_tablespace IS 'Tablespace for indexes'; COMMENT ON TEXT SEARCH CONFIGURATION my_config IS 'Special word filtering'; diff --git a/doc/src/sgml/ref/create_statistics.sgml b/doc/src/sgml/ref/create_statistics.sgml new file mode 100644 index 0000000000..60184a347b --- /dev/null +++ b/doc/src/sgml/ref/create_statistics.sgml @@ -0,0 +1,155 @@ + + + + + CREATE STATISTICS + + + + CREATE STATISTICS + 7 + SQL - Language Statements + + + + CREATE STATISTICS + define extended statistics + + + + +CREATE STATISTICS [ IF NOT EXISTS ] statistics_name ON ( + column_name, column_name [, ...]) + FROM table_name + + + + + + Description + + + CREATE STATISTICS will create a new extended statistics + object on the specified table. + The statistics will be created in the current database and + will be owned by the user issuing the command. + + + + If a schema name is given (for example, CREATE STATISTICS + myschema.mystat ...) then the statistics is created in the specified + schema. Otherwise it is created in the current schema. The name of + the statistics must be distinct from the name of any other statistics in the + same schema. + + + + + Parameters + + + + + IF NOT EXISTS + + + Do not throw an error if a statistics with the same name already exists. + A notice is issued in this case. Note that only the name of the + statistics object is considered here. The definition of the statistics is + not considered. + + + + + + statistics_name + + + The name (optionally schema-qualified) of the statistics to be created. + + + + + + column_name + + + The name of a column to be included in the statistics. + + + + + + table_name + + + The name (optionally schema-qualified) of the table the statistics should + be created on. + + + + + + + + + + Notes + + + You must be the owner of a table to create or change statistics on it. + + + + + Examples + + + Create table t1 with two functionally dependent columns, i.e. + knowledge of a value in the first column is sufficient for determining the + value in the other column. Then functional dependencies are built on those + columns: + + +CREATE TABLE t1 ( + a int, + b int +); + +INSERT INTO t1 SELECT i/100, i/500 + FROM generate_series(1,1000000) s(i); + +CREATE STATISTICS s1 ON (a, b) FROM t1; + +ANALYZE t1; + +-- valid combination of values +EXPLAIN ANALYZE SELECT * FROM t1 WHERE (a = 1) AND (b = 0); + +-- invalid combination of values +EXPLAIN ANALYZE SELECT * FROM t1 WHERE (a = 1) AND (b = 1); + + + + + + + Compatibility + + + There's no CREATE STATISTICS command in the SQL standard. + + + + + See Also + + + + + + + diff --git a/doc/src/sgml/ref/drop_statistics.sgml b/doc/src/sgml/ref/drop_statistics.sgml new file mode 100644 index 0000000000..98c338182b --- /dev/null +++ b/doc/src/sgml/ref/drop_statistics.sgml @@ -0,0 +1,98 @@ + + + + + DROP STATISTICS + + + + DROP STATISTICS + 7 + SQL - Language Statements + + + + DROP STATISTICS + remove extended statistics + + + + +DROP STATISTICS [ IF EXISTS ] name [, ...] + + + + + Description + + + DROP STATISTICS removes statistics from the database. + Only the statistics owner, the schema owner, and superuser can drop a + statistics. + + + + + + Parameters + + + + IF EXISTS + + + Do not throw an error if the statistics do not exist. A notice is + issued in this case. + + + + + + name + + + The name (optionally schema-qualified) of the statistics to drop. + + + + + + + + + Examples + + + To destroy two statistics objects on different schemas, without failing + if they don't exist: + + +DROP STATISTICS IF EXISTS + accounting.users_uid_creation, + public.grants_user_role; + + + + + + + Compatibility + + + There's no DROP STATISTICS command in the SQL standard. + + + + + See Also + + + + + + + + diff --git a/doc/src/sgml/reference.sgml b/doc/src/sgml/reference.sgml index 3d8ad232fa..9000b3aaaa 100644 --- a/doc/src/sgml/reference.sgml +++ b/doc/src/sgml/reference.sgml @@ -60,6 +60,7 @@ &alterSchema; &alterSequence; &alterServer; + &alterStatistics; &alterSubscription; &alterSystem; &alterTable; @@ -108,6 +109,7 @@ &createSchema; &createSequence; &createServer; + &createStatistics; &createSubscription; &createTable; &createTableAs; @@ -154,6 +156,7 @@ &dropSchema; &dropSequence; &dropServer; + &dropStatistics; &dropSubscription; &dropTable; &dropTableSpace; diff --git a/src/backend/Makefile b/src/backend/Makefile index fffb0d95ba..bce9d2c3eb 100644 --- a/src/backend/Makefile +++ b/src/backend/Makefile @@ -19,7 +19,7 @@ include $(top_builddir)/src/Makefile.global SUBDIRS = access bootstrap catalog parser commands executor foreign lib libpq \ main nodes optimizer port postmaster regex replication rewrite \ - storage tcop tsearch utils $(top_builddir)/src/timezone + statistics storage tcop tsearch utils $(top_builddir)/src/timezone include $(srcdir)/common.mk diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile index 159cab5c18..fd33426bad 100644 --- a/src/backend/catalog/Makefile +++ b/src/backend/catalog/Makefile @@ -33,6 +33,7 @@ POSTGRES_BKI_SRCS = $(addprefix $(top_srcdir)/src/include/catalog/,\ pg_attrdef.h pg_constraint.h pg_inherits.h pg_index.h pg_operator.h \ pg_opfamily.h pg_opclass.h pg_am.h pg_amop.h pg_amproc.h \ pg_language.h pg_largeobject_metadata.h pg_largeobject.h pg_aggregate.h \ + pg_statistic_ext.h \ pg_statistic.h pg_rewrite.h pg_trigger.h pg_event_trigger.h pg_description.h \ pg_cast.h pg_enum.h pg_namespace.h pg_conversion.h pg_depend.h \ pg_database.h pg_db_role_setting.h pg_tablespace.h pg_pltemplate.h \ diff --git a/src/backend/catalog/aclchk.c b/src/backend/catalog/aclchk.c index be86d76a59..d01930f4a8 100644 --- a/src/backend/catalog/aclchk.c +++ b/src/backend/catalog/aclchk.c @@ -48,6 +48,7 @@ #include "catalog/pg_operator.h" #include "catalog/pg_opfamily.h" #include "catalog/pg_proc.h" +#include "catalog/pg_statistic_ext.h" #include "catalog/pg_subscription.h" #include "catalog/pg_tablespace.h" #include "catalog/pg_type.h" @@ -3302,6 +3303,8 @@ static const char *const no_priv_msg[MAX_ACL_KIND] = gettext_noop("permission denied for collation %s"), /* ACL_KIND_CONVERSION */ gettext_noop("permission denied for conversion %s"), + /* ACL_KIND_STATISTICS */ + gettext_noop("permission denied for statistics %s"), /* ACL_KIND_TABLESPACE */ gettext_noop("permission denied for tablespace %s"), /* ACL_KIND_TSDICTIONARY */ @@ -3352,6 +3355,8 @@ static const char *const not_owner_msg[MAX_ACL_KIND] = gettext_noop("must be owner of collation %s"), /* ACL_KIND_CONVERSION */ gettext_noop("must be owner of conversion %s"), + /* ACL_KIND_STATISTICS */ + gettext_noop("must be owner of statistics %s"), /* ACL_KIND_TABLESPACE */ gettext_noop("must be owner of tablespace %s"), /* ACL_KIND_TSDICTIONARY */ @@ -3467,6 +3472,10 @@ pg_aclmask(AclObjectKind objkind, Oid table_oid, AttrNumber attnum, Oid roleid, mask, how, NULL); case ACL_KIND_NAMESPACE: return pg_namespace_aclmask(table_oid, roleid, mask, how); + case ACL_KIND_STATISTICS: + elog(ERROR, "grantable rights not supported for statistics"); + /* not reached, but keep compiler quiet */ + return ACL_NO_RIGHTS; case ACL_KIND_TABLESPACE: return pg_tablespace_aclmask(table_oid, roleid, mask, how); case ACL_KIND_FDW: @@ -5103,6 +5112,32 @@ pg_subscription_ownercheck(Oid sub_oid, Oid roleid) return has_privs_of_role(roleid, ownerId); } +/* + * Ownership check for a extended statistics (specified by OID). + */ +bool +pg_statistics_ownercheck(Oid stat_oid, Oid roleid) +{ + HeapTuple tuple; + Oid ownerId; + + /* Superusers bypass all permission checking. */ + if (superuser_arg(roleid)) + return true; + + tuple = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(stat_oid)); + if (!HeapTupleIsValid(tuple)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("statistics with OID %u do not exist", stat_oid))); + + ownerId = ((Form_pg_statistic_ext) GETSTRUCT(tuple))->staowner; + + ReleaseSysCache(tuple); + + return has_privs_of_role(roleid, ownerId); +} + /* * Check whether specified role has CREATEROLE privilege (or is a superuser) * diff --git a/src/backend/catalog/dependency.c b/src/backend/catalog/dependency.c index fc088b2165..ee27cae7df 100644 --- a/src/backend/catalog/dependency.c +++ b/src/backend/catalog/dependency.c @@ -51,6 +51,7 @@ #include "catalog/pg_publication.h" #include "catalog/pg_publication_rel.h" #include "catalog/pg_rewrite.h" +#include "catalog/pg_statistic_ext.h" #include "catalog/pg_subscription.h" #include "catalog/pg_tablespace.h" #include "catalog/pg_transform.h" @@ -154,6 +155,7 @@ static const Oid object_classes[] = { RewriteRelationId, /* OCLASS_REWRITE */ TriggerRelationId, /* OCLASS_TRIGGER */ NamespaceRelationId, /* OCLASS_SCHEMA */ + StatisticExtRelationId, /* OCLASS_STATISTIC_EXT */ TSParserRelationId, /* OCLASS_TSPARSER */ TSDictionaryRelationId, /* OCLASS_TSDICT */ TSTemplateRelationId, /* OCLASS_TSTEMPLATE */ @@ -1263,6 +1265,10 @@ doDeletion(const ObjectAddress *object, int flags) DropTransformById(object->objectId); break; + case OCLASS_STATISTIC_EXT: + RemoveStatisticsById(object->objectId); + break; + default: elog(ERROR, "unrecognized object class: %u", object->classId); @@ -2377,6 +2383,9 @@ getObjectClass(const ObjectAddress *object) case NamespaceRelationId: return OCLASS_SCHEMA; + case StatisticExtRelationId: + return OCLASS_STATISTIC_EXT; + case TSParserRelationId: return OCLASS_TSPARSER; diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index d49dcdc015..eee5e2f6ca 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -52,6 +52,7 @@ #include "catalog/pg_opclass.h" #include "catalog/pg_partitioned_table.h" #include "catalog/pg_statistic.h" +#include "catalog/pg_statistic_ext.h" #include "catalog/pg_subscription_rel.h" #include "catalog/pg_tablespace.h" #include "catalog/pg_type.h" @@ -1609,7 +1610,10 @@ RemoveAttributeById(Oid relid, AttrNumber attnum) heap_close(attr_rel, RowExclusiveLock); if (attnum > 0) + { RemoveStatistics(relid, attnum); + RemoveStatisticsExt(relid, attnum); + } relation_close(rel, NoLock); } @@ -1860,6 +1864,7 @@ heap_drop_with_catalog(Oid relid) * delete statistics */ RemoveStatistics(relid, 0); + RemoveStatisticsExt(relid, 0); /* * delete attribute tuples @@ -2771,6 +2776,75 @@ RemoveStatistics(Oid relid, AttrNumber attnum) } +/* + * RemoveStatisticsExt --- remove entries in pg_statistic_ext for a relation + * + * If attnum is zero, remove all entries for rel; else remove only the + * one(s) involving that column. + */ +void +RemoveStatisticsExt(Oid relid, AttrNumber attnum) +{ + Relation pgstatisticext; + SysScanDesc scan; + ScanKeyData key; + HeapTuple tuple; + + /* + * Scan pg_statistic_ext to delete relevant tuples + */ + pgstatisticext = heap_open(StatisticExtRelationId, RowExclusiveLock); + + ScanKeyInit(&key, + Anum_pg_statistic_ext_starelid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(relid)); + + scan = systable_beginscan(pgstatisticext, + StatisticExtRelidIndexId, + true, NULL, 1, &key); + + while (HeapTupleIsValid(tuple = systable_getnext(scan))) + { + bool delete = false; + + if (attnum == 0) + delete = true; + else if (attnum != 0) + { + Form_pg_statistic_ext staForm; + int i; + + /* + * Decode the stakeys array and delete any stats that involve the + * specified column. + */ + staForm = (Form_pg_statistic_ext) GETSTRUCT(tuple); + for (i = 0; i < staForm->stakeys.dim1; i++) + { + if (staForm->stakeys.values[i] == attnum) + { + delete = true; + break; + } + } + } + + if (delete) + { + CatalogTupleDelete(pgstatisticext, &tuple->t_self); + deleteDependencyRecordsFor(StatisticExtRelationId, + HeapTupleGetOid(tuple), + false); + } + } + + systable_endscan(scan); + + heap_close(pgstatisticext, RowExclusiveLock); +} + + /* * RelationTruncateIndexes - truncate all indexes associated * with the heap relation to zero tuples. diff --git a/src/backend/catalog/namespace.c b/src/backend/catalog/namespace.c index a38da3047f..e521bd908a 100644 --- a/src/backend/catalog/namespace.c +++ b/src/backend/catalog/namespace.c @@ -2085,6 +2085,62 @@ ConversionIsVisible(Oid conid) return visible; } +/* + * get_statistics_oid - find a statistics by possibly qualified name + * + * If not found, returns InvalidOid if missing_ok, else throws error + */ +Oid +get_statistics_oid(List *names, bool missing_ok) +{ + char *schemaname; + char *stats_name; + Oid namespaceId; + Oid stats_oid = InvalidOid; + ListCell *l; + + /* deconstruct the name list */ + DeconstructQualifiedName(names, &schemaname, &stats_name); + + if (schemaname) + { + /* use exact schema given */ + namespaceId = LookupExplicitNamespace(schemaname, missing_ok); + if (missing_ok && !OidIsValid(namespaceId)) + stats_oid = InvalidOid; + else + stats_oid = GetSysCacheOid2(STATEXTNAMENSP, + PointerGetDatum(stats_name), + ObjectIdGetDatum(namespaceId)); + } + else + { + /* search for it in search path */ + recomputeNamespacePath(); + + foreach(l, activeSearchPath) + { + namespaceId = lfirst_oid(l); + + if (namespaceId == myTempNamespace) + continue; /* do not look in temp namespace */ + stats_oid = GetSysCacheOid2(STATEXTNAMENSP, + PointerGetDatum(stats_name), + ObjectIdGetDatum(namespaceId)); + if (OidIsValid(stats_oid)) + break; + } + } + + if (!OidIsValid(stats_oid) && !missing_ok) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("statistics \"%s\" do not exist", + NameListToString(names)))); + + return stats_oid; +} + /* * get_ts_parser_oid - find a TS parser by possibly qualified name * diff --git a/src/backend/catalog/objectaddress.c b/src/backend/catalog/objectaddress.c index 61a831b403..2948d64fa7 100644 --- a/src/backend/catalog/objectaddress.c +++ b/src/backend/catalog/objectaddress.c @@ -48,6 +48,7 @@ #include "catalog/pg_publication.h" #include "catalog/pg_publication_rel.h" #include "catalog/pg_rewrite.h" +#include "catalog/pg_statistic_ext.h" #include "catalog/pg_subscription.h" #include "catalog/pg_tablespace.h" #include "catalog/pg_transform.h" @@ -478,6 +479,18 @@ static const ObjectPropertyType ObjectProperty[] = InvalidAttrNumber, ACL_KIND_SUBSCRIPTION, true + }, + { + StatisticExtRelationId, + StatisticExtOidIndexId, + STATEXTOID, + STATEXTNAMENSP, + Anum_pg_statistic_ext_staname, + Anum_pg_statistic_ext_stanamespace, + Anum_pg_statistic_ext_staowner, + InvalidAttrNumber, /* no ACL (same as relation) */ + ACL_KIND_STATISTICS, + true } }; @@ -696,6 +709,10 @@ static const struct object_type_map /* OCLASS_TRANSFORM */ { "transform", OBJECT_TRANSFORM + }, + /* OBJECT_STATISTIC_EXT */ + { + "statistics", OBJECT_STATISTIC_EXT } }; @@ -974,6 +991,12 @@ get_object_address(ObjectType objtype, Node *object, address = get_object_address_defacl(castNode(List, object), missing_ok); break; + case OBJECT_STATISTIC_EXT: + address.classId = StatisticExtRelationId; + address.objectId = get_statistics_oid(castNode(List, object), + missing_ok); + address.objectSubId = 0; + break; default: elog(ERROR, "unrecognized objtype: %d", (int) objtype); /* placate compiler, in case it thinks elog might return */ @@ -2083,6 +2106,7 @@ pg_get_object_address(PG_FUNCTION_ARGS) case OBJECT_ATTRIBUTE: case OBJECT_COLLATION: case OBJECT_CONVERSION: + case OBJECT_STATISTIC_EXT: case OBJECT_TSPARSER: case OBJECT_TSDICTIONARY: case OBJECT_TSTEMPLATE: @@ -2370,6 +2394,10 @@ check_object_ownership(Oid roleid, ObjectType objtype, ObjectAddress address, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("must be superuser"))); break; + case OBJECT_STATISTIC_EXT: + if (!pg_statistics_ownercheck(address.objectId, roleid)) + aclcheck_error_type(ACLCHECK_NOT_OWNER, address.objectId); + break; default: elog(ERROR, "unrecognized object type: %d", (int) objtype); @@ -3857,6 +3885,10 @@ getObjectTypeDescription(const ObjectAddress *object) appendStringInfoString(&buffer, "subscription"); break; + case OCLASS_STATISTIC_EXT: + appendStringInfoString(&buffer, "statistics"); + break; + default: appendStringInfo(&buffer, "unrecognized %u", object->classId); break; @@ -4880,6 +4912,29 @@ getObjectIdentityParts(const ObjectAddress *object, break; } + case OCLASS_STATISTIC_EXT: + { + HeapTuple tup; + Form_pg_statistic_ext formStatistic; + char *schema; + + tup = SearchSysCache1(STATEXTOID, + ObjectIdGetDatum(object->objectId)); + if (!HeapTupleIsValid(tup)) + elog(ERROR, "cache lookup failed for statistics %u", + object->objectId); + formStatistic = (Form_pg_statistic_ext) GETSTRUCT(tup); + schema = get_namespace_name_or_temp(formStatistic->stanamespace); + appendStringInfoString(&buffer, + quote_qualified_identifier(schema, + NameStr(formStatistic->staname))); + if (objname) + *objname = list_make2(schema, + pstrdup(NameStr(formStatistic->staname))); + ReleaseSysCache(tup); + } + break; + default: appendStringInfo(&buffer, "unrecognized object %u %u %d", object->classId, diff --git a/src/backend/catalog/pg_shdepend.c b/src/backend/catalog/pg_shdepend.c index 8d946ff44c..d28a8afb47 100644 --- a/src/backend/catalog/pg_shdepend.c +++ b/src/backend/catalog/pg_shdepend.c @@ -39,6 +39,7 @@ #include "catalog/pg_opfamily.h" #include "catalog/pg_proc.h" #include "catalog/pg_shdepend.h" +#include "catalog/pg_statistic_ext.h" #include "catalog/pg_subscription.h" #include "catalog/pg_tablespace.h" #include "catalog/pg_ts_config.h" @@ -1415,6 +1416,7 @@ shdepReassignOwned(List *roleids, Oid newrole) case OperatorFamilyRelationId: case OperatorClassRelationId: case ExtensionRelationId: + case StatisticExtRelationId: case TableSpaceRelationId: case DatabaseRelationId: case TSConfigRelationId: diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 80d14296de..b41882aa52 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -186,6 +186,16 @@ CREATE OR REPLACE VIEW pg_sequences AS WHERE NOT pg_is_other_temp_schema(N.oid) AND relkind = 'S'; +CREATE VIEW pg_stats_ext AS + SELECT + N.nspname AS schemaname, + C.relname AS tablename, + S.staname AS staname, + S.stakeys AS attnums, + length(s.standistinct) AS ndistbytes + FROM (pg_statistic_ext S JOIN pg_class C ON (C.oid = S.starelid)) + LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace); + CREATE VIEW pg_stats WITH (security_barrier) AS SELECT nspname AS schemaname, diff --git a/src/backend/commands/Makefile b/src/backend/commands/Makefile index e0fab38cbe..4a6c99e090 100644 --- a/src/backend/commands/Makefile +++ b/src/backend/commands/Makefile @@ -18,8 +18,8 @@ OBJS = amcmds.o aggregatecmds.o alter.o analyze.o async.o cluster.o comment.o \ event_trigger.o explain.o extension.o foreigncmds.o functioncmds.o \ indexcmds.o lockcmds.o matview.o operatorcmds.o opclasscmds.o \ policy.o portalcmds.o prepare.o proclang.o publicationcmds.o \ - schemacmds.o seclabel.o sequence.o subscriptioncmds.o tablecmds.o \ - tablespace.o trigger.o tsearchcmds.o typecmds.o user.o vacuum.o \ - vacuumlazy.o variable.o view.o + schemacmds.o seclabel.o sequence.o statscmds.o subscriptioncmds.o \ + tablecmds.o tablespace.o trigger.o tsearchcmds.o typecmds.o user.o \ + vacuum.o vacuumlazy.o variable.o view.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/commands/alter.c b/src/backend/commands/alter.c index cf1391c2e6..2c6435b759 100644 --- a/src/backend/commands/alter.c +++ b/src/backend/commands/alter.c @@ -33,6 +33,7 @@ #include "catalog/pg_opfamily.h" #include "catalog/pg_proc.h" #include "catalog/pg_subscription.h" +#include "catalog/pg_statistic_ext.h" #include "catalog/pg_ts_config.h" #include "catalog/pg_ts_dict.h" #include "catalog/pg_ts_parser.h" @@ -120,6 +121,10 @@ report_namespace_conflict(Oid classId, const char *name, Oid nspOid) Assert(OidIsValid(nspOid)); msgfmt = gettext_noop("conversion \"%s\" already exists in schema \"%s\""); break; + case StatisticExtRelationId: + Assert(OidIsValid(nspOid)); + msgfmt = gettext_noop("statistics \"%s\" already exists in schema \"%s\""); + break; case TSParserRelationId: Assert(OidIsValid(nspOid)); msgfmt = gettext_noop("text search parser \"%s\" already exists in schema \"%s\""); @@ -373,6 +378,7 @@ ExecRenameStmt(RenameStmt *stmt) case OBJECT_OPCLASS: case OBJECT_OPFAMILY: case OBJECT_LANGUAGE: + case OBJECT_STATISTIC_EXT: case OBJECT_TSCONFIGURATION: case OBJECT_TSDICTIONARY: case OBJECT_TSPARSER: @@ -489,6 +495,7 @@ ExecAlterObjectSchemaStmt(AlterObjectSchemaStmt *stmt, case OBJECT_OPERATOR: case OBJECT_OPCLASS: case OBJECT_OPFAMILY: + case OBJECT_STATISTIC_EXT: case OBJECT_TSCONFIGURATION: case OBJECT_TSDICTIONARY: case OBJECT_TSPARSER: @@ -803,6 +810,7 @@ ExecAlterOwnerStmt(AlterOwnerStmt *stmt) case OBJECT_OPERATOR: case OBJECT_OPCLASS: case OBJECT_OPFAMILY: + case OBJECT_STATISTIC_EXT: case OBJECT_TABLESPACE: case OBJECT_TSDICTIONARY: case OBJECT_TSCONFIGURATION: diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c index 055338fdff..c5b5c54bab 100644 --- a/src/backend/commands/analyze.c +++ b/src/backend/commands/analyze.c @@ -17,6 +17,7 @@ #include #include "access/multixact.h" +#include "access/sysattr.h" #include "access/transam.h" #include "access/tupconvert.h" #include "access/tuptoaster.h" @@ -28,6 +29,7 @@ #include "catalog/pg_collation.h" #include "catalog/pg_inherits_fn.h" #include "catalog/pg_namespace.h" +#include "catalog/pg_statistic_ext.h" #include "commands/dbcommands.h" #include "commands/tablecmds.h" #include "commands/vacuum.h" @@ -39,13 +41,17 @@ #include "parser/parse_relation.h" #include "pgstat.h" #include "postmaster/autovacuum.h" +#include "statistics/extended_stats_internal.h" +#include "statistics/statistics.h" #include "storage/bufmgr.h" #include "storage/lmgr.h" #include "storage/proc.h" #include "storage/procarray.h" #include "utils/acl.h" #include "utils/attoptcache.h" +#include "utils/builtins.h" #include "utils/datum.h" +#include "utils/fmgroids.h" #include "utils/guc.h" #include "utils/lsyscache.h" #include "utils/memutils.h" @@ -566,6 +572,10 @@ do_analyze_rel(Relation onerel, int options, VacuumParams *params, update_attstats(RelationGetRelid(Irel[ind]), false, thisdata->attr_cnt, thisdata->vacattrstats); } + + /* Build extended statistics (if there are any). */ + BuildRelationExtStatistics(onerel, totalrows, numrows, rows, attr_cnt, + vacattrstats); } /* @@ -1681,19 +1691,6 @@ ind_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull) /* * Extra information used by the default analysis routines */ -typedef struct -{ - Oid eqopr; /* '=' operator for datatype, if any */ - Oid eqfunc; /* and associated function */ - Oid ltopr; /* '<' operator for datatype, if any */ -} StdAnalyzeData; - -typedef struct -{ - Datum value; /* a data value */ - int tupno; /* position index for tuple it came from */ -} ScalarItem; - typedef struct { int count; /* # of duplicates */ diff --git a/src/backend/commands/dropcmds.c b/src/backend/commands/dropcmds.c index ab73fbf961..cb948f0204 100644 --- a/src/backend/commands/dropcmds.c +++ b/src/backend/commands/dropcmds.c @@ -286,6 +286,13 @@ does_not_exist_skipping(ObjectType objtype, Node *object) msg = gettext_noop("schema \"%s\" does not exist, skipping"); name = strVal((Value *) object); break; + case OBJECT_STATISTIC_EXT: + if (!schema_does_not_exist_skipping(castNode(List, object), &msg, &name)) + { + msg = gettext_noop("extended statistics \"%s\" do not exist, skipping"); + name = NameListToString(castNode(List, object)); + } + break; case OBJECT_TSPARSER: if (!schema_does_not_exist_skipping(castNode(List, object), &msg, &name)) { diff --git a/src/backend/commands/event_trigger.c b/src/backend/commands/event_trigger.c index 346b347ae1..7366fc74be 100644 --- a/src/backend/commands/event_trigger.c +++ b/src/backend/commands/event_trigger.c @@ -112,6 +112,7 @@ static event_trigger_support_data event_trigger_support[] = { {"SCHEMA", true}, {"SEQUENCE", true}, {"SERVER", true}, + {"STATISTICS", true}, {"SUBSCRIPTION", true}, {"TABLE", true}, {"TABLESPACE", false}, @@ -1108,6 +1109,7 @@ EventTriggerSupportsObjectType(ObjectType obtype) case OBJECT_SCHEMA: case OBJECT_SEQUENCE: case OBJECT_SUBSCRIPTION: + case OBJECT_STATISTIC_EXT: case OBJECT_TABCONSTRAINT: case OBJECT_TABLE: case OBJECT_TRANSFORM: @@ -1173,6 +1175,7 @@ EventTriggerSupportsObjectClass(ObjectClass objclass) case OCLASS_PUBLICATION: case OCLASS_PUBLICATION_REL: case OCLASS_SUBSCRIPTION: + case OCLASS_STATISTIC_EXT: return true; } diff --git a/src/backend/commands/statscmds.c b/src/backend/commands/statscmds.c new file mode 100644 index 0000000000..416309106a --- /dev/null +++ b/src/backend/commands/statscmds.c @@ -0,0 +1,296 @@ +/*------------------------------------------------------------------------- + * + * statscmds.c + * Commands for creating and altering extended statistics + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/commands/statscmds.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/relscan.h" +#include "catalog/dependency.h" +#include "catalog/indexing.h" +#include "catalog/namespace.h" +#include "catalog/pg_namespace.h" +#include "catalog/pg_statistic_ext.h" +#include "commands/defrem.h" +#include "miscadmin.h" +#include "statistics/statistics.h" +#include "utils/builtins.h" +#include "utils/inval.h" +#include "utils/memutils.h" +#include "utils/rel.h" +#include "utils/syscache.h" +#include "utils/typcache.h" + + +/* used for sorting the attnums in CreateStatistics */ +static int +compare_int16(const void *a, const void *b) +{ + return memcmp(a, b, sizeof(int16)); +} + +/* + * CREATE STATISTICS + */ +ObjectAddress +CreateStatistics(CreateStatsStmt *stmt) +{ + int i; + ListCell *l; + int16 attnums[STATS_MAX_DIMENSIONS]; + int numcols = 0; + ObjectAddress address = InvalidObjectAddress; + char *namestr; + NameData staname; + Oid statoid; + Oid namespaceId; + HeapTuple htup; + Datum values[Natts_pg_statistic_ext]; + bool nulls[Natts_pg_statistic_ext]; + int2vector *stakeys; + Relation statrel; + Relation rel; + Oid relid; + ObjectAddress parentobject, + childobject; + Datum types[1]; /* only ndistinct defined now */ + int ntypes; + ArrayType *staenabled; + bool build_ndistinct; + bool requested_type = false; + + Assert(IsA(stmt, CreateStatsStmt)); + + /* resolve the pieces of the name (namespace etc.) */ + namespaceId = QualifiedNameGetCreationNamespace(stmt->defnames, &namestr); + namestrcpy(&staname, namestr); + + /* + * If if_not_exists was given and the statistics already exists, bail out. + */ + if (SearchSysCacheExists2(STATEXTNAMENSP, + PointerGetDatum(&staname), + ObjectIdGetDatum(namespaceId))) + { + if (stmt->if_not_exists) + { + ereport(NOTICE, + (errcode(ERRCODE_DUPLICATE_OBJECT), + errmsg("statistics \"%s\" already exist, skipping", + namestr))); + return InvalidObjectAddress; + } + + ereport(ERROR, + (errcode(ERRCODE_DUPLICATE_OBJECT), + errmsg("statistics \"%s\" already exist", namestr))); + } + + rel = heap_openrv(stmt->relation, AccessExclusiveLock); + relid = RelationGetRelid(rel); + + if (rel->rd_rel->relkind != RELKIND_RELATION && + rel->rd_rel->relkind != RELKIND_MATVIEW) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("relation \"%s\" is not a table or materialized view", + RelationGetRelationName(rel)))); + + /* + * Transform column names to array of attnums. While at it, enforce some + * constraints. + */ + foreach(l, stmt->keys) + { + char *attname = strVal(lfirst(l)); + HeapTuple atttuple; + Form_pg_attribute attForm; + TypeCacheEntry *type; + + atttuple = SearchSysCacheAttName(relid, attname); + if (!HeapTupleIsValid(atttuple)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_COLUMN), + errmsg("column \"%s\" referenced in statistics does not exist", + attname))); + attForm = (Form_pg_attribute) GETSTRUCT(atttuple); + + /* Disallow use of system attributes in extended stats */ + if (attForm->attnum < 0) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("statistic creation on system columns is not supported"))); + + /* Disallow data types without a less-than operator */ + type = lookup_type_cache(attForm->atttypid, TYPECACHE_LT_OPR); + if (type->lt_opr == InvalidOid) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("only scalar types can be used in extended statistics"))); + + /* Make sure no more than STATS_MAX_DIMENSIONS columns are used */ + if (numcols >= STATS_MAX_DIMENSIONS) + ereport(ERROR, + (errcode(ERRCODE_TOO_MANY_COLUMNS), + errmsg("cannot have more than %d keys in statistics", + STATS_MAX_DIMENSIONS))); + + attnums[numcols] = ((Form_pg_attribute) GETSTRUCT(atttuple))->attnum; + ReleaseSysCache(atttuple); + numcols++; + } + + /* + * Check that at least two columns were specified in the statement. The + * upper bound was already checked in the loop above. + */ + if (numcols < 2) + ereport(ERROR, + (errcode(ERRCODE_TOO_MANY_COLUMNS), + errmsg("statistics require at least 2 columns"))); + + /* + * Sort the attnums, which makes detecting duplicies somewhat easier, and + * it does not hurt (it does not affect the efficiency, unlike for + * indexes, for example). + */ + qsort(attnums, numcols, sizeof(int16), compare_int16); + + /* + * Look for duplicities in the list of columns. The attnums are sorted so + * just check consecutive elements. + */ + for (i = 1; i < numcols; i++) + if (attnums[i] == attnums[i - 1]) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_COLUMN), + errmsg("duplicate column name in statistics definition"))); + + stakeys = buildint2vector(attnums, numcols); + + /* + * Parse the statistics options. Currently only statistics types are + * recognized. + */ + build_ndistinct = false; + foreach(l, stmt->options) + { + DefElem *opt = (DefElem *) lfirst(l); + + if (strcmp(opt->defname, "ndistinct") == 0) + { + build_ndistinct = defGetBoolean(opt); + requested_type = true; + } + else + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("unrecognized STATISTICS option \"%s\"", + opt->defname))); + } + /* If no statistic type was specified, build them all. */ + if (!requested_type) + build_ndistinct = true; + + /* construct the char array of enabled statistic types */ + ntypes = 0; + if (build_ndistinct) + types[ntypes++] = CharGetDatum(STATS_EXT_NDISTINCT); + Assert(ntypes > 0); + staenabled = construct_array(types, ntypes, CHAROID, 1, true, 'c'); + + /* + * Everything seems fine, so let's build the pg_statistic_ext tuple. + */ + memset(values, 0, sizeof(values)); + memset(nulls, false, sizeof(nulls)); + values[Anum_pg_statistic_ext_starelid - 1] = ObjectIdGetDatum(relid); + values[Anum_pg_statistic_ext_staname - 1] = NameGetDatum(&staname); + values[Anum_pg_statistic_ext_stanamespace - 1] = ObjectIdGetDatum(namespaceId); + values[Anum_pg_statistic_ext_staowner - 1] = ObjectIdGetDatum(GetUserId()); + values[Anum_pg_statistic_ext_stakeys - 1] = PointerGetDatum(stakeys); + values[Anum_pg_statistic_ext_staenabled - 1] = PointerGetDatum(staenabled); + + /* no statistics build yet */ + nulls[Anum_pg_statistic_ext_standistinct - 1] = true; + + /* insert it into pg_statistic_ext */ + statrel = heap_open(StatisticExtRelationId, RowExclusiveLock); + htup = heap_form_tuple(statrel->rd_att, values, nulls); + CatalogTupleInsert(statrel, htup); + statoid = HeapTupleGetOid(htup); + heap_freetuple(htup); + heap_close(statrel, RowExclusiveLock); + relation_close(rel, NoLock); + + /* + * Add a dependency on a table, so that stats get dropped on DROP TABLE. + */ + ObjectAddressSet(parentobject, RelationRelationId, relid); + ObjectAddressSet(childobject, StatisticExtRelationId, statoid); + recordDependencyOn(&childobject, &parentobject, DEPENDENCY_AUTO); + + /* + * Also add dependency on the schema. This is required to ensure that we + * drop the statistics on DROP SCHEMA. This is not handled automatically + * by DROP TABLE because the statistics are not an object in the table's + * schema. + */ + ObjectAddressSet(parentobject, NamespaceRelationId, namespaceId); + recordDependencyOn(&childobject, &parentobject, DEPENDENCY_AUTO); + + ObjectAddressSet(address, StatisticExtRelationId, statoid); + + /* + * Invalidate relcache so that others see the new statistics. + */ + CacheInvalidateRelcache(rel); + + return address; +} + +/* + * Guts of statistics deletion. + */ +void +RemoveStatisticsById(Oid statsOid) +{ + Relation relation; + Oid relid; + Relation rel; + HeapTuple tup; + Form_pg_statistic_ext statext; + + /* + * Delete the pg_statistic_ext tuple. + */ + relation = heap_open(StatisticExtRelationId, RowExclusiveLock); + + tup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(statsOid)); + + if (!HeapTupleIsValid(tup)) /* should not happen */ + elog(ERROR, "cache lookup failed for statistics %u", statsOid); + + statext = (Form_pg_statistic_ext) GETSTRUCT(tup); + relid = statext->starelid; + + rel = heap_open(relid, AccessExclusiveLock); + + simple_heap_delete(relation, &tup->t_self); + + CacheInvalidateRelcache(rel); + + ReleaseSysCache(tup); + + heap_close(relation, RowExclusiveLock); + heap_close(rel, NoLock); +} diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index 93bda42715..c23d5c5285 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -3337,6 +3337,20 @@ _copyIndexStmt(const IndexStmt *from) return newnode; } +static CreateStatsStmt * +_copyCreateStatsStmt(const CreateStatsStmt *from) +{ + CreateStatsStmt *newnode = makeNode(CreateStatsStmt); + + COPY_NODE_FIELD(defnames); + COPY_NODE_FIELD(relation); + COPY_NODE_FIELD(keys); + COPY_NODE_FIELD(options); + COPY_SCALAR_FIELD(if_not_exists); + + return newnode; +} + static CreateFunctionStmt * _copyCreateFunctionStmt(const CreateFunctionStmt *from) { @@ -5050,6 +5064,9 @@ copyObject(const void *from) case T_IndexStmt: retval = _copyIndexStmt(from); break; + case T_CreateStatsStmt: + retval = _copyCreateStatsStmt(from); + break; case T_CreateFunctionStmt: retval = _copyCreateFunctionStmt(from); break; diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index 0d12636d92..5941b7a2bf 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -1334,6 +1334,18 @@ _equalIndexStmt(const IndexStmt *a, const IndexStmt *b) return true; } +static bool +_equalCreateStatsStmt(const CreateStatsStmt *a, const CreateStatsStmt *b) +{ + COMPARE_NODE_FIELD(defnames); + COMPARE_NODE_FIELD(relation); + COMPARE_NODE_FIELD(keys); + COMPARE_NODE_FIELD(options); + COMPARE_SCALAR_FIELD(if_not_exists); + + return true; +} + static bool _equalCreateFunctionStmt(const CreateFunctionStmt *a, const CreateFunctionStmt *b) { @@ -3236,6 +3248,9 @@ equal(const void *a, const void *b) case T_IndexStmt: retval = _equalIndexStmt(a, b); break; + case T_CreateStatsStmt: + retval = _equalCreateStatsStmt(a, b); + break; case T_CreateFunctionStmt: retval = _equalCreateFunctionStmt(a, b); break; diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 1b9005fa53..541af02935 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -2202,6 +2202,7 @@ _outRelOptInfo(StringInfo str, const RelOptInfo *node) WRITE_NODE_FIELD(lateral_vars); WRITE_BITMAPSET_FIELD(lateral_referencers); WRITE_NODE_FIELD(indexlist); + WRITE_NODE_FIELD(statlist); WRITE_UINT_FIELD(pages); WRITE_FLOAT_FIELD(tuples, "%.0f"); WRITE_FLOAT_FIELD(allvisfrac, "%.6f"); @@ -2274,6 +2275,18 @@ _outForeignKeyOptInfo(StringInfo str, const ForeignKeyOptInfo *node) appendStringInfo(str, " %d", list_length(node->rinfos[i])); } +static void +_outStatisticExtInfo(StringInfo str, const StatisticExtInfo *node) +{ + WRITE_NODE_TYPE("STATISTICEXTINFO"); + + /* NB: this isn't a complete set of fields */ + WRITE_OID_FIELD(statOid); + /* don't write rel, leads to infinite recursion in plan tree dump */ + WRITE_CHAR_FIELD(kind); + WRITE_BITMAPSET_FIELD(keys); +} + static void _outEquivalenceClass(StringInfo str, const EquivalenceClass *node) { @@ -2577,6 +2590,18 @@ _outIndexStmt(StringInfo str, const IndexStmt *node) WRITE_BOOL_FIELD(if_not_exists); } +static void +_outCreateStatsStmt(StringInfo str, const CreateStatsStmt *node) +{ + WRITE_NODE_TYPE("CREATESTATSSTMT"); + + WRITE_NODE_FIELD(defnames); + WRITE_NODE_FIELD(relation); + WRITE_NODE_FIELD(keys); + WRITE_NODE_FIELD(options); + WRITE_BOOL_FIELD(if_not_exists); +} + static void _outNotifyStmt(StringInfo str, const NotifyStmt *node) { @@ -3936,6 +3961,9 @@ outNode(StringInfo str, const void *obj) case T_PlannerParamItem: _outPlannerParamItem(str, obj); break; + case T_StatisticExtInfo: + _outStatisticExtInfo(str, obj); + break; case T_ExtensibleNode: _outExtensibleNode(str, obj); @@ -3953,6 +3981,9 @@ outNode(StringInfo str, const void *obj) case T_IndexStmt: _outIndexStmt(str, obj); break; + case T_CreateStatsStmt: + _outCreateStatsStmt(str, obj); + break; case T_NotifyStmt: _outNotifyStmt(str, obj); break; diff --git a/src/backend/optimizer/util/plancat.c b/src/backend/optimizer/util/plancat.c index 463f806467..cc88dcc28e 100644 --- a/src/backend/optimizer/util/plancat.c +++ b/src/backend/optimizer/util/plancat.c @@ -29,6 +29,7 @@ #include "catalog/heap.h" #include "catalog/partition.h" #include "catalog/pg_am.h" +#include "catalog/pg_statistic_ext.h" #include "foreign/fdwapi.h" #include "miscadmin.h" #include "nodes/makefuncs.h" @@ -40,8 +41,11 @@ #include "parser/parse_relation.h" #include "parser/parsetree.h" #include "rewrite/rewriteManip.h" +#include "statistics/statistics.h" #include "storage/bufmgr.h" +#include "utils/builtins.h" #include "utils/lsyscache.h" +#include "utils/syscache.h" #include "utils/rel.h" #include "utils/snapmgr.h" @@ -63,7 +67,7 @@ static List *get_relation_constraints(PlannerInfo *root, bool include_notnull); static List *build_index_tlist(PlannerInfo *root, IndexOptInfo *index, Relation heapRelation); - +static List *get_relation_statistics(RelOptInfo *rel, Relation relation); /* * get_relation_info - @@ -398,6 +402,8 @@ get_relation_info(PlannerInfo *root, Oid relationObjectId, bool inhparent, rel->indexlist = indexinfos; + rel->statlist = get_relation_statistics(rel, relation); + /* Grab foreign-table info using the relcache, while we have it */ if (relation->rd_rel->relkind == RELKIND_FOREIGN_TABLE) { @@ -1251,6 +1257,65 @@ get_relation_constraints(PlannerInfo *root, return result; } +/* + * get_relation_statistics + * Retrieve extended statistics defined on the table. + * + * Returns a List (possibly empty) of StatisticExtInfo objects describing + * the statistics. Note that this doesn't load the actual statistics data, + * just the identifying metadata. Only stats actually built are considered. + */ +static List * +get_relation_statistics(RelOptInfo *rel, Relation relation) +{ + List *statoidlist; + List *stainfos = NIL; + ListCell *l; + + statoidlist = RelationGetStatExtList(relation); + + foreach(l, statoidlist) + { + Oid statOid = lfirst_oid(l); + Form_pg_statistic_ext staForm; + HeapTuple htup; + Bitmapset *keys = NULL; + int i; + + htup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(statOid)); + if (!htup) + elog(ERROR, "cache lookup failed for statistics %u", statOid); + staForm = (Form_pg_statistic_ext) GETSTRUCT(htup); + + /* + * First, build the array of columns covered. This is ultimately + * wasted if no stats are actually built, but it doesn't seem worth + * troubling over that case. + */ + for (i = 0; i < staForm->stakeys.dim1; i++) + keys = bms_add_member(keys, staForm->stakeys.values[i]); + + /* add one StatisticExtInfo for each kind built */ + if (statext_is_kind_built(htup, STATS_EXT_NDISTINCT)) + { + StatisticExtInfo *info = makeNode(StatisticExtInfo); + + info->statOid = statOid; + info->rel = rel; + info->kind = STATS_EXT_NDISTINCT; + info->keys = bms_copy(keys); + + stainfos = lcons(info, stainfos); + } + + ReleaseSysCache(htup); + bms_free(keys); + } + + list_free(statoidlist); + + return stainfos; +} /* * relation_excluded_by_constraints diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 82844a0399..bbcfc1fb4f 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -257,7 +257,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); ConstraintsSetStmt CopyStmt CreateAsStmt CreateCastStmt CreateDomainStmt CreateExtensionStmt CreateGroupStmt CreateOpClassStmt CreateOpFamilyStmt AlterOpFamilyStmt CreatePLangStmt - CreateSchemaStmt CreateSeqStmt CreateStmt CreateTableSpaceStmt + CreateSchemaStmt CreateSeqStmt CreateStmt CreateStatsStmt CreateTableSpaceStmt CreateFdwStmt CreateForeignServerStmt CreateForeignTableStmt CreateAssertStmt CreateTransformStmt CreateTrigStmt CreateEventTrigStmt CreateUserStmt CreateUserMappingStmt CreateRoleStmt CreatePolicyStmt @@ -874,6 +874,7 @@ stmt : | CreateSeqStmt | CreateStmt | CreateSubscriptionStmt + | CreateStatsStmt | CreateTableSpaceStmt | CreateTransformStmt | CreateTrigStmt @@ -3747,6 +3748,35 @@ OptConsTableSpace: USING INDEX TABLESPACE name { $$ = $4; } ExistingIndex: USING INDEX index_name { $$ = $3; } ; +/***************************************************************************** + * + * QUERY : + * CREATE STATISTICS stats_name WITH (options) ON (columns) FROM relname + * + *****************************************************************************/ + + +CreateStatsStmt: CREATE STATISTICS any_name opt_reloptions ON '(' columnList ')' FROM qualified_name + { + CreateStatsStmt *n = makeNode(CreateStatsStmt); + n->defnames = $3; + n->relation = $10; + n->keys = $7; + n->options = $4; + n->if_not_exists = false; + $$ = (Node *)n; + } + | CREATE STATISTICS IF_P NOT EXISTS any_name opt_reloptions ON '(' columnList ')' FROM qualified_name + { + CreateStatsStmt *n = makeNode(CreateStatsStmt); + n->defnames = $6; + n->relation = $13; + n->keys = $10; + n->options = $7; + n->if_not_exists = true; + $$ = (Node *)n; + } + ; /***************************************************************************** * @@ -6042,6 +6072,7 @@ drop_type_any_name: | FOREIGN TABLE { $$ = OBJECT_FOREIGN_TABLE; } | COLLATION { $$ = OBJECT_COLLATION; } | CONVERSION_P { $$ = OBJECT_CONVERSION; } + | STATISTICS { $$ = OBJECT_STATISTIC_EXT; } | TEXT_P SEARCH PARSER { $$ = OBJECT_TSPARSER; } | TEXT_P SEARCH DICTIONARY { $$ = OBJECT_TSDICTIONARY; } | TEXT_P SEARCH TEMPLATE { $$ = OBJECT_TSTEMPLATE; } @@ -6119,7 +6150,7 @@ opt_restart_seqs: * EXTENSION | EVENT TRIGGER | FOREIGN DATA WRAPPER | * FOREIGN TABLE | INDEX | [PROCEDURAL] LANGUAGE | * MATERIALIZED VIEW | POLICY | ROLE | SCHEMA | SEQUENCE | - * SERVER | TABLE | TABLESPACE | + * SERVER | STATISTICS | TABLE | TABLESPACE | * TEXT SEARCH CONFIGURATION | TEXT SEARCH DICTIONARY | * TEXT SEARCH PARSER | TEXT SEARCH TEMPLATE | TYPE | * VIEW] | @@ -6288,6 +6319,7 @@ comment_type_any_name: COLUMN { $$ = OBJECT_COLUMN; } | INDEX { $$ = OBJECT_INDEX; } | SEQUENCE { $$ = OBJECT_SEQUENCE; } + | STATISTICS { $$ = OBJECT_STATISTIC_EXT; } | TABLE { $$ = OBJECT_TABLE; } | VIEW { $$ = OBJECT_VIEW; } | MATERIALIZED VIEW { $$ = OBJECT_MATVIEW; } @@ -8428,6 +8460,15 @@ RenameStmt: ALTER AGGREGATE aggregate_with_argtypes RENAME TO name n->missing_ok = false; $$ = (Node *)n; } + | ALTER STATISTICS any_name RENAME TO name + { + RenameStmt *n = makeNode(RenameStmt); + n->renameType = OBJECT_STATISTIC_EXT; + n->object = (Node *) $3; + n->newname = $6; + n->missing_ok = false; + $$ = (Node *)n; + } | ALTER TEXT_P SEARCH PARSER any_name RENAME TO name { RenameStmt *n = makeNode(RenameStmt); @@ -8643,6 +8684,15 @@ AlterObjectSchemaStmt: n->missing_ok = true; $$ = (Node *)n; } + | ALTER STATISTICS any_name SET SCHEMA name + { + AlterObjectSchemaStmt *n = makeNode(AlterObjectSchemaStmt); + n->objectType = OBJECT_STATISTIC_EXT; + n->object = (Node *) $3; + n->newschema = $6; + n->missing_ok = false; + $$ = (Node *)n; + } | ALTER TEXT_P SEARCH PARSER any_name SET SCHEMA name { AlterObjectSchemaStmt *n = makeNode(AlterObjectSchemaStmt); @@ -8906,6 +8956,14 @@ AlterOwnerStmt: ALTER AGGREGATE aggregate_with_argtypes OWNER TO RoleSpec n->newowner = $6; $$ = (Node *)n; } + | ALTER STATISTICS any_name OWNER TO RoleSpec + { + AlterOwnerStmt *n = makeNode(AlterOwnerStmt); + n->objectType = OBJECT_STATISTIC_EXT; + n->object = (Node *) $3; + n->newowner = $6; + $$ = (Node *)n; + } | ALTER TEXT_P SEARCH DICTIONARY any_name OWNER TO RoleSpec { AlterOwnerStmt *n = makeNode(AlterOwnerStmt); diff --git a/src/backend/statistics/Makefile b/src/backend/statistics/Makefile new file mode 100644 index 0000000000..b3615bd69a --- /dev/null +++ b/src/backend/statistics/Makefile @@ -0,0 +1,17 @@ +#------------------------------------------------------------------------- +# +# Makefile-- +# Makefile for statistics +# +# IDENTIFICATION +# src/backend/statistics/Makefile +# +#------------------------------------------------------------------------- + +subdir = src/backend/statistics +top_builddir = ../../.. +include $(top_builddir)/src/Makefile.global + +OBJS = extended_stats.o mvdistinct.o + +include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/statistics/README b/src/backend/statistics/README new file mode 100644 index 0000000000..beb7c2485f --- /dev/null +++ b/src/backend/statistics/README @@ -0,0 +1,34 @@ +Extended statistics +=================== + +When estimating various quantities (e.g. condition selectivities) the default +approach relies on the assumption of independence. In practice that's often +not true, resulting in estimation errors. + +Extended statistics track different types of dependencies between the columns, +hopefully improving the estimates and producing better plans. + +Currently we only have one type of extended statistics - ndistinct +coefficients, and we use it to improve estimates of grouping queries. See +README.ndistinct for details. + + +Size of sample in ANALYZE +------------------------- +When performing ANALYZE, the number of rows to sample is determined as + + (300 * statistics_target) + +That works reasonably well for statistics on individual columns, but perhaps +it's not enough for extended statistics. Papers analyzing estimation errors +all use samples proportional to the table (usually finding that 1-3% of the +table is enough to build accurate stats). + +The requested accuracy (number of MCV items or histogram bins) should also +be considered when determining the sample size, and in extended statistics +those are not necessarily limited by statistics_target. + +This however merits further discussion, because collecting the sample is quite +expensive and increasing it further would make ANALYZE even more painful. +Judging by the experiments with the current implementation, the fixed size +seems to work reasonably well for now, so we leave this as a future work. diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c new file mode 100644 index 0000000000..d2b9f6a7c7 --- /dev/null +++ b/src/backend/statistics/extended_stats.c @@ -0,0 +1,389 @@ +/*------------------------------------------------------------------------- + * + * extended_stats.c + * POSTGRES extended statistics + * + * Generic code supporting statistic objects created via CREATE STATISTICS. + * + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/statistics/extended_stats.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/genam.h" +#include "access/heapam.h" +#include "access/htup_details.h" +#include "catalog/indexing.h" +#include "catalog/pg_collation.h" +#include "catalog/pg_statistic_ext.h" +#include "nodes/relation.h" +#include "statistics/extended_stats_internal.h" +#include "statistics/statistics.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" +#include "utils/lsyscache.h" +#include "utils/rel.h" +#include "utils/syscache.h" + + +/* + * Used internally to refer to an individual pg_statistic_ext entry. + */ +typedef struct StatExtEntry +{ + Oid statOid; /* OID of pg_statistic_ext entry */ + Bitmapset *columns; /* attribute numbers covered by the statistics */ + List *types; /* 'char' list of enabled statistic kinds */ +} StatExtEntry; + + +static List *fetch_statentries_for_relation(Relation pg_statext, Oid relid); +static VacAttrStats **lookup_var_attr_stats(Relation rel, Bitmapset *attrs, + int natts, VacAttrStats **vacattrstats); +static void statext_store(Relation pg_stext, Oid relid, + MVNDistinct *ndistinct, + VacAttrStats **stats); + + +/* + * Compute requested extended stats, using the rows sampled for the plain + * (single-column) stats. + * + * This fetches a list of stats from pg_statistic_ext, computes the stats + * and serializes them back into the catalog (as bytea values). + */ +void +BuildRelationExtStatistics(Relation onerel, double totalrows, + int numrows, HeapTuple *rows, + int natts, VacAttrStats **vacattrstats) +{ + Relation pg_stext; + ListCell *lc; + List *stats; + + pg_stext = heap_open(StatisticExtRelationId, RowExclusiveLock); + stats = fetch_statentries_for_relation(pg_stext, RelationGetRelid(onerel)); + + foreach(lc, stats) + { + StatExtEntry *stat = (StatExtEntry *) lfirst(lc); + MVNDistinct *ndistinct = NULL; + VacAttrStats **stats; + ListCell *lc2; + + /* filter only the interesting vacattrstats records */ + stats = lookup_var_attr_stats(onerel, stat->columns, + natts, vacattrstats); + + /* check allowed number of dimensions */ + Assert(bms_num_members(stat->columns) >= 2 && + bms_num_members(stat->columns) <= STATS_MAX_DIMENSIONS); + + /* compute statistic of each type */ + foreach(lc2, stat->types) + { + char t = (char) lfirst_int(lc2); + + if (t == STATS_EXT_NDISTINCT) + ndistinct = statext_ndistinct_build(totalrows, numrows, rows, + stat->columns, stats); + } + + /* store the statistics in the catalog */ + statext_store(pg_stext, stat->statOid, ndistinct, stats); + } + + heap_close(pg_stext, RowExclusiveLock); +} + +/* + * statext_is_kind_built + * Is this stat kind built in the given pg_statistic_ext tuple? + */ +bool +statext_is_kind_built(HeapTuple htup, char type) +{ + AttrNumber attnum; + + switch (type) + { + case STATS_EXT_NDISTINCT: + attnum = Anum_pg_statistic_ext_standistinct; + break; + + default: + elog(ERROR, "unexpected statistics type requested: %d", type); + } + + return !heap_attisnull(htup, attnum); +} + +/* + * Return a list (of StatExtEntry) of statistics for the given relation. + */ +static List * +fetch_statentries_for_relation(Relation pg_statext, Oid relid) +{ + SysScanDesc scan; + ScanKeyData skey; + HeapTuple htup; + List *result = NIL; + + /* + * Prepare to scan pg_statistic_ext for entries having indrelid = this + * rel. + */ + ScanKeyInit(&skey, + Anum_pg_statistic_ext_starelid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(relid)); + + scan = systable_beginscan(pg_statext, StatisticExtRelidIndexId, true, + NULL, 1, &skey); + + while (HeapTupleIsValid(htup = systable_getnext(scan))) + { + StatExtEntry *entry; + Datum datum; + bool isnull; + int i; + ArrayType *arr; + char *enabled; + Form_pg_statistic_ext staForm; + + entry = palloc0(sizeof(StatExtEntry)); + entry->statOid = HeapTupleGetOid(htup); + staForm = (Form_pg_statistic_ext) GETSTRUCT(htup); + for (i = 0; i < staForm->stakeys.dim1; i++) + { + entry->columns = bms_add_member(entry->columns, + staForm->stakeys.values[i]); + } + + /* decode the staenabled char array into a list of chars */ + datum = SysCacheGetAttr(STATEXTOID, htup, + Anum_pg_statistic_ext_staenabled, &isnull); + Assert(!isnull); + arr = DatumGetArrayTypeP(datum); + if (ARR_NDIM(arr) != 1 || + ARR_HASNULL(arr) || + ARR_ELEMTYPE(arr) != CHAROID) + elog(ERROR, "staenabled is not a 1-D char array"); + enabled = (char *) ARR_DATA_PTR(arr); + for (i = 0; i < ARR_DIMS(arr)[0]; i++) + { + Assert(enabled[i] == STATS_EXT_NDISTINCT); + entry->types = lappend_int(entry->types, (int) enabled[i]); + } + + result = lappend(result, entry); + } + + systable_endscan(scan); + + return result; +} + +/* + * Using 'vacattrstats' of size 'natts' as input data, return a newly built + * VacAttrStats array which includes only the items corresponding to attributes + * indicated by 'attrs'. + */ +static VacAttrStats ** +lookup_var_attr_stats(Relation rel, Bitmapset *attrs, int natts, + VacAttrStats **vacattrstats) +{ + int i = 0; + int x = -1; + VacAttrStats **stats; + Bitmapset *matched = NULL; + + stats = (VacAttrStats **) + palloc(bms_num_members(attrs) * sizeof(VacAttrStats *)); + + /* lookup VacAttrStats info for the requested columns (same attnum) */ + while ((x = bms_next_member(attrs, x)) >= 0) + { + int j; + + stats[i] = NULL; + for (j = 0; j < natts; j++) + { + if (x == vacattrstats[j]->tupattnum) + { + stats[i] = vacattrstats[j]; + break; + } + } + + if (!stats[i]) + ereport(ERROR, + (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), + errmsg("extended statistics could not be collected for column \"%s\" of relation %s.%s", + NameStr(RelationGetDescr(rel)->attrs[x - 1]->attname), + get_namespace_name(rel->rd_rel->relnamespace), + RelationGetRelationName(rel)), + errhint("Consider ALTER TABLE \"%s\".\"%s\" ALTER \"%s\" SET STATISTICS -1", + get_namespace_name(rel->rd_rel->relnamespace), + RelationGetRelationName(rel), + NameStr(RelationGetDescr(rel)->attrs[x - 1]->attname)))); + + /* + * Check that we found a non-dropped column and that the attnum + * matches. + */ + Assert(!stats[i]->attr->attisdropped); + matched = bms_add_member(matched, stats[i]->tupattnum); + + i++; + } + if (bms_subset_compare(matched, attrs) != BMS_EQUAL) + elog(ERROR, "could not find all attributes in attribute stats array"); + bms_free(matched); + + return stats; +} + +/* + * statext_store + * Serializes the statistics and stores them into the pg_statistic_ext tuple. + */ +static void +statext_store(Relation pg_stext, Oid statOid, + MVNDistinct *ndistinct, + VacAttrStats **stats) +{ + HeapTuple stup, + oldtup; + Datum values[Natts_pg_statistic_ext]; + bool nulls[Natts_pg_statistic_ext]; + bool replaces[Natts_pg_statistic_ext]; + + memset(nulls, 1, Natts_pg_statistic_ext * sizeof(bool)); + memset(replaces, 0, Natts_pg_statistic_ext * sizeof(bool)); + memset(values, 0, Natts_pg_statistic_ext * sizeof(Datum)); + + /* + * Construct a new pg_statistic_ext tuple, replacing the calculated stats. + */ + if (ndistinct != NULL) + { + bytea *data = statext_ndistinct_serialize(ndistinct); + + nulls[Anum_pg_statistic_ext_standistinct - 1] = (data == NULL); + values[Anum_pg_statistic_ext_standistinct - 1] = PointerGetDatum(data); + } + + /* always replace the value (either by bytea or NULL) */ + replaces[Anum_pg_statistic_ext_standistinct - 1] = true; + + /* there should already be a pg_statistic_ext tuple */ + oldtup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(statOid)); + if (!HeapTupleIsValid(oldtup)) + elog(ERROR, "cache lookup failed for extended statistics %u", statOid); + + /* replace it */ + stup = heap_modify_tuple(oldtup, + RelationGetDescr(pg_stext), + values, + nulls, + replaces); + ReleaseSysCache(oldtup); + CatalogTupleUpdate(pg_stext, &stup->t_self, stup); + + heap_freetuple(stup); +} + +/* initialize multi-dimensional sort */ +MultiSortSupport +multi_sort_init(int ndims) +{ + MultiSortSupport mss; + + Assert(ndims >= 2); + + mss = (MultiSortSupport) palloc0(offsetof(MultiSortSupportData, ssup) + +sizeof(SortSupportData) * ndims); + + mss->ndims = ndims; + + return mss; +} + +/* + * Prepare sort support info using the given sort operator + * at the position 'sortdim' + */ +void +multi_sort_add_dimension(MultiSortSupport mss, int sortdim, Oid oper) +{ + SortSupport ssup = &mss->ssup[sortdim]; + + ssup->ssup_cxt = CurrentMemoryContext; + ssup->ssup_collation = DEFAULT_COLLATION_OID; + ssup->ssup_nulls_first = false; + ssup->ssup_cxt = CurrentMemoryContext; + + PrepareSortSupportFromOrderingOp(oper, ssup); +} + +/* compare all the dimensions in the selected order */ +int +multi_sort_compare(const void *a, const void *b, void *arg) +{ + MultiSortSupport mss = (MultiSortSupport) arg; + SortItem *ia = (SortItem *) a; + SortItem *ib = (SortItem *) b; + int i; + + for (i = 0; i < mss->ndims; i++) + { + int compare; + + compare = ApplySortComparator(ia->values[i], ia->isnull[i], + ib->values[i], ib->isnull[i], + &mss->ssup[i]); + + if (compare != 0) + return compare; + } + + /* equal by default */ + return 0; +} + +/* compare selected dimension */ +int +multi_sort_compare_dim(int dim, const SortItem *a, const SortItem *b, + MultiSortSupport mss) +{ + return ApplySortComparator(a->values[dim], a->isnull[dim], + b->values[dim], b->isnull[dim], + &mss->ssup[dim]); +} + +int +multi_sort_compare_dims(int start, int end, + const SortItem *a, const SortItem *b, + MultiSortSupport mss) +{ + int dim; + + for (dim = start; dim <= end; dim++) + { + int r = ApplySortComparator(a->values[dim], a->isnull[dim], + b->values[dim], b->isnull[dim], + &mss->ssup[dim]); + + if (r != 0) + return r; + } + + return 0; +} diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c new file mode 100644 index 0000000000..5df4e29eec --- /dev/null +++ b/src/backend/statistics/mvdistinct.c @@ -0,0 +1,671 @@ +/*------------------------------------------------------------------------- + * + * mvdistinct.c + * POSTGRES multivariate ndistinct coefficients + * + * Estimating number of groups in a combination of columns (e.g. for GROUP BY) + * is tricky, and the estimation error is often significant. + + * The multivariate ndistinct coefficients address this by storing ndistinct + * estimates for combinations of the user-specified columns. So for example + * given a statistics object on three columns (a,b,c), this module estimates + * and store n-distinct for (a,b), (a,c), (b,c) and (a,b,c). The per-column + * estimates are already available in pg_statistic. + * + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/statistics/mvdistinct.c + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include + +#include "access/htup_details.h" +#include "catalog/pg_statistic_ext.h" +#include "utils/fmgrprotos.h" +#include "utils/lsyscache.h" +#include "lib/stringinfo.h" +#include "utils/syscache.h" +#include "utils/typcache.h" +#include "statistics/extended_stats_internal.h" +#include "statistics/statistics.h" + + +static double ndistinct_for_combination(double totalrows, int numrows, + HeapTuple *rows, VacAttrStats **stats, + int k, int *combination); +static double estimate_ndistinct(double totalrows, int numrows, int d, int f1); +static int n_choose_k(int n, int k); +static int num_combinations(int n); + +/* Combination generator API */ + +/* internal state for generator of k-combinations of n elements */ +typedef struct CombinationGenerator +{ + int k; /* size of the combination */ + int n; /* total number of elements */ + int current; /* index of the next combination to return */ + int ncombinations; /* number of combinations (size of array) */ + int *combinations; /* array of pre-built combinations */ +} CombinationGenerator; + +static CombinationGenerator *generator_init(int n, int k); +static void generator_free(CombinationGenerator *state); +static int *generator_next(CombinationGenerator *state); +static void generate_combinations(CombinationGenerator *state); + + +/* + * statext_ndistinct_build + * Compute ndistinct coefficient for the combination of attributes. + * + * This computes the ndistinct estimate using the same estimator used + * in analyze.c and then computes the coefficient. + */ +MVNDistinct * +statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows, + Bitmapset *attrs, VacAttrStats **stats) +{ + MVNDistinct *result; + int k; + int itemcnt; + int numattrs = bms_num_members(attrs); + int numcombs = num_combinations(numattrs); + + result = palloc(offsetof(MVNDistinct, items) + + numcombs * sizeof(MVNDistinctItem)); + result->magic = STATS_NDISTINCT_MAGIC; + result->type = STATS_NDISTINCT_TYPE_BASIC; + result->nitems = numcombs; + + itemcnt = 0; + for (k = 2; k <= numattrs; k++) + { + int *combination; + CombinationGenerator *generator; + + /* generate combinations of K out of N elements */ + generator = generator_init(numattrs, k); + + while ((combination = generator_next(generator))) + { + MVNDistinctItem *item = &result->items[itemcnt]; + int j; + + item->attrs = NULL; + for (j = 0; j < k; j++) + item->attrs = bms_add_member(item->attrs, + stats[combination[j]]->attr->attnum); + item->ndistinct = + ndistinct_for_combination(totalrows, numrows, rows, + stats, k, combination); + + itemcnt++; + Assert(itemcnt <= result->nitems); + } + + generator_free(generator); + } + + /* must consume exactly the whole output array */ + Assert(itemcnt == result->nitems); + + return result; +} + +/* + * statext_ndistinct_load + * Load the ndistinct value for the indicated pg_statistic_ext tuple + */ +MVNDistinct * +statext_ndistinct_load(Oid mvoid) +{ + bool isnull = false; + Datum ndist; + HeapTuple htup; + + htup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(mvoid)); + if (!htup) + elog(ERROR, "cache lookup failed for statistics %u", mvoid); + + ndist = SysCacheGetAttr(STATEXTOID, htup, + Anum_pg_statistic_ext_standistinct, &isnull); + if (isnull) + elog(ERROR, + "requested statistic kind %c not yet built for statistics %u", + STATS_EXT_NDISTINCT, mvoid); + + ReleaseSysCache(htup); + + return statext_ndistinct_deserialize(DatumGetByteaP(ndist)); +} + +/* + * statext_ndistinct_serialize + * serialize ndistinct to the on-disk bytea format + */ +bytea * +statext_ndistinct_serialize(MVNDistinct *ndistinct) +{ + int i; + bytea *output; + char *tmp; + Size len; + + Assert(ndistinct->magic == STATS_NDISTINCT_MAGIC); + Assert(ndistinct->type == STATS_NDISTINCT_TYPE_BASIC); + + /* + * Base size is base struct size, plus one base struct for each items, + * including number of items for each. + */ + len = VARHDRSZ + offsetof(MVNDistinct, items) + + ndistinct->nitems * (offsetof(MVNDistinctItem, attrs) + sizeof(int)); + + /* and also include space for the actual attribute numbers */ + for (i = 0; i < ndistinct->nitems; i++) + { + int nmembers; + + nmembers = bms_num_members(ndistinct->items[i].attrs); + Assert(nmembers >= 2); + len += sizeof(AttrNumber) * nmembers; + } + + output = (bytea *) palloc(len); + SET_VARSIZE(output, len); + + tmp = VARDATA(output); + + /* Store the base struct values */ + memcpy(tmp, ndistinct, offsetof(MVNDistinct, items)); + tmp += offsetof(MVNDistinct, items); + + /* + * store number of attributes and attribute numbers for each ndistinct + * entry + */ + for (i = 0; i < ndistinct->nitems; i++) + { + MVNDistinctItem item = ndistinct->items[i]; + int nmembers = bms_num_members(item.attrs); + int x; + + memcpy(tmp, &item.ndistinct, sizeof(double)); + tmp += sizeof(double); + memcpy(tmp, &nmembers, sizeof(int)); + tmp += sizeof(int); + + x = -1; + while ((x = bms_next_member(item.attrs, x)) >= 0) + { + AttrNumber value = (AttrNumber) x; + + memcpy(tmp, &value, sizeof(AttrNumber)); + tmp += sizeof(AttrNumber); + } + + Assert(tmp <= ((char *) output + len)); + } + + return output; +} + +/* + * statext_ndistinct_deserialize + * Read an on-disk bytea format MVNDistinct to in-memory format + */ +MVNDistinct * +statext_ndistinct_deserialize(bytea *data) +{ + int i; + Size expected_size; + MVNDistinct *ndistinct; + char *tmp; + + if (data == NULL) + return NULL; + + if (VARSIZE_ANY_EXHDR(data) < offsetof(MVNDistinct, items)) + elog(ERROR, "invalid MVNDistinct size %ld (expected at least %ld)", + VARSIZE_ANY_EXHDR(data), offsetof(MVNDistinct, items)); + + /* read the MVNDistinct header */ + ndistinct = (MVNDistinct *) palloc(sizeof(MVNDistinct)); + + /* initialize pointer to the data part (skip the varlena header) */ + tmp = VARDATA_ANY(data); + + /* get the header and perform basic sanity checks */ + memcpy(ndistinct, tmp, offsetof(MVNDistinct, items)); + tmp += offsetof(MVNDistinct, items); + + if (ndistinct->magic != STATS_NDISTINCT_MAGIC) + elog(ERROR, "invalid ndistinct magic %d (expected %d)", + ndistinct->magic, STATS_NDISTINCT_MAGIC); + + if (ndistinct->type != STATS_NDISTINCT_TYPE_BASIC) + elog(ERROR, "invalid ndistinct type %d (expected %d)", + ndistinct->type, STATS_NDISTINCT_TYPE_BASIC); + + Assert(ndistinct->nitems > 0); + + /* what minimum bytea size do we expect for those parameters */ + expected_size = offsetof(MVNDistinct, items) + + ndistinct->nitems * (offsetof(MVNDistinctItem, attrs) + + sizeof(AttrNumber) * 2); + + if (VARSIZE_ANY_EXHDR(data) < expected_size) + elog(ERROR, "invalid dependencies size %ld (expected at least %ld)", + VARSIZE_ANY_EXHDR(data), expected_size); + + /* allocate space for the ndistinct items */ + ndistinct = repalloc(ndistinct, offsetof(MVNDistinct, items) + + (ndistinct->nitems * sizeof(MVNDistinctItem))); + + for (i = 0; i < ndistinct->nitems; i++) + { + MVNDistinctItem *item = &ndistinct->items[i]; + int nelems; + + item->attrs = NULL; + + /* ndistinct value */ + memcpy(&item->ndistinct, tmp, sizeof(double)); + tmp += sizeof(double); + + /* number of attributes */ + memcpy(&nelems, tmp, sizeof(int)); + tmp += sizeof(int); + Assert((nelems >= 2) && (nelems <= STATS_MAX_DIMENSIONS)); + + while (nelems-- > 0) + { + AttrNumber attno; + + memcpy(&attno, tmp, sizeof(AttrNumber)); + tmp += sizeof(AttrNumber); + item->attrs = bms_add_member(item->attrs, attno); + } + + /* still within the bytea */ + Assert(tmp <= ((char *) data + VARSIZE_ANY(data))); + } + + /* we should have consumed the whole bytea exactly */ + Assert(tmp == ((char *) data + VARSIZE_ANY(data))); + + return ndistinct; +} + +/* + * pg_ndistinct_in + * input routine for type pg_ndistinct + * + * pg_ndistinct is real enough to be a table column, but it has no + * operations of its own, and disallows input (jus like pg_node_tree). + */ +Datum +pg_ndistinct_in(PG_FUNCTION_ARGS) +{ + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot accept a value of type %s", "pg_ndistinct"))); + + PG_RETURN_VOID(); /* keep compiler quiet */ +} + +/* + * pg_ndistinct + * output routine for type pg_ndistinct + * + * Produces a human-readable representation of the value. + */ +Datum +pg_ndistinct_out(PG_FUNCTION_ARGS) +{ + bytea *data = PG_GETARG_BYTEA_PP(0); + MVNDistinct *ndist = statext_ndistinct_deserialize(data); + int i; + StringInfoData str; + + initStringInfo(&str); + appendStringInfoChar(&str, '['); + + for (i = 0; i < ndist->nitems; i++) + { + MVNDistinctItem item = ndist->items[i]; + + if (i > 0) + appendStringInfoString(&str, ", "); + + appendStringInfoChar(&str, '{'); + outBitmapset(&str, item.attrs); + appendStringInfo(&str, ", %f}", item.ndistinct); + } + + appendStringInfoChar(&str, ']'); + + PG_RETURN_CSTRING(str.data); +} + +/* + * pg_ndistinct_recv + * binary input routine for type pg_ndistinct + */ +Datum +pg_ndistinct_recv(PG_FUNCTION_ARGS) +{ + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot accept a value of type %s", "pg_ndistinct"))); + + PG_RETURN_VOID(); /* keep compiler quiet */ +} + +/* + * pg_ndistinct_send + * binary output routine for type pg_ndistinct + * + * n-distinct is serialized into a bytea value, so let's send that. + */ +Datum +pg_ndistinct_send(PG_FUNCTION_ARGS) +{ + return byteasend(fcinfo); +} + +/* + * ndistinct_for_combination + * Estimates number of distinct values in a combination of columns. + * + * This uses the same ndistinct estimator as compute_scalar_stats() in + * ANALYZE, i.e., + * n*d / (n - f1 + f1*n/N) + * + * except that instead of values in a single column we are dealing with + * combination of multiple columns. + */ +static double +ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows, + VacAttrStats **stats, int k, int *combination) +{ + int i, + j; + int f1, + cnt, + d; + bool *isnull; + Datum *values; + SortItem *items; + MultiSortSupport mss; + + mss = multi_sort_init(k); + + /* + * In order to determine the number of distinct elements, create separate + * values[]/isnull[] arrays with all the data we have, then sort them + * using the specified column combination as dimensions. We could try to + * sort in place, but it'd probably be more complex and bug-prone. + */ + items = (SortItem *) palloc(numrows * sizeof(SortItem)); + values = (Datum *) palloc0(sizeof(Datum) * numrows * k); + isnull = (bool *) palloc0(sizeof(bool) * numrows * k); + + for (i = 0; i < numrows; i++) + { + items[i].values = &values[i * k]; + items[i].isnull = &isnull[i * k]; + } + + /* + * For each dimension, set up sort-support and fill in the values from + * the sample data. + */ + for (i = 0; i < k; i++) + { + VacAttrStats *colstat = stats[combination[i]]; + TypeCacheEntry *type; + + type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR); + if (type->lt_opr == InvalidOid) /* shouldn't happen */ + elog(ERROR, "cache lookup failed for ordering operator for type %u", + colstat->attrtypid); + + /* prepare the sort function for this dimension */ + multi_sort_add_dimension(mss, i, type->lt_opr); + + /* accumulate all the data for this dimension into the arrays */ + for (j = 0; j < numrows; j++) + { + items[j].values[i] = + heap_getattr(rows[j], + colstat->attr->attnum, + colstat->tupDesc, + &items[j].isnull[i]); + } + } + + /* We can sort the array now ... */ + qsort_arg((void *) items, numrows, sizeof(SortItem), + multi_sort_compare, mss); + + /* ... and count the number of distinct combinations */ + + f1 = 0; + cnt = 1; + d = 1; + for (i = 1; i < numrows; i++) + { + if (multi_sort_compare(&items[i], &items[i - 1], mss) != 0) + { + if (cnt == 1) + f1 += 1; + + d++; + cnt = 0; + } + + cnt += 1; + } + + if (cnt == 1) + f1 += 1; + + return estimate_ndistinct(totalrows, numrows, d, f1); +} + +/* The Duj1 estimator (already used in analyze.c). */ +static double +estimate_ndistinct(double totalrows, int numrows, int d, int f1) +{ + double numer, + denom, + ndistinct; + + numer = (double) numrows * (double) d; + + denom = (double) (numrows - f1) + + (double) f1 *(double) numrows / totalrows; + + ndistinct = numer / denom; + + /* Clamp to sane range in case of roundoff error */ + if (ndistinct < (double) d) + ndistinct = (double) d; + + if (ndistinct > totalrows) + ndistinct = totalrows; + + return floor(ndistinct + 0.5); +} + +/* + * n_choose_k + * computes binomial coefficients using an algorithm that is both + * efficient and prevents overflows + */ +static int +n_choose_k(int n, int k) +{ + int d, + r; + + Assert((k > 0) && (n >= k)); + + /* use symmetry of the binomial coefficients */ + k = Min(k, n - k); + + r = 1; + for (d = 1; d <= k; ++d) + { + r *= n--; + r /= d; + } + + return r; +} + +/* + * num_combinations + * number of combinations, excluding single-value combinations + */ +static int +num_combinations(int n) +{ + int k; + int ncombs = 1; + + for (k = 1; k <= n; k++) + ncombs *= 2; + + ncombs -= (n + 1); + + return ncombs; +} + +/* + * generator_init + * initialize the generator of combinations + * + * The generator produces combinations of K elements in the interval (0..N). + * We prebuild all the combinations in this method, which is simpler than + * generating them on the fly. + */ +static CombinationGenerator * +generator_init(int n, int k) +{ + CombinationGenerator *state; + + Assert((n >= k) && (k > 0)); + + /* allocate the generator state as a single chunk of memory */ + state = (CombinationGenerator *) palloc(sizeof(CombinationGenerator)); + + state->ncombinations = n_choose_k(n, k); + + /* pre-allocate space for all combinations*/ + state->combinations = (int *) palloc(sizeof(int) * k * state->ncombinations); + + state->current = 0; + state->k = k; + state->n = n; + + /* now actually pre-generate all the combinations of K elements */ + generate_combinations(state); + + /* make sure we got the expected number of combinations */ + Assert(state->current == state->ncombinations); + + /* reset the number, so we start with the first one */ + state->current = 0; + + return state; +} + +/* + * generator_next + * returns the next combination from the prebuilt list + * + * Returns a combination of K array indexes (0 .. N), as specified to + * generator_init), or NULL when there are no more combination. + */ +static int * +generator_next(CombinationGenerator *state) +{ + if (state->current == state->ncombinations) + return NULL; + + return &state->combinations[state->k * state->current++]; +} + +/* + * generator_free + * free the internal state of the generator + * + * Releases the generator internal state (pre-built combinations). + */ +static void +generator_free(CombinationGenerator *state) +{ + pfree(state->combinations); + pfree(state); +} + +/* + * generate_combinations_recurse + * given a prefix, generate all possible combinations + * + * Given a prefix (first few elements of the combination), generate following + * elements recursively. We generate the combinations in lexicographic order, + * which eliminates permutations of the same combination. + */ +static void +generate_combinations_recurse(CombinationGenerator *state, + int index, int start, int *current) +{ + /* If we haven't filled all the elements, simply recurse. */ + if (index < state->k) + { + int i; + + /* + * The values have to be in ascending order, so make sure we start + * with the value passed by parameter. + */ + + for (i = start; i < state->n; i++) + { + current[index] = i; + generate_combinations_recurse(state, (index + 1), (i + 1), current); + } + + return; + } + else + { + /* we got a valid combination, add it to the array */ + memcpy(&state->combinations[(state->k * state->current)], + current, state->k * sizeof(int)); + state->current++; + } +} + +/* + * generate_combinations + * generate all k-combinations of N elements + */ +static void +generate_combinations(CombinationGenerator *state) +{ + int *current = (int *) palloc0(sizeof(int) * state->k); + + generate_combinations_recurse(state, 0, 0, current); + + pfree(current); +} diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index c8d20fffea..b59821bf97 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -1623,6 +1623,10 @@ ProcessUtilitySlow(ParseState *pstate, commandCollected = true; break; + case T_CreateStatsStmt: + address = CreateStatistics((CreateStatsStmt *) parsetree); + break; + case T_AlterCollationStmt: address = AlterCollation((AlterCollationStmt *) parsetree); break; @@ -1992,6 +1996,8 @@ AlterObjectTypeCommandTag(ObjectType objtype) break; case OBJECT_SUBSCRIPTION: tag = "ALTER SUBSCRIPTION"; + case OBJECT_STATISTIC_EXT: + tag = "ALTER STATISTICS"; break; default: tag = "???"; @@ -2286,6 +2292,8 @@ CreateCommandTag(Node *parsetree) break; case OBJECT_PUBLICATION: tag = "DROP PUBLICATION"; + case OBJECT_STATISTIC_EXT: + tag = "DROP STATISTICS"; break; default: tag = "???"; @@ -2689,6 +2697,10 @@ CreateCommandTag(Node *parsetree) tag = "EXECUTE"; break; + case T_CreateStatsStmt: + tag = "CREATE STATISTICS"; + break; + case T_DeallocateStmt: { DeallocateStmt *stmt = (DeallocateStmt *) parsetree; diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c index 5c823250bc..81c91039e4 100644 --- a/src/backend/utils/adt/ruleutils.c +++ b/src/backend/utils/adt/ruleutils.c @@ -35,6 +35,7 @@ #include "catalog/pg_operator.h" #include "catalog/pg_partitioned_table.h" #include "catalog/pg_proc.h" +#include "catalog/pg_statistic_ext.h" #include "catalog/pg_trigger.h" #include "catalog/pg_type.h" #include "commands/defrem.h" @@ -317,6 +318,7 @@ static char *pg_get_indexdef_worker(Oid indexrelid, int colno, const Oid *excludeOps, bool attrsOnly, bool showTblSpc, int prettyFlags, bool missing_ok); +static char *pg_get_statisticsext_worker(Oid statextid, bool missing_ok); static char *pg_get_partkeydef_worker(Oid relid, int prettyFlags, bool attrsOnly); static char *pg_get_constraintdef_worker(Oid constraintId, bool fullCommand, @@ -1421,6 +1423,85 @@ pg_get_indexdef_worker(Oid indexrelid, int colno, return buf.data; } +/* + * pg_get_statisticsextdef + * Get the definition of an extended statistics object + */ +Datum +pg_get_statisticsextdef(PG_FUNCTION_ARGS) +{ + Oid statextid = PG_GETARG_OID(0); + char *res; + + res = pg_get_statisticsext_worker(statextid, true); + + if (res == NULL) + PG_RETURN_NULL(); + + PG_RETURN_TEXT_P(string_to_text(res)); +} + +/* + * Internal workhorse to decompile an extended statistics object. + */ +static char * +pg_get_statisticsext_worker(Oid statextid, bool missing_ok) +{ + Form_pg_statistic_ext statextrec; + Form_pg_class pgclassrec; + HeapTuple statexttup; + HeapTuple pgclasstup; + StringInfoData buf; + int colno; + + statexttup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(statextid)); + + if (!HeapTupleIsValid(statexttup)) + { + if (missing_ok) + return NULL; + elog(ERROR, "cache lookup failed for extended statistics %u", statextid); + } + + statextrec = (Form_pg_statistic_ext) GETSTRUCT(statexttup); + + pgclasstup = SearchSysCache1(RELOID, ObjectIdGetDatum(statextrec->starelid)); + + if (!HeapTupleIsValid(statexttup)) + { + ReleaseSysCache(statexttup); + elog(ERROR, "cache lookup failed for relation %u", statextrec->starelid); + } + + pgclassrec = (Form_pg_class) GETSTRUCT(pgclasstup); + + initStringInfo(&buf); + + appendStringInfo(&buf, "CREATE STATISTICS %s ON (", + quote_identifier(NameStr(statextrec->staname))); + + for (colno = 0; colno < statextrec->stakeys.dim1; colno++) + { + AttrNumber attnum = statextrec->stakeys.values[colno]; + char *attname; + + if (colno > 0) + appendStringInfoString(&buf, ", "); + + attname = get_relid_attribute_name(statextrec->starelid, attnum); + + appendStringInfoString(&buf, quote_identifier(attname)); + } + + appendStringInfo(&buf, ") FROM %s", + quote_identifier(NameStr(pgclassrec->relname))); + + ReleaseSysCache(statexttup); + ReleaseSysCache(pgclasstup); + + return buf.data; +} + /* * pg_get_partkeydef * diff --git a/src/backend/utils/adt/selfuncs.c b/src/backend/utils/adt/selfuncs.c index f8b28fe0e6..cc24c8aeb5 100644 --- a/src/backend/utils/adt/selfuncs.c +++ b/src/backend/utils/adt/selfuncs.c @@ -110,6 +110,7 @@ #include "catalog/pg_operator.h" #include "catalog/pg_opfamily.h" #include "catalog/pg_statistic.h" +#include "catalog/pg_statistic_ext.h" #include "catalog/pg_type.h" #include "executor/executor.h" #include "mb/pg_wchar.h" @@ -126,6 +127,7 @@ #include "parser/parse_clause.h" #include "parser/parse_coerce.h" #include "parser/parsetree.h" +#include "statistics/statistics.h" #include "utils/builtins.h" #include "utils/bytea.h" #include "utils/date.h" @@ -164,6 +166,8 @@ static double eqjoinsel_inner(Oid operator, static double eqjoinsel_semi(Oid operator, VariableStatData *vardata1, VariableStatData *vardata2, RelOptInfo *inner_rel); +static bool estimate_multivariate_ndistinct(PlannerInfo *root, + RelOptInfo *rel, List **varinfos, double *ndistinct); static bool convert_to_scalar(Datum value, Oid valuetypid, double *scaledvalue, Datum lobound, Datum hibound, Oid boundstypid, double *scaledlobound, double *scaledhibound); @@ -3398,25 +3402,25 @@ estimate_num_groups(PlannerInfo *root, List *groupExprs, double input_rows, { GroupVarInfo *varinfo1 = (GroupVarInfo *) linitial(varinfos); RelOptInfo *rel = varinfo1->rel; - double reldistinct = varinfo1->ndistinct; + double reldistinct = 1; double relmaxndistinct = reldistinct; int relvarcount = 1; List *newvarinfos = NIL; + List *relvarinfos = NIL; /* - * Get the product of numdistinct estimates of the Vars for this rel. - * Also, construct new varinfos list of remaining Vars. + * Split the list of varinfos in two - one for the current rel, + * one for remaining Vars on other rels. */ + relvarinfos = lcons(varinfo1, relvarinfos); for_each_cell(l, lnext(list_head(varinfos))) { GroupVarInfo *varinfo2 = (GroupVarInfo *) lfirst(l); if (varinfo2->rel == varinfo1->rel) { - reldistinct *= varinfo2->ndistinct; - if (relmaxndistinct < varinfo2->ndistinct) - relmaxndistinct = varinfo2->ndistinct; - relvarcount++; + /* varinfos on current rel */ + relvarinfos = lcons(varinfo2, relvarinfos); } else { @@ -3425,6 +3429,43 @@ estimate_num_groups(PlannerInfo *root, List *groupExprs, double input_rows, } } + /* + * Get the numdistinct estimate for the Vars of this rel. We + * iteratively search for multivariate n-distinct with maximum number + * of vars; assuming that each var group is independent of the others, + * we multiply them together. Any remaining relvarinfos after + * no more multivariate matches are found are assumed independent too, + * so their individual ndistinct estimates are multiplied also. + */ + while (relvarinfos) + { + double mvndistinct; + + if (estimate_multivariate_ndistinct(root, rel, &relvarinfos, + &mvndistinct)) + { + reldistinct *= mvndistinct; + if (relmaxndistinct < mvndistinct) + relmaxndistinct = mvndistinct; + relvarcount++; /* inaccurate, but doesn't matter */ + } + else + { + foreach (l, relvarinfos) + { + GroupVarInfo *varinfo2 = (GroupVarInfo *) lfirst(l); + + reldistinct *= varinfo2->ndistinct; + if (relmaxndistinct < varinfo2->ndistinct) + relmaxndistinct = varinfo2->ndistinct; + relvarcount++; + } + + /* we're done with this relation */ + relvarinfos = NIL; + } + } + /* * Sanity check --- don't divide by zero if empty relation. */ @@ -3667,6 +3708,132 @@ estimate_hash_bucketsize(PlannerInfo *root, Node *hashkey, double nbuckets) *------------------------------------------------------------------------- */ +/* + * Find applicable ndistinct statistics for the given list of VarInfos (which + * must all belong to the given rel), and update *ndistinct to the estimate of + * the MVNDistinctItem that best matches. If a match it found, *varinfos is + * updated to remove the list of matched varinfos. + * + * Varinfos that aren't for simple Vars are ignored. + * + * Return TRUE if we're able to find a match, FALSE otherwise. + */ +static bool +estimate_multivariate_ndistinct(PlannerInfo *root, RelOptInfo *rel, + List **varinfos, double *ndistinct) +{ + ListCell *lc; + Bitmapset *attnums = NULL; + int nmatches; + Oid statOid = InvalidOid; + MVNDistinct *stats; + Bitmapset *matched = NULL; + + /* bail out immediately if the table has no extended statistics */ + if (!rel->statlist) + return false; + + /* Determine the attnums we're looking for */ + foreach(lc, *varinfos) + { + GroupVarInfo *varinfo = (GroupVarInfo *) lfirst(lc); + + Assert(varinfo->rel == rel); + + if (IsA(varinfo->var, Var)) + { + attnums = bms_add_member(attnums, + ((Var *) varinfo->var)->varattno); + } + } + + /* look for the ndistinct statistics matching the most vars */ + nmatches = 1; /* we require at least two matches */ + foreach(lc, rel->statlist) + { + StatisticExtInfo *info = (StatisticExtInfo *) lfirst(lc); + Bitmapset *shared; + + /* skip statistics of other kinds */ + if (info->kind != STATS_EXT_NDISTINCT) + continue; + + /* compute attnums shared by the vars and the statistic */ + shared = bms_intersect(info->keys, attnums); + + /* + * Does this statistics matches more columns than the currently + * best statistic? If so, use this one instead. + * + * XXX This should break ties using name of the statistic, or + * something like that, to make the outcome stable. + */ + if (bms_num_members(shared) > nmatches) + { + statOid = info->statOid; + nmatches = bms_num_members(shared); + matched = shared; + } + } + + /* No match? */ + if (statOid == InvalidOid) + return false; + Assert(nmatches > 1 && matched != NULL); + + stats = statext_ndistinct_load(statOid); + + /* + * If we have a match, search it for the specific item that matches (there + * must be one), and construct the output values. + */ + if (stats) + { + int i; + List *newlist = NIL; + MVNDistinctItem *item = NULL; + + /* Find the specific item that exactly matches the combination */ + for (i = 0; i < stats->nitems; i++) + { + MVNDistinctItem *tmpitem = &stats->items[i]; + + if (bms_subset_compare(tmpitem->attrs, matched) == BMS_EQUAL) + { + item = tmpitem; + break; + } + } + + /* make sure we found an item */ + if (!item) + elog(ERROR, "corrupt MVNDistinct entry"); + + /* Form the output varinfo list, keeping only unmatched ones */ + foreach(lc, *varinfos) + { + GroupVarInfo *varinfo = (GroupVarInfo *) lfirst(lc); + AttrNumber attnum; + + if (!IsA(varinfo->var, Var)) + { + newlist = lappend(newlist, varinfo); + continue; + } + + attnum = ((Var *) varinfo->var)->varattno; + if (!bms_is_member(attnum, matched)) + newlist = lappend(newlist, varinfo); + } + + *varinfos = newlist; + *ndistinct = item->ndistinct; + return true; + } + + return false; +} + /* * convert_to_scalar * Convert non-NULL values of the indicated types to the comparison diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c index ce55fc5277..a6b60c67ca 100644 --- a/src/backend/utils/cache/relcache.c +++ b/src/backend/utils/cache/relcache.c @@ -56,6 +56,7 @@ #include "catalog/pg_publication.h" #include "catalog/pg_rewrite.h" #include "catalog/pg_shseclabel.h" +#include "catalog/pg_statistic_ext.h" #include "catalog/pg_subscription.h" #include "catalog/pg_tablespace.h" #include "catalog/pg_trigger.h" @@ -4451,6 +4452,82 @@ RelationGetIndexList(Relation relation) return result; } +/* + * RelationGetStatExtList + * get a list of OIDs of extended statistics on this relation + * + * The statistics list is created only if someone requests it, in a way + * similar to RelationGetIndexList(). We scan pg_statistic_ext to find + * relevant statistics, and add the list to the relcache entry so that we + * won't have to compute it again. Note that shared cache inval of a + * relcache entry will delete the old list and set rd_statvalid to 0, + * so that we must recompute the statistics list on next request. This + * handles creation or deletion of a statistic. + * + * The returned list is guaranteed to be sorted in order by OID, although + * this is not currently needed. + * + * Since shared cache inval causes the relcache's copy of the list to go away, + * we return a copy of the list palloc'd in the caller's context. The caller + * may list_free() the returned list after scanning it. This is necessary + * since the caller will typically be doing syscache lookups on the relevant + * statistics, and syscache lookup could cause SI messages to be processed! + */ +List * +RelationGetStatExtList(Relation relation) +{ + Relation indrel; + SysScanDesc indscan; + ScanKeyData skey; + HeapTuple htup; + List *result; + List *oldlist; + MemoryContext oldcxt; + + /* Quick exit if we already computed the list. */ + if (relation->rd_statvalid != 0) + return list_copy(relation->rd_statlist); + + /* + * We build the list we intend to return (in the caller's context) while + * doing the scan. After successfully completing the scan, we copy that + * list into the relcache entry. This avoids cache-context memory leakage + * if we get some sort of error partway through. + */ + result = NIL; + + /* Prepare to scan pg_statistic_ext for entries having starelid = this rel. */ + ScanKeyInit(&skey, + Anum_pg_statistic_ext_starelid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(RelationGetRelid(relation))); + + indrel = heap_open(StatisticExtRelationId, AccessShareLock); + indscan = systable_beginscan(indrel, StatisticExtRelidIndexId, true, + NULL, 1, &skey); + + while (HeapTupleIsValid(htup = systable_getnext(indscan))) + /* TODO maybe include only already built statistics? */ + result = insert_ordered_oid(result, HeapTupleGetOid(htup)); + + systable_endscan(indscan); + + heap_close(indrel, AccessShareLock); + + /* Now save a copy of the completed list in the relcache entry. */ + oldcxt = MemoryContextSwitchTo(CacheMemoryContext); + oldlist = relation->rd_statlist; + relation->rd_statlist = list_copy(result); + + relation->rd_statvalid = true; + MemoryContextSwitchTo(oldcxt); + + /* Don't leak the old list, if there is one */ + list_free(oldlist); + + return result; +} + /* * insert_ordered_oid * Insert a new Oid into a sorted list of Oids, preserving ordering @@ -5560,6 +5637,8 @@ load_relcache_init_file(bool shared) rel->rd_pkattr = NULL; rel->rd_idattr = NULL; rel->rd_pubactions = NULL; + rel->rd_statvalid = false; + rel->rd_statlist = NIL; rel->rd_createSubid = InvalidSubTransactionId; rel->rd_newRelfilenodeSubid = InvalidSubTransactionId; rel->rd_amcache = NULL; diff --git a/src/backend/utils/cache/syscache.c b/src/backend/utils/cache/syscache.c index d5a376406f..d8c823f42b 100644 --- a/src/backend/utils/cache/syscache.c +++ b/src/backend/utils/cache/syscache.c @@ -61,6 +61,7 @@ #include "catalog/pg_shseclabel.h" #include "catalog/pg_replication_origin.h" #include "catalog/pg_statistic.h" +#include "catalog/pg_statistic_ext.h" #include "catalog/pg_subscription.h" #include "catalog/pg_subscription_rel.h" #include "catalog/pg_tablespace.h" @@ -726,6 +727,28 @@ static const struct cachedesc cacheinfo[] = { }, 32 }, + {StatisticExtRelationId, /* STATEXTNAMENSP */ + StatisticExtNameIndexId, + 2, + { + Anum_pg_statistic_ext_staname, + Anum_pg_statistic_ext_stanamespace, + 0, + 0 + }, + 4 + }, + {StatisticExtRelationId, /* STATEXTOID */ + StatisticExtOidIndexId, + 1, + { + ObjectIdAttributeNumber, + 0, + 0, + 0 + }, + 4 + }, {StatisticRelationId, /* STATRELATTINH */ StatisticRelidAttnumInhIndexId, 3, diff --git a/src/bin/pg_dump/common.c b/src/bin/pg_dump/common.c index 89530a9f0f..e2bc3576dc 100644 --- a/src/bin/pg_dump/common.c +++ b/src/bin/pg_dump/common.c @@ -272,6 +272,10 @@ getSchemaData(Archive *fout, int *numTablesPtr) write_msg(NULL, "reading indexes\n"); getIndexes(fout, tblinfo, numTables); + if (g_verbose) + write_msg(NULL, "reading extended statistics\n"); + getExtendedStatistics(fout, tblinfo, numTables); + if (g_verbose) write_msg(NULL, "reading constraints\n"); getConstraints(fout, tblinfo, numTables); diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index dd0892539a..f77581d6ec 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -3540,7 +3540,8 @@ _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData, bool acl_pass) strcmp(te->desc, "TRIGGER") == 0 || strcmp(te->desc, "ROW SECURITY") == 0 || strcmp(te->desc, "POLICY") == 0 || - strcmp(te->desc, "USER MAPPING") == 0) + strcmp(te->desc, "USER MAPPING") == 0 || + strcmp(te->desc, "STATISTICS") == 0) { /* these object types don't have separate owners */ } diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index b3d95d7f6e..ba34cc163e 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -192,6 +192,7 @@ static void dumpAttrDef(Archive *fout, AttrDefInfo *adinfo); static void dumpSequence(Archive *fout, TableInfo *tbinfo); static void dumpSequenceData(Archive *fout, TableDataInfo *tdinfo); static void dumpIndex(Archive *fout, IndxInfo *indxinfo); +static void dumpStatisticsExt(Archive *fout, StatsExtInfo *statsextinfo); static void dumpConstraint(Archive *fout, ConstraintInfo *coninfo); static void dumpTableConstraintComment(Archive *fout, ConstraintInfo *coninfo); static void dumpTSParser(Archive *fout, TSParserInfo *prsinfo); @@ -6582,6 +6583,99 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) destroyPQExpBuffer(query); } +/* + * getExtendedStatistics + * get information about extended statistics on a dumpable table + * or materialized view. + * + * Note: extended statistics data is not returned directly to the caller, but + * it does get entered into the DumpableObject tables. + */ +void +getExtendedStatistics(Archive *fout, TableInfo tblinfo[], int numTables) +{ + int i, + j; + PQExpBuffer query; + PGresult *res; + StatsExtInfo *statsextinfo; + int ntups; + int i_tableoid; + int i_oid; + int i_staname; + int i_stadef; + + /* Extended statistics were new in v10 */ + if (fout->remoteVersion < 100000) + return; + + query = createPQExpBuffer(); + + for (i = 0; i < numTables; i++) + { + TableInfo *tbinfo = &tblinfo[i]; + + /* Only plain tables and materialized views can have extended statistics. */ + if (tbinfo->relkind != RELKIND_RELATION && + tbinfo->relkind != RELKIND_MATVIEW) + continue; + + /* + * Ignore extended statistics of tables whose definitions are not to + * be dumped. + */ + if (!(tbinfo->dobj.dump & DUMP_COMPONENT_DEFINITION)) + continue; + + if (g_verbose) + write_msg(NULL, "reading extended statistics for table \"%s.%s\"\n", + tbinfo->dobj.namespace->dobj.name, + tbinfo->dobj.name); + + /* Make sure we are in proper schema so stadef is right */ + selectSourceSchema(fout, tbinfo->dobj.namespace->dobj.name); + + resetPQExpBuffer(query); + + appendPQExpBuffer(query, + "SELECT " + "tableoid, " + "oid, " + "staname, " + "pg_catalog.pg_get_statisticsextdef(oid) AS stadef " + "FROM pg_statistic_ext " + "WHERE starelid = '%u' " + "ORDER BY staname", tbinfo->dobj.catId.oid); + + res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); + + ntups = PQntuples(res); + + i_tableoid = PQfnumber(res, "tableoid"); + i_oid = PQfnumber(res, "oid"); + i_staname = PQfnumber(res, "staname"); + i_stadef = PQfnumber(res, "stadef"); + + statsextinfo = (StatsExtInfo *) pg_malloc(ntups * sizeof(StatsExtInfo)); + + for (j = 0; j < ntups; j++) + { + statsextinfo[j].dobj.objType = DO_STATSEXT; + statsextinfo[j].dobj.catId.tableoid = atooid(PQgetvalue(res, j, i_tableoid)); + statsextinfo[j].dobj.catId.oid = atooid(PQgetvalue(res, j, i_oid)); + AssignDumpId(&statsextinfo[j].dobj); + statsextinfo[j].dobj.name = pg_strdup(PQgetvalue(res, j, i_staname)); + statsextinfo[j].dobj.namespace = tbinfo->dobj.namespace; + statsextinfo[j].statsexttable = tbinfo; + statsextinfo[j].statsextdef = pg_strdup(PQgetvalue(res, j, i_stadef)); + } + + PQclear(res); + } + + destroyPQExpBuffer(query); +} + /* * getConstraints * @@ -9234,6 +9328,9 @@ dumpDumpableObject(Archive *fout, DumpableObject *dobj) case DO_INDEX: dumpIndex(fout, (IndxInfo *) dobj); break; + case DO_STATSEXT: + dumpStatisticsExt(fout, (StatsExtInfo *) dobj); + break; case DO_REFRESH_MATVIEW: refreshMatViewData(fout, (TableDataInfo *) dobj); break; @@ -15728,6 +15825,61 @@ dumpIndex(Archive *fout, IndxInfo *indxinfo) destroyPQExpBuffer(labelq); } +/* + * dumpStatisticsExt + * write out to fout an extended statistics object + */ +static void +dumpStatisticsExt(Archive *fout, StatsExtInfo *statsextinfo) +{ + DumpOptions *dopt = fout->dopt; + TableInfo *tbinfo = statsextinfo->statsexttable; + PQExpBuffer q; + PQExpBuffer delq; + PQExpBuffer labelq; + + if (dopt->dataOnly) + return; + + q = createPQExpBuffer(); + delq = createPQExpBuffer(); + labelq = createPQExpBuffer(); + + appendPQExpBuffer(labelq, "STATISTICS %s", + fmtId(statsextinfo->dobj.name)); + + appendPQExpBuffer(q, "%s;\n", statsextinfo->statsextdef); + + appendPQExpBuffer(delq, "DROP STATISTICS %s.", + fmtId(tbinfo->dobj.namespace->dobj.name)); + appendPQExpBuffer(delq, "%s;\n", + fmtId(statsextinfo->dobj.name)); + + if (statsextinfo->dobj.dump & DUMP_COMPONENT_DEFINITION) + ArchiveEntry(fout, statsextinfo->dobj.catId, + statsextinfo->dobj.dumpId, + statsextinfo->dobj.name, + tbinfo->dobj.namespace->dobj.name, + NULL, + tbinfo->rolname, false, + "STATISTICS", SECTION_POST_DATA, + q->data, delq->data, NULL, + NULL, 0, + NULL, NULL); + + /* Dump Statistics Comments */ + if (statsextinfo->dobj.dump & DUMP_COMPONENT_COMMENT) + dumpComment(fout, labelq->data, + tbinfo->dobj.namespace->dobj.name, + tbinfo->rolname, + statsextinfo->dobj.catId, 0, + statsextinfo->dobj.dumpId); + + destroyPQExpBuffer(q); + destroyPQExpBuffer(delq); + destroyPQExpBuffer(labelq); +} + /* * dumpConstraint * write out to fout a user-defined constraint @@ -17266,6 +17418,7 @@ addBoundaryDependencies(DumpableObject **dobjs, int numObjs, addObjectDependency(postDataBound, dobj->dumpId); break; case DO_INDEX: + case DO_STATSEXT: case DO_REFRESH_MATVIEW: case DO_TRIGGER: case DO_EVENT_TRIGGER: diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h index a466527ec6..cb22f63bd6 100644 --- a/src/bin/pg_dump/pg_dump.h +++ b/src/bin/pg_dump/pg_dump.h @@ -56,6 +56,7 @@ typedef enum DO_TABLE, DO_ATTRDEF, DO_INDEX, + DO_STATSEXT, DO_RULE, DO_TRIGGER, DO_CONSTRAINT, @@ -362,6 +363,13 @@ typedef struct _indxInfo int relpages; /* relpages of the underlying table */ } IndxInfo; +typedef struct _statsExtInfo +{ + DumpableObject dobj; + TableInfo *statsexttable; /* link to table the stats ext is for */ + char *statsextdef; +} StatsExtInfo; + typedef struct _ruleInfo { DumpableObject dobj; @@ -682,6 +690,7 @@ extern void getOwnedSeqs(Archive *fout, TableInfo tblinfo[], int numTables); extern InhInfo *getInherits(Archive *fout, int *numInherits); extern PartInfo *getPartitions(Archive *fout, int *numPartitions); extern void getIndexes(Archive *fout, TableInfo tblinfo[], int numTables); +extern void getExtendedStatistics(Archive *fout, TableInfo tblinfo[], int numTables); extern void getConstraints(Archive *fout, TableInfo tblinfo[], int numTables); extern RuleInfo *getRules(Archive *fout, int *numRules); extern void getTriggers(Archive *fout, TableInfo tblinfo[], int numTables); diff --git a/src/bin/pg_dump/pg_dump_sort.c b/src/bin/pg_dump/pg_dump_sort.c index e555de8857..5c19b05ca4 100644 --- a/src/bin/pg_dump/pg_dump_sort.c +++ b/src/bin/pg_dump/pg_dump_sort.c @@ -53,10 +53,11 @@ static const int dbObjectTypePriority[] = 18, /* DO_TABLE */ 20, /* DO_ATTRDEF */ 28, /* DO_INDEX */ - 29, /* DO_RULE */ - 30, /* DO_TRIGGER */ + 29, /* DO_STATSEXT */ + 30, /* DO_RULE */ + 31, /* DO_TRIGGER */ 27, /* DO_CONSTRAINT */ - 31, /* DO_FK_CONSTRAINT */ + 32, /* DO_FK_CONSTRAINT */ 2, /* DO_PROCLANG */ 10, /* DO_CAST */ 23, /* DO_TABLE_DATA */ @@ -1291,6 +1292,11 @@ describeDumpableObject(DumpableObject *obj, char *buf, int bufsize) "INDEX %s (ID %d OID %u)", obj->name, obj->dumpId, obj->catId.oid); return; + case DO_STATSEXT: + snprintf(buf, bufsize, + "STATISTICS %s (ID %d OID %u)", + obj->name, obj->dumpId, obj->catId.oid); + return; case DO_REFRESH_MATVIEW: snprintf(buf, bufsize, "REFRESH MATERIALIZED VIEW %s (ID %d OID %u)", diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index 8c583127fd..3cf1742020 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -2320,6 +2320,57 @@ describeOneTableDetails(const char *schemaname, PQclear(result); } + /* print any extended statistics */ + if (pset.sversion >= 100000) + { + printfPQExpBuffer(&buf, + "SELECT oid, stanamespace::regnamespace AS nsp, staname, stakeys,\n" + " (SELECT pg_catalog.string_agg(pg_catalog.quote_ident(attname::text),', ') \n" + " FROM ((SELECT pg_catalog.unnest(stakeys) AS attnum) s\n" + " JOIN pg_catalog.pg_attribute a ON (starelid = a.attrelid AND\n" + "a.attnum = s.attnum AND not attisdropped))) AS columns,\n" + " (staenabled::char[] @> '{d}'::char[]) AS ndist_enabled\n" + "FROM pg_catalog.pg_statistic_ext stat WHERE starelid = '%s'\n" + "ORDER BY 1;", + oid); + + result = PSQLexec(buf.data); + if (!result) + goto error_return; + else + tuples = PQntuples(result); + + if (tuples > 0) + { + printTableAddFooter(&cont, _("Statistics:")); + + for (i = 0; i < tuples; i++) + { + int cnt = 0; + + printfPQExpBuffer(&buf, " "); + + /* statistics name (qualified with namespace) */ + appendPQExpBuffer(&buf, "\"%s.%s\" WITH (", + PQgetvalue(result, i, 1), + PQgetvalue(result, i, 2)); + + /* options */ + if (strcmp(PQgetvalue(result, i, 5), "t") == 0) + { + appendPQExpBufferStr(&buf, "ndistinct"); + cnt++; + } + + appendPQExpBuffer(&buf, ") ON (%s)", + PQgetvalue(result, i, 4)); + + printTableAddFooter(&cont, buf.data); + } + } + PQclear(result); + } + /* print rules */ if (tableinfo.hasrules && tableinfo.relkind != RELKIND_MATVIEW) { diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index c9c9a18777..b8fa18ae2e 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201703241 +#define CATALOG_VERSION_NO 201703242 #endif diff --git a/src/include/catalog/dependency.h b/src/include/catalog/dependency.h index 10759c7c58..9effbce2f1 100644 --- a/src/include/catalog/dependency.h +++ b/src/include/catalog/dependency.h @@ -147,6 +147,7 @@ typedef enum ObjectClass OCLASS_REWRITE, /* pg_rewrite */ OCLASS_TRIGGER, /* pg_trigger */ OCLASS_SCHEMA, /* pg_namespace */ + OCLASS_STATISTIC_EXT, /* pg_statistic_ext */ OCLASS_TSPARSER, /* pg_ts_parser */ OCLASS_TSDICT, /* pg_ts_dict */ OCLASS_TSTEMPLATE, /* pg_ts_template */ diff --git a/src/include/catalog/heap.h b/src/include/catalog/heap.h index 1187797fd9..473fe177ba 100644 --- a/src/include/catalog/heap.h +++ b/src/include/catalog/heap.h @@ -119,6 +119,7 @@ extern void RemoveAttrDefault(Oid relid, AttrNumber attnum, DropBehavior behavior, bool complain, bool internal); extern void RemoveAttrDefaultById(Oid attrdefId); extern void RemoveStatistics(Oid relid, AttrNumber attnum); +extern void RemoveStatisticsExt(Oid relid, AttrNumber attnum); extern Form_pg_attribute SystemAttributeDefinition(AttrNumber attno, bool relhasoids); diff --git a/src/include/catalog/indexing.h b/src/include/catalog/indexing.h index 5d4190c05e..a7266860ce 100644 --- a/src/include/catalog/indexing.h +++ b/src/include/catalog/indexing.h @@ -182,6 +182,13 @@ DECLARE_UNIQUE_INDEX(pg_largeobject_loid_pn_index, 2683, on pg_largeobject using DECLARE_UNIQUE_INDEX(pg_largeobject_metadata_oid_index, 2996, on pg_largeobject_metadata using btree(oid oid_ops)); #define LargeObjectMetadataOidIndexId 2996 +DECLARE_UNIQUE_INDEX(pg_statistic_ext_oid_index, 3380, on pg_statistic_ext using btree(oid oid_ops)); +#define StatisticExtOidIndexId 3380 +DECLARE_UNIQUE_INDEX(pg_statistic_ext_name_index, 3997, on pg_statistic_ext using btree(staname name_ops, stanamespace oid_ops)); +#define StatisticExtNameIndexId 3997 +DECLARE_INDEX(pg_statistic_ext_relid_index, 3379, on pg_statistic_ext using btree(starelid oid_ops)); +#define StatisticExtRelidIndexId 3379 + DECLARE_UNIQUE_INDEX(pg_namespace_nspname_index, 2684, on pg_namespace using btree(nspname name_ops)); #define NamespaceNameIndexId 2684 DECLARE_UNIQUE_INDEX(pg_namespace_oid_index, 2685, on pg_namespace using btree(oid oid_ops)); diff --git a/src/include/catalog/namespace.h b/src/include/catalog/namespace.h index dbeb25b1ac..35e0e2b089 100644 --- a/src/include/catalog/namespace.h +++ b/src/include/catalog/namespace.h @@ -141,6 +141,8 @@ extern Oid get_collation_oid(List *collname, bool missing_ok); extern Oid get_conversion_oid(List *conname, bool missing_ok); extern Oid FindDefaultConversionProc(int32 for_encoding, int32 to_encoding); +extern Oid get_statistics_oid(List *names, bool missing_ok); + /* initialization & transaction cleanup code */ extern void InitializeSearchPath(void); extern void AtEOXact_Namespace(bool isCommit, bool parallel); diff --git a/src/include/catalog/pg_cast.h b/src/include/catalog/pg_cast.h index ce8dc59e5a..bc5d28a4fa 100644 --- a/src/include/catalog/pg_cast.h +++ b/src/include/catalog/pg_cast.h @@ -254,6 +254,10 @@ DATA(insert ( 23 18 78 e f )); /* pg_node_tree can be coerced to, but not from, text */ DATA(insert ( 194 25 0 i b )); +/* pg_ndistinct can be coerced to, but not from, bytea and text */ +DATA(insert ( 3361 17 0 i b )); +DATA(insert ( 3361 25 0 i i )); + /* * Datetime category */ diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index a66d045100..ee67459c32 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -1983,6 +1983,8 @@ DESCR("select statement of a view"); DATA(insert OID = 1642 ( pg_get_userbyid PGNSP PGUID 12 1 0 0 0 f f f f t f s s 1 0 19 "26" _null_ _null_ _null_ _null_ _null_ pg_get_userbyid _null_ _null_ _null_ )); DESCR("role name by OID (with fallback)"); DATA(insert OID = 1643 ( pg_get_indexdef PGNSP PGUID 12 1 0 0 0 f f f f t f s s 1 0 25 "26" _null_ _null_ _null_ _null_ _null_ pg_get_indexdef _null_ _null_ _null_ )); +DESCR("extended statistics description"); +DATA(insert OID = 3415 ( pg_get_statisticsextdef PGNSP PGUID 12 1 0 0 0 f f f f t f s s 1 0 25 "26" _null_ _null_ _null_ _null_ _null_ pg_get_statisticsextdef _null_ _null_ _null_ )); DESCR("index description"); DATA(insert OID = 3352 ( pg_get_partkeydef PGNSP PGUID 12 1 0 0 0 f f f f t f s s 1 0 25 "26" _null_ _null_ _null_ _null_ _null_ pg_get_partkeydef _null_ _null_ _null_ )); DESCR("partition key description"); @@ -2758,6 +2760,15 @@ DESCR("current user privilege on any column by rel name"); DATA(insert OID = 3029 ( has_any_column_privilege PGNSP PGUID 12 10 0 0 0 f f f f t f s s 2 0 16 "26 25" _null_ _null_ _null_ _null_ _null_ has_any_column_privilege_id _null_ _null_ _null_ )); DESCR("current user privilege on any column by rel oid"); +DATA(insert OID = 3355 ( pg_ndistinct_in PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 3361 "2275" _null_ _null_ _null_ _null_ _null_ pg_ndistinct_in _null_ _null_ _null_ )); +DESCR("I/O"); +DATA(insert OID = 3356 ( pg_ndistinct_out PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 2275 "3361" _null_ _null_ _null_ _null_ _null_ pg_ndistinct_out _null_ _null_ _null_ )); +DESCR("I/O"); +DATA(insert OID = 3357 ( pg_ndistinct_recv PGNSP PGUID 12 1 0 0 0 f f f f t f s s 1 0 3361 "2281" _null_ _null_ _null_ _null_ _null_ pg_ndistinct_recv _null_ _null_ _null_ )); +DESCR("I/O"); +DATA(insert OID = 3358 ( pg_ndistinct_send PGNSP PGUID 12 1 0 0 0 f f f f t f s s 1 0 17 "3361" _null_ _null_ _null_ _null_ _null_ pg_ndistinct_send _null_ _null_ _null_ )); +DESCR("I/O"); + DATA(insert OID = 1928 ( pg_stat_get_numscans PGNSP PGUID 12 1 0 0 0 f f f f t f s r 1 0 20 "26" _null_ _null_ _null_ _null_ _null_ pg_stat_get_numscans _null_ _null_ _null_ )); DESCR("statistics: number of scans done for table/index"); DATA(insert OID = 1929 ( pg_stat_get_tuples_returned PGNSP PGUID 12 1 0 0 0 f f f f t f s r 1 0 20 "26" _null_ _null_ _null_ _null_ _null_ pg_stat_get_tuples_returned _null_ _null_ _null_ )); diff --git a/src/include/catalog/pg_statistic_ext.h b/src/include/catalog/pg_statistic_ext.h new file mode 100644 index 0000000000..5f67fe7fc8 --- /dev/null +++ b/src/include/catalog/pg_statistic_ext.h @@ -0,0 +1,75 @@ +/*------------------------------------------------------------------------- + * + * pg_statistic_ext.h + * definition of the system "extended statistic" relation (pg_statistic_ext) + * along with the relation's initial contents. + * + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/catalog/pg_statistic_ext.h + * + * NOTES + * the genbki.pl script reads this file and generates .bki + * information from the DATA() statements. + * + *------------------------------------------------------------------------- + */ +#ifndef PG_STATISTIC_EXT_H +#define PG_STATISTIC_EXT_H + +#include "catalog/genbki.h" + +/* ---------------- + * pg_statistic_ext definition. cpp turns this into + * typedef struct FormData_pg_statistic_ext + * ---------------- + */ +#define StatisticExtRelationId 3381 + +CATALOG(pg_statistic_ext,3381) +{ + /* These fields form the unique key for the entry: */ + Oid starelid; /* relation containing attributes */ + NameData staname; /* statistics name */ + Oid stanamespace; /* OID of namespace containing this statistics */ + Oid staowner; /* statistics owner */ + + /* + * variable-length fields start here, but we allow direct access to + * stakeys + */ + int2vector stakeys; /* array of column keys */ + +#ifdef CATALOG_VARLEN + char staenabled[1] BKI_FORCE_NOT_NULL; /* statistic types + * requested to build */ + pg_ndistinct standistinct; /* ndistinct coefficients (serialized) */ +#endif + +} FormData_pg_statistic_ext; + +/* ---------------- + * Form_pg_statistic_ext corresponds to a pointer to a tuple with + * the format of pg_statistic_ext relation. + * ---------------- + */ +typedef FormData_pg_statistic_ext *Form_pg_statistic_ext; + +/* ---------------- + * compiler constants for pg_statistic_ext + * ---------------- + */ +#define Natts_pg_statistic_ext 7 +#define Anum_pg_statistic_ext_starelid 1 +#define Anum_pg_statistic_ext_staname 2 +#define Anum_pg_statistic_ext_stanamespace 3 +#define Anum_pg_statistic_ext_staowner 4 +#define Anum_pg_statistic_ext_stakeys 5 +#define Anum_pg_statistic_ext_staenabled 6 +#define Anum_pg_statistic_ext_standistinct 7 + +#define STATS_EXT_NDISTINCT 'd' + +#endif /* PG_STATISTIC_EXT_H */ diff --git a/src/include/catalog/pg_type.h b/src/include/catalog/pg_type.h index 9f61238179..9ad67258fe 100644 --- a/src/include/catalog/pg_type.h +++ b/src/include/catalog/pg_type.h @@ -364,6 +364,10 @@ DATA(insert OID = 194 ( pg_node_tree PGNSP PGUID -1 f b S f t \054 0 0 0 pg_node DESCR("string representing an internal node tree"); #define PGNODETREEOID 194 +DATA(insert OID = 3361 ( pg_ndistinct PGNSP PGUID -1 f b S f t \054 0 0 0 pg_ndistinct_in pg_ndistinct_out pg_ndistinct_recv pg_ndistinct_send - - - i x f 0 -1 0 100 _null_ _null_ _null_ )); +DESCR("multivariate ndistinct coefficients"); +#define PGNDISTINCTOID 3361 + DATA(insert OID = 32 ( pg_ddl_command PGNSP PGUID SIZEOF_POINTER t p P f t \054 0 0 0 pg_ddl_command_in pg_ddl_command_out pg_ddl_command_recv pg_ddl_command_send - - - ALIGNOF_POINTER p f 0 -1 0 0 _null_ _null_ _null_ )); DESCR("internal type for passing CollectedCommand"); #define PGDDLCOMMANDOID 32 diff --git a/src/include/catalog/toasting.h b/src/include/catalog/toasting.h index db7f145b5f..00d0a8326f 100644 --- a/src/include/catalog/toasting.h +++ b/src/include/catalog/toasting.h @@ -53,6 +53,7 @@ DECLARE_TOAST(pg_proc, 2836, 2837); DECLARE_TOAST(pg_rewrite, 2838, 2839); DECLARE_TOAST(pg_seclabel, 3598, 3599); DECLARE_TOAST(pg_statistic, 2840, 2841); +DECLARE_TOAST(pg_statistic_ext, 3439, 3440); DECLARE_TOAST(pg_trigger, 2336, 2337); /* shared catalogs */ diff --git a/src/include/commands/defrem.h b/src/include/commands/defrem.h index 8740cee944..c323e81e6c 100644 --- a/src/include/commands/defrem.h +++ b/src/include/commands/defrem.h @@ -77,6 +77,10 @@ extern ObjectAddress DefineOperator(List *names, List *parameters); extern void RemoveOperatorById(Oid operOid); extern ObjectAddress AlterOperator(AlterOperatorStmt *stmt); +/* commands/statscmds.c */ +extern ObjectAddress CreateStatistics(CreateStatsStmt *stmt); +extern void RemoveStatisticsById(Oid statsOid); + /* commands/aggregatecmds.c */ extern ObjectAddress DefineAggregate(ParseState *pstate, List *name, List *args, bool oldstyle, List *parameters); diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index b2d8514f89..fc883a6f3e 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -279,6 +279,7 @@ typedef enum NodeTag T_PlaceHolderInfo, T_MinMaxAggInfo, T_PlannerParamItem, + T_StatisticExtInfo, /* * TAGS FOR MEMORY NODES (memnodes.h) @@ -424,6 +425,7 @@ typedef enum NodeTag T_CreateSubscriptionStmt, T_AlterSubscriptionStmt, T_DropSubscriptionStmt, + T_CreateStatsStmt, T_AlterCollationStmt, /* diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index f3773ca929..3a71dd5b37 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -1593,6 +1593,7 @@ typedef enum ObjectType OBJECT_SCHEMA, OBJECT_SEQUENCE, OBJECT_SUBSCRIPTION, + OBJECT_STATISTIC_EXT, OBJECT_TABCONSTRAINT, OBJECT_TABLE, OBJECT_TABLESPACE, @@ -2656,6 +2657,20 @@ typedef struct IndexStmt bool if_not_exists; /* just do nothing if index already exists? */ } IndexStmt; +/* ---------------------- + * Create Statistics Statement + * ---------------------- + */ +typedef struct CreateStatsStmt +{ + NodeTag type; + List *defnames; /* qualified name (list of Value strings) */ + RangeVar *relation; /* relation to build statistics on */ + List *keys; /* String nodes naming referenced columns */ + List *options; /* list of DefElem */ + bool if_not_exists; /* do nothing if statistics already exists */ +} CreateStatsStmt; + /* ---------------------- * Create Function Statement * ---------------------- diff --git a/src/include/nodes/relation.h b/src/include/nodes/relation.h index 1c88a79a21..0a5187cef3 100644 --- a/src/include/nodes/relation.h +++ b/src/include/nodes/relation.h @@ -529,6 +529,7 @@ typedef struct RelOptInfo List *lateral_vars; /* LATERAL Vars and PHVs referenced by rel */ Relids lateral_referencers; /* rels that reference me laterally */ List *indexlist; /* list of IndexOptInfo */ + List *statlist; /* list of StatisticExtInfo */ BlockNumber pages; /* size estimates derived from pg_class */ double tuples; double allvisfrac; @@ -668,6 +669,24 @@ typedef struct ForeignKeyOptInfo List *rinfos[INDEX_MAX_KEYS]; } ForeignKeyOptInfo; +/* + * StatisticExtInfo + * Information about extended statistics for planning/optimization + * + * This contains information about which columns are covered by the + * statistics (stakeys), which options were requested while adding the + * statistics (*_enabled), and which kinds of statistics were actually + * built and are available for the optimizer (*_built). + */ +typedef struct StatisticExtInfo +{ + NodeTag type; + + Oid statOid; /* OID of the statistics row */ + RelOptInfo *rel; /* back-link to index's table */ + char kind; /* statistic kind of this entry */ + Bitmapset *keys; /* attnums of the columns covered */ +} StatisticExtInfo; /* * EquivalenceClasses diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h new file mode 100644 index 0000000000..961f1f78c0 --- /dev/null +++ b/src/include/statistics/extended_stats_internal.h @@ -0,0 +1,64 @@ +/*------------------------------------------------------------------------- + * + * extended_stats_internal.h + * POSTGRES extended statistics internal declarations + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/include/statistics/extended_stats_internal.h + * + *------------------------------------------------------------------------- + */ +#ifndef EXTENDED_STATS_INTERNAL_H +#define EXTENDED_STATS_INTERNAL_H + +#include "utils/sortsupport.h" +#include "statistics/statistics.h" + + +typedef struct +{ + Oid eqopr; /* '=' operator for datatype, if any */ + Oid eqfunc; /* and associated function */ + Oid ltopr; /* '<' operator for datatype, if any */ +} StdAnalyzeData; + +typedef struct +{ + Datum value; /* a data value */ + int tupno; /* position index for tuple it came from */ +} ScalarItem; + +/* multi-sort */ +typedef struct MultiSortSupportData +{ + int ndims; /* number of dimensions supported by the */ + SortSupportData ssup[1]; /* sort support data for each dimension */ +} MultiSortSupportData; + +typedef MultiSortSupportData *MultiSortSupport; + +typedef struct SortItem +{ + Datum *values; + bool *isnull; +} SortItem; + +extern MVNDistinct *statext_ndistinct_build(double totalrows, + int numrows, HeapTuple *rows, + Bitmapset *attrs, VacAttrStats **stats); +extern bytea *statext_ndistinct_serialize(MVNDistinct *ndistinct); +extern MVNDistinct *statext_ndistinct_deserialize(bytea *data); + +extern MultiSortSupport multi_sort_init(int ndims); +extern void multi_sort_add_dimension(MultiSortSupport mss, int sortdim, + Oid oper); +extern int multi_sort_compare(const void *a, const void *b, void *arg); +extern int multi_sort_compare_dim(int dim, const SortItem * a, + const SortItem * b, MultiSortSupport mss); +extern int multi_sort_compare_dims(int start, int end, const SortItem * a, + const SortItem * b, MultiSortSupport mss); + +#endif /* EXTENDED_STATS_INTERNAL_H */ diff --git a/src/include/statistics/statistics.h b/src/include/statistics/statistics.h new file mode 100644 index 0000000000..a15e39e1a3 --- /dev/null +++ b/src/include/statistics/statistics.h @@ -0,0 +1,47 @@ +/*------------------------------------------------------------------------- + * + * statistics.h + * Extended statistics and selectivity estimation functions. + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/statistics/statistics.h + * + *------------------------------------------------------------------------- + */ +#ifndef STATISTICS_H +#define STATISTICS_H + +#include "commands/vacuum.h" + +#define STATS_MAX_DIMENSIONS 8 /* max number of attributes */ + +/* Multivariate distinct coefficients */ +#define STATS_NDISTINCT_MAGIC 0xA352BFA4 /* struct identifier */ +#define STATS_NDISTINCT_TYPE_BASIC 1 /* struct version */ + +/* MVDistinctItem represents a single combination of columns */ +typedef struct MVNDistinctItem +{ + double ndistinct; /* ndistinct value for this combination */ + Bitmapset *attrs; /* attr numbers of items */ +} MVNDistinctItem; + +/* A MVNDistinct object, comprising all possible combinations of columns */ +typedef struct MVNDistinct +{ + uint32 magic; /* magic constant marker */ + uint32 type; /* type of ndistinct (BASIC) */ + uint32 nitems; /* number of items in the statistic */ + MVNDistinctItem items[FLEXIBLE_ARRAY_MEMBER]; +} MVNDistinct; + +extern MVNDistinct *statext_ndistinct_load(Oid mvoid); + +extern void BuildRelationExtStatistics(Relation onerel, double totalrows, + int numrows, HeapTuple *rows, + int natts, VacAttrStats **vacattrstats); +extern bool statext_is_kind_built(HeapTuple htup, char kind); + +#endif /* STATISTICS_H */ diff --git a/src/include/utils/acl.h b/src/include/utils/acl.h index 0d118525c9..c957d8e170 100644 --- a/src/include/utils/acl.h +++ b/src/include/utils/acl.h @@ -192,6 +192,7 @@ typedef enum AclObjectKind ACL_KIND_OPFAMILY, /* pg_opfamily */ ACL_KIND_COLLATION, /* pg_collation */ ACL_KIND_CONVERSION, /* pg_conversion */ + ACL_KIND_STATISTICS, /* pg_statistic_ext */ ACL_KIND_TABLESPACE, /* pg_tablespace */ ACL_KIND_TSDICTIONARY, /* pg_ts_dict */ ACL_KIND_TSCONFIGURATION, /* pg_ts_config */ @@ -326,6 +327,7 @@ extern bool pg_event_trigger_ownercheck(Oid et_oid, Oid roleid); extern bool pg_extension_ownercheck(Oid ext_oid, Oid roleid); extern bool pg_publication_ownercheck(Oid pub_oid, Oid roleid); extern bool pg_subscription_ownercheck(Oid sub_oid, Oid roleid); +extern bool pg_statistics_ownercheck(Oid stat_oid, Oid roleid); extern bool has_createrole_privilege(Oid roleid); extern bool has_bypassrls_privilege(Oid roleid); diff --git a/src/include/utils/rel.h b/src/include/utils/rel.h index a617a7cf56..ab875bb9d7 100644 --- a/src/include/utils/rel.h +++ b/src/include/utils/rel.h @@ -92,6 +92,7 @@ typedef struct RelationData bool rd_isvalid; /* relcache entry is valid */ char rd_indexvalid; /* state of rd_indexlist: 0 = not valid, 1 = * valid, 2 = temporarily forced */ + bool rd_statvalid; /* is rd_statlist valid? */ /* * rd_createSubid is the ID of the highest subtransaction the rel has @@ -136,6 +137,9 @@ typedef struct RelationData Oid rd_pkindex; /* OID of primary key, if any */ Oid rd_replidindex; /* OID of replica identity index, if any */ + /* data managed by RelationGetStatExtList: */ + List *rd_statlist; /* list of OIDs of extended stats */ + /* data managed by RelationGetIndexAttrBitmap: */ Bitmapset *rd_indexattr; /* identifies columns used in indexes */ Bitmapset *rd_keyattr; /* cols that can be ref'd by foreign keys */ diff --git a/src/include/utils/relcache.h b/src/include/utils/relcache.h index da36b6774f..81af3aebb8 100644 --- a/src/include/utils/relcache.h +++ b/src/include/utils/relcache.h @@ -39,6 +39,7 @@ extern void RelationClose(Relation relation); */ extern List *RelationGetFKeyList(Relation relation); extern List *RelationGetIndexList(Relation relation); +extern List *RelationGetStatExtList(Relation relation); extern Oid RelationGetOidIndex(Relation relation); extern Oid RelationGetPrimaryKeyIndex(Relation relation); extern Oid RelationGetReplicaIndex(Relation relation); diff --git a/src/include/utils/syscache.h b/src/include/utils/syscache.h index b35faf81b9..36805ebefb 100644 --- a/src/include/utils/syscache.h +++ b/src/include/utils/syscache.h @@ -86,6 +86,8 @@ enum SysCacheIdentifier PUBLICATIONRELMAP, RULERELNAME, SEQRELID, + STATEXTNAMENSP, + STATEXTOID, STATRELATTINH, SUBSCRIPTIONOID, SUBSCRIPTIONNAME, diff --git a/src/test/regress/expected/alter_generic.out b/src/test/regress/expected/alter_generic.out index b01be59bbb..ce581bb93d 100644 --- a/src/test/regress/expected/alter_generic.out +++ b/src/test/regress/expected/alter_generic.out @@ -496,6 +496,48 @@ ALTER OPERATOR FAMILY alt_opf18 USING btree ADD ALTER OPERATOR FAMILY alt_opf18 USING btree DROP FUNCTION 2 (int4, int4); ERROR: function 2(integer,integer) does not exist in operator family "alt_opf18" DROP OPERATOR FAMILY alt_opf18 USING btree; +-- +-- Statistics +-- +SET SESSION AUTHORIZATION regress_alter_user1; +CREATE TABLE alt_regress_1 (a INTEGER, b INTEGER); +CREATE STATISTICS alt_stat1 ON (a, b) FROM alt_regress_1; +CREATE STATISTICS alt_stat2 ON (a, b) FROM alt_regress_1; +ALTER STATISTICS alt_stat1 RENAME TO alt_stat2; -- failed (name conflict) +ERROR: statistics "alt_stat2" already exists in schema "alt_nsp1" +ALTER STATISTICS alt_stat1 RENAME TO alt_stat3; -- failed (name conflict) +ALTER STATISTICS alt_stat2 OWNER TO regress_alter_user2; -- failed (no role membership) +ERROR: must be member of role "regress_alter_user2" +ALTER STATISTICS alt_stat2 OWNER TO regress_alter_user3; -- OK +ALTER STATISTICS alt_stat2 SET SCHEMA alt_nsp2; -- OK +SET SESSION AUTHORIZATION regress_alter_user2; +CREATE STATISTICS alt_stat1 ON (a, b) FROM alt_regress_1; +CREATE STATISTICS alt_stat2 ON (a, b) FROM alt_regress_1; +ALTER STATISTICS alt_stat3 RENAME TO alt_stat4; -- failed (not owner) +ERROR: must be owner of statistics alt_stat3 +ALTER STATISTICS alt_stat1 RENAME TO alt_stat4; -- OK +ALTER STATISTICS alt_stat3 OWNER TO regress_alter_user2; -- failed (not owner) +ERROR: must be owner of statistics alt_stat3 +ALTER STATISTICS alt_stat2 OWNER TO regress_alter_user3; -- failed (no role membership) +ERROR: must be member of role "regress_alter_user3" +ALTER STATISTICS alt_stat3 SET SCHEMA alt_nsp2; -- failed (not owner) +ERROR: must be owner of statistics alt_stat3 +ALTER STATISTICS alt_stat2 SET SCHEMA alt_nsp2; -- failed (name conflict) +ERROR: statistics "alt_stat2" already exists in schema "alt_nsp2" +RESET SESSION AUTHORIZATION; +SELECT nspname, staname, rolname + FROM pg_statistic_ext s, pg_namespace n, pg_authid a + WHERE s.stanamespace = n.oid AND s.staowner = a.oid + AND n.nspname in ('alt_nsp1', 'alt_nsp2') + ORDER BY nspname, staname; + nspname | staname | rolname +----------+-----------+--------------------- + alt_nsp1 | alt_stat2 | regress_alter_user2 + alt_nsp1 | alt_stat3 | regress_alter_user1 + alt_nsp1 | alt_stat4 | regress_alter_user2 + alt_nsp2 | alt_stat2 | regress_alter_user3 +(4 rows) + -- -- Text Search Dictionary -- @@ -639,7 +681,7 @@ DROP LANGUAGE alt_lang3 CASCADE; DROP LANGUAGE alt_lang4 CASCADE; ERROR: language "alt_lang4" does not exist DROP SCHEMA alt_nsp1 CASCADE; -NOTICE: drop cascades to 26 other objects +NOTICE: drop cascades to 27 other objects DETAIL: drop cascades to function alt_func3(integer) drop cascades to function alt_agg3(integer) drop cascades to function alt_func4(integer) @@ -656,6 +698,7 @@ drop cascades to operator family alt_opc1 for access method hash drop cascades to operator family alt_opc2 for access method hash drop cascades to operator family alt_opf4 for access method hash drop cascades to operator family alt_opf2 for access method hash +drop cascades to table alt_regress_1 drop cascades to text search dictionary alt_ts_dict3 drop cascades to text search dictionary alt_ts_dict4 drop cascades to text search dictionary alt_ts_dict2 diff --git a/src/test/regress/expected/object_address.out b/src/test/regress/expected/object_address.out index 978d9a9a0f..814e05e4ef 100644 --- a/src/test/regress/expected/object_address.out +++ b/src/test/regress/expected/object_address.out @@ -39,6 +39,7 @@ CREATE TRANSFORM FOR int LANGUAGE SQL ( CREATE PUBLICATION addr_pub FOR TABLE addr_nsp.gentable; CREATE SUBSCRIPTION addr_sub CONNECTION '' PUBLICATION bar WITH (DISABLED, NOCONNECT); WARNING: tables were not subscribed, you will have to run ALTER SUBSCRIPTION ... REFRESH PUBLICATION to subscribe the tables +CREATE STATISTICS addr_nsp.gentable_stat ON (a,b) FROM addr_nsp.gentable; -- test some error cases SELECT pg_get_object_address('stone', '{}', '{}'); ERROR: unrecognized object type "stone" @@ -409,7 +410,8 @@ WITH objects (type, name, args) AS (VALUES ('access method', '{btree}', '{}'), ('publication', '{addr_pub}', '{}'), ('publication relation', '{addr_nsp, gentable}', '{addr_pub}'), - ('subscription', '{addr_sub}', '{}') + ('subscription', '{addr_sub}', '{}'), + ('statistics', '{addr_nsp, gentable_stat}', '{}') ) SELECT (pg_identify_object(addr1.classid, addr1.objid, addr1.objsubid)).*, -- test roundtrip through pg_identify_object_as_address @@ -457,6 +459,7 @@ SELECT (pg_identify_object(addr1.classid, addr1.objid, addr1.objsubid)).*, trigger | | | t on addr_nsp.gentable | t operator family | pg_catalog | integer_ops | pg_catalog.integer_ops USING btree | t policy | | | genpol on addr_nsp.gentable | t + statistics | addr_nsp | gentable_stat | addr_nsp.gentable_stat | t collation | pg_catalog | "default" | pg_catalog."default" | t transform | | | for integer on language sql | t text search dictionary | addr_nsp | addr_ts_dict | addr_nsp.addr_ts_dict | t @@ -466,7 +469,7 @@ SELECT (pg_identify_object(addr1.classid, addr1.objid, addr1.objsubid)).*, subscription | | addr_sub | addr_sub | t publication | | addr_pub | addr_pub | t publication relation | | | gentable in publication addr_pub | t -(45 rows) +(46 rows) --- --- Cleanup resources diff --git a/src/test/regress/expected/opr_sanity.out b/src/test/regress/expected/opr_sanity.out index 64d9dd605f..262036ac4f 100644 --- a/src/test/regress/expected/opr_sanity.out +++ b/src/test/regress/expected/opr_sanity.out @@ -823,11 +823,12 @@ WHERE c.castmethod = 'b' AND text | character | 0 | i character varying | character | 0 | i pg_node_tree | text | 0 | i + pg_ndistinct | bytea | 0 | i cidr | inet | 0 | i xml | text | 0 | a xml | character varying | 0 | a xml | character | 0 | a -(7 rows) +(8 rows) -- **************** pg_conversion **************** -- Look for illegal values in pg_conversion fields. diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index c4c8450b83..7f04c7a7cc 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2164,6 +2164,14 @@ pg_stats| SELECT n.nspname AS schemaname, JOIN pg_attribute a ON (((c.oid = a.attrelid) AND (a.attnum = s.staattnum)))) LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace))) WHERE ((NOT a.attisdropped) AND has_column_privilege(c.oid, a.attnum, 'select'::text) AND ((c.relrowsecurity = false) OR (NOT row_security_active(c.oid)))); +pg_stats_ext| SELECT n.nspname AS schemaname, + c.relname AS tablename, + s.staname, + s.stakeys AS attnums, + length((s.standistinct)::text) AS ndistbytes + FROM ((pg_statistic_ext s + JOIN pg_class c ON ((c.oid = s.starelid))) + LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace))); pg_tables| SELECT n.nspname AS schemaname, c.relname AS tablename, pg_get_userbyid(c.relowner) AS tableowner, diff --git a/src/test/regress/expected/sanity_check.out b/src/test/regress/expected/sanity_check.out index 8e3028edaa..753ad81e43 100644 --- a/src/test/regress/expected/sanity_check.out +++ b/src/test/regress/expected/sanity_check.out @@ -142,6 +142,7 @@ pg_shdepend|t pg_shdescription|t pg_shseclabel|t pg_statistic|t +pg_statistic_ext|t pg_subscription|t pg_subscription_rel|t pg_tablespace|t diff --git a/src/test/regress/expected/stats_ext.out b/src/test/regress/expected/stats_ext.out new file mode 100644 index 0000000000..83d70bf9b9 --- /dev/null +++ b/src/test/regress/expected/stats_ext.out @@ -0,0 +1,155 @@ +-- Generic extended statistics support +-- Ensure stats are dropped sanely +CREATE TABLE ab1 (a INTEGER, b INTEGER, c INTEGER); +CREATE STATISTICS ab1_a_b_stats ON (a, b) FROM ab1; +DROP STATISTICS ab1_a_b_stats; +CREATE SCHEMA regress_schema_2; +CREATE STATISTICS regress_schema_2.ab1_a_b_stats ON (a, b) FROM ab1; +DROP STATISTICS regress_schema_2.ab1_a_b_stats; +-- Ensure statistics are dropped when columns are +CREATE STATISTICS ab1_b_c_stats ON (b, c) FROM ab1; +CREATE STATISTICS ab1_a_b_c_stats ON (a, b, c) FROM ab1; +CREATE STATISTICS ab1_a_b_stats ON (a, b) FROM ab1; +ALTER TABLE ab1 DROP COLUMN a; +\d ab1 + Table "public.ab1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + b | integer | | | + c | integer | | | +Statistics: + "public.ab1_b_c_stats" WITH (ndistinct) ON (b, c) + +DROP TABLE ab1; +-- Ensure things work sanely with SET STATISTICS 0 +CREATE TABLE ab1 (a INTEGER, b INTEGER); +ALTER TABLE ab1 ALTER a SET STATISTICS 0; +INSERT INTO ab1 SELECT a, a%23 FROM generate_series(1, 1000) a; +CREATE STATISTICS ab1_a_b_stats ON (a, b) FROM ab1; +ANALYZE ab1; +ERROR: extended statistics could not be collected for column "a" of relation public.ab1 +HINT: Consider ALTER TABLE "public"."ab1" ALTER "a" SET STATISTICS -1 +ALTER TABLE ab1 ALTER a SET STATISTICS -1; +ANALYZE ab1; +DROP TABLE ab1; +-- n-distinct tests +CREATE TABLE ndistinct ( + filler1 TEXT, + filler2 NUMERIC, + a INT, + b INT, + filler3 DATE, + c INT, + d INT +); +-- unknown column +CREATE STATISTICS s10 ON (unknown_column) FROM ndistinct; +ERROR: column "unknown_column" referenced in statistics does not exist +-- single column +CREATE STATISTICS s10 ON (a) FROM ndistinct; +ERROR: statistics require at least 2 columns +-- single column, duplicated +CREATE STATISTICS s10 ON (a,a) FROM ndistinct; +ERROR: duplicate column name in statistics definition +-- two columns, one duplicated +CREATE STATISTICS s10 ON (a, a, b) FROM ndistinct; +ERROR: duplicate column name in statistics definition +-- correct command +CREATE STATISTICS s10 ON (a, b, c) FROM ndistinct; +-- perfectly correlated groups +INSERT INTO ndistinct (a, b, c, filler1) + SELECT i/100, i/100, i/100, cash_words(i::money) + FROM generate_series(1,10000) s(i); +ANALYZE ndistinct; +SELECT staenabled, standistinct + FROM pg_statistic_ext WHERE starelid = 'ndistinct'::regclass; + staenabled | standistinct +------------+------------------------------------------------------------------------------------------------ + {d} | [{(b 3 4), 101.000000}, {(b 3 6), 101.000000}, {(b 4 6), 101.000000}, {(b 3 4 6), 101.000000}] +(1 row) + +EXPLAIN (COSTS off) + SELECT COUNT(*) FROM ndistinct GROUP BY a, b; + QUERY PLAN +----------------------------- + HashAggregate + Group Key: a, b + -> Seq Scan on ndistinct +(3 rows) + +EXPLAIN (COSTS off) + SELECT COUNT(*) FROM ndistinct GROUP BY a, b, c; + QUERY PLAN +----------------------------- + HashAggregate + Group Key: a, b, c + -> Seq Scan on ndistinct +(3 rows) + +EXPLAIN (COSTS off) + SELECT COUNT(*) FROM ndistinct GROUP BY a, b, c, d; + QUERY PLAN +----------------------------- + HashAggregate + Group Key: a, b, c, d + -> Seq Scan on ndistinct +(3 rows) + +TRUNCATE TABLE ndistinct; +-- partially correlated groups +INSERT INTO ndistinct (a, b, c) + SELECT i/50, i/100, i/200 FROM generate_series(1,10000) s(i); +ANALYZE ndistinct; +SELECT staenabled, standistinct + FROM pg_statistic_ext WHERE starelid = 'ndistinct'::regclass; + staenabled | standistinct +------------+------------------------------------------------------------------------------------------------ + {d} | [{(b 3 4), 201.000000}, {(b 3 6), 201.000000}, {(b 4 6), 101.000000}, {(b 3 4 6), 201.000000}] +(1 row) + +EXPLAIN + SELECT COUNT(*) FROM ndistinct GROUP BY a, b; + QUERY PLAN +--------------------------------------------------------------------- + HashAggregate (cost=230.00..232.01 rows=201 width=16) + Group Key: a, b + -> Seq Scan on ndistinct (cost=0.00..155.00 rows=10000 width=8) +(3 rows) + +EXPLAIN + SELECT COUNT(*) FROM ndistinct GROUP BY a, b, c; + QUERY PLAN +---------------------------------------------------------------------- + HashAggregate (cost=255.00..257.01 rows=201 width=20) + Group Key: a, b, c + -> Seq Scan on ndistinct (cost=0.00..155.00 rows=10000 width=12) +(3 rows) + +EXPLAIN + SELECT COUNT(*) FROM ndistinct GROUP BY a, b, c, d; + QUERY PLAN +---------------------------------------------------------------------- + HashAggregate (cost=280.00..290.00 rows=1000 width=24) + Group Key: a, b, c, d + -> Seq Scan on ndistinct (cost=0.00..155.00 rows=10000 width=16) +(3 rows) + +EXPLAIN + SELECT COUNT(*) FROM ndistinct GROUP BY b, c, d; + QUERY PLAN +---------------------------------------------------------------------- + HashAggregate (cost=255.00..265.00 rows=1000 width=20) + Group Key: b, c, d + -> Seq Scan on ndistinct (cost=0.00..155.00 rows=10000 width=12) +(3 rows) + +EXPLAIN + SELECT COUNT(*) FROM ndistinct GROUP BY a, d; + QUERY PLAN +--------------------------------------------------------------------- + HashAggregate (cost=230.00..240.00 rows=1000 width=16) + Group Key: a, d + -> Seq Scan on ndistinct (cost=0.00..155.00 rows=10000 width=8) +(3 rows) + +DROP TABLE ndistinct; diff --git a/src/test/regress/expected/type_sanity.out b/src/test/regress/expected/type_sanity.out index 8d75bbfab3..84022f6a29 100644 --- a/src/test/regress/expected/type_sanity.out +++ b/src/test/regress/expected/type_sanity.out @@ -59,7 +59,7 @@ WHERE (p1.typtype = 'c' AND p1.typrelid = 0) OR -- Look for types that should have an array type according to their typtype, -- but don't. We exclude composites here because we have not bothered to -- make array types corresponding to the system catalogs' rowtypes. --- NOTE: as of v10, this check finds pg_node_tree and smgr. +-- NOTE: as of v10, this check finds pg_node_tree, pg_ndistinct, smgr. SELECT p1.oid, p1.typname FROM pg_type as p1 WHERE p1.typtype not in ('c','d','p') AND p1.typname NOT LIKE E'\\_%' @@ -67,11 +67,12 @@ WHERE p1.typtype not in ('c','d','p') AND p1.typname NOT LIKE E'\\_%' (SELECT 1 FROM pg_type as p2 WHERE p2.typname = ('_' || p1.typname)::name AND p2.typelem = p1.oid and p1.typarray = p2.oid); - oid | typname ------+-------------- - 194 | pg_node_tree - 210 | smgr -(2 rows) + oid | typname +------+-------------- + 194 | pg_node_tree + 3361 | pg_ndistinct + 210 | smgr +(3 rows) -- Make sure typarray points to a varlena array type of our own base SELECT p1.oid, p1.typname as basetype, p2.typname as arraytype, diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 38743d98c3..c283bdcb37 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -89,7 +89,7 @@ test: brin gin gist spgist privileges init_privs security_label collate matview # ---------- # Another group of parallel tests # ---------- -test: alter_generic alter_operator misc psql async dbsize misc_functions sysviews tsrf tidscan +test: alter_generic alter_operator misc psql async dbsize misc_functions sysviews tsrf tidscan stats_ext # rules cannot run concurrently with any test that creates a view test: rules psql_crosstab amutils diff --git a/src/test/regress/serial_schedule b/src/test/regress/serial_schedule index d9f64c2873..3a0d536a2b 100644 --- a/src/test/regress/serial_schedule +++ b/src/test/regress/serial_schedule @@ -130,6 +130,7 @@ test: misc_functions test: sysviews test: tsrf test: tidscan +test: stats_ext test: rules test: psql_crosstab test: select_parallel diff --git a/src/test/regress/sql/alter_generic.sql b/src/test/regress/sql/alter_generic.sql index c9ea479967..f6fa8d8bfd 100644 --- a/src/test/regress/sql/alter_generic.sql +++ b/src/test/regress/sql/alter_generic.sql @@ -433,6 +433,37 @@ ALTER OPERATOR FAMILY alt_opf18 USING btree ADD ALTER OPERATOR FAMILY alt_opf18 USING btree DROP FUNCTION 2 (int4, int4); DROP OPERATOR FAMILY alt_opf18 USING btree; +-- +-- Statistics +-- +SET SESSION AUTHORIZATION regress_alter_user1; +CREATE TABLE alt_regress_1 (a INTEGER, b INTEGER); +CREATE STATISTICS alt_stat1 ON (a, b) FROM alt_regress_1; +CREATE STATISTICS alt_stat2 ON (a, b) FROM alt_regress_1; + +ALTER STATISTICS alt_stat1 RENAME TO alt_stat2; -- failed (name conflict) +ALTER STATISTICS alt_stat1 RENAME TO alt_stat3; -- failed (name conflict) +ALTER STATISTICS alt_stat2 OWNER TO regress_alter_user2; -- failed (no role membership) +ALTER STATISTICS alt_stat2 OWNER TO regress_alter_user3; -- OK +ALTER STATISTICS alt_stat2 SET SCHEMA alt_nsp2; -- OK + +SET SESSION AUTHORIZATION regress_alter_user2; +CREATE STATISTICS alt_stat1 ON (a, b) FROM alt_regress_1; +CREATE STATISTICS alt_stat2 ON (a, b) FROM alt_regress_1; + +ALTER STATISTICS alt_stat3 RENAME TO alt_stat4; -- failed (not owner) +ALTER STATISTICS alt_stat1 RENAME TO alt_stat4; -- OK +ALTER STATISTICS alt_stat3 OWNER TO regress_alter_user2; -- failed (not owner) +ALTER STATISTICS alt_stat2 OWNER TO regress_alter_user3; -- failed (no role membership) +ALTER STATISTICS alt_stat3 SET SCHEMA alt_nsp2; -- failed (not owner) +ALTER STATISTICS alt_stat2 SET SCHEMA alt_nsp2; -- failed (name conflict) + +RESET SESSION AUTHORIZATION; +SELECT nspname, staname, rolname + FROM pg_statistic_ext s, pg_namespace n, pg_authid a + WHERE s.stanamespace = n.oid AND s.staowner = a.oid + AND n.nspname in ('alt_nsp1', 'alt_nsp2') + ORDER BY nspname, staname; -- -- Text Search Dictionary diff --git a/src/test/regress/sql/object_address.sql b/src/test/regress/sql/object_address.sql index 28476daff1..c9219e47c4 100644 --- a/src/test/regress/sql/object_address.sql +++ b/src/test/regress/sql/object_address.sql @@ -41,6 +41,7 @@ CREATE TRANSFORM FOR int LANGUAGE SQL ( TO SQL WITH FUNCTION int4recv(internal)); CREATE PUBLICATION addr_pub FOR TABLE addr_nsp.gentable; CREATE SUBSCRIPTION addr_sub CONNECTION '' PUBLICATION bar WITH (DISABLED, NOCONNECT); +CREATE STATISTICS addr_nsp.gentable_stat ON (a,b) FROM addr_nsp.gentable; -- test some error cases SELECT pg_get_object_address('stone', '{}', '{}'); @@ -185,7 +186,8 @@ WITH objects (type, name, args) AS (VALUES ('access method', '{btree}', '{}'), ('publication', '{addr_pub}', '{}'), ('publication relation', '{addr_nsp, gentable}', '{addr_pub}'), - ('subscription', '{addr_sub}', '{}') + ('subscription', '{addr_sub}', '{}'), + ('statistics', '{addr_nsp, gentable_stat}', '{}') ) SELECT (pg_identify_object(addr1.classid, addr1.objid, addr1.objsubid)).*, -- test roundtrip through pg_identify_object_as_address diff --git a/src/test/regress/sql/stats_ext.sql b/src/test/regress/sql/stats_ext.sql new file mode 100644 index 0000000000..946cb84853 --- /dev/null +++ b/src/test/regress/sql/stats_ext.sql @@ -0,0 +1,102 @@ +-- Generic extended statistics support + +-- Ensure stats are dropped sanely +CREATE TABLE ab1 (a INTEGER, b INTEGER, c INTEGER); +CREATE STATISTICS ab1_a_b_stats ON (a, b) FROM ab1; +DROP STATISTICS ab1_a_b_stats; + +CREATE SCHEMA regress_schema_2; +CREATE STATISTICS regress_schema_2.ab1_a_b_stats ON (a, b) FROM ab1; +DROP STATISTICS regress_schema_2.ab1_a_b_stats; + +-- Ensure statistics are dropped when columns are +CREATE STATISTICS ab1_b_c_stats ON (b, c) FROM ab1; +CREATE STATISTICS ab1_a_b_c_stats ON (a, b, c) FROM ab1; +CREATE STATISTICS ab1_a_b_stats ON (a, b) FROM ab1; +ALTER TABLE ab1 DROP COLUMN a; +\d ab1 +DROP TABLE ab1; + +-- Ensure things work sanely with SET STATISTICS 0 +CREATE TABLE ab1 (a INTEGER, b INTEGER); +ALTER TABLE ab1 ALTER a SET STATISTICS 0; +INSERT INTO ab1 SELECT a, a%23 FROM generate_series(1, 1000) a; +CREATE STATISTICS ab1_a_b_stats ON (a, b) FROM ab1; +ANALYZE ab1; +ALTER TABLE ab1 ALTER a SET STATISTICS -1; +ANALYZE ab1; +DROP TABLE ab1; + + +-- n-distinct tests +CREATE TABLE ndistinct ( + filler1 TEXT, + filler2 NUMERIC, + a INT, + b INT, + filler3 DATE, + c INT, + d INT +); + +-- unknown column +CREATE STATISTICS s10 ON (unknown_column) FROM ndistinct; + +-- single column +CREATE STATISTICS s10 ON (a) FROM ndistinct; + +-- single column, duplicated +CREATE STATISTICS s10 ON (a,a) FROM ndistinct; + +-- two columns, one duplicated +CREATE STATISTICS s10 ON (a, a, b) FROM ndistinct; + +-- correct command +CREATE STATISTICS s10 ON (a, b, c) FROM ndistinct; + +-- perfectly correlated groups +INSERT INTO ndistinct (a, b, c, filler1) + SELECT i/100, i/100, i/100, cash_words(i::money) + FROM generate_series(1,10000) s(i); + +ANALYZE ndistinct; + +SELECT staenabled, standistinct + FROM pg_statistic_ext WHERE starelid = 'ndistinct'::regclass; + +EXPLAIN (COSTS off) + SELECT COUNT(*) FROM ndistinct GROUP BY a, b; + +EXPLAIN (COSTS off) + SELECT COUNT(*) FROM ndistinct GROUP BY a, b, c; + +EXPLAIN (COSTS off) + SELECT COUNT(*) FROM ndistinct GROUP BY a, b, c, d; + +TRUNCATE TABLE ndistinct; + +-- partially correlated groups +INSERT INTO ndistinct (a, b, c) + SELECT i/50, i/100, i/200 FROM generate_series(1,10000) s(i); + +ANALYZE ndistinct; + +SELECT staenabled, standistinct + FROM pg_statistic_ext WHERE starelid = 'ndistinct'::regclass; + +EXPLAIN + SELECT COUNT(*) FROM ndistinct GROUP BY a, b; + +EXPLAIN + SELECT COUNT(*) FROM ndistinct GROUP BY a, b, c; + +EXPLAIN + SELECT COUNT(*) FROM ndistinct GROUP BY a, b, c, d; + +EXPLAIN + SELECT COUNT(*) FROM ndistinct GROUP BY b, c, d; + +EXPLAIN + SELECT COUNT(*) FROM ndistinct GROUP BY a, d; + +DROP TABLE ndistinct; diff --git a/src/test/regress/sql/type_sanity.sql b/src/test/regress/sql/type_sanity.sql index 0a31249f5d..4c65814008 100644 --- a/src/test/regress/sql/type_sanity.sql +++ b/src/test/regress/sql/type_sanity.sql @@ -53,7 +53,7 @@ WHERE (p1.typtype = 'c' AND p1.typrelid = 0) OR -- Look for types that should have an array type according to their typtype, -- but don't. We exclude composites here because we have not bothered to -- make array types corresponding to the system catalogs' rowtypes. --- NOTE: as of v10, this check finds pg_node_tree and smgr. +-- NOTE: as of v10, this check finds pg_node_tree, pg_ndistinct, smgr. SELECT p1.oid, p1.typname FROM pg_type as p1 -- cgit v1.2.3 From 70adf2fbe18f83f34b576ee83f42ea9d28375bf0 Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Sat, 25 Mar 2017 02:39:44 +0900 Subject: Make VACUUM VERBOSE report the number of skipped frozen pages. Previously manual VACUUM did not report the number of skipped frozen pages even when VERBOSE option is specified. But this information is helpful to monitor the VACUUM activity, and also autovacuum reports that number in the log file when the condition of log_autovacuum_min_duration is met. This commit changes VACUUM VERBOSE so that it reports the number of frozen pages that it skips. Author: Masahiko Sawada Reviewed-by: Yugo Nagata and Jim Nasby Discussion: http://postgr.es/m/CAD21AoDZQKCxo0L39Mrq08cONNkXQKXuh=2DP1Q8ebmt35SoaA@mail.gmail.com --- doc/src/sgml/ref/vacuum.sgml | 2 +- src/backend/commands/vacuumlazy.c | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/vacuum.sgml b/doc/src/sgml/ref/vacuum.sgml index 543ebcf649..8ec5b40b71 100644 --- a/doc/src/sgml/ref/vacuum.sgml +++ b/doc/src/sgml/ref/vacuum.sgml @@ -273,7 +273,7 @@ DETAIL: CPU: user: 0.06 s, system: 0.01 s, elapsed: 0.07 s. INFO: "onek": found 3000 removable, 1000 nonremovable tuples in 143 pages DETAIL: 0 dead tuples cannot be removed yet. There were 0 unused item pointers. -Skipped 0 pages due to buffer pins. +Skipped 0 pages due to buffer pins, 0 frozen pages. 0 pages are entirely empty. CPU: user: 0.39 s, system: 0.07 s, elapsed: 1.56 s. INFO: analyzing "public.onek" diff --git a/src/backend/commands/vacuumlazy.c b/src/backend/commands/vacuumlazy.c index b74e4934ec..5b43a66bdc 100644 --- a/src/backend/commands/vacuumlazy.c +++ b/src/backend/commands/vacuumlazy.c @@ -1341,10 +1341,14 @@ lazy_scan_heap(Relation onerel, int options, LVRelStats *vacrelstats, nkeep, OldestXmin); appendStringInfo(&buf, _("There were %.0f unused item pointers.\n"), nunused); - appendStringInfo(&buf, ngettext("Skipped %u page due to buffer pins.\n", - "Skipped %u pages due to buffer pins.\n", + appendStringInfo(&buf, ngettext("Skipped %u page due to buffer pins, ", + "Skipped %u pages due to buffer pins, ", vacrelstats->pinskipped_pages), vacrelstats->pinskipped_pages); + appendStringInfo(&buf, ngettext("%u frozen page.\n", + "%u frozen pages.\n", + vacrelstats->frozenskipped_pages), + vacrelstats->frozenskipped_pages); appendStringInfo(&buf, ngettext("%u page is entirely empty.\n", "%u pages are entirely empty.\n", empty_pages), -- cgit v1.2.3 From 87dee41f3ed6d6c2a93e7ff359776cfe24f145e0 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 24 Mar 2017 23:25:24 -0400 Subject: Add COMMENT and SECURITY LABEL support for publications and subscriptions --- doc/src/sgml/ref/comment.sgml | 2 ++ doc/src/sgml/ref/security_label.sgml | 2 ++ src/backend/catalog/system_views.sql | 22 +++++++++++++++ src/backend/parser/gram.y | 4 +++ .../dummy_seclabel/expected/dummy_seclabel.out | 32 ++++++++++++++-------- .../modules/dummy_seclabel/sql/dummy_seclabel.sql | 7 +++++ src/test/regress/expected/publication.out | 7 +++++ src/test/regress/expected/rules.out | 23 ++++++++++++++++ src/test/regress/expected/subscription.out | 7 +++++ src/test/regress/sql/publication.sql | 3 ++ src/test/regress/sql/subscription.sql | 3 ++ 11 files changed, 100 insertions(+), 12 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/comment.sgml b/doc/src/sgml/ref/comment.sgml index 8fe17a5767..df328117f1 100644 --- a/doc/src/sgml/ref/comment.sgml +++ b/doc/src/sgml/ref/comment.sgml @@ -46,12 +46,14 @@ COMMENT ON OPERATOR FAMILY object_name USING index_method | POLICY policy_name ON table_name | [ PROCEDURAL ] LANGUAGE object_name | + PUBLICATION object_name | ROLE object_name | RULE rule_name ON table_name | SCHEMA object_name | SEQUENCE object_name | SERVER object_name | STATISTICS object_name | + SUBSCRIPTION object_name | TABLE object_name | TABLESPACE object_name | TEXT SEARCH CONFIGURATION object_name | diff --git a/doc/src/sgml/ref/security_label.sgml b/doc/src/sgml/ref/security_label.sgml index afd86aff3a..aa8be473bd 100644 --- a/doc/src/sgml/ref/security_label.sgml +++ b/doc/src/sgml/ref/security_label.sgml @@ -34,9 +34,11 @@ SECURITY LABEL [ FOR provider ] ON LARGE OBJECT large_object_oid | MATERIALIZED VIEW object_name | [ PROCEDURAL ] LANGUAGE object_name | + PUBLICATION object_name | ROLE object_name | SCHEMA object_name | SEQUENCE object_name | + SUBSCRIPTION object_name | TABLESPACE object_name | TYPE object_name | VIEW object_name diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index b41882aa52..d8b762ee3f 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -424,6 +424,28 @@ FROM WHERE l.objsubid = 0 UNION ALL +SELECT + l.objoid, l.classoid, l.objsubid, + 'publication'::text AS objtype, + NULL::oid AS objnamespace, + quote_ident(p.pubname) AS objname, + l.provider, l.label +FROM + pg_seclabel l + JOIN pg_publication p ON l.classoid = p.tableoid AND l.objoid = p.oid +WHERE + l.objsubid = 0 +UNION ALL +SELECT + l.objoid, l.classoid, 0::int4 AS objsubid, + 'subscription'::text AS objtype, + NULL::oid AS objnamespace, + quote_ident(s.subname) AS objname, + l.provider, l.label +FROM + pg_shseclabel l + JOIN pg_subscription s ON l.classoid = s.tableoid AND l.objoid = s.oid +UNION ALL SELECT l.objoid, l.classoid, 0::int4 AS objsubid, 'database'::text AS objtype, diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index bbcfc1fb4f..19dd77d787 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -6340,9 +6340,11 @@ comment_type_name: | EXTENSION { $$ = OBJECT_EXTENSION; } | FOREIGN DATA_P WRAPPER { $$ = OBJECT_FDW; } | opt_procedural LANGUAGE { $$ = OBJECT_LANGUAGE; } + | PUBLICATION { $$ = OBJECT_PUBLICATION; } | ROLE { $$ = OBJECT_ROLE; } | SCHEMA { $$ = OBJECT_SCHEMA; } | SERVER { $$ = OBJECT_FOREIGN_SERVER; } + | SUBSCRIPTION { $$ = OBJECT_SUBSCRIPTION; } | TABLESPACE { $$ = OBJECT_TABLESPACE; } ; @@ -6453,8 +6455,10 @@ security_label_type_name: DATABASE { $$ = OBJECT_DATABASE; } | EVENT TRIGGER { $$ = OBJECT_EVENT_TRIGGER; } | opt_procedural LANGUAGE { $$ = OBJECT_LANGUAGE; } + | PUBLICATION { $$ = OBJECT_PUBLICATION; } | ROLE { $$ = OBJECT_ROLE; } | SCHEMA { $$ = OBJECT_SCHEMA; } + | SUBSCRIPTION { $$ = OBJECT_SUBSCRIPTION; } | TABLESPACE { $$ = OBJECT_TABLESPACE; } ; diff --git a/src/test/modules/dummy_seclabel/expected/dummy_seclabel.out b/src/test/modules/dummy_seclabel/expected/dummy_seclabel.out index 9c0c9cd815..7273df17b2 100644 --- a/src/test/modules/dummy_seclabel/expected/dummy_seclabel.out +++ b/src/test/modules/dummy_seclabel/expected/dummy_seclabel.out @@ -67,20 +67,28 @@ SECURITY LABEL ON FUNCTION dummy_seclabel_four() IS 'classified'; -- OK SECURITY LABEL ON DOMAIN dummy_seclabel_domain IS 'classified'; -- OK CREATE SCHEMA dummy_seclabel_test; SECURITY LABEL ON SCHEMA dummy_seclabel_test IS 'unclassified'; -- OK +SET client_min_messages = error; +CREATE PUBLICATION dummy_pub; +CREATE SUBSCRIPTION dummy_sub CONNECTION '' PUBLICATION foo WITH (NOCONNECT); +RESET client_min_messages; +SECURITY LABEL ON PUBLICATION dummy_pub IS 'classified'; +SECURITY LABEL ON SUBSCRIPTION dummy_sub IS 'classified'; SELECT objtype, objname, provider, label FROM pg_seclabels ORDER BY objtype, objname; - objtype | objname | provider | label -----------+------------------------------+----------+-------------- - column | dummy_seclabel_tbl1.a | dummy | unclassified - domain | dummy_seclabel_domain | dummy | classified - function | dummy_seclabel_four() | dummy | classified - role | regress_dummy_seclabel_user1 | dummy | classified - role | regress_dummy_seclabel_user2 | dummy | unclassified - schema | dummy_seclabel_test | dummy | unclassified - table | dummy_seclabel_tbl1 | dummy | top secret - table | dummy_seclabel_tbl2 | dummy | classified - view | dummy_seclabel_view1 | dummy | classified -(9 rows) + objtype | objname | provider | label +--------------+------------------------------+----------+-------------- + column | dummy_seclabel_tbl1.a | dummy | unclassified + domain | dummy_seclabel_domain | dummy | classified + function | dummy_seclabel_four() | dummy | classified + publication | dummy_pub | dummy | classified + role | regress_dummy_seclabel_user1 | dummy | classified + role | regress_dummy_seclabel_user2 | dummy | unclassified + schema | dummy_seclabel_test | dummy | unclassified + subscription | dummy_sub | dummy | classified + table | dummy_seclabel_tbl1 | dummy | top secret + table | dummy_seclabel_tbl2 | dummy | classified + view | dummy_seclabel_view1 | dummy | classified +(11 rows) -- check for event trigger CREATE FUNCTION event_trigger_test() diff --git a/src/test/modules/dummy_seclabel/sql/dummy_seclabel.sql b/src/test/modules/dummy_seclabel/sql/dummy_seclabel.sql index 854906f3ed..6b0d0acbff 100644 --- a/src/test/modules/dummy_seclabel/sql/dummy_seclabel.sql +++ b/src/test/modules/dummy_seclabel/sql/dummy_seclabel.sql @@ -71,6 +71,13 @@ SECURITY LABEL ON DOMAIN dummy_seclabel_domain IS 'classified'; -- OK CREATE SCHEMA dummy_seclabel_test; SECURITY LABEL ON SCHEMA dummy_seclabel_test IS 'unclassified'; -- OK +SET client_min_messages = error; +CREATE PUBLICATION dummy_pub; +CREATE SUBSCRIPTION dummy_sub CONNECTION '' PUBLICATION foo WITH (NOCONNECT); +RESET client_min_messages; +SECURITY LABEL ON PUBLICATION dummy_pub IS 'classified'; +SECURITY LABEL ON SUBSCRIPTION dummy_sub IS 'classified'; + SELECT objtype, objname, provider, label FROM pg_seclabels ORDER BY objtype, objname; diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out index 5a7c0edf7d..0964718a60 100644 --- a/src/test/regress/expected/publication.out +++ b/src/test/regress/expected/publication.out @@ -6,6 +6,13 @@ CREATE ROLE regress_publication_user2; CREATE ROLE regress_publication_user_dummy LOGIN NOSUPERUSER; SET SESSION AUTHORIZATION 'regress_publication_user'; CREATE PUBLICATION testpub_default; +COMMENT ON PUBLICATION testpub_default IS 'test publication'; +SELECT obj_description(p.oid, 'pg_publication') FROM pg_publication p; + obj_description +------------------ + test publication +(1 row) + CREATE PUBLICATION testpib_ins_trunct WITH (nopublish delete, nopublish update); ALTER PUBLICATION testpub_default WITH (nopublish insert, nopublish delete); \dRp diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 7f04c7a7cc..e8f8726c53 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1605,6 +1605,29 @@ UNION ALL FROM (pg_seclabel l JOIN pg_event_trigger evt ON (((l.classoid = evt.tableoid) AND (l.objoid = evt.oid)))) WHERE (l.objsubid = 0) +UNION ALL + SELECT l.objoid, + l.classoid, + l.objsubid, + 'publication'::text AS objtype, + NULL::oid AS objnamespace, + quote_ident((p.pubname)::text) AS objname, + l.provider, + l.label + FROM (pg_seclabel l + JOIN pg_publication p ON (((l.classoid = p.tableoid) AND (l.objoid = p.oid)))) + WHERE (l.objsubid = 0) +UNION ALL + SELECT l.objoid, + l.classoid, + 0 AS objsubid, + 'subscription'::text AS objtype, + NULL::oid AS objnamespace, + quote_ident((s.subname)::text) AS objname, + l.provider, + l.label + FROM (pg_shseclabel l + JOIN pg_subscription s ON (((l.classoid = s.tableoid) AND (l.objoid = s.oid)))) UNION ALL SELECT l.objoid, l.classoid, diff --git a/src/test/regress/expected/subscription.out b/src/test/regress/expected/subscription.out index 74a5255e2a..41f8def2f7 100644 --- a/src/test/regress/expected/subscription.out +++ b/src/test/regress/expected/subscription.out @@ -30,6 +30,13 @@ ERROR: publication name "foo" used more than once -- ok CREATE SUBSCRIPTION testsub CONNECTION 'dbname=doesnotexist' PUBLICATION testpub WITH (NOCONNECT); WARNING: tables were not subscribed, you will have to run ALTER SUBSCRIPTION ... REFRESH PUBLICATION to subscribe the tables +COMMENT ON SUBSCRIPTION testsub IS 'test subscription'; +SELECT obj_description(s.oid, 'pg_subscription') FROM pg_subscription s; + obj_description +------------------- + test subscription +(1 row) + -- fail - name already exists CREATE SUBSCRIPTION testsub CONNECTION 'dbname=doesnotexist' PUBLICATION testpub WITH (NOCONNECT); ERROR: subscription "testsub" already exists diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql index cff9931a77..85530bec0e 100644 --- a/src/test/regress/sql/publication.sql +++ b/src/test/regress/sql/publication.sql @@ -8,6 +8,9 @@ SET SESSION AUTHORIZATION 'regress_publication_user'; CREATE PUBLICATION testpub_default; +COMMENT ON PUBLICATION testpub_default IS 'test publication'; +SELECT obj_description(p.oid, 'pg_publication') FROM pg_publication p; + CREATE PUBLICATION testpib_ins_trunct WITH (nopublish delete, nopublish update); ALTER PUBLICATION testpub_default WITH (nopublish insert, nopublish delete); diff --git a/src/test/regress/sql/subscription.sql b/src/test/regress/sql/subscription.sql index b0eac18785..2db97db2f5 100644 --- a/src/test/regress/sql/subscription.sql +++ b/src/test/regress/sql/subscription.sql @@ -27,6 +27,9 @@ CREATE SUBSCRIPTION testsub CONNECTION 'dbname=doesnotexist' PUBLICATION foo, te -- ok CREATE SUBSCRIPTION testsub CONNECTION 'dbname=doesnotexist' PUBLICATION testpub WITH (NOCONNECT); +COMMENT ON SUBSCRIPTION testsub IS 'test subscription'; +SELECT obj_description(s.oid, 'pg_subscription') FROM pg_subscription s; + -- fail - name already exists CREATE SUBSCRIPTION testsub CONNECTION 'dbname=doesnotexist' PUBLICATION testpub WITH (NOCONNECT); -- cgit v1.2.3 From e148009740f91ecc9555227d0d52ac43df30e6ed Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Sat, 25 Mar 2017 00:28:28 -0400 Subject: Remove ICU tests from default run These tests require the test database to be in UTF8 encoding. Until there is a better solution, take them out of the default test set and treat them like the existing collate.linux.utf8 test, meaning it has to be selected manually. --- doc/src/sgml/regress.sgml | 6 +- src/test/regress/GNUmakefile | 3 - src/test/regress/expected/collate.icu.out | 1126 ------------------------ src/test/regress/expected/collate.icu.utf8.out | 1126 ++++++++++++++++++++++++ src/test/regress/sql/collate.icu.sql | 433 --------- src/test/regress/sql/collate.icu.utf8.sql | 433 +++++++++ 6 files changed, 1563 insertions(+), 1564 deletions(-) delete mode 100644 src/test/regress/expected/collate.icu.out create mode 100644 src/test/regress/expected/collate.icu.utf8.out delete mode 100644 src/test/regress/sql/collate.icu.sql create mode 100644 src/test/regress/sql/collate.icu.utf8.sql (limited to 'doc/src') diff --git a/doc/src/sgml/regress.sgml b/doc/src/sgml/regress.sgml index 89e765ea3a..b9d1af8707 100644 --- a/doc/src/sgml/regress.sgml +++ b/doc/src/sgml/regress.sgml @@ -264,10 +264,12 @@ make check EXTRA_TESTS=numeric_big To run the collation tests: -make check EXTRA_TESTS=collate.linux.utf8 LANG=en_US.utf8 +make check EXTRA_TESTS='collate.icu.utf8 collate.linux.utf8' LANG=en_US.utf8 The collate.linux.utf8 test works only on Linux/glibc - platforms, and only when run in a database that uses UTF-8 encoding. + platforms. The collate.icu.utf8 test only works when + support for ICU was built. Both tests will only succeed when run in a + database that uses UTF-8 encoding. diff --git a/src/test/regress/GNUmakefile b/src/test/regress/GNUmakefile index a747facb9a..b923ea1420 100644 --- a/src/test/regress/GNUmakefile +++ b/src/test/regress/GNUmakefile @@ -125,9 +125,6 @@ tablespace-setup: ## REGRESS_OPTS = --dlpath=. $(EXTRA_REGRESS_OPTS) -ifeq ($(with_icu),yes) -override EXTRA_TESTS := collate.icu $(EXTRA_TESTS) -endif check: all tablespace-setup $(pg_regress_check) $(REGRESS_OPTS) --schedule=$(srcdir)/parallel_schedule $(MAXCONNOPT) $(EXTRA_TESTS) diff --git a/src/test/regress/expected/collate.icu.out b/src/test/regress/expected/collate.icu.out deleted file mode 100644 index e1fc9984f2..0000000000 --- a/src/test/regress/expected/collate.icu.out +++ /dev/null @@ -1,1126 +0,0 @@ -/* - * This test is for ICU collations. - */ -SET client_encoding TO UTF8; -CREATE SCHEMA collate_tests; -SET search_path = collate_tests; -CREATE TABLE collate_test1 ( - a int, - b text COLLATE "en-x-icu" NOT NULL -); -\d collate_test1 - Table "collate_tests.collate_test1" - Column | Type | Collation | Nullable | Default ---------+---------+-----------+----------+--------- - a | integer | | | - b | text | en-x-icu | not null | - -CREATE TABLE collate_test_fail ( - a int, - b text COLLATE "ja_JP.eucjp-x-icu" -); -ERROR: collation "ja_JP.eucjp-x-icu" for encoding "UTF8" does not exist -LINE 3: b text COLLATE "ja_JP.eucjp-x-icu" - ^ -CREATE TABLE collate_test_fail ( - a int, - b text COLLATE "foo-x-icu" -); -ERROR: collation "foo-x-icu" for encoding "UTF8" does not exist -LINE 3: b text COLLATE "foo-x-icu" - ^ -CREATE TABLE collate_test_fail ( - a int COLLATE "en-x-icu", - b text -); -ERROR: collations are not supported by type integer -LINE 2: a int COLLATE "en-x-icu", - ^ -CREATE TABLE collate_test_like ( - LIKE collate_test1 -); -\d collate_test_like - Table "collate_tests.collate_test_like" - Column | Type | Collation | Nullable | Default ---------+---------+-----------+----------+--------- - a | integer | | | - b | text | en-x-icu | not null | - -CREATE TABLE collate_test2 ( - a int, - b text COLLATE "sv-x-icu" -); -CREATE TABLE collate_test3 ( - a int, - b text COLLATE "C" -); -INSERT INTO collate_test1 VALUES (1, 'abc'), (2, 'äbc'), (3, 'bbc'), (4, 'ABC'); -INSERT INTO collate_test2 SELECT * FROM collate_test1; -INSERT INTO collate_test3 SELECT * FROM collate_test1; -SELECT * FROM collate_test1 WHERE b >= 'bbc'; - a | b ----+----- - 3 | bbc -(1 row) - -SELECT * FROM collate_test2 WHERE b >= 'bbc'; - a | b ----+----- - 2 | äbc - 3 | bbc -(2 rows) - -SELECT * FROM collate_test3 WHERE b >= 'bbc'; - a | b ----+----- - 2 | äbc - 3 | bbc -(2 rows) - -SELECT * FROM collate_test3 WHERE b >= 'BBC'; - a | b ----+----- - 1 | abc - 2 | äbc - 3 | bbc -(3 rows) - -SELECT * FROM collate_test1 WHERE b COLLATE "C" >= 'bbc'; - a | b ----+----- - 2 | äbc - 3 | bbc -(2 rows) - -SELECT * FROM collate_test1 WHERE b >= 'bbc' COLLATE "C"; - a | b ----+----- - 2 | äbc - 3 | bbc -(2 rows) - -SELECT * FROM collate_test1 WHERE b COLLATE "C" >= 'bbc' COLLATE "C"; - a | b ----+----- - 2 | äbc - 3 | bbc -(2 rows) - -SELECT * FROM collate_test1 WHERE b COLLATE "C" >= 'bbc' COLLATE "en-x-icu"; -ERROR: collation mismatch between explicit collations "C" and "en-x-icu" -LINE 1: ...* FROM collate_test1 WHERE b COLLATE "C" >= 'bbc' COLLATE "e... - ^ -CREATE DOMAIN testdomain_sv AS text COLLATE "sv-x-icu"; -CREATE DOMAIN testdomain_i AS int COLLATE "sv-x-icu"; -- fails -ERROR: collations are not supported by type integer -CREATE TABLE collate_test4 ( - a int, - b testdomain_sv -); -INSERT INTO collate_test4 SELECT * FROM collate_test1; -SELECT a, b FROM collate_test4 ORDER BY b; - a | b ----+----- - 1 | abc - 4 | ABC - 3 | bbc - 2 | äbc -(4 rows) - -CREATE TABLE collate_test5 ( - a int, - b testdomain_sv COLLATE "en-x-icu" -); -INSERT INTO collate_test5 SELECT * FROM collate_test1; -SELECT a, b FROM collate_test5 ORDER BY b; - a | b ----+----- - 1 | abc - 4 | ABC - 2 | äbc - 3 | bbc -(4 rows) - -SELECT a, b FROM collate_test1 ORDER BY b; - a | b ----+----- - 1 | abc - 4 | ABC - 2 | äbc - 3 | bbc -(4 rows) - -SELECT a, b FROM collate_test2 ORDER BY b; - a | b ----+----- - 1 | abc - 4 | ABC - 3 | bbc - 2 | äbc -(4 rows) - -SELECT a, b FROM collate_test3 ORDER BY b; - a | b ----+----- - 4 | ABC - 1 | abc - 3 | bbc - 2 | äbc -(4 rows) - -SELECT a, b FROM collate_test1 ORDER BY b COLLATE "C"; - a | b ----+----- - 4 | ABC - 1 | abc - 3 | bbc - 2 | äbc -(4 rows) - --- star expansion -SELECT * FROM collate_test1 ORDER BY b; - a | b ----+----- - 1 | abc - 4 | ABC - 2 | äbc - 3 | bbc -(4 rows) - -SELECT * FROM collate_test2 ORDER BY b; - a | b ----+----- - 1 | abc - 4 | ABC - 3 | bbc - 2 | äbc -(4 rows) - -SELECT * FROM collate_test3 ORDER BY b; - a | b ----+----- - 4 | ABC - 1 | abc - 3 | bbc - 2 | äbc -(4 rows) - --- constant expression folding -SELECT 'bbc' COLLATE "en-x-icu" > 'äbc' COLLATE "en-x-icu" AS "true"; - true ------- - t -(1 row) - -SELECT 'bbc' COLLATE "sv-x-icu" > 'äbc' COLLATE "sv-x-icu" AS "false"; - false -------- - f -(1 row) - --- upper/lower -CREATE TABLE collate_test10 ( - a int, - x text COLLATE "en-x-icu", - y text COLLATE "tr-x-icu" -); -INSERT INTO collate_test10 VALUES (1, 'hij', 'hij'), (2, 'HIJ', 'HIJ'); -SELECT a, lower(x), lower(y), upper(x), upper(y), initcap(x), initcap(y) FROM collate_test10; - a | lower | lower | upper | upper | initcap | initcap ----+-------+-------+-------+-------+---------+--------- - 1 | hij | hij | HIJ | HİJ | Hij | Hij - 2 | hij | hıj | HIJ | HIJ | Hij | Hıj -(2 rows) - -SELECT a, lower(x COLLATE "C"), lower(y COLLATE "C") FROM collate_test10; - a | lower | lower ----+-------+------- - 1 | hij | hij - 2 | hij | hij -(2 rows) - -SELECT a, x, y FROM collate_test10 ORDER BY lower(y), a; - a | x | y ----+-----+----- - 2 | HIJ | HIJ - 1 | hij | hij -(2 rows) - --- LIKE/ILIKE -SELECT * FROM collate_test1 WHERE b LIKE 'abc'; - a | b ----+----- - 1 | abc -(1 row) - -SELECT * FROM collate_test1 WHERE b LIKE 'abc%'; - a | b ----+----- - 1 | abc -(1 row) - -SELECT * FROM collate_test1 WHERE b LIKE '%bc%'; - a | b ----+----- - 1 | abc - 2 | äbc - 3 | bbc -(3 rows) - -SELECT * FROM collate_test1 WHERE b ILIKE 'abc'; - a | b ----+----- - 1 | abc - 4 | ABC -(2 rows) - -SELECT * FROM collate_test1 WHERE b ILIKE 'abc%'; - a | b ----+----- - 1 | abc - 4 | ABC -(2 rows) - -SELECT * FROM collate_test1 WHERE b ILIKE '%bc%'; - a | b ----+----- - 1 | abc - 2 | äbc - 3 | bbc - 4 | ABC -(4 rows) - -SELECT 'Türkiye' COLLATE "en-x-icu" ILIKE '%KI%' AS "true"; - true ------- - t -(1 row) - -SELECT 'Türkiye' COLLATE "tr-x-icu" ILIKE '%KI%' AS "false"; - false -------- - f -(1 row) - -SELECT 'bıt' ILIKE 'BIT' COLLATE "en-x-icu" AS "false"; - false -------- - f -(1 row) - -SELECT 'bıt' ILIKE 'BIT' COLLATE "tr-x-icu" AS "true"; - true ------- - t -(1 row) - --- The following actually exercises the selectivity estimation for ILIKE. -SELECT relname FROM pg_class WHERE relname ILIKE 'abc%'; - relname ---------- -(0 rows) - --- regular expressions -SELECT * FROM collate_test1 WHERE b ~ '^abc$'; - a | b ----+----- - 1 | abc -(1 row) - -SELECT * FROM collate_test1 WHERE b ~ '^abc'; - a | b ----+----- - 1 | abc -(1 row) - -SELECT * FROM collate_test1 WHERE b ~ 'bc'; - a | b ----+----- - 1 | abc - 2 | äbc - 3 | bbc -(3 rows) - -SELECT * FROM collate_test1 WHERE b ~* '^abc$'; - a | b ----+----- - 1 | abc - 4 | ABC -(2 rows) - -SELECT * FROM collate_test1 WHERE b ~* '^abc'; - a | b ----+----- - 1 | abc - 4 | ABC -(2 rows) - -SELECT * FROM collate_test1 WHERE b ~* 'bc'; - a | b ----+----- - 1 | abc - 2 | äbc - 3 | bbc - 4 | ABC -(4 rows) - -CREATE TABLE collate_test6 ( - a int, - b text COLLATE "en-x-icu" -); -INSERT INTO collate_test6 VALUES (1, 'abc'), (2, 'ABC'), (3, '123'), (4, 'ab1'), - (5, 'a1!'), (6, 'a c'), (7, '!.;'), (8, ' '), - (9, 'äbç'), (10, 'ÄBÇ'); -SELECT b, - b ~ '^[[:alpha:]]+$' AS is_alpha, - b ~ '^[[:upper:]]+$' AS is_upper, - b ~ '^[[:lower:]]+$' AS is_lower, - b ~ '^[[:digit:]]+$' AS is_digit, - b ~ '^[[:alnum:]]+$' AS is_alnum, - b ~ '^[[:graph:]]+$' AS is_graph, - b ~ '^[[:print:]]+$' AS is_print, - b ~ '^[[:punct:]]+$' AS is_punct, - b ~ '^[[:space:]]+$' AS is_space -FROM collate_test6; - b | is_alpha | is_upper | is_lower | is_digit | is_alnum | is_graph | is_print | is_punct | is_space ------+----------+----------+----------+----------+----------+----------+----------+----------+---------- - abc | t | f | t | f | t | t | t | f | f - ABC | t | t | f | f | t | t | t | f | f - 123 | f | f | f | t | t | t | t | f | f - ab1 | f | f | f | f | t | t | t | f | f - a1! | f | f | f | f | f | t | t | f | f - a c | f | f | f | f | f | f | t | f | f - !.; | f | f | f | f | f | t | t | t | f - | f | f | f | f | f | f | t | f | t - äbç | t | f | t | f | t | t | t | f | f - ÄBÇ | t | t | f | f | t | t | t | f | f -(10 rows) - -SELECT 'Türkiye' COLLATE "en-x-icu" ~* 'KI' AS "true"; - true ------- - t -(1 row) - -SELECT 'Türkiye' COLLATE "tr-x-icu" ~* 'KI' AS "true"; -- true with ICU - true ------- - t -(1 row) - -SELECT 'bıt' ~* 'BIT' COLLATE "en-x-icu" AS "false"; - false -------- - f -(1 row) - -SELECT 'bıt' ~* 'BIT' COLLATE "tr-x-icu" AS "false"; -- false with ICU - false -------- - f -(1 row) - --- The following actually exercises the selectivity estimation for ~*. -SELECT relname FROM pg_class WHERE relname ~* '^abc'; - relname ---------- -(0 rows) - -/* not run by default because it requires tr_TR system locale --- to_char - -SET lc_time TO 'tr_TR'; -SELECT to_char(date '2010-04-01', 'DD TMMON YYYY'); -SELECT to_char(date '2010-04-01', 'DD TMMON YYYY' COLLATE "tr-x-icu"); -*/ --- backwards parsing -CREATE VIEW collview1 AS SELECT * FROM collate_test1 WHERE b COLLATE "C" >= 'bbc'; -CREATE VIEW collview2 AS SELECT a, b FROM collate_test1 ORDER BY b COLLATE "C"; -CREATE VIEW collview3 AS SELECT a, lower((x || x) COLLATE "C") FROM collate_test10; -SELECT table_name, view_definition FROM information_schema.views - WHERE table_name LIKE 'collview%' ORDER BY 1; - table_name | view_definition -------------+-------------------------------------------------------------------------- - collview1 | SELECT collate_test1.a, + - | collate_test1.b + - | FROM collate_test1 + - | WHERE ((collate_test1.b COLLATE "C") >= 'bbc'::text); - collview2 | SELECT collate_test1.a, + - | collate_test1.b + - | FROM collate_test1 + - | ORDER BY (collate_test1.b COLLATE "C"); - collview3 | SELECT collate_test10.a, + - | lower(((collate_test10.x || collate_test10.x) COLLATE "C")) AS lower+ - | FROM collate_test10; -(3 rows) - --- collation propagation in various expression types -SELECT a, coalesce(b, 'foo') FROM collate_test1 ORDER BY 2; - a | coalesce ----+---------- - 1 | abc - 4 | ABC - 2 | äbc - 3 | bbc -(4 rows) - -SELECT a, coalesce(b, 'foo') FROM collate_test2 ORDER BY 2; - a | coalesce ----+---------- - 1 | abc - 4 | ABC - 3 | bbc - 2 | äbc -(4 rows) - -SELECT a, coalesce(b, 'foo') FROM collate_test3 ORDER BY 2; - a | coalesce ----+---------- - 4 | ABC - 1 | abc - 3 | bbc - 2 | äbc -(4 rows) - -SELECT a, lower(coalesce(x, 'foo')), lower(coalesce(y, 'foo')) FROM collate_test10; - a | lower | lower ----+-------+------- - 1 | hij | hij - 2 | hij | hıj -(2 rows) - -SELECT a, b, greatest(b, 'CCC') FROM collate_test1 ORDER BY 3; - a | b | greatest ----+-----+---------- - 1 | abc | CCC - 2 | äbc | CCC - 3 | bbc | CCC - 4 | ABC | CCC -(4 rows) - -SELECT a, b, greatest(b, 'CCC') FROM collate_test2 ORDER BY 3; - a | b | greatest ----+-----+---------- - 1 | abc | CCC - 3 | bbc | CCC - 4 | ABC | CCC - 2 | äbc | äbc -(4 rows) - -SELECT a, b, greatest(b, 'CCC') FROM collate_test3 ORDER BY 3; - a | b | greatest ----+-----+---------- - 4 | ABC | CCC - 1 | abc | abc - 3 | bbc | bbc - 2 | äbc | äbc -(4 rows) - -SELECT a, x, y, lower(greatest(x, 'foo')), lower(greatest(y, 'foo')) FROM collate_test10; - a | x | y | lower | lower ----+-----+-----+-------+------- - 1 | hij | hij | hij | hij - 2 | HIJ | HIJ | hij | hıj -(2 rows) - -SELECT a, nullif(b, 'abc') FROM collate_test1 ORDER BY 2; - a | nullif ----+-------- - 4 | ABC - 2 | äbc - 3 | bbc - 1 | -(4 rows) - -SELECT a, nullif(b, 'abc') FROM collate_test2 ORDER BY 2; - a | nullif ----+-------- - 4 | ABC - 3 | bbc - 2 | äbc - 1 | -(4 rows) - -SELECT a, nullif(b, 'abc') FROM collate_test3 ORDER BY 2; - a | nullif ----+-------- - 4 | ABC - 3 | bbc - 2 | äbc - 1 | -(4 rows) - -SELECT a, lower(nullif(x, 'foo')), lower(nullif(y, 'foo')) FROM collate_test10; - a | lower | lower ----+-------+------- - 1 | hij | hij - 2 | hij | hıj -(2 rows) - -SELECT a, CASE b WHEN 'abc' THEN 'abcd' ELSE b END FROM collate_test1 ORDER BY 2; - a | b ----+------ - 4 | ABC - 2 | äbc - 1 | abcd - 3 | bbc -(4 rows) - -SELECT a, CASE b WHEN 'abc' THEN 'abcd' ELSE b END FROM collate_test2 ORDER BY 2; - a | b ----+------ - 4 | ABC - 1 | abcd - 3 | bbc - 2 | äbc -(4 rows) - -SELECT a, CASE b WHEN 'abc' THEN 'abcd' ELSE b END FROM collate_test3 ORDER BY 2; - a | b ----+------ - 4 | ABC - 1 | abcd - 3 | bbc - 2 | äbc -(4 rows) - -CREATE DOMAIN testdomain AS text; -SELECT a, b::testdomain FROM collate_test1 ORDER BY 2; - a | b ----+----- - 1 | abc - 4 | ABC - 2 | äbc - 3 | bbc -(4 rows) - -SELECT a, b::testdomain FROM collate_test2 ORDER BY 2; - a | b ----+----- - 1 | abc - 4 | ABC - 3 | bbc - 2 | äbc -(4 rows) - -SELECT a, b::testdomain FROM collate_test3 ORDER BY 2; - a | b ----+----- - 4 | ABC - 1 | abc - 3 | bbc - 2 | äbc -(4 rows) - -SELECT a, b::testdomain_sv FROM collate_test3 ORDER BY 2; - a | b ----+----- - 1 | abc - 4 | ABC - 3 | bbc - 2 | äbc -(4 rows) - -SELECT a, lower(x::testdomain), lower(y::testdomain) FROM collate_test10; - a | lower | lower ----+-------+------- - 1 | hij | hij - 2 | hij | hıj -(2 rows) - -SELECT min(b), max(b) FROM collate_test1; - min | max ------+----- - abc | bbc -(1 row) - -SELECT min(b), max(b) FROM collate_test2; - min | max ------+----- - abc | äbc -(1 row) - -SELECT min(b), max(b) FROM collate_test3; - min | max ------+----- - ABC | äbc -(1 row) - -SELECT array_agg(b ORDER BY b) FROM collate_test1; - array_agg -------------------- - {abc,ABC,äbc,bbc} -(1 row) - -SELECT array_agg(b ORDER BY b) FROM collate_test2; - array_agg -------------------- - {abc,ABC,bbc,äbc} -(1 row) - -SELECT array_agg(b ORDER BY b) FROM collate_test3; - array_agg -------------------- - {ABC,abc,bbc,äbc} -(1 row) - -SELECT a, b FROM collate_test1 UNION ALL SELECT a, b FROM collate_test1 ORDER BY 2; - a | b ----+----- - 1 | abc - 1 | abc - 4 | ABC - 4 | ABC - 2 | äbc - 2 | äbc - 3 | bbc - 3 | bbc -(8 rows) - -SELECT a, b FROM collate_test2 UNION SELECT a, b FROM collate_test2 ORDER BY 2; - a | b ----+----- - 1 | abc - 4 | ABC - 3 | bbc - 2 | äbc -(4 rows) - -SELECT a, b FROM collate_test3 WHERE a < 4 INTERSECT SELECT a, b FROM collate_test3 WHERE a > 1 ORDER BY 2; - a | b ----+----- - 3 | bbc - 2 | äbc -(2 rows) - -SELECT a, b FROM collate_test3 EXCEPT SELECT a, b FROM collate_test3 WHERE a < 2 ORDER BY 2; - a | b ----+----- - 4 | ABC - 3 | bbc - 2 | äbc -(3 rows) - -SELECT a, b FROM collate_test1 UNION ALL SELECT a, b FROM collate_test3 ORDER BY 2; -- fail -ERROR: could not determine which collation to use for string comparison -HINT: Use the COLLATE clause to set the collation explicitly. -SELECT a, b FROM collate_test1 UNION ALL SELECT a, b FROM collate_test3; -- ok - a | b ----+----- - 1 | abc - 2 | äbc - 3 | bbc - 4 | ABC - 1 | abc - 2 | äbc - 3 | bbc - 4 | ABC -(8 rows) - -SELECT a, b FROM collate_test1 UNION SELECT a, b FROM collate_test3 ORDER BY 2; -- fail -ERROR: collation mismatch between implicit collations "en-x-icu" and "C" -LINE 1: SELECT a, b FROM collate_test1 UNION SELECT a, b FROM collat... - ^ -HINT: You can choose the collation by applying the COLLATE clause to one or both expressions. -SELECT a, b COLLATE "C" FROM collate_test1 UNION SELECT a, b FROM collate_test3 ORDER BY 2; -- ok - a | b ----+----- - 4 | ABC - 1 | abc - 3 | bbc - 2 | äbc -(4 rows) - -SELECT a, b FROM collate_test1 INTERSECT SELECT a, b FROM collate_test3 ORDER BY 2; -- fail -ERROR: collation mismatch between implicit collations "en-x-icu" and "C" -LINE 1: ...ELECT a, b FROM collate_test1 INTERSECT SELECT a, b FROM col... - ^ -HINT: You can choose the collation by applying the COLLATE clause to one or both expressions. -SELECT a, b FROM collate_test1 EXCEPT SELECT a, b FROM collate_test3 ORDER BY 2; -- fail -ERROR: collation mismatch between implicit collations "en-x-icu" and "C" -LINE 1: SELECT a, b FROM collate_test1 EXCEPT SELECT a, b FROM colla... - ^ -HINT: You can choose the collation by applying the COLLATE clause to one or both expressions. -CREATE TABLE test_u AS SELECT a, b FROM collate_test1 UNION ALL SELECT a, b FROM collate_test3; -- fail -ERROR: no collation was derived for column "b" with collatable type text -HINT: Use the COLLATE clause to set the collation explicitly. --- ideally this would be a parse-time error, but for now it must be run-time: -select x < y from collate_test10; -- fail -ERROR: could not determine which collation to use for string comparison -HINT: Use the COLLATE clause to set the collation explicitly. -select x || y from collate_test10; -- ok, because || is not collation aware - ?column? ----------- - hijhij - HIJHIJ -(2 rows) - -select x, y from collate_test10 order by x || y; -- not so ok -ERROR: collation mismatch between implicit collations "en-x-icu" and "tr-x-icu" -LINE 1: select x, y from collate_test10 order by x || y; - ^ -HINT: You can choose the collation by applying the COLLATE clause to one or both expressions. --- collation mismatch between recursive and non-recursive term -WITH RECURSIVE foo(x) AS - (SELECT x FROM (VALUES('a' COLLATE "en-x-icu"),('b')) t(x) - UNION ALL - SELECT (x || 'c') COLLATE "de-x-icu" FROM foo WHERE length(x) < 10) -SELECT * FROM foo; -ERROR: recursive query "foo" column 1 has collation "en-x-icu" in non-recursive term but collation "de-x-icu" overall -LINE 2: (SELECT x FROM (VALUES('a' COLLATE "en-x-icu"),('b')) t(x... - ^ -HINT: Use the COLLATE clause to set the collation of the non-recursive term. --- casting -SELECT CAST('42' AS text COLLATE "C"); -ERROR: syntax error at or near "COLLATE" -LINE 1: SELECT CAST('42' AS text COLLATE "C"); - ^ -SELECT a, CAST(b AS varchar) FROM collate_test1 ORDER BY 2; - a | b ----+----- - 1 | abc - 4 | ABC - 2 | äbc - 3 | bbc -(4 rows) - -SELECT a, CAST(b AS varchar) FROM collate_test2 ORDER BY 2; - a | b ----+----- - 1 | abc - 4 | ABC - 3 | bbc - 2 | äbc -(4 rows) - -SELECT a, CAST(b AS varchar) FROM collate_test3 ORDER BY 2; - a | b ----+----- - 4 | ABC - 1 | abc - 3 | bbc - 2 | äbc -(4 rows) - --- propagation of collation in SQL functions (inlined and non-inlined cases) --- and plpgsql functions too -CREATE FUNCTION mylt (text, text) RETURNS boolean LANGUAGE sql - AS $$ select $1 < $2 $$; -CREATE FUNCTION mylt_noninline (text, text) RETURNS boolean LANGUAGE sql - AS $$ select $1 < $2 limit 1 $$; -CREATE FUNCTION mylt_plpgsql (text, text) RETURNS boolean LANGUAGE plpgsql - AS $$ begin return $1 < $2; end $$; -SELECT a.b AS a, b.b AS b, a.b < b.b AS lt, - mylt(a.b, b.b), mylt_noninline(a.b, b.b), mylt_plpgsql(a.b, b.b) -FROM collate_test1 a, collate_test1 b -ORDER BY a.b, b.b; - a | b | lt | mylt | mylt_noninline | mylt_plpgsql ------+-----+----+------+----------------+-------------- - abc | abc | f | f | f | f - abc | ABC | t | t | t | t - abc | äbc | t | t | t | t - abc | bbc | t | t | t | t - ABC | abc | f | f | f | f - ABC | ABC | f | f | f | f - ABC | äbc | t | t | t | t - ABC | bbc | t | t | t | t - äbc | abc | f | f | f | f - äbc | ABC | f | f | f | f - äbc | äbc | f | f | f | f - äbc | bbc | t | t | t | t - bbc | abc | f | f | f | f - bbc | ABC | f | f | f | f - bbc | äbc | f | f | f | f - bbc | bbc | f | f | f | f -(16 rows) - -SELECT a.b AS a, b.b AS b, a.b < b.b COLLATE "C" AS lt, - mylt(a.b, b.b COLLATE "C"), mylt_noninline(a.b, b.b COLLATE "C"), - mylt_plpgsql(a.b, b.b COLLATE "C") -FROM collate_test1 a, collate_test1 b -ORDER BY a.b, b.b; - a | b | lt | mylt | mylt_noninline | mylt_plpgsql ------+-----+----+------+----------------+-------------- - abc | abc | f | f | f | f - abc | ABC | f | f | f | f - abc | äbc | t | t | t | t - abc | bbc | t | t | t | t - ABC | abc | t | t | t | t - ABC | ABC | f | f | f | f - ABC | äbc | t | t | t | t - ABC | bbc | t | t | t | t - äbc | abc | f | f | f | f - äbc | ABC | f | f | f | f - äbc | äbc | f | f | f | f - äbc | bbc | f | f | f | f - bbc | abc | f | f | f | f - bbc | ABC | f | f | f | f - bbc | äbc | t | t | t | t - bbc | bbc | f | f | f | f -(16 rows) - --- collation override in plpgsql -CREATE FUNCTION mylt2 (x text, y text) RETURNS boolean LANGUAGE plpgsql AS $$ -declare - xx text := x; - yy text := y; -begin - return xx < yy; -end -$$; -SELECT mylt2('a', 'B' collate "en-x-icu") as t, mylt2('a', 'B' collate "C") as f; - t | f ----+--- - t | f -(1 row) - -CREATE OR REPLACE FUNCTION - mylt2 (x text, y text) RETURNS boolean LANGUAGE plpgsql AS $$ -declare - xx text COLLATE "POSIX" := x; - yy text := y; -begin - return xx < yy; -end -$$; -SELECT mylt2('a', 'B') as f; - f ---- - f -(1 row) - -SELECT mylt2('a', 'B' collate "C") as fail; -- conflicting collations -ERROR: could not determine which collation to use for string comparison -HINT: Use the COLLATE clause to set the collation explicitly. -CONTEXT: PL/pgSQL function mylt2(text,text) line 6 at RETURN -SELECT mylt2('a', 'B' collate "POSIX") as f; - f ---- - f -(1 row) - --- polymorphism -SELECT * FROM unnest((SELECT array_agg(b ORDER BY b) FROM collate_test1)) ORDER BY 1; - unnest --------- - abc - ABC - äbc - bbc -(4 rows) - -SELECT * FROM unnest((SELECT array_agg(b ORDER BY b) FROM collate_test2)) ORDER BY 1; - unnest --------- - abc - ABC - bbc - äbc -(4 rows) - -SELECT * FROM unnest((SELECT array_agg(b ORDER BY b) FROM collate_test3)) ORDER BY 1; - unnest --------- - ABC - abc - bbc - äbc -(4 rows) - -CREATE FUNCTION dup (anyelement) RETURNS anyelement - AS 'select $1' LANGUAGE sql; -SELECT a, dup(b) FROM collate_test1 ORDER BY 2; - a | dup ----+----- - 1 | abc - 4 | ABC - 2 | äbc - 3 | bbc -(4 rows) - -SELECT a, dup(b) FROM collate_test2 ORDER BY 2; - a | dup ----+----- - 1 | abc - 4 | ABC - 3 | bbc - 2 | äbc -(4 rows) - -SELECT a, dup(b) FROM collate_test3 ORDER BY 2; - a | dup ----+----- - 4 | ABC - 1 | abc - 3 | bbc - 2 | äbc -(4 rows) - --- indexes -CREATE INDEX collate_test1_idx1 ON collate_test1 (b); -CREATE INDEX collate_test1_idx2 ON collate_test1 (b COLLATE "C"); -CREATE INDEX collate_test1_idx3 ON collate_test1 ((b COLLATE "C")); -- this is different grammatically -CREATE INDEX collate_test1_idx4 ON collate_test1 (((b||'foo') COLLATE "POSIX")); -CREATE INDEX collate_test1_idx5 ON collate_test1 (a COLLATE "C"); -- fail -ERROR: collations are not supported by type integer -CREATE INDEX collate_test1_idx6 ON collate_test1 ((a COLLATE "C")); -- fail -ERROR: collations are not supported by type integer -LINE 1: ...ATE INDEX collate_test1_idx6 ON collate_test1 ((a COLLATE "C... - ^ -SELECT relname, pg_get_indexdef(oid) FROM pg_class WHERE relname LIKE 'collate_test%_idx%' ORDER BY 1; - relname | pg_get_indexdef ---------------------+----------------------------------------------------------------------------------------------------- - collate_test1_idx1 | CREATE INDEX collate_test1_idx1 ON collate_test1 USING btree (b) - collate_test1_idx2 | CREATE INDEX collate_test1_idx2 ON collate_test1 USING btree (b COLLATE "C") - collate_test1_idx3 | CREATE INDEX collate_test1_idx3 ON collate_test1 USING btree (b COLLATE "C") - collate_test1_idx4 | CREATE INDEX collate_test1_idx4 ON collate_test1 USING btree (((b || 'foo'::text)) COLLATE "POSIX") -(4 rows) - --- schema manipulation commands -CREATE ROLE regress_test_role; -CREATE SCHEMA test_schema; --- We need to do this this way to cope with varying names for encodings: -do $$ -BEGIN - EXECUTE 'CREATE COLLATION test0 (provider = icu, locale = ' || - quote_literal(current_setting('lc_collate')) || ');'; -END -$$; -CREATE COLLATION test0 FROM "C"; -- fail, duplicate name -ERROR: collation "test0" already exists -do $$ -BEGIN - EXECUTE 'CREATE COLLATION test1 (provider = icu, lc_collate = ' || - quote_literal(current_setting('lc_collate')) || - ', lc_ctype = ' || - quote_literal(current_setting('lc_ctype')) || ');'; -END -$$; -CREATE COLLATION test3 (provider = icu, lc_collate = 'en_US.utf8'); -- fail, need lc_ctype -ERROR: parameter "lc_ctype" must be specified -CREATE COLLATION testx (provider = icu, locale = 'nonsense'); /* never fails with ICU */ DROP COLLATION testx; -CREATE COLLATION test4 FROM nonsense; -ERROR: collation "nonsense" for encoding "UTF8" does not exist -CREATE COLLATION test5 FROM test0; -SELECT collname FROM pg_collation WHERE collname LIKE 'test%' ORDER BY 1; - collname ----------- - test0 - test1 - test5 -(3 rows) - -ALTER COLLATION test1 RENAME TO test11; -ALTER COLLATION test0 RENAME TO test11; -- fail -ERROR: collation "test11" already exists in schema "collate_tests" -ALTER COLLATION test1 RENAME TO test22; -- fail -ERROR: collation "test1" for encoding "UTF8" does not exist -ALTER COLLATION test11 OWNER TO regress_test_role; -ALTER COLLATION test11 OWNER TO nonsense; -ERROR: role "nonsense" does not exist -ALTER COLLATION test11 SET SCHEMA test_schema; -COMMENT ON COLLATION test0 IS 'US English'; -SELECT collname, nspname, obj_description(pg_collation.oid, 'pg_collation') - FROM pg_collation JOIN pg_namespace ON (collnamespace = pg_namespace.oid) - WHERE collname LIKE 'test%' - ORDER BY 1; - collname | nspname | obj_description -----------+---------------+----------------- - test0 | collate_tests | US English - test11 | test_schema | - test5 | collate_tests | -(3 rows) - -DROP COLLATION test0, test_schema.test11, test5; -DROP COLLATION test0; -- fail -ERROR: collation "test0" for encoding "UTF8" does not exist -DROP COLLATION IF EXISTS test0; -NOTICE: collation "test0" does not exist, skipping -SELECT collname FROM pg_collation WHERE collname LIKE 'test%'; - collname ----------- -(0 rows) - -DROP SCHEMA test_schema; -DROP ROLE regress_test_role; --- ALTER -ALTER COLLATION "en-x-icu" REFRESH VERSION; -NOTICE: version has not changed --- dependencies -CREATE COLLATION test0 FROM "C"; -CREATE TABLE collate_dep_test1 (a int, b text COLLATE test0); -CREATE DOMAIN collate_dep_dom1 AS text COLLATE test0; -CREATE TYPE collate_dep_test2 AS (x int, y text COLLATE test0); -CREATE VIEW collate_dep_test3 AS SELECT text 'foo' COLLATE test0 AS foo; -CREATE TABLE collate_dep_test4t (a int, b text); -CREATE INDEX collate_dep_test4i ON collate_dep_test4t (b COLLATE test0); -DROP COLLATION test0 RESTRICT; -- fail -ERROR: cannot drop collation test0 because other objects depend on it -DETAIL: table collate_dep_test1 column b depends on collation test0 -type collate_dep_dom1 depends on collation test0 -composite type collate_dep_test2 column y depends on collation test0 -view collate_dep_test3 depends on collation test0 -index collate_dep_test4i depends on collation test0 -HINT: Use DROP ... CASCADE to drop the dependent objects too. -DROP COLLATION test0 CASCADE; -NOTICE: drop cascades to 5 other objects -DETAIL: drop cascades to table collate_dep_test1 column b -drop cascades to type collate_dep_dom1 -drop cascades to composite type collate_dep_test2 column y -drop cascades to view collate_dep_test3 -drop cascades to index collate_dep_test4i -\d collate_dep_test1 - Table "collate_tests.collate_dep_test1" - Column | Type | Collation | Nullable | Default ---------+---------+-----------+----------+--------- - a | integer | | | - -\d collate_dep_test2 - Composite type "collate_tests.collate_dep_test2" - Column | Type | Collation | Nullable | Default ---------+---------+-----------+----------+--------- - x | integer | | | - -DROP TABLE collate_dep_test1, collate_dep_test4t; -DROP TYPE collate_dep_test2; --- test range types and collations -create type textrange_c as range(subtype=text, collation="C"); -create type textrange_en_us as range(subtype=text, collation="en-x-icu"); -select textrange_c('A','Z') @> 'b'::text; - ?column? ----------- - f -(1 row) - -select textrange_en_us('A','Z') @> 'b'::text; - ?column? ----------- - t -(1 row) - -drop type textrange_c; -drop type textrange_en_us; --- cleanup -DROP SCHEMA collate_tests CASCADE; -NOTICE: drop cascades to 18 other objects -DETAIL: drop cascades to table collate_test1 -drop cascades to table collate_test_like -drop cascades to table collate_test2 -drop cascades to table collate_test3 -drop cascades to type testdomain_sv -drop cascades to table collate_test4 -drop cascades to table collate_test5 -drop cascades to table collate_test10 -drop cascades to table collate_test6 -drop cascades to view collview1 -drop cascades to view collview2 -drop cascades to view collview3 -drop cascades to type testdomain -drop cascades to function mylt(text,text) -drop cascades to function mylt_noninline(text,text) -drop cascades to function mylt_plpgsql(text,text) -drop cascades to function mylt2(text,text) -drop cascades to function dup(anyelement) -RESET search_path; --- leave a collation for pg_upgrade test -CREATE COLLATION coll_icu_upgrade FROM "und-x-icu"; diff --git a/src/test/regress/expected/collate.icu.utf8.out b/src/test/regress/expected/collate.icu.utf8.out new file mode 100644 index 0000000000..e1fc9984f2 --- /dev/null +++ b/src/test/regress/expected/collate.icu.utf8.out @@ -0,0 +1,1126 @@ +/* + * This test is for ICU collations. + */ +SET client_encoding TO UTF8; +CREATE SCHEMA collate_tests; +SET search_path = collate_tests; +CREATE TABLE collate_test1 ( + a int, + b text COLLATE "en-x-icu" NOT NULL +); +\d collate_test1 + Table "collate_tests.collate_test1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | text | en-x-icu | not null | + +CREATE TABLE collate_test_fail ( + a int, + b text COLLATE "ja_JP.eucjp-x-icu" +); +ERROR: collation "ja_JP.eucjp-x-icu" for encoding "UTF8" does not exist +LINE 3: b text COLLATE "ja_JP.eucjp-x-icu" + ^ +CREATE TABLE collate_test_fail ( + a int, + b text COLLATE "foo-x-icu" +); +ERROR: collation "foo-x-icu" for encoding "UTF8" does not exist +LINE 3: b text COLLATE "foo-x-icu" + ^ +CREATE TABLE collate_test_fail ( + a int COLLATE "en-x-icu", + b text +); +ERROR: collations are not supported by type integer +LINE 2: a int COLLATE "en-x-icu", + ^ +CREATE TABLE collate_test_like ( + LIKE collate_test1 +); +\d collate_test_like + Table "collate_tests.collate_test_like" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | text | en-x-icu | not null | + +CREATE TABLE collate_test2 ( + a int, + b text COLLATE "sv-x-icu" +); +CREATE TABLE collate_test3 ( + a int, + b text COLLATE "C" +); +INSERT INTO collate_test1 VALUES (1, 'abc'), (2, 'äbc'), (3, 'bbc'), (4, 'ABC'); +INSERT INTO collate_test2 SELECT * FROM collate_test1; +INSERT INTO collate_test3 SELECT * FROM collate_test1; +SELECT * FROM collate_test1 WHERE b >= 'bbc'; + a | b +---+----- + 3 | bbc +(1 row) + +SELECT * FROM collate_test2 WHERE b >= 'bbc'; + a | b +---+----- + 2 | äbc + 3 | bbc +(2 rows) + +SELECT * FROM collate_test3 WHERE b >= 'bbc'; + a | b +---+----- + 2 | äbc + 3 | bbc +(2 rows) + +SELECT * FROM collate_test3 WHERE b >= 'BBC'; + a | b +---+----- + 1 | abc + 2 | äbc + 3 | bbc +(3 rows) + +SELECT * FROM collate_test1 WHERE b COLLATE "C" >= 'bbc'; + a | b +---+----- + 2 | äbc + 3 | bbc +(2 rows) + +SELECT * FROM collate_test1 WHERE b >= 'bbc' COLLATE "C"; + a | b +---+----- + 2 | äbc + 3 | bbc +(2 rows) + +SELECT * FROM collate_test1 WHERE b COLLATE "C" >= 'bbc' COLLATE "C"; + a | b +---+----- + 2 | äbc + 3 | bbc +(2 rows) + +SELECT * FROM collate_test1 WHERE b COLLATE "C" >= 'bbc' COLLATE "en-x-icu"; +ERROR: collation mismatch between explicit collations "C" and "en-x-icu" +LINE 1: ...* FROM collate_test1 WHERE b COLLATE "C" >= 'bbc' COLLATE "e... + ^ +CREATE DOMAIN testdomain_sv AS text COLLATE "sv-x-icu"; +CREATE DOMAIN testdomain_i AS int COLLATE "sv-x-icu"; -- fails +ERROR: collations are not supported by type integer +CREATE TABLE collate_test4 ( + a int, + b testdomain_sv +); +INSERT INTO collate_test4 SELECT * FROM collate_test1; +SELECT a, b FROM collate_test4 ORDER BY b; + a | b +---+----- + 1 | abc + 4 | ABC + 3 | bbc + 2 | äbc +(4 rows) + +CREATE TABLE collate_test5 ( + a int, + b testdomain_sv COLLATE "en-x-icu" +); +INSERT INTO collate_test5 SELECT * FROM collate_test1; +SELECT a, b FROM collate_test5 ORDER BY b; + a | b +---+----- + 1 | abc + 4 | ABC + 2 | äbc + 3 | bbc +(4 rows) + +SELECT a, b FROM collate_test1 ORDER BY b; + a | b +---+----- + 1 | abc + 4 | ABC + 2 | äbc + 3 | bbc +(4 rows) + +SELECT a, b FROM collate_test2 ORDER BY b; + a | b +---+----- + 1 | abc + 4 | ABC + 3 | bbc + 2 | äbc +(4 rows) + +SELECT a, b FROM collate_test3 ORDER BY b; + a | b +---+----- + 4 | ABC + 1 | abc + 3 | bbc + 2 | äbc +(4 rows) + +SELECT a, b FROM collate_test1 ORDER BY b COLLATE "C"; + a | b +---+----- + 4 | ABC + 1 | abc + 3 | bbc + 2 | äbc +(4 rows) + +-- star expansion +SELECT * FROM collate_test1 ORDER BY b; + a | b +---+----- + 1 | abc + 4 | ABC + 2 | äbc + 3 | bbc +(4 rows) + +SELECT * FROM collate_test2 ORDER BY b; + a | b +---+----- + 1 | abc + 4 | ABC + 3 | bbc + 2 | äbc +(4 rows) + +SELECT * FROM collate_test3 ORDER BY b; + a | b +---+----- + 4 | ABC + 1 | abc + 3 | bbc + 2 | äbc +(4 rows) + +-- constant expression folding +SELECT 'bbc' COLLATE "en-x-icu" > 'äbc' COLLATE "en-x-icu" AS "true"; + true +------ + t +(1 row) + +SELECT 'bbc' COLLATE "sv-x-icu" > 'äbc' COLLATE "sv-x-icu" AS "false"; + false +------- + f +(1 row) + +-- upper/lower +CREATE TABLE collate_test10 ( + a int, + x text COLLATE "en-x-icu", + y text COLLATE "tr-x-icu" +); +INSERT INTO collate_test10 VALUES (1, 'hij', 'hij'), (2, 'HIJ', 'HIJ'); +SELECT a, lower(x), lower(y), upper(x), upper(y), initcap(x), initcap(y) FROM collate_test10; + a | lower | lower | upper | upper | initcap | initcap +---+-------+-------+-------+-------+---------+--------- + 1 | hij | hij | HIJ | HİJ | Hij | Hij + 2 | hij | hıj | HIJ | HIJ | Hij | Hıj +(2 rows) + +SELECT a, lower(x COLLATE "C"), lower(y COLLATE "C") FROM collate_test10; + a | lower | lower +---+-------+------- + 1 | hij | hij + 2 | hij | hij +(2 rows) + +SELECT a, x, y FROM collate_test10 ORDER BY lower(y), a; + a | x | y +---+-----+----- + 2 | HIJ | HIJ + 1 | hij | hij +(2 rows) + +-- LIKE/ILIKE +SELECT * FROM collate_test1 WHERE b LIKE 'abc'; + a | b +---+----- + 1 | abc +(1 row) + +SELECT * FROM collate_test1 WHERE b LIKE 'abc%'; + a | b +---+----- + 1 | abc +(1 row) + +SELECT * FROM collate_test1 WHERE b LIKE '%bc%'; + a | b +---+----- + 1 | abc + 2 | äbc + 3 | bbc +(3 rows) + +SELECT * FROM collate_test1 WHERE b ILIKE 'abc'; + a | b +---+----- + 1 | abc + 4 | ABC +(2 rows) + +SELECT * FROM collate_test1 WHERE b ILIKE 'abc%'; + a | b +---+----- + 1 | abc + 4 | ABC +(2 rows) + +SELECT * FROM collate_test1 WHERE b ILIKE '%bc%'; + a | b +---+----- + 1 | abc + 2 | äbc + 3 | bbc + 4 | ABC +(4 rows) + +SELECT 'Türkiye' COLLATE "en-x-icu" ILIKE '%KI%' AS "true"; + true +------ + t +(1 row) + +SELECT 'Türkiye' COLLATE "tr-x-icu" ILIKE '%KI%' AS "false"; + false +------- + f +(1 row) + +SELECT 'bıt' ILIKE 'BIT' COLLATE "en-x-icu" AS "false"; + false +------- + f +(1 row) + +SELECT 'bıt' ILIKE 'BIT' COLLATE "tr-x-icu" AS "true"; + true +------ + t +(1 row) + +-- The following actually exercises the selectivity estimation for ILIKE. +SELECT relname FROM pg_class WHERE relname ILIKE 'abc%'; + relname +--------- +(0 rows) + +-- regular expressions +SELECT * FROM collate_test1 WHERE b ~ '^abc$'; + a | b +---+----- + 1 | abc +(1 row) + +SELECT * FROM collate_test1 WHERE b ~ '^abc'; + a | b +---+----- + 1 | abc +(1 row) + +SELECT * FROM collate_test1 WHERE b ~ 'bc'; + a | b +---+----- + 1 | abc + 2 | äbc + 3 | bbc +(3 rows) + +SELECT * FROM collate_test1 WHERE b ~* '^abc$'; + a | b +---+----- + 1 | abc + 4 | ABC +(2 rows) + +SELECT * FROM collate_test1 WHERE b ~* '^abc'; + a | b +---+----- + 1 | abc + 4 | ABC +(2 rows) + +SELECT * FROM collate_test1 WHERE b ~* 'bc'; + a | b +---+----- + 1 | abc + 2 | äbc + 3 | bbc + 4 | ABC +(4 rows) + +CREATE TABLE collate_test6 ( + a int, + b text COLLATE "en-x-icu" +); +INSERT INTO collate_test6 VALUES (1, 'abc'), (2, 'ABC'), (3, '123'), (4, 'ab1'), + (5, 'a1!'), (6, 'a c'), (7, '!.;'), (8, ' '), + (9, 'äbç'), (10, 'ÄBÇ'); +SELECT b, + b ~ '^[[:alpha:]]+$' AS is_alpha, + b ~ '^[[:upper:]]+$' AS is_upper, + b ~ '^[[:lower:]]+$' AS is_lower, + b ~ '^[[:digit:]]+$' AS is_digit, + b ~ '^[[:alnum:]]+$' AS is_alnum, + b ~ '^[[:graph:]]+$' AS is_graph, + b ~ '^[[:print:]]+$' AS is_print, + b ~ '^[[:punct:]]+$' AS is_punct, + b ~ '^[[:space:]]+$' AS is_space +FROM collate_test6; + b | is_alpha | is_upper | is_lower | is_digit | is_alnum | is_graph | is_print | is_punct | is_space +-----+----------+----------+----------+----------+----------+----------+----------+----------+---------- + abc | t | f | t | f | t | t | t | f | f + ABC | t | t | f | f | t | t | t | f | f + 123 | f | f | f | t | t | t | t | f | f + ab1 | f | f | f | f | t | t | t | f | f + a1! | f | f | f | f | f | t | t | f | f + a c | f | f | f | f | f | f | t | f | f + !.; | f | f | f | f | f | t | t | t | f + | f | f | f | f | f | f | t | f | t + äbç | t | f | t | f | t | t | t | f | f + ÄBÇ | t | t | f | f | t | t | t | f | f +(10 rows) + +SELECT 'Türkiye' COLLATE "en-x-icu" ~* 'KI' AS "true"; + true +------ + t +(1 row) + +SELECT 'Türkiye' COLLATE "tr-x-icu" ~* 'KI' AS "true"; -- true with ICU + true +------ + t +(1 row) + +SELECT 'bıt' ~* 'BIT' COLLATE "en-x-icu" AS "false"; + false +------- + f +(1 row) + +SELECT 'bıt' ~* 'BIT' COLLATE "tr-x-icu" AS "false"; -- false with ICU + false +------- + f +(1 row) + +-- The following actually exercises the selectivity estimation for ~*. +SELECT relname FROM pg_class WHERE relname ~* '^abc'; + relname +--------- +(0 rows) + +/* not run by default because it requires tr_TR system locale +-- to_char + +SET lc_time TO 'tr_TR'; +SELECT to_char(date '2010-04-01', 'DD TMMON YYYY'); +SELECT to_char(date '2010-04-01', 'DD TMMON YYYY' COLLATE "tr-x-icu"); +*/ +-- backwards parsing +CREATE VIEW collview1 AS SELECT * FROM collate_test1 WHERE b COLLATE "C" >= 'bbc'; +CREATE VIEW collview2 AS SELECT a, b FROM collate_test1 ORDER BY b COLLATE "C"; +CREATE VIEW collview3 AS SELECT a, lower((x || x) COLLATE "C") FROM collate_test10; +SELECT table_name, view_definition FROM information_schema.views + WHERE table_name LIKE 'collview%' ORDER BY 1; + table_name | view_definition +------------+-------------------------------------------------------------------------- + collview1 | SELECT collate_test1.a, + + | collate_test1.b + + | FROM collate_test1 + + | WHERE ((collate_test1.b COLLATE "C") >= 'bbc'::text); + collview2 | SELECT collate_test1.a, + + | collate_test1.b + + | FROM collate_test1 + + | ORDER BY (collate_test1.b COLLATE "C"); + collview3 | SELECT collate_test10.a, + + | lower(((collate_test10.x || collate_test10.x) COLLATE "C")) AS lower+ + | FROM collate_test10; +(3 rows) + +-- collation propagation in various expression types +SELECT a, coalesce(b, 'foo') FROM collate_test1 ORDER BY 2; + a | coalesce +---+---------- + 1 | abc + 4 | ABC + 2 | äbc + 3 | bbc +(4 rows) + +SELECT a, coalesce(b, 'foo') FROM collate_test2 ORDER BY 2; + a | coalesce +---+---------- + 1 | abc + 4 | ABC + 3 | bbc + 2 | äbc +(4 rows) + +SELECT a, coalesce(b, 'foo') FROM collate_test3 ORDER BY 2; + a | coalesce +---+---------- + 4 | ABC + 1 | abc + 3 | bbc + 2 | äbc +(4 rows) + +SELECT a, lower(coalesce(x, 'foo')), lower(coalesce(y, 'foo')) FROM collate_test10; + a | lower | lower +---+-------+------- + 1 | hij | hij + 2 | hij | hıj +(2 rows) + +SELECT a, b, greatest(b, 'CCC') FROM collate_test1 ORDER BY 3; + a | b | greatest +---+-----+---------- + 1 | abc | CCC + 2 | äbc | CCC + 3 | bbc | CCC + 4 | ABC | CCC +(4 rows) + +SELECT a, b, greatest(b, 'CCC') FROM collate_test2 ORDER BY 3; + a | b | greatest +---+-----+---------- + 1 | abc | CCC + 3 | bbc | CCC + 4 | ABC | CCC + 2 | äbc | äbc +(4 rows) + +SELECT a, b, greatest(b, 'CCC') FROM collate_test3 ORDER BY 3; + a | b | greatest +---+-----+---------- + 4 | ABC | CCC + 1 | abc | abc + 3 | bbc | bbc + 2 | äbc | äbc +(4 rows) + +SELECT a, x, y, lower(greatest(x, 'foo')), lower(greatest(y, 'foo')) FROM collate_test10; + a | x | y | lower | lower +---+-----+-----+-------+------- + 1 | hij | hij | hij | hij + 2 | HIJ | HIJ | hij | hıj +(2 rows) + +SELECT a, nullif(b, 'abc') FROM collate_test1 ORDER BY 2; + a | nullif +---+-------- + 4 | ABC + 2 | äbc + 3 | bbc + 1 | +(4 rows) + +SELECT a, nullif(b, 'abc') FROM collate_test2 ORDER BY 2; + a | nullif +---+-------- + 4 | ABC + 3 | bbc + 2 | äbc + 1 | +(4 rows) + +SELECT a, nullif(b, 'abc') FROM collate_test3 ORDER BY 2; + a | nullif +---+-------- + 4 | ABC + 3 | bbc + 2 | äbc + 1 | +(4 rows) + +SELECT a, lower(nullif(x, 'foo')), lower(nullif(y, 'foo')) FROM collate_test10; + a | lower | lower +---+-------+------- + 1 | hij | hij + 2 | hij | hıj +(2 rows) + +SELECT a, CASE b WHEN 'abc' THEN 'abcd' ELSE b END FROM collate_test1 ORDER BY 2; + a | b +---+------ + 4 | ABC + 2 | äbc + 1 | abcd + 3 | bbc +(4 rows) + +SELECT a, CASE b WHEN 'abc' THEN 'abcd' ELSE b END FROM collate_test2 ORDER BY 2; + a | b +---+------ + 4 | ABC + 1 | abcd + 3 | bbc + 2 | äbc +(4 rows) + +SELECT a, CASE b WHEN 'abc' THEN 'abcd' ELSE b END FROM collate_test3 ORDER BY 2; + a | b +---+------ + 4 | ABC + 1 | abcd + 3 | bbc + 2 | äbc +(4 rows) + +CREATE DOMAIN testdomain AS text; +SELECT a, b::testdomain FROM collate_test1 ORDER BY 2; + a | b +---+----- + 1 | abc + 4 | ABC + 2 | äbc + 3 | bbc +(4 rows) + +SELECT a, b::testdomain FROM collate_test2 ORDER BY 2; + a | b +---+----- + 1 | abc + 4 | ABC + 3 | bbc + 2 | äbc +(4 rows) + +SELECT a, b::testdomain FROM collate_test3 ORDER BY 2; + a | b +---+----- + 4 | ABC + 1 | abc + 3 | bbc + 2 | äbc +(4 rows) + +SELECT a, b::testdomain_sv FROM collate_test3 ORDER BY 2; + a | b +---+----- + 1 | abc + 4 | ABC + 3 | bbc + 2 | äbc +(4 rows) + +SELECT a, lower(x::testdomain), lower(y::testdomain) FROM collate_test10; + a | lower | lower +---+-------+------- + 1 | hij | hij + 2 | hij | hıj +(2 rows) + +SELECT min(b), max(b) FROM collate_test1; + min | max +-----+----- + abc | bbc +(1 row) + +SELECT min(b), max(b) FROM collate_test2; + min | max +-----+----- + abc | äbc +(1 row) + +SELECT min(b), max(b) FROM collate_test3; + min | max +-----+----- + ABC | äbc +(1 row) + +SELECT array_agg(b ORDER BY b) FROM collate_test1; + array_agg +------------------- + {abc,ABC,äbc,bbc} +(1 row) + +SELECT array_agg(b ORDER BY b) FROM collate_test2; + array_agg +------------------- + {abc,ABC,bbc,äbc} +(1 row) + +SELECT array_agg(b ORDER BY b) FROM collate_test3; + array_agg +------------------- + {ABC,abc,bbc,äbc} +(1 row) + +SELECT a, b FROM collate_test1 UNION ALL SELECT a, b FROM collate_test1 ORDER BY 2; + a | b +---+----- + 1 | abc + 1 | abc + 4 | ABC + 4 | ABC + 2 | äbc + 2 | äbc + 3 | bbc + 3 | bbc +(8 rows) + +SELECT a, b FROM collate_test2 UNION SELECT a, b FROM collate_test2 ORDER BY 2; + a | b +---+----- + 1 | abc + 4 | ABC + 3 | bbc + 2 | äbc +(4 rows) + +SELECT a, b FROM collate_test3 WHERE a < 4 INTERSECT SELECT a, b FROM collate_test3 WHERE a > 1 ORDER BY 2; + a | b +---+----- + 3 | bbc + 2 | äbc +(2 rows) + +SELECT a, b FROM collate_test3 EXCEPT SELECT a, b FROM collate_test3 WHERE a < 2 ORDER BY 2; + a | b +---+----- + 4 | ABC + 3 | bbc + 2 | äbc +(3 rows) + +SELECT a, b FROM collate_test1 UNION ALL SELECT a, b FROM collate_test3 ORDER BY 2; -- fail +ERROR: could not determine which collation to use for string comparison +HINT: Use the COLLATE clause to set the collation explicitly. +SELECT a, b FROM collate_test1 UNION ALL SELECT a, b FROM collate_test3; -- ok + a | b +---+----- + 1 | abc + 2 | äbc + 3 | bbc + 4 | ABC + 1 | abc + 2 | äbc + 3 | bbc + 4 | ABC +(8 rows) + +SELECT a, b FROM collate_test1 UNION SELECT a, b FROM collate_test3 ORDER BY 2; -- fail +ERROR: collation mismatch between implicit collations "en-x-icu" and "C" +LINE 1: SELECT a, b FROM collate_test1 UNION SELECT a, b FROM collat... + ^ +HINT: You can choose the collation by applying the COLLATE clause to one or both expressions. +SELECT a, b COLLATE "C" FROM collate_test1 UNION SELECT a, b FROM collate_test3 ORDER BY 2; -- ok + a | b +---+----- + 4 | ABC + 1 | abc + 3 | bbc + 2 | äbc +(4 rows) + +SELECT a, b FROM collate_test1 INTERSECT SELECT a, b FROM collate_test3 ORDER BY 2; -- fail +ERROR: collation mismatch between implicit collations "en-x-icu" and "C" +LINE 1: ...ELECT a, b FROM collate_test1 INTERSECT SELECT a, b FROM col... + ^ +HINT: You can choose the collation by applying the COLLATE clause to one or both expressions. +SELECT a, b FROM collate_test1 EXCEPT SELECT a, b FROM collate_test3 ORDER BY 2; -- fail +ERROR: collation mismatch between implicit collations "en-x-icu" and "C" +LINE 1: SELECT a, b FROM collate_test1 EXCEPT SELECT a, b FROM colla... + ^ +HINT: You can choose the collation by applying the COLLATE clause to one or both expressions. +CREATE TABLE test_u AS SELECT a, b FROM collate_test1 UNION ALL SELECT a, b FROM collate_test3; -- fail +ERROR: no collation was derived for column "b" with collatable type text +HINT: Use the COLLATE clause to set the collation explicitly. +-- ideally this would be a parse-time error, but for now it must be run-time: +select x < y from collate_test10; -- fail +ERROR: could not determine which collation to use for string comparison +HINT: Use the COLLATE clause to set the collation explicitly. +select x || y from collate_test10; -- ok, because || is not collation aware + ?column? +---------- + hijhij + HIJHIJ +(2 rows) + +select x, y from collate_test10 order by x || y; -- not so ok +ERROR: collation mismatch between implicit collations "en-x-icu" and "tr-x-icu" +LINE 1: select x, y from collate_test10 order by x || y; + ^ +HINT: You can choose the collation by applying the COLLATE clause to one or both expressions. +-- collation mismatch between recursive and non-recursive term +WITH RECURSIVE foo(x) AS + (SELECT x FROM (VALUES('a' COLLATE "en-x-icu"),('b')) t(x) + UNION ALL + SELECT (x || 'c') COLLATE "de-x-icu" FROM foo WHERE length(x) < 10) +SELECT * FROM foo; +ERROR: recursive query "foo" column 1 has collation "en-x-icu" in non-recursive term but collation "de-x-icu" overall +LINE 2: (SELECT x FROM (VALUES('a' COLLATE "en-x-icu"),('b')) t(x... + ^ +HINT: Use the COLLATE clause to set the collation of the non-recursive term. +-- casting +SELECT CAST('42' AS text COLLATE "C"); +ERROR: syntax error at or near "COLLATE" +LINE 1: SELECT CAST('42' AS text COLLATE "C"); + ^ +SELECT a, CAST(b AS varchar) FROM collate_test1 ORDER BY 2; + a | b +---+----- + 1 | abc + 4 | ABC + 2 | äbc + 3 | bbc +(4 rows) + +SELECT a, CAST(b AS varchar) FROM collate_test2 ORDER BY 2; + a | b +---+----- + 1 | abc + 4 | ABC + 3 | bbc + 2 | äbc +(4 rows) + +SELECT a, CAST(b AS varchar) FROM collate_test3 ORDER BY 2; + a | b +---+----- + 4 | ABC + 1 | abc + 3 | bbc + 2 | äbc +(4 rows) + +-- propagation of collation in SQL functions (inlined and non-inlined cases) +-- and plpgsql functions too +CREATE FUNCTION mylt (text, text) RETURNS boolean LANGUAGE sql + AS $$ select $1 < $2 $$; +CREATE FUNCTION mylt_noninline (text, text) RETURNS boolean LANGUAGE sql + AS $$ select $1 < $2 limit 1 $$; +CREATE FUNCTION mylt_plpgsql (text, text) RETURNS boolean LANGUAGE plpgsql + AS $$ begin return $1 < $2; end $$; +SELECT a.b AS a, b.b AS b, a.b < b.b AS lt, + mylt(a.b, b.b), mylt_noninline(a.b, b.b), mylt_plpgsql(a.b, b.b) +FROM collate_test1 a, collate_test1 b +ORDER BY a.b, b.b; + a | b | lt | mylt | mylt_noninline | mylt_plpgsql +-----+-----+----+------+----------------+-------------- + abc | abc | f | f | f | f + abc | ABC | t | t | t | t + abc | äbc | t | t | t | t + abc | bbc | t | t | t | t + ABC | abc | f | f | f | f + ABC | ABC | f | f | f | f + ABC | äbc | t | t | t | t + ABC | bbc | t | t | t | t + äbc | abc | f | f | f | f + äbc | ABC | f | f | f | f + äbc | äbc | f | f | f | f + äbc | bbc | t | t | t | t + bbc | abc | f | f | f | f + bbc | ABC | f | f | f | f + bbc | äbc | f | f | f | f + bbc | bbc | f | f | f | f +(16 rows) + +SELECT a.b AS a, b.b AS b, a.b < b.b COLLATE "C" AS lt, + mylt(a.b, b.b COLLATE "C"), mylt_noninline(a.b, b.b COLLATE "C"), + mylt_plpgsql(a.b, b.b COLLATE "C") +FROM collate_test1 a, collate_test1 b +ORDER BY a.b, b.b; + a | b | lt | mylt | mylt_noninline | mylt_plpgsql +-----+-----+----+------+----------------+-------------- + abc | abc | f | f | f | f + abc | ABC | f | f | f | f + abc | äbc | t | t | t | t + abc | bbc | t | t | t | t + ABC | abc | t | t | t | t + ABC | ABC | f | f | f | f + ABC | äbc | t | t | t | t + ABC | bbc | t | t | t | t + äbc | abc | f | f | f | f + äbc | ABC | f | f | f | f + äbc | äbc | f | f | f | f + äbc | bbc | f | f | f | f + bbc | abc | f | f | f | f + bbc | ABC | f | f | f | f + bbc | äbc | t | t | t | t + bbc | bbc | f | f | f | f +(16 rows) + +-- collation override in plpgsql +CREATE FUNCTION mylt2 (x text, y text) RETURNS boolean LANGUAGE plpgsql AS $$ +declare + xx text := x; + yy text := y; +begin + return xx < yy; +end +$$; +SELECT mylt2('a', 'B' collate "en-x-icu") as t, mylt2('a', 'B' collate "C") as f; + t | f +---+--- + t | f +(1 row) + +CREATE OR REPLACE FUNCTION + mylt2 (x text, y text) RETURNS boolean LANGUAGE plpgsql AS $$ +declare + xx text COLLATE "POSIX" := x; + yy text := y; +begin + return xx < yy; +end +$$; +SELECT mylt2('a', 'B') as f; + f +--- + f +(1 row) + +SELECT mylt2('a', 'B' collate "C") as fail; -- conflicting collations +ERROR: could not determine which collation to use for string comparison +HINT: Use the COLLATE clause to set the collation explicitly. +CONTEXT: PL/pgSQL function mylt2(text,text) line 6 at RETURN +SELECT mylt2('a', 'B' collate "POSIX") as f; + f +--- + f +(1 row) + +-- polymorphism +SELECT * FROM unnest((SELECT array_agg(b ORDER BY b) FROM collate_test1)) ORDER BY 1; + unnest +-------- + abc + ABC + äbc + bbc +(4 rows) + +SELECT * FROM unnest((SELECT array_agg(b ORDER BY b) FROM collate_test2)) ORDER BY 1; + unnest +-------- + abc + ABC + bbc + äbc +(4 rows) + +SELECT * FROM unnest((SELECT array_agg(b ORDER BY b) FROM collate_test3)) ORDER BY 1; + unnest +-------- + ABC + abc + bbc + äbc +(4 rows) + +CREATE FUNCTION dup (anyelement) RETURNS anyelement + AS 'select $1' LANGUAGE sql; +SELECT a, dup(b) FROM collate_test1 ORDER BY 2; + a | dup +---+----- + 1 | abc + 4 | ABC + 2 | äbc + 3 | bbc +(4 rows) + +SELECT a, dup(b) FROM collate_test2 ORDER BY 2; + a | dup +---+----- + 1 | abc + 4 | ABC + 3 | bbc + 2 | äbc +(4 rows) + +SELECT a, dup(b) FROM collate_test3 ORDER BY 2; + a | dup +---+----- + 4 | ABC + 1 | abc + 3 | bbc + 2 | äbc +(4 rows) + +-- indexes +CREATE INDEX collate_test1_idx1 ON collate_test1 (b); +CREATE INDEX collate_test1_idx2 ON collate_test1 (b COLLATE "C"); +CREATE INDEX collate_test1_idx3 ON collate_test1 ((b COLLATE "C")); -- this is different grammatically +CREATE INDEX collate_test1_idx4 ON collate_test1 (((b||'foo') COLLATE "POSIX")); +CREATE INDEX collate_test1_idx5 ON collate_test1 (a COLLATE "C"); -- fail +ERROR: collations are not supported by type integer +CREATE INDEX collate_test1_idx6 ON collate_test1 ((a COLLATE "C")); -- fail +ERROR: collations are not supported by type integer +LINE 1: ...ATE INDEX collate_test1_idx6 ON collate_test1 ((a COLLATE "C... + ^ +SELECT relname, pg_get_indexdef(oid) FROM pg_class WHERE relname LIKE 'collate_test%_idx%' ORDER BY 1; + relname | pg_get_indexdef +--------------------+----------------------------------------------------------------------------------------------------- + collate_test1_idx1 | CREATE INDEX collate_test1_idx1 ON collate_test1 USING btree (b) + collate_test1_idx2 | CREATE INDEX collate_test1_idx2 ON collate_test1 USING btree (b COLLATE "C") + collate_test1_idx3 | CREATE INDEX collate_test1_idx3 ON collate_test1 USING btree (b COLLATE "C") + collate_test1_idx4 | CREATE INDEX collate_test1_idx4 ON collate_test1 USING btree (((b || 'foo'::text)) COLLATE "POSIX") +(4 rows) + +-- schema manipulation commands +CREATE ROLE regress_test_role; +CREATE SCHEMA test_schema; +-- We need to do this this way to cope with varying names for encodings: +do $$ +BEGIN + EXECUTE 'CREATE COLLATION test0 (provider = icu, locale = ' || + quote_literal(current_setting('lc_collate')) || ');'; +END +$$; +CREATE COLLATION test0 FROM "C"; -- fail, duplicate name +ERROR: collation "test0" already exists +do $$ +BEGIN + EXECUTE 'CREATE COLLATION test1 (provider = icu, lc_collate = ' || + quote_literal(current_setting('lc_collate')) || + ', lc_ctype = ' || + quote_literal(current_setting('lc_ctype')) || ');'; +END +$$; +CREATE COLLATION test3 (provider = icu, lc_collate = 'en_US.utf8'); -- fail, need lc_ctype +ERROR: parameter "lc_ctype" must be specified +CREATE COLLATION testx (provider = icu, locale = 'nonsense'); /* never fails with ICU */ DROP COLLATION testx; +CREATE COLLATION test4 FROM nonsense; +ERROR: collation "nonsense" for encoding "UTF8" does not exist +CREATE COLLATION test5 FROM test0; +SELECT collname FROM pg_collation WHERE collname LIKE 'test%' ORDER BY 1; + collname +---------- + test0 + test1 + test5 +(3 rows) + +ALTER COLLATION test1 RENAME TO test11; +ALTER COLLATION test0 RENAME TO test11; -- fail +ERROR: collation "test11" already exists in schema "collate_tests" +ALTER COLLATION test1 RENAME TO test22; -- fail +ERROR: collation "test1" for encoding "UTF8" does not exist +ALTER COLLATION test11 OWNER TO regress_test_role; +ALTER COLLATION test11 OWNER TO nonsense; +ERROR: role "nonsense" does not exist +ALTER COLLATION test11 SET SCHEMA test_schema; +COMMENT ON COLLATION test0 IS 'US English'; +SELECT collname, nspname, obj_description(pg_collation.oid, 'pg_collation') + FROM pg_collation JOIN pg_namespace ON (collnamespace = pg_namespace.oid) + WHERE collname LIKE 'test%' + ORDER BY 1; + collname | nspname | obj_description +----------+---------------+----------------- + test0 | collate_tests | US English + test11 | test_schema | + test5 | collate_tests | +(3 rows) + +DROP COLLATION test0, test_schema.test11, test5; +DROP COLLATION test0; -- fail +ERROR: collation "test0" for encoding "UTF8" does not exist +DROP COLLATION IF EXISTS test0; +NOTICE: collation "test0" does not exist, skipping +SELECT collname FROM pg_collation WHERE collname LIKE 'test%'; + collname +---------- +(0 rows) + +DROP SCHEMA test_schema; +DROP ROLE regress_test_role; +-- ALTER +ALTER COLLATION "en-x-icu" REFRESH VERSION; +NOTICE: version has not changed +-- dependencies +CREATE COLLATION test0 FROM "C"; +CREATE TABLE collate_dep_test1 (a int, b text COLLATE test0); +CREATE DOMAIN collate_dep_dom1 AS text COLLATE test0; +CREATE TYPE collate_dep_test2 AS (x int, y text COLLATE test0); +CREATE VIEW collate_dep_test3 AS SELECT text 'foo' COLLATE test0 AS foo; +CREATE TABLE collate_dep_test4t (a int, b text); +CREATE INDEX collate_dep_test4i ON collate_dep_test4t (b COLLATE test0); +DROP COLLATION test0 RESTRICT; -- fail +ERROR: cannot drop collation test0 because other objects depend on it +DETAIL: table collate_dep_test1 column b depends on collation test0 +type collate_dep_dom1 depends on collation test0 +composite type collate_dep_test2 column y depends on collation test0 +view collate_dep_test3 depends on collation test0 +index collate_dep_test4i depends on collation test0 +HINT: Use DROP ... CASCADE to drop the dependent objects too. +DROP COLLATION test0 CASCADE; +NOTICE: drop cascades to 5 other objects +DETAIL: drop cascades to table collate_dep_test1 column b +drop cascades to type collate_dep_dom1 +drop cascades to composite type collate_dep_test2 column y +drop cascades to view collate_dep_test3 +drop cascades to index collate_dep_test4i +\d collate_dep_test1 + Table "collate_tests.collate_dep_test1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + +\d collate_dep_test2 + Composite type "collate_tests.collate_dep_test2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + x | integer | | | + +DROP TABLE collate_dep_test1, collate_dep_test4t; +DROP TYPE collate_dep_test2; +-- test range types and collations +create type textrange_c as range(subtype=text, collation="C"); +create type textrange_en_us as range(subtype=text, collation="en-x-icu"); +select textrange_c('A','Z') @> 'b'::text; + ?column? +---------- + f +(1 row) + +select textrange_en_us('A','Z') @> 'b'::text; + ?column? +---------- + t +(1 row) + +drop type textrange_c; +drop type textrange_en_us; +-- cleanup +DROP SCHEMA collate_tests CASCADE; +NOTICE: drop cascades to 18 other objects +DETAIL: drop cascades to table collate_test1 +drop cascades to table collate_test_like +drop cascades to table collate_test2 +drop cascades to table collate_test3 +drop cascades to type testdomain_sv +drop cascades to table collate_test4 +drop cascades to table collate_test5 +drop cascades to table collate_test10 +drop cascades to table collate_test6 +drop cascades to view collview1 +drop cascades to view collview2 +drop cascades to view collview3 +drop cascades to type testdomain +drop cascades to function mylt(text,text) +drop cascades to function mylt_noninline(text,text) +drop cascades to function mylt_plpgsql(text,text) +drop cascades to function mylt2(text,text) +drop cascades to function dup(anyelement) +RESET search_path; +-- leave a collation for pg_upgrade test +CREATE COLLATION coll_icu_upgrade FROM "und-x-icu"; diff --git a/src/test/regress/sql/collate.icu.sql b/src/test/regress/sql/collate.icu.sql deleted file mode 100644 index ef39445b30..0000000000 --- a/src/test/regress/sql/collate.icu.sql +++ /dev/null @@ -1,433 +0,0 @@ -/* - * This test is for ICU collations. - */ - -SET client_encoding TO UTF8; - -CREATE SCHEMA collate_tests; -SET search_path = collate_tests; - - -CREATE TABLE collate_test1 ( - a int, - b text COLLATE "en-x-icu" NOT NULL -); - -\d collate_test1 - -CREATE TABLE collate_test_fail ( - a int, - b text COLLATE "ja_JP.eucjp-x-icu" -); - -CREATE TABLE collate_test_fail ( - a int, - b text COLLATE "foo-x-icu" -); - -CREATE TABLE collate_test_fail ( - a int COLLATE "en-x-icu", - b text -); - -CREATE TABLE collate_test_like ( - LIKE collate_test1 -); - -\d collate_test_like - -CREATE TABLE collate_test2 ( - a int, - b text COLLATE "sv-x-icu" -); - -CREATE TABLE collate_test3 ( - a int, - b text COLLATE "C" -); - -INSERT INTO collate_test1 VALUES (1, 'abc'), (2, 'äbc'), (3, 'bbc'), (4, 'ABC'); -INSERT INTO collate_test2 SELECT * FROM collate_test1; -INSERT INTO collate_test3 SELECT * FROM collate_test1; - -SELECT * FROM collate_test1 WHERE b >= 'bbc'; -SELECT * FROM collate_test2 WHERE b >= 'bbc'; -SELECT * FROM collate_test3 WHERE b >= 'bbc'; -SELECT * FROM collate_test3 WHERE b >= 'BBC'; - -SELECT * FROM collate_test1 WHERE b COLLATE "C" >= 'bbc'; -SELECT * FROM collate_test1 WHERE b >= 'bbc' COLLATE "C"; -SELECT * FROM collate_test1 WHERE b COLLATE "C" >= 'bbc' COLLATE "C"; -SELECT * FROM collate_test1 WHERE b COLLATE "C" >= 'bbc' COLLATE "en-x-icu"; - - -CREATE DOMAIN testdomain_sv AS text COLLATE "sv-x-icu"; -CREATE DOMAIN testdomain_i AS int COLLATE "sv-x-icu"; -- fails -CREATE TABLE collate_test4 ( - a int, - b testdomain_sv -); -INSERT INTO collate_test4 SELECT * FROM collate_test1; -SELECT a, b FROM collate_test4 ORDER BY b; - -CREATE TABLE collate_test5 ( - a int, - b testdomain_sv COLLATE "en-x-icu" -); -INSERT INTO collate_test5 SELECT * FROM collate_test1; -SELECT a, b FROM collate_test5 ORDER BY b; - - -SELECT a, b FROM collate_test1 ORDER BY b; -SELECT a, b FROM collate_test2 ORDER BY b; -SELECT a, b FROM collate_test3 ORDER BY b; - -SELECT a, b FROM collate_test1 ORDER BY b COLLATE "C"; - --- star expansion -SELECT * FROM collate_test1 ORDER BY b; -SELECT * FROM collate_test2 ORDER BY b; -SELECT * FROM collate_test3 ORDER BY b; - --- constant expression folding -SELECT 'bbc' COLLATE "en-x-icu" > 'äbc' COLLATE "en-x-icu" AS "true"; -SELECT 'bbc' COLLATE "sv-x-icu" > 'äbc' COLLATE "sv-x-icu" AS "false"; - --- upper/lower - -CREATE TABLE collate_test10 ( - a int, - x text COLLATE "en-x-icu", - y text COLLATE "tr-x-icu" -); - -INSERT INTO collate_test10 VALUES (1, 'hij', 'hij'), (2, 'HIJ', 'HIJ'); - -SELECT a, lower(x), lower(y), upper(x), upper(y), initcap(x), initcap(y) FROM collate_test10; -SELECT a, lower(x COLLATE "C"), lower(y COLLATE "C") FROM collate_test10; - -SELECT a, x, y FROM collate_test10 ORDER BY lower(y), a; - --- LIKE/ILIKE - -SELECT * FROM collate_test1 WHERE b LIKE 'abc'; -SELECT * FROM collate_test1 WHERE b LIKE 'abc%'; -SELECT * FROM collate_test1 WHERE b LIKE '%bc%'; -SELECT * FROM collate_test1 WHERE b ILIKE 'abc'; -SELECT * FROM collate_test1 WHERE b ILIKE 'abc%'; -SELECT * FROM collate_test1 WHERE b ILIKE '%bc%'; - -SELECT 'Türkiye' COLLATE "en-x-icu" ILIKE '%KI%' AS "true"; -SELECT 'Türkiye' COLLATE "tr-x-icu" ILIKE '%KI%' AS "false"; - -SELECT 'bıt' ILIKE 'BIT' COLLATE "en-x-icu" AS "false"; -SELECT 'bıt' ILIKE 'BIT' COLLATE "tr-x-icu" AS "true"; - --- The following actually exercises the selectivity estimation for ILIKE. -SELECT relname FROM pg_class WHERE relname ILIKE 'abc%'; - --- regular expressions - -SELECT * FROM collate_test1 WHERE b ~ '^abc$'; -SELECT * FROM collate_test1 WHERE b ~ '^abc'; -SELECT * FROM collate_test1 WHERE b ~ 'bc'; -SELECT * FROM collate_test1 WHERE b ~* '^abc$'; -SELECT * FROM collate_test1 WHERE b ~* '^abc'; -SELECT * FROM collate_test1 WHERE b ~* 'bc'; - -CREATE TABLE collate_test6 ( - a int, - b text COLLATE "en-x-icu" -); -INSERT INTO collate_test6 VALUES (1, 'abc'), (2, 'ABC'), (3, '123'), (4, 'ab1'), - (5, 'a1!'), (6, 'a c'), (7, '!.;'), (8, ' '), - (9, 'äbç'), (10, 'ÄBÇ'); -SELECT b, - b ~ '^[[:alpha:]]+$' AS is_alpha, - b ~ '^[[:upper:]]+$' AS is_upper, - b ~ '^[[:lower:]]+$' AS is_lower, - b ~ '^[[:digit:]]+$' AS is_digit, - b ~ '^[[:alnum:]]+$' AS is_alnum, - b ~ '^[[:graph:]]+$' AS is_graph, - b ~ '^[[:print:]]+$' AS is_print, - b ~ '^[[:punct:]]+$' AS is_punct, - b ~ '^[[:space:]]+$' AS is_space -FROM collate_test6; - -SELECT 'Türkiye' COLLATE "en-x-icu" ~* 'KI' AS "true"; -SELECT 'Türkiye' COLLATE "tr-x-icu" ~* 'KI' AS "true"; -- true with ICU - -SELECT 'bıt' ~* 'BIT' COLLATE "en-x-icu" AS "false"; -SELECT 'bıt' ~* 'BIT' COLLATE "tr-x-icu" AS "false"; -- false with ICU - --- The following actually exercises the selectivity estimation for ~*. -SELECT relname FROM pg_class WHERE relname ~* '^abc'; - - -/* not run by default because it requires tr_TR system locale --- to_char - -SET lc_time TO 'tr_TR'; -SELECT to_char(date '2010-04-01', 'DD TMMON YYYY'); -SELECT to_char(date '2010-04-01', 'DD TMMON YYYY' COLLATE "tr-x-icu"); -*/ - - --- backwards parsing - -CREATE VIEW collview1 AS SELECT * FROM collate_test1 WHERE b COLLATE "C" >= 'bbc'; -CREATE VIEW collview2 AS SELECT a, b FROM collate_test1 ORDER BY b COLLATE "C"; -CREATE VIEW collview3 AS SELECT a, lower((x || x) COLLATE "C") FROM collate_test10; - -SELECT table_name, view_definition FROM information_schema.views - WHERE table_name LIKE 'collview%' ORDER BY 1; - - --- collation propagation in various expression types - -SELECT a, coalesce(b, 'foo') FROM collate_test1 ORDER BY 2; -SELECT a, coalesce(b, 'foo') FROM collate_test2 ORDER BY 2; -SELECT a, coalesce(b, 'foo') FROM collate_test3 ORDER BY 2; -SELECT a, lower(coalesce(x, 'foo')), lower(coalesce(y, 'foo')) FROM collate_test10; - -SELECT a, b, greatest(b, 'CCC') FROM collate_test1 ORDER BY 3; -SELECT a, b, greatest(b, 'CCC') FROM collate_test2 ORDER BY 3; -SELECT a, b, greatest(b, 'CCC') FROM collate_test3 ORDER BY 3; -SELECT a, x, y, lower(greatest(x, 'foo')), lower(greatest(y, 'foo')) FROM collate_test10; - -SELECT a, nullif(b, 'abc') FROM collate_test1 ORDER BY 2; -SELECT a, nullif(b, 'abc') FROM collate_test2 ORDER BY 2; -SELECT a, nullif(b, 'abc') FROM collate_test3 ORDER BY 2; -SELECT a, lower(nullif(x, 'foo')), lower(nullif(y, 'foo')) FROM collate_test10; - -SELECT a, CASE b WHEN 'abc' THEN 'abcd' ELSE b END FROM collate_test1 ORDER BY 2; -SELECT a, CASE b WHEN 'abc' THEN 'abcd' ELSE b END FROM collate_test2 ORDER BY 2; -SELECT a, CASE b WHEN 'abc' THEN 'abcd' ELSE b END FROM collate_test3 ORDER BY 2; - -CREATE DOMAIN testdomain AS text; -SELECT a, b::testdomain FROM collate_test1 ORDER BY 2; -SELECT a, b::testdomain FROM collate_test2 ORDER BY 2; -SELECT a, b::testdomain FROM collate_test3 ORDER BY 2; -SELECT a, b::testdomain_sv FROM collate_test3 ORDER BY 2; -SELECT a, lower(x::testdomain), lower(y::testdomain) FROM collate_test10; - -SELECT min(b), max(b) FROM collate_test1; -SELECT min(b), max(b) FROM collate_test2; -SELECT min(b), max(b) FROM collate_test3; - -SELECT array_agg(b ORDER BY b) FROM collate_test1; -SELECT array_agg(b ORDER BY b) FROM collate_test2; -SELECT array_agg(b ORDER BY b) FROM collate_test3; - -SELECT a, b FROM collate_test1 UNION ALL SELECT a, b FROM collate_test1 ORDER BY 2; -SELECT a, b FROM collate_test2 UNION SELECT a, b FROM collate_test2 ORDER BY 2; -SELECT a, b FROM collate_test3 WHERE a < 4 INTERSECT SELECT a, b FROM collate_test3 WHERE a > 1 ORDER BY 2; -SELECT a, b FROM collate_test3 EXCEPT SELECT a, b FROM collate_test3 WHERE a < 2 ORDER BY 2; - -SELECT a, b FROM collate_test1 UNION ALL SELECT a, b FROM collate_test3 ORDER BY 2; -- fail -SELECT a, b FROM collate_test1 UNION ALL SELECT a, b FROM collate_test3; -- ok -SELECT a, b FROM collate_test1 UNION SELECT a, b FROM collate_test3 ORDER BY 2; -- fail -SELECT a, b COLLATE "C" FROM collate_test1 UNION SELECT a, b FROM collate_test3 ORDER BY 2; -- ok -SELECT a, b FROM collate_test1 INTERSECT SELECT a, b FROM collate_test3 ORDER BY 2; -- fail -SELECT a, b FROM collate_test1 EXCEPT SELECT a, b FROM collate_test3 ORDER BY 2; -- fail - -CREATE TABLE test_u AS SELECT a, b FROM collate_test1 UNION ALL SELECT a, b FROM collate_test3; -- fail - --- ideally this would be a parse-time error, but for now it must be run-time: -select x < y from collate_test10; -- fail -select x || y from collate_test10; -- ok, because || is not collation aware -select x, y from collate_test10 order by x || y; -- not so ok - --- collation mismatch between recursive and non-recursive term -WITH RECURSIVE foo(x) AS - (SELECT x FROM (VALUES('a' COLLATE "en-x-icu"),('b')) t(x) - UNION ALL - SELECT (x || 'c') COLLATE "de-x-icu" FROM foo WHERE length(x) < 10) -SELECT * FROM foo; - - --- casting - -SELECT CAST('42' AS text COLLATE "C"); - -SELECT a, CAST(b AS varchar) FROM collate_test1 ORDER BY 2; -SELECT a, CAST(b AS varchar) FROM collate_test2 ORDER BY 2; -SELECT a, CAST(b AS varchar) FROM collate_test3 ORDER BY 2; - - --- propagation of collation in SQL functions (inlined and non-inlined cases) --- and plpgsql functions too - -CREATE FUNCTION mylt (text, text) RETURNS boolean LANGUAGE sql - AS $$ select $1 < $2 $$; - -CREATE FUNCTION mylt_noninline (text, text) RETURNS boolean LANGUAGE sql - AS $$ select $1 < $2 limit 1 $$; - -CREATE FUNCTION mylt_plpgsql (text, text) RETURNS boolean LANGUAGE plpgsql - AS $$ begin return $1 < $2; end $$; - -SELECT a.b AS a, b.b AS b, a.b < b.b AS lt, - mylt(a.b, b.b), mylt_noninline(a.b, b.b), mylt_plpgsql(a.b, b.b) -FROM collate_test1 a, collate_test1 b -ORDER BY a.b, b.b; - -SELECT a.b AS a, b.b AS b, a.b < b.b COLLATE "C" AS lt, - mylt(a.b, b.b COLLATE "C"), mylt_noninline(a.b, b.b COLLATE "C"), - mylt_plpgsql(a.b, b.b COLLATE "C") -FROM collate_test1 a, collate_test1 b -ORDER BY a.b, b.b; - - --- collation override in plpgsql - -CREATE FUNCTION mylt2 (x text, y text) RETURNS boolean LANGUAGE plpgsql AS $$ -declare - xx text := x; - yy text := y; -begin - return xx < yy; -end -$$; - -SELECT mylt2('a', 'B' collate "en-x-icu") as t, mylt2('a', 'B' collate "C") as f; - -CREATE OR REPLACE FUNCTION - mylt2 (x text, y text) RETURNS boolean LANGUAGE plpgsql AS $$ -declare - xx text COLLATE "POSIX" := x; - yy text := y; -begin - return xx < yy; -end -$$; - -SELECT mylt2('a', 'B') as f; -SELECT mylt2('a', 'B' collate "C") as fail; -- conflicting collations -SELECT mylt2('a', 'B' collate "POSIX") as f; - - --- polymorphism - -SELECT * FROM unnest((SELECT array_agg(b ORDER BY b) FROM collate_test1)) ORDER BY 1; -SELECT * FROM unnest((SELECT array_agg(b ORDER BY b) FROM collate_test2)) ORDER BY 1; -SELECT * FROM unnest((SELECT array_agg(b ORDER BY b) FROM collate_test3)) ORDER BY 1; - -CREATE FUNCTION dup (anyelement) RETURNS anyelement - AS 'select $1' LANGUAGE sql; - -SELECT a, dup(b) FROM collate_test1 ORDER BY 2; -SELECT a, dup(b) FROM collate_test2 ORDER BY 2; -SELECT a, dup(b) FROM collate_test3 ORDER BY 2; - - --- indexes - -CREATE INDEX collate_test1_idx1 ON collate_test1 (b); -CREATE INDEX collate_test1_idx2 ON collate_test1 (b COLLATE "C"); -CREATE INDEX collate_test1_idx3 ON collate_test1 ((b COLLATE "C")); -- this is different grammatically -CREATE INDEX collate_test1_idx4 ON collate_test1 (((b||'foo') COLLATE "POSIX")); - -CREATE INDEX collate_test1_idx5 ON collate_test1 (a COLLATE "C"); -- fail -CREATE INDEX collate_test1_idx6 ON collate_test1 ((a COLLATE "C")); -- fail - -SELECT relname, pg_get_indexdef(oid) FROM pg_class WHERE relname LIKE 'collate_test%_idx%' ORDER BY 1; - - --- schema manipulation commands - -CREATE ROLE regress_test_role; -CREATE SCHEMA test_schema; - --- We need to do this this way to cope with varying names for encodings: -do $$ -BEGIN - EXECUTE 'CREATE COLLATION test0 (provider = icu, locale = ' || - quote_literal(current_setting('lc_collate')) || ');'; -END -$$; -CREATE COLLATION test0 FROM "C"; -- fail, duplicate name -do $$ -BEGIN - EXECUTE 'CREATE COLLATION test1 (provider = icu, lc_collate = ' || - quote_literal(current_setting('lc_collate')) || - ', lc_ctype = ' || - quote_literal(current_setting('lc_ctype')) || ');'; -END -$$; -CREATE COLLATION test3 (provider = icu, lc_collate = 'en_US.utf8'); -- fail, need lc_ctype -CREATE COLLATION testx (provider = icu, locale = 'nonsense'); /* never fails with ICU */ DROP COLLATION testx; - -CREATE COLLATION test4 FROM nonsense; -CREATE COLLATION test5 FROM test0; - -SELECT collname FROM pg_collation WHERE collname LIKE 'test%' ORDER BY 1; - -ALTER COLLATION test1 RENAME TO test11; -ALTER COLLATION test0 RENAME TO test11; -- fail -ALTER COLLATION test1 RENAME TO test22; -- fail - -ALTER COLLATION test11 OWNER TO regress_test_role; -ALTER COLLATION test11 OWNER TO nonsense; -ALTER COLLATION test11 SET SCHEMA test_schema; - -COMMENT ON COLLATION test0 IS 'US English'; - -SELECT collname, nspname, obj_description(pg_collation.oid, 'pg_collation') - FROM pg_collation JOIN pg_namespace ON (collnamespace = pg_namespace.oid) - WHERE collname LIKE 'test%' - ORDER BY 1; - -DROP COLLATION test0, test_schema.test11, test5; -DROP COLLATION test0; -- fail -DROP COLLATION IF EXISTS test0; - -SELECT collname FROM pg_collation WHERE collname LIKE 'test%'; - -DROP SCHEMA test_schema; -DROP ROLE regress_test_role; - - --- ALTER - -ALTER COLLATION "en-x-icu" REFRESH VERSION; - - --- dependencies - -CREATE COLLATION test0 FROM "C"; - -CREATE TABLE collate_dep_test1 (a int, b text COLLATE test0); -CREATE DOMAIN collate_dep_dom1 AS text COLLATE test0; -CREATE TYPE collate_dep_test2 AS (x int, y text COLLATE test0); -CREATE VIEW collate_dep_test3 AS SELECT text 'foo' COLLATE test0 AS foo; -CREATE TABLE collate_dep_test4t (a int, b text); -CREATE INDEX collate_dep_test4i ON collate_dep_test4t (b COLLATE test0); - -DROP COLLATION test0 RESTRICT; -- fail -DROP COLLATION test0 CASCADE; - -\d collate_dep_test1 -\d collate_dep_test2 - -DROP TABLE collate_dep_test1, collate_dep_test4t; -DROP TYPE collate_dep_test2; - --- test range types and collations - -create type textrange_c as range(subtype=text, collation="C"); -create type textrange_en_us as range(subtype=text, collation="en-x-icu"); - -select textrange_c('A','Z') @> 'b'::text; -select textrange_en_us('A','Z') @> 'b'::text; - -drop type textrange_c; -drop type textrange_en_us; - - --- cleanup -DROP SCHEMA collate_tests CASCADE; -RESET search_path; - --- leave a collation for pg_upgrade test -CREATE COLLATION coll_icu_upgrade FROM "und-x-icu"; diff --git a/src/test/regress/sql/collate.icu.utf8.sql b/src/test/regress/sql/collate.icu.utf8.sql new file mode 100644 index 0000000000..ef39445b30 --- /dev/null +++ b/src/test/regress/sql/collate.icu.utf8.sql @@ -0,0 +1,433 @@ +/* + * This test is for ICU collations. + */ + +SET client_encoding TO UTF8; + +CREATE SCHEMA collate_tests; +SET search_path = collate_tests; + + +CREATE TABLE collate_test1 ( + a int, + b text COLLATE "en-x-icu" NOT NULL +); + +\d collate_test1 + +CREATE TABLE collate_test_fail ( + a int, + b text COLLATE "ja_JP.eucjp-x-icu" +); + +CREATE TABLE collate_test_fail ( + a int, + b text COLLATE "foo-x-icu" +); + +CREATE TABLE collate_test_fail ( + a int COLLATE "en-x-icu", + b text +); + +CREATE TABLE collate_test_like ( + LIKE collate_test1 +); + +\d collate_test_like + +CREATE TABLE collate_test2 ( + a int, + b text COLLATE "sv-x-icu" +); + +CREATE TABLE collate_test3 ( + a int, + b text COLLATE "C" +); + +INSERT INTO collate_test1 VALUES (1, 'abc'), (2, 'äbc'), (3, 'bbc'), (4, 'ABC'); +INSERT INTO collate_test2 SELECT * FROM collate_test1; +INSERT INTO collate_test3 SELECT * FROM collate_test1; + +SELECT * FROM collate_test1 WHERE b >= 'bbc'; +SELECT * FROM collate_test2 WHERE b >= 'bbc'; +SELECT * FROM collate_test3 WHERE b >= 'bbc'; +SELECT * FROM collate_test3 WHERE b >= 'BBC'; + +SELECT * FROM collate_test1 WHERE b COLLATE "C" >= 'bbc'; +SELECT * FROM collate_test1 WHERE b >= 'bbc' COLLATE "C"; +SELECT * FROM collate_test1 WHERE b COLLATE "C" >= 'bbc' COLLATE "C"; +SELECT * FROM collate_test1 WHERE b COLLATE "C" >= 'bbc' COLLATE "en-x-icu"; + + +CREATE DOMAIN testdomain_sv AS text COLLATE "sv-x-icu"; +CREATE DOMAIN testdomain_i AS int COLLATE "sv-x-icu"; -- fails +CREATE TABLE collate_test4 ( + a int, + b testdomain_sv +); +INSERT INTO collate_test4 SELECT * FROM collate_test1; +SELECT a, b FROM collate_test4 ORDER BY b; + +CREATE TABLE collate_test5 ( + a int, + b testdomain_sv COLLATE "en-x-icu" +); +INSERT INTO collate_test5 SELECT * FROM collate_test1; +SELECT a, b FROM collate_test5 ORDER BY b; + + +SELECT a, b FROM collate_test1 ORDER BY b; +SELECT a, b FROM collate_test2 ORDER BY b; +SELECT a, b FROM collate_test3 ORDER BY b; + +SELECT a, b FROM collate_test1 ORDER BY b COLLATE "C"; + +-- star expansion +SELECT * FROM collate_test1 ORDER BY b; +SELECT * FROM collate_test2 ORDER BY b; +SELECT * FROM collate_test3 ORDER BY b; + +-- constant expression folding +SELECT 'bbc' COLLATE "en-x-icu" > 'äbc' COLLATE "en-x-icu" AS "true"; +SELECT 'bbc' COLLATE "sv-x-icu" > 'äbc' COLLATE "sv-x-icu" AS "false"; + +-- upper/lower + +CREATE TABLE collate_test10 ( + a int, + x text COLLATE "en-x-icu", + y text COLLATE "tr-x-icu" +); + +INSERT INTO collate_test10 VALUES (1, 'hij', 'hij'), (2, 'HIJ', 'HIJ'); + +SELECT a, lower(x), lower(y), upper(x), upper(y), initcap(x), initcap(y) FROM collate_test10; +SELECT a, lower(x COLLATE "C"), lower(y COLLATE "C") FROM collate_test10; + +SELECT a, x, y FROM collate_test10 ORDER BY lower(y), a; + +-- LIKE/ILIKE + +SELECT * FROM collate_test1 WHERE b LIKE 'abc'; +SELECT * FROM collate_test1 WHERE b LIKE 'abc%'; +SELECT * FROM collate_test1 WHERE b LIKE '%bc%'; +SELECT * FROM collate_test1 WHERE b ILIKE 'abc'; +SELECT * FROM collate_test1 WHERE b ILIKE 'abc%'; +SELECT * FROM collate_test1 WHERE b ILIKE '%bc%'; + +SELECT 'Türkiye' COLLATE "en-x-icu" ILIKE '%KI%' AS "true"; +SELECT 'Türkiye' COLLATE "tr-x-icu" ILIKE '%KI%' AS "false"; + +SELECT 'bıt' ILIKE 'BIT' COLLATE "en-x-icu" AS "false"; +SELECT 'bıt' ILIKE 'BIT' COLLATE "tr-x-icu" AS "true"; + +-- The following actually exercises the selectivity estimation for ILIKE. +SELECT relname FROM pg_class WHERE relname ILIKE 'abc%'; + +-- regular expressions + +SELECT * FROM collate_test1 WHERE b ~ '^abc$'; +SELECT * FROM collate_test1 WHERE b ~ '^abc'; +SELECT * FROM collate_test1 WHERE b ~ 'bc'; +SELECT * FROM collate_test1 WHERE b ~* '^abc$'; +SELECT * FROM collate_test1 WHERE b ~* '^abc'; +SELECT * FROM collate_test1 WHERE b ~* 'bc'; + +CREATE TABLE collate_test6 ( + a int, + b text COLLATE "en-x-icu" +); +INSERT INTO collate_test6 VALUES (1, 'abc'), (2, 'ABC'), (3, '123'), (4, 'ab1'), + (5, 'a1!'), (6, 'a c'), (7, '!.;'), (8, ' '), + (9, 'äbç'), (10, 'ÄBÇ'); +SELECT b, + b ~ '^[[:alpha:]]+$' AS is_alpha, + b ~ '^[[:upper:]]+$' AS is_upper, + b ~ '^[[:lower:]]+$' AS is_lower, + b ~ '^[[:digit:]]+$' AS is_digit, + b ~ '^[[:alnum:]]+$' AS is_alnum, + b ~ '^[[:graph:]]+$' AS is_graph, + b ~ '^[[:print:]]+$' AS is_print, + b ~ '^[[:punct:]]+$' AS is_punct, + b ~ '^[[:space:]]+$' AS is_space +FROM collate_test6; + +SELECT 'Türkiye' COLLATE "en-x-icu" ~* 'KI' AS "true"; +SELECT 'Türkiye' COLLATE "tr-x-icu" ~* 'KI' AS "true"; -- true with ICU + +SELECT 'bıt' ~* 'BIT' COLLATE "en-x-icu" AS "false"; +SELECT 'bıt' ~* 'BIT' COLLATE "tr-x-icu" AS "false"; -- false with ICU + +-- The following actually exercises the selectivity estimation for ~*. +SELECT relname FROM pg_class WHERE relname ~* '^abc'; + + +/* not run by default because it requires tr_TR system locale +-- to_char + +SET lc_time TO 'tr_TR'; +SELECT to_char(date '2010-04-01', 'DD TMMON YYYY'); +SELECT to_char(date '2010-04-01', 'DD TMMON YYYY' COLLATE "tr-x-icu"); +*/ + + +-- backwards parsing + +CREATE VIEW collview1 AS SELECT * FROM collate_test1 WHERE b COLLATE "C" >= 'bbc'; +CREATE VIEW collview2 AS SELECT a, b FROM collate_test1 ORDER BY b COLLATE "C"; +CREATE VIEW collview3 AS SELECT a, lower((x || x) COLLATE "C") FROM collate_test10; + +SELECT table_name, view_definition FROM information_schema.views + WHERE table_name LIKE 'collview%' ORDER BY 1; + + +-- collation propagation in various expression types + +SELECT a, coalesce(b, 'foo') FROM collate_test1 ORDER BY 2; +SELECT a, coalesce(b, 'foo') FROM collate_test2 ORDER BY 2; +SELECT a, coalesce(b, 'foo') FROM collate_test3 ORDER BY 2; +SELECT a, lower(coalesce(x, 'foo')), lower(coalesce(y, 'foo')) FROM collate_test10; + +SELECT a, b, greatest(b, 'CCC') FROM collate_test1 ORDER BY 3; +SELECT a, b, greatest(b, 'CCC') FROM collate_test2 ORDER BY 3; +SELECT a, b, greatest(b, 'CCC') FROM collate_test3 ORDER BY 3; +SELECT a, x, y, lower(greatest(x, 'foo')), lower(greatest(y, 'foo')) FROM collate_test10; + +SELECT a, nullif(b, 'abc') FROM collate_test1 ORDER BY 2; +SELECT a, nullif(b, 'abc') FROM collate_test2 ORDER BY 2; +SELECT a, nullif(b, 'abc') FROM collate_test3 ORDER BY 2; +SELECT a, lower(nullif(x, 'foo')), lower(nullif(y, 'foo')) FROM collate_test10; + +SELECT a, CASE b WHEN 'abc' THEN 'abcd' ELSE b END FROM collate_test1 ORDER BY 2; +SELECT a, CASE b WHEN 'abc' THEN 'abcd' ELSE b END FROM collate_test2 ORDER BY 2; +SELECT a, CASE b WHEN 'abc' THEN 'abcd' ELSE b END FROM collate_test3 ORDER BY 2; + +CREATE DOMAIN testdomain AS text; +SELECT a, b::testdomain FROM collate_test1 ORDER BY 2; +SELECT a, b::testdomain FROM collate_test2 ORDER BY 2; +SELECT a, b::testdomain FROM collate_test3 ORDER BY 2; +SELECT a, b::testdomain_sv FROM collate_test3 ORDER BY 2; +SELECT a, lower(x::testdomain), lower(y::testdomain) FROM collate_test10; + +SELECT min(b), max(b) FROM collate_test1; +SELECT min(b), max(b) FROM collate_test2; +SELECT min(b), max(b) FROM collate_test3; + +SELECT array_agg(b ORDER BY b) FROM collate_test1; +SELECT array_agg(b ORDER BY b) FROM collate_test2; +SELECT array_agg(b ORDER BY b) FROM collate_test3; + +SELECT a, b FROM collate_test1 UNION ALL SELECT a, b FROM collate_test1 ORDER BY 2; +SELECT a, b FROM collate_test2 UNION SELECT a, b FROM collate_test2 ORDER BY 2; +SELECT a, b FROM collate_test3 WHERE a < 4 INTERSECT SELECT a, b FROM collate_test3 WHERE a > 1 ORDER BY 2; +SELECT a, b FROM collate_test3 EXCEPT SELECT a, b FROM collate_test3 WHERE a < 2 ORDER BY 2; + +SELECT a, b FROM collate_test1 UNION ALL SELECT a, b FROM collate_test3 ORDER BY 2; -- fail +SELECT a, b FROM collate_test1 UNION ALL SELECT a, b FROM collate_test3; -- ok +SELECT a, b FROM collate_test1 UNION SELECT a, b FROM collate_test3 ORDER BY 2; -- fail +SELECT a, b COLLATE "C" FROM collate_test1 UNION SELECT a, b FROM collate_test3 ORDER BY 2; -- ok +SELECT a, b FROM collate_test1 INTERSECT SELECT a, b FROM collate_test3 ORDER BY 2; -- fail +SELECT a, b FROM collate_test1 EXCEPT SELECT a, b FROM collate_test3 ORDER BY 2; -- fail + +CREATE TABLE test_u AS SELECT a, b FROM collate_test1 UNION ALL SELECT a, b FROM collate_test3; -- fail + +-- ideally this would be a parse-time error, but for now it must be run-time: +select x < y from collate_test10; -- fail +select x || y from collate_test10; -- ok, because || is not collation aware +select x, y from collate_test10 order by x || y; -- not so ok + +-- collation mismatch between recursive and non-recursive term +WITH RECURSIVE foo(x) AS + (SELECT x FROM (VALUES('a' COLLATE "en-x-icu"),('b')) t(x) + UNION ALL + SELECT (x || 'c') COLLATE "de-x-icu" FROM foo WHERE length(x) < 10) +SELECT * FROM foo; + + +-- casting + +SELECT CAST('42' AS text COLLATE "C"); + +SELECT a, CAST(b AS varchar) FROM collate_test1 ORDER BY 2; +SELECT a, CAST(b AS varchar) FROM collate_test2 ORDER BY 2; +SELECT a, CAST(b AS varchar) FROM collate_test3 ORDER BY 2; + + +-- propagation of collation in SQL functions (inlined and non-inlined cases) +-- and plpgsql functions too + +CREATE FUNCTION mylt (text, text) RETURNS boolean LANGUAGE sql + AS $$ select $1 < $2 $$; + +CREATE FUNCTION mylt_noninline (text, text) RETURNS boolean LANGUAGE sql + AS $$ select $1 < $2 limit 1 $$; + +CREATE FUNCTION mylt_plpgsql (text, text) RETURNS boolean LANGUAGE plpgsql + AS $$ begin return $1 < $2; end $$; + +SELECT a.b AS a, b.b AS b, a.b < b.b AS lt, + mylt(a.b, b.b), mylt_noninline(a.b, b.b), mylt_plpgsql(a.b, b.b) +FROM collate_test1 a, collate_test1 b +ORDER BY a.b, b.b; + +SELECT a.b AS a, b.b AS b, a.b < b.b COLLATE "C" AS lt, + mylt(a.b, b.b COLLATE "C"), mylt_noninline(a.b, b.b COLLATE "C"), + mylt_plpgsql(a.b, b.b COLLATE "C") +FROM collate_test1 a, collate_test1 b +ORDER BY a.b, b.b; + + +-- collation override in plpgsql + +CREATE FUNCTION mylt2 (x text, y text) RETURNS boolean LANGUAGE plpgsql AS $$ +declare + xx text := x; + yy text := y; +begin + return xx < yy; +end +$$; + +SELECT mylt2('a', 'B' collate "en-x-icu") as t, mylt2('a', 'B' collate "C") as f; + +CREATE OR REPLACE FUNCTION + mylt2 (x text, y text) RETURNS boolean LANGUAGE plpgsql AS $$ +declare + xx text COLLATE "POSIX" := x; + yy text := y; +begin + return xx < yy; +end +$$; + +SELECT mylt2('a', 'B') as f; +SELECT mylt2('a', 'B' collate "C") as fail; -- conflicting collations +SELECT mylt2('a', 'B' collate "POSIX") as f; + + +-- polymorphism + +SELECT * FROM unnest((SELECT array_agg(b ORDER BY b) FROM collate_test1)) ORDER BY 1; +SELECT * FROM unnest((SELECT array_agg(b ORDER BY b) FROM collate_test2)) ORDER BY 1; +SELECT * FROM unnest((SELECT array_agg(b ORDER BY b) FROM collate_test3)) ORDER BY 1; + +CREATE FUNCTION dup (anyelement) RETURNS anyelement + AS 'select $1' LANGUAGE sql; + +SELECT a, dup(b) FROM collate_test1 ORDER BY 2; +SELECT a, dup(b) FROM collate_test2 ORDER BY 2; +SELECT a, dup(b) FROM collate_test3 ORDER BY 2; + + +-- indexes + +CREATE INDEX collate_test1_idx1 ON collate_test1 (b); +CREATE INDEX collate_test1_idx2 ON collate_test1 (b COLLATE "C"); +CREATE INDEX collate_test1_idx3 ON collate_test1 ((b COLLATE "C")); -- this is different grammatically +CREATE INDEX collate_test1_idx4 ON collate_test1 (((b||'foo') COLLATE "POSIX")); + +CREATE INDEX collate_test1_idx5 ON collate_test1 (a COLLATE "C"); -- fail +CREATE INDEX collate_test1_idx6 ON collate_test1 ((a COLLATE "C")); -- fail + +SELECT relname, pg_get_indexdef(oid) FROM pg_class WHERE relname LIKE 'collate_test%_idx%' ORDER BY 1; + + +-- schema manipulation commands + +CREATE ROLE regress_test_role; +CREATE SCHEMA test_schema; + +-- We need to do this this way to cope with varying names for encodings: +do $$ +BEGIN + EXECUTE 'CREATE COLLATION test0 (provider = icu, locale = ' || + quote_literal(current_setting('lc_collate')) || ');'; +END +$$; +CREATE COLLATION test0 FROM "C"; -- fail, duplicate name +do $$ +BEGIN + EXECUTE 'CREATE COLLATION test1 (provider = icu, lc_collate = ' || + quote_literal(current_setting('lc_collate')) || + ', lc_ctype = ' || + quote_literal(current_setting('lc_ctype')) || ');'; +END +$$; +CREATE COLLATION test3 (provider = icu, lc_collate = 'en_US.utf8'); -- fail, need lc_ctype +CREATE COLLATION testx (provider = icu, locale = 'nonsense'); /* never fails with ICU */ DROP COLLATION testx; + +CREATE COLLATION test4 FROM nonsense; +CREATE COLLATION test5 FROM test0; + +SELECT collname FROM pg_collation WHERE collname LIKE 'test%' ORDER BY 1; + +ALTER COLLATION test1 RENAME TO test11; +ALTER COLLATION test0 RENAME TO test11; -- fail +ALTER COLLATION test1 RENAME TO test22; -- fail + +ALTER COLLATION test11 OWNER TO regress_test_role; +ALTER COLLATION test11 OWNER TO nonsense; +ALTER COLLATION test11 SET SCHEMA test_schema; + +COMMENT ON COLLATION test0 IS 'US English'; + +SELECT collname, nspname, obj_description(pg_collation.oid, 'pg_collation') + FROM pg_collation JOIN pg_namespace ON (collnamespace = pg_namespace.oid) + WHERE collname LIKE 'test%' + ORDER BY 1; + +DROP COLLATION test0, test_schema.test11, test5; +DROP COLLATION test0; -- fail +DROP COLLATION IF EXISTS test0; + +SELECT collname FROM pg_collation WHERE collname LIKE 'test%'; + +DROP SCHEMA test_schema; +DROP ROLE regress_test_role; + + +-- ALTER + +ALTER COLLATION "en-x-icu" REFRESH VERSION; + + +-- dependencies + +CREATE COLLATION test0 FROM "C"; + +CREATE TABLE collate_dep_test1 (a int, b text COLLATE test0); +CREATE DOMAIN collate_dep_dom1 AS text COLLATE test0; +CREATE TYPE collate_dep_test2 AS (x int, y text COLLATE test0); +CREATE VIEW collate_dep_test3 AS SELECT text 'foo' COLLATE test0 AS foo; +CREATE TABLE collate_dep_test4t (a int, b text); +CREATE INDEX collate_dep_test4i ON collate_dep_test4t (b COLLATE test0); + +DROP COLLATION test0 RESTRICT; -- fail +DROP COLLATION test0 CASCADE; + +\d collate_dep_test1 +\d collate_dep_test2 + +DROP TABLE collate_dep_test1, collate_dep_test4t; +DROP TYPE collate_dep_test2; + +-- test range types and collations + +create type textrange_c as range(subtype=text, collation="C"); +create type textrange_en_us as range(subtype=text, collation="en-x-icu"); + +select textrange_c('A','Z') @> 'b'::text; +select textrange_en_us('A','Z') @> 'b'::text; + +drop type textrange_c; +drop type textrange_en_us; + + +-- cleanup +DROP SCHEMA collate_tests CASCADE; +RESET search_path; + +-- leave a collation for pg_upgrade test +CREATE COLLATION coll_icu_upgrade FROM "und-x-icu"; -- cgit v1.2.3 From 5737c12df0581b3298e3e9586bdef170811ce176 Mon Sep 17 00:00:00 2001 From: Simon Riggs Date: Sat, 25 Mar 2017 14:07:27 +0000 Subject: Report catalog_xmin separately in hot_standby_feedback If the upstream walsender is using a physical replication slot, store the catalog_xmin in the slot's catalog_xmin field. If the upstream doesn't use a slot and has only a PGPROC entry behaviour doesn't change, as we store the combined xmin and catalog_xmin in the PGPROC entry. Author: Craig Ringer --- doc/src/sgml/protocol.sgml | 33 ++++++- src/backend/replication/walreceiver.c | 43 ++++++-- src/backend/replication/walsender.c | 108 +++++++++++++++------ src/backend/storage/ipc/procarray.c | 12 ++- src/include/storage/proc.h | 5 + src/include/storage/procarray.h | 11 +++ .../recovery/t/010_logical_decoding_timelines.pl | 38 +++++++- 7 files changed, 199 insertions(+), 51 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index 48ca414031..b3a50261c3 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -1916,10 +1916,11 @@ The commands accepted in walsender mode are: - The standby's current xmin. This may be 0, if the standby is - sending notification that Hot Standby feedback will no longer - be sent on this connection. Later non-zero messages may - reinitiate the feedback mechanism. + The standby's current global xmin, excluding the catalog_xmin from any + replication slots. If both this value and the following + catalog_xmin are 0 this is treated as a notification that Hot Standby + feedback will no longer be sent on this connection. Later non-zero + messages may reinitiate the feedback mechanism. @@ -1929,7 +1930,29 @@ The commands accepted in walsender mode are: - The standby's current epoch. + The epoch of the global xmin xid on the standby. + + + + + + Int32 + + + + The lowest catalog_xmin of any replication slots on the standby. Set to 0 + if no catalog_xmin exists on the standby or if hot standby feedback is being + disabled. + + + + + + Int32 + + + + The epoch of the catalog_xmin xid on the standby. diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index 31c567b37e..771ac305c3 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -1175,8 +1175,8 @@ XLogWalRcvSendHSFeedback(bool immed) { TimestampTz now; TransactionId nextXid; - uint32 nextEpoch; - TransactionId xmin; + uint32 xmin_epoch, catalog_xmin_epoch; + TransactionId xmin, catalog_xmin; static TimestampTz sendTime = 0; /* initially true so we always send at least one feedback message */ static bool master_has_standby_xmin = true; @@ -1221,29 +1221,54 @@ XLogWalRcvSendHSFeedback(bool immed) * everything else has been checked. */ if (hot_standby_feedback) - xmin = GetOldestXmin(NULL, PROCARRAY_FLAGS_DEFAULT); + { + TransactionId slot_xmin; + + /* + * Usually GetOldestXmin() would include both global replication slot + * xmin and catalog_xmin in its calculations, but we want to derive + * separate values for each of those. So we ask for an xmin that + * excludes the catalog_xmin. + */ + xmin = GetOldestXmin(NULL, + PROCARRAY_FLAGS_DEFAULT|PROCARRAY_SLOTS_XMIN); + + ProcArrayGetReplicationSlotXmin(&slot_xmin, &catalog_xmin); + + if (TransactionIdIsValid(slot_xmin) && + TransactionIdPrecedes(slot_xmin, xmin)) + xmin = slot_xmin; + } else + { xmin = InvalidTransactionId; + catalog_xmin = InvalidTransactionId; + } /* * Get epoch and adjust if nextXid and oldestXmin are different sides of * the epoch boundary. */ - GetNextXidAndEpoch(&nextXid, &nextEpoch); + GetNextXidAndEpoch(&nextXid, &xmin_epoch); + catalog_xmin_epoch = xmin_epoch; if (nextXid < xmin) - nextEpoch--; + xmin_epoch --; + if (nextXid < catalog_xmin) + catalog_xmin_epoch --; - elog(DEBUG2, "sending hot standby feedback xmin %u epoch %u", - xmin, nextEpoch); + elog(DEBUG2, "sending hot standby feedback xmin %u epoch %u catalog_xmin %u catalog_xmin_epoch %u", + xmin, xmin_epoch, catalog_xmin, catalog_xmin_epoch); /* Construct the message and send it. */ resetStringInfo(&reply_message); pq_sendbyte(&reply_message, 'h'); pq_sendint64(&reply_message, GetCurrentTimestamp()); pq_sendint(&reply_message, xmin, 4); - pq_sendint(&reply_message, nextEpoch, 4); + pq_sendint(&reply_message, xmin_epoch, 4); + pq_sendint(&reply_message, catalog_xmin, 4); + pq_sendint(&reply_message, catalog_xmin_epoch, 4); walrcv_send(wrconn, reply_message.data, reply_message.len); - if (TransactionIdIsValid(xmin)) + if (TransactionIdIsValid(xmin) || TransactionIdIsValid(catalog_xmin)) master_has_standby_xmin = true; else master_has_standby_xmin = false; diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index a29d0e7cf4..59ae22df8c 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -242,6 +242,7 @@ static void WalSndPrepareWrite(LogicalDecodingContext *ctx, XLogRecPtr lsn, Tran static void WalSndWriteData(LogicalDecodingContext *ctx, XLogRecPtr lsn, TransactionId xid, bool last_write); static XLogRecPtr WalSndWaitForWal(XLogRecPtr loc); static TimeOffset LagTrackerRead(int head, XLogRecPtr lsn, TimestampTz now); +static bool TransactionIdInRecentPast(TransactionId xid, uint32 epoch); static void XLogRead(char *buf, XLogRecPtr startptr, Size count); @@ -1756,7 +1757,7 @@ ProcessStandbyReplyMessage(void) /* compute new replication slot xmin horizon if needed */ static void -PhysicalReplicationSlotNewXmin(TransactionId feedbackXmin) +PhysicalReplicationSlotNewXmin(TransactionId feedbackXmin, TransactionId feedbackCatalogXmin) { bool changed = false; ReplicationSlot *slot = MyReplicationSlot; @@ -1777,6 +1778,14 @@ PhysicalReplicationSlotNewXmin(TransactionId feedbackXmin) slot->data.xmin = feedbackXmin; slot->effective_xmin = feedbackXmin; } + if (!TransactionIdIsNormal(slot->data.catalog_xmin) || + !TransactionIdIsNormal(feedbackCatalogXmin) || + TransactionIdPrecedes(slot->data.catalog_xmin, feedbackCatalogXmin)) + { + changed = true; + slot->data.catalog_xmin = feedbackCatalogXmin; + slot->effective_catalog_xmin = feedbackCatalogXmin; + } SpinLockRelease(&slot->mutex); if (changed) @@ -1786,60 +1795,93 @@ PhysicalReplicationSlotNewXmin(TransactionId feedbackXmin) } } +/* + * Check that the provided xmin/epoch are sane, that is, not in the future + * and not so far back as to be already wrapped around. + * + * Epoch of nextXid should be same as standby, or if the counter has + * wrapped, then one greater than standby. + * + * This check doesn't care about whether clog exists for these xids + * at all. + */ +static bool +TransactionIdInRecentPast(TransactionId xid, uint32 epoch) +{ + TransactionId nextXid; + uint32 nextEpoch; + + GetNextXidAndEpoch(&nextXid, &nextEpoch); + + if (xid <= nextXid) + { + if (epoch != nextEpoch) + return false; + } + else + { + if (epoch + 1 != nextEpoch) + return false; + } + + if (!TransactionIdPrecedesOrEquals(xid, nextXid)) + return false; /* epoch OK, but it's wrapped around */ + + return true; +} + /* * Hot Standby feedback */ static void ProcessStandbyHSFeedbackMessage(void) { - TransactionId nextXid; - uint32 nextEpoch; TransactionId feedbackXmin; uint32 feedbackEpoch; + TransactionId feedbackCatalogXmin; + uint32 feedbackCatalogEpoch; /* * Decipher the reply message. The caller already consumed the msgtype - * byte. + * byte. See XLogWalRcvSendHSFeedback() in walreceiver.c for the creation + * of this message. */ (void) pq_getmsgint64(&reply_message); /* sendTime; not used ATM */ feedbackXmin = pq_getmsgint(&reply_message, 4); feedbackEpoch = pq_getmsgint(&reply_message, 4); + feedbackCatalogXmin = pq_getmsgint(&reply_message, 4); + feedbackCatalogEpoch = pq_getmsgint(&reply_message, 4); - elog(DEBUG2, "hot standby feedback xmin %u epoch %u", + elog(DEBUG2, "hot standby feedback xmin %u epoch %u, catalog_xmin %u epoch %u", feedbackXmin, - feedbackEpoch); + feedbackEpoch, + feedbackCatalogXmin, + feedbackCatalogEpoch); - /* Unset WalSender's xmin if the feedback message value is invalid */ - if (!TransactionIdIsNormal(feedbackXmin)) + /* + * Unset WalSender's xmins if the feedback message values are invalid. + * This happens when the downstream turned hot_standby_feedback off. + */ + if (!TransactionIdIsNormal(feedbackXmin) + && !TransactionIdIsNormal(feedbackCatalogXmin)) { MyPgXact->xmin = InvalidTransactionId; if (MyReplicationSlot != NULL) - PhysicalReplicationSlotNewXmin(feedbackXmin); + PhysicalReplicationSlotNewXmin(feedbackXmin, feedbackCatalogXmin); return; } /* * Check that the provided xmin/epoch are sane, that is, not in the future * and not so far back as to be already wrapped around. Ignore if not. - * - * Epoch of nextXid should be same as standby, or if the counter has - * wrapped, then one greater than standby. */ - GetNextXidAndEpoch(&nextXid, &nextEpoch); - - if (feedbackXmin <= nextXid) - { - if (feedbackEpoch != nextEpoch) - return; - } - else - { - if (feedbackEpoch + 1 != nextEpoch) - return; - } + if (TransactionIdIsNormal(feedbackXmin) && + !TransactionIdInRecentPast(feedbackXmin, feedbackEpoch)) + return; - if (!TransactionIdPrecedesOrEquals(feedbackXmin, nextXid)) - return; /* epoch OK, but it's wrapped around */ + if (TransactionIdIsNormal(feedbackCatalogXmin) && + !TransactionIdInRecentPast(feedbackCatalogXmin, feedbackCatalogEpoch)) + return; /* * Set the WalSender's xmin equal to the standby's requested xmin, so that @@ -1864,15 +1906,23 @@ ProcessStandbyHSFeedbackMessage(void) * already since a VACUUM could have just finished calling GetOldestXmin.) * * If we're using a replication slot we reserve the xmin via that, - * otherwise via the walsender's PGXACT entry. + * otherwise via the walsender's PGXACT entry. We can only track the + * catalog xmin separately when using a slot, so we store the least + * of the two provided when not using a slot. * * XXX: It might make sense to generalize the ephemeral slot concept and * always use the slot mechanism to handle the feedback xmin. */ if (MyReplicationSlot != NULL) /* XXX: persistency configurable? */ - PhysicalReplicationSlotNewXmin(feedbackXmin); + PhysicalReplicationSlotNewXmin(feedbackXmin, feedbackCatalogXmin); else - MyPgXact->xmin = feedbackXmin; + { + if (TransactionIdIsNormal(feedbackCatalogXmin) + && TransactionIdPrecedes(feedbackCatalogXmin, feedbackXmin)) + MyPgXact->xmin = feedbackCatalogXmin; + else + MyPgXact->xmin = feedbackXmin; + } } /* diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c index 40c3247d4b..7c2e1e1c85 100644 --- a/src/backend/storage/ipc/procarray.c +++ b/src/backend/storage/ipc/procarray.c @@ -1264,6 +1264,10 @@ TransactionIdIsActive(TransactionId xid) * corresponding flags is set. Typically, if you want to ignore ones with * PROC_IN_VACUUM flag, you can use PROCARRAY_FLAGS_VACUUM. * + * PROCARRAY_SLOTS_XMIN causes GetOldestXmin to ignore the xmin and + * catalog_xmin of any replication slots that exist in the system when + * calculating the oldest xmin. + * * This is used by VACUUM to decide which deleted tuples must be preserved in * the passed in table. For shared relations backends in all databases must be * considered, but for non-shared relations that's not required, since only @@ -1342,7 +1346,7 @@ GetOldestXmin(Relation rel, int flags) volatile PGPROC *proc = &allProcs[pgprocno]; volatile PGXACT *pgxact = &allPgXact[pgprocno]; - if (pgxact->vacuumFlags & flags) + if (pgxact->vacuumFlags & (flags & PROCARRAY_PROC_FLAGS_MASK)) continue; if (allDbs || @@ -1418,7 +1422,8 @@ GetOldestXmin(Relation rel, int flags) /* * Check whether there are replication slots requiring an older xmin. */ - if (TransactionIdIsValid(replication_slot_xmin) && + if (!(flags & PROCARRAY_SLOTS_XMIN) && + TransactionIdIsValid(replication_slot_xmin) && NormalTransactionIdPrecedes(replication_slot_xmin, result)) result = replication_slot_xmin; @@ -1428,7 +1433,8 @@ GetOldestXmin(Relation rel, int flags) * possible. We need to do so if we're computing the global limit (rel = * NULL) or if the passed relation is a catalog relation of some kind. */ - if ((rel == NULL || + if (!(flags & PROCARRAY_SLOTS_XMIN) && + (rel == NULL || RelationIsAccessibleInLogicalDecoding(rel)) && TransactionIdIsValid(replication_slot_catalog_xmin) && NormalTransactionIdPrecedes(replication_slot_catalog_xmin, result)) diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h index 945dd1d592..1b345faa2d 100644 --- a/src/include/storage/proc.h +++ b/src/include/storage/proc.h @@ -44,6 +44,10 @@ struct XidCache * * Note: If you modify these flags, you need to modify PROCARRAY_XXX flags * in src/include/storage/procarray.h. + * + * PROC_RESERVED may later be assigned for use in vacuumFlags, but its value is + * used for PROCARRAY_SLOTS_XMIN in procarray.h, so GetOldestXmin won't be able + * to match and ignore processes with this flag set. */ #define PROC_IS_AUTOVACUUM 0x01 /* is it an autovac worker? */ #define PROC_IN_VACUUM 0x02 /* currently running lazy vacuum */ @@ -51,6 +55,7 @@ struct XidCache #define PROC_VACUUM_FOR_WRAPAROUND 0x08 /* set by autovac only */ #define PROC_IN_LOGICAL_DECODING 0x10 /* currently doing logical * decoding outside xact */ +#define PROC_RESERVED 0x20 /* reserved for procarray */ /* flags reset at EOXact */ #define PROC_VACUUM_STATE_MASK \ diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h index c8e1ae517c..9b42e49524 100644 --- a/src/include/storage/procarray.h +++ b/src/include/storage/procarray.h @@ -32,6 +32,17 @@ #define PROCARRAY_LOGICAL_DECODING_FLAG 0x10 /* currently doing logical * decoding outside xact */ +#define PROCARRAY_SLOTS_XMIN 0x20 /* replication slot xmin, + * catalog_xmin */ +/* + * Only flags in PROCARRAY_PROC_FLAGS_MASK are considered when matching + * PGXACT->vacuumFlags. Other flags are used for different purposes and + * have no corresponding PROC flag equivalent. + */ +#define PROCARRAY_PROC_FLAGS_MASK (PROCARRAY_VACUUM_FLAG | \ + PROCARRAY_ANALYZE_FLAG | \ + PROCARRAY_LOGICAL_DECODING_FLAG) + /* Use the following flags as an input "flags" to GetOldestXmin function */ /* Consider all backends except for logical decoding ones which manage xmin separately */ #define PROCARRAY_FLAGS_DEFAULT PROCARRAY_LOGICAL_DECODING_FLAG diff --git a/src/test/recovery/t/010_logical_decoding_timelines.pl b/src/test/recovery/t/010_logical_decoding_timelines.pl index 09830dc39c..4561a06143 100644 --- a/src/test/recovery/t/010_logical_decoding_timelines.pl +++ b/src/test/recovery/t/010_logical_decoding_timelines.pl @@ -20,7 +20,7 @@ use warnings; use PostgresNode; use TestLib; -use Test::More tests => 7; +use Test::More tests => 10; use RecursiveCopy; use File::Copy; use IPC::Run (); @@ -31,10 +31,14 @@ my ($stdout, $stderr, $ret); # Initialize master node my $node_master = get_new_node('master'); $node_master->init(allows_streaming => 1, has_archiving => 1); -$node_master->append_conf('postgresql.conf', "wal_level = 'logical'\n"); -$node_master->append_conf('postgresql.conf', "max_replication_slots = 2\n"); -$node_master->append_conf('postgresql.conf', "max_wal_senders = 2\n"); -$node_master->append_conf('postgresql.conf', "log_min_messages = 'debug2'\n"); +$node_master->append_conf('postgresql.conf', q[ +wal_level = 'logical' +max_replication_slots = 3 +max_wal_senders = 2 +log_min_messages = 'debug2' +hot_standby_feedback = on +wal_receiver_status_interval = 1 +]); $node_master->dump_info; $node_master->start; @@ -51,11 +55,17 @@ $node_master->safe_psql('postgres', 'CHECKPOINT;'); my $backup_name = 'b1'; $node_master->backup_fs_hot($backup_name); +$node_master->safe_psql('postgres', + q[SELECT pg_create_physical_replication_slot('phys_slot');]); + my $node_replica = get_new_node('replica'); $node_replica->init_from_backup( $node_master, $backup_name, has_streaming => 1, has_restoring => 1); +$node_replica->append_conf( + 'recovery.conf', q[primary_slot_name = 'phys_slot']); + $node_replica->start; $node_master->safe_psql('postgres', @@ -71,6 +81,24 @@ $stdout = $node_replica->safe_psql('postgres', is($stdout, 'before_basebackup', 'Expected to find only slot before_basebackup on replica'); +# Examine the physical slot the replica uses to stream changes +# from the master to make sure its hot_standby_feedback +# has locked in a catalog_xmin on the physical slot, and that +# any xmin is < the catalog_xmin +$node_master->poll_query_until('postgres', q[ + SELECT catalog_xmin IS NOT NULL + FROM pg_replication_slots + WHERE slot_name = 'phys_slot' + ]); +my $phys_slot = $node_master->slot('phys_slot'); +isnt($phys_slot->{'xmin'}, '', + 'xmin assigned on physical slot of master'); +isnt($phys_slot->{'catalog_xmin'}, '', + 'catalog_xmin assigned on physical slot of master'); +# Ignore wrap-around here, we're on a new cluster: +cmp_ok($phys_slot->{'xmin'}, '>=', $phys_slot->{'catalog_xmin'}, + 'xmin on physical slot must not be lower than catalog_xmin'); + # Boom, crash $node_master->stop('immediate'); -- cgit v1.2.3 From d63762452434a3a046e8c7d130d5a77c594176e4 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Sun, 26 Mar 2017 14:54:56 -0400 Subject: doc: Clean up bibliography rendering for XSLT In the DSSSL stylesheets, we had an extensive customization of the bibliography rendering. Since the bibliography isn't that used much, it doesn't seem worth doing an elaborate porting of that to XSLT. So this just moves some things around, removes some unused things, and does some minimal XSLT stylesheet customizations to make things look clean. --- doc/src/sgml/biblio.sgml | 120 ++++++++++++----------------------------- doc/src/sgml/stylesheet-fo.xsl | 17 ++++++ doc/src/sgml/stylesheet.xsl | 21 ++++++++ 3 files changed, 71 insertions(+), 87 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/biblio.sgml b/doc/src/sgml/biblio.sgml index ab5af16aee..5462bc38e4 100644 --- a/doc/src/sgml/biblio.sgml +++ b/doc/src/sgml/biblio.sgml @@ -12,8 +12,7 @@ Some white papers and technical reports from the original POSTGRES development team are available at the University of California, Berkeley, Computer Science - Department - web site. + Department web site.     @@ -21,7 +20,6 @@ The Practical <acronym>SQL</acronym> Handbook - Bowman et al, 2001 Using SQL Variants Fourth Edition @@ -39,18 +37,14 @@ 0-201-70309-2 - 2001 Addison-Wesley Professional - - 2001 - + 2001 A Guide to the <acronym>SQL</acronym> Standard - Date and Darwen, 1997 A user's guide to the standard database language SQL Fourth Edition @@ -64,19 +58,14 @@ 0-201-96426-0 - 1997 Addison-Wesley - - 1997 - Addison-Wesley Longman, Inc. - + 1997 An Introduction to Database Systems - Date, 2004 Eighth Edition @@ -85,14 +74,10 @@ 0-321-19784-4 - 2003 Addison-Wesley - - 2004 - Pearson Education, Inc. - + 2003 @@ -109,18 +94,14 @@ 0-321-12226-7 - 2003 Addison-Wesley - - 2004 - + 2003 Understanding the New <acronym>SQL</acronym> - Melton and Simon, 1993 A complete guide @@ -133,20 +114,15 @@ 1-55860-245-3 - 1993 Morgan Kaufmann - - 1993 - Morgan Kaufmann Publishers, Inc. - + 1993 Principles of Database and Knowledge Base Systems - Ullman, 1988 Jeffrey D. @@ -167,7 +143,6 @@ Enhancement of the ANSI SQL Implementation of PostgreSQL - Simkovics, 1998 Stefan @@ -203,16 +178,15 @@ ssimkovi@ag.or.at     - November 29, 1998 Department of Information Systems, Vienna University of Technology
    Vienna, Austria
     + November 29, 1998 The <productname>Postgres95</productname> User Manual - Yu and Chen, 1995 A. @@ -223,24 +197,17 @@ ssimkovi@ag.or.at Chen - - - The POSTGRES Group - - - - Sept. 5, 1995 University of California
    Berkeley, California
     + Sept. 5, 1995     - - <ulink url="http://db.cs.berkeley.edu/papers/UCB-MS-zfong.pdf"> - The design and implementation of the <productname>POSTGRES</productname> query optimizer - </ulink> + <ulink url="http://db.cs.berkeley.edu/papers/UCB-MS-zfong.pdf">The + design and implementation of the <productname>POSTGRES</productname> query + optimizer</ulink> Zelaine Fong @@ -257,25 +224,23 @@ ssimkovi@ag.or.at Partial indexing in POSTGRES: research project - Olson, 1993 Nels Olson - 1993 UCB Engin T7.49.1993 O676 University of California
    Berkeley, California
     + 1993     A Unified Framework for Version Modeling Using Production Rules in a Database System - Ong and Goh, 1990 L. @@ -289,20 +254,18 @@ ssimkovi@ag.or.at ERL Technical Memorandum M90/33 - April, 1990 University of California
    Berkeley, California
     + April, 1990     - <ulink url="http://db.cs.berkeley.edu/papers/ERL-M87-13.pdf"> - The <productname>POSTGRES</productname> data model - </ulink> - Rowe and Stonebraker, 1987 + <ulink url="http://db.cs.berkeley.edu/papers/ERL-M87-13.pdf">The <productname>POSTGRES</productname> + data model</ulink> L. @@ -323,14 +286,8 @@ ssimkovi@ag.or.at - Generalized Partial Indexes - <ulink url="http://citeseer.ist.psu.edu/seshadri95generalized.html">(cached version) -<!-- - Original URL: http://citeseer.ist.psu.edu/seshadri95generalized.html ---> - </ulink> - - Seshardri, 1995 + <ulink url="http://citeseer.ist.psu.edu/seshadri95generalized.html">Generalized + Partial Indexes</ulink> P. @@ -347,21 +304,19 @@ ssimkovi@ag.or.at 6-10 March 1995
    Taipeh, Taiwan
    - 1995 Cat. No.95CH35724 IEEE Computer Society Press
    Los Alamitos, California
     + 1995 420-7     - <ulink url="http://db.cs.berkeley.edu/papers/ERL-M85-95.pdf"> - The design of <productname>POSTGRES</productname> - </ulink> - Stonebraker and Rowe, 1986 + <ulink url="http://db.cs.berkeley.edu/papers/ERL-M85-95.pdf">The + design of <productname>POSTGRES</productname></ulink> M. @@ -383,7 +338,6 @@ ssimkovi@ag.or.at The design of the <productname>POSTGRES</productname> rules system - Stonebraker, Hanson, Hong, 1987 M. @@ -408,10 +362,9 @@ ssimkovi@ag.or.at - <ulink url="http://db.cs.berkeley.edu/papers/ERL-M87-06.pdf"> - The design of the <productname>POSTGRES</productname> storage system - </ulink> - Stonebraker, 1987 + <ulink url="http://db.cs.berkeley.edu/papers/ERL-M87-06.pdf">The + design of the <productname>POSTGRES</productname> storage + system</ulink> M. @@ -428,10 +381,9 @@ ssimkovi@ag.or.at - <ulink url="http://db.cs.berkeley.edu/papers/ERL-M89-82.pdf"> - A commentary on the <productname>POSTGRES</productname> rules system - </ulink> - Stonebraker et al, 1989 + <ulink url="http://db.cs.berkeley.edu/papers/ERL-M89-82.pdf">A + commentary on the <productname>POSTGRES</productname> rules + system</ulink> M. @@ -455,10 +407,8 @@ ssimkovi@ag.or.at - <ulink url="http://db.cs.berkeley.edu/papers/ERL-M89-17.pdf"> - The case for partial indexes - </ulink> - Stonebraker, M, 1989b + <ulink url="http://db.cs.berkeley.edu/papers/ERL-M89-17.pdf">The + case for partial indexes</ulink> M. @@ -468,17 +418,15 @@ ssimkovi@ag.or.at SIGMOD Record 18(4) - 4-11 Dec. 1989 + 4-11 - <ulink url="http://db.cs.berkeley.edu/papers/ERL-M90-34.pdf"> - The implementation of <productname>POSTGRES</productname> - </ulink> - Stonebraker, Rowe, Hirohama, 1990 + <ulink url="http://db.cs.berkeley.edu/papers/ERL-M90-34.pdf">The + implementation of <productname>POSTGRES</productname></ulink> M. @@ -505,10 +453,8 @@ ssimkovi@ag.or.at - <ulink url="http://db.cs.berkeley.edu/papers/ERL-M90-36.pdf"> - On Rules, Procedures, Caching and Views in Database Systems - </ulink> - Stonebraker et al, ACM, 1990 + <ulink url="http://db.cs.berkeley.edu/papers/ERL-M90-36.pdf">On + Rules, Procedures, Caching and Views in Database Systems</ulink> M. diff --git a/doc/src/sgml/stylesheet-fo.xsl b/doc/src/sgml/stylesheet-fo.xsl index 434e69d8e3..69536f92cd 100644 --- a/doc/src/sgml/stylesheet-fo.xsl +++ b/doc/src/sgml/stylesheet-fo.xsl @@ -24,6 +24,23 @@ + + + + , + + + + + + + + ISBN + + + + + diff --git a/doc/src/sgml/stylesheet.xsl b/doc/src/sgml/stylesheet.xsl index efcb80ffca..e36e8cc5cc 100644 --- a/doc/src/sgml/stylesheet.xsl +++ b/doc/src/sgml/stylesheet.xsl @@ -40,6 +40,27 @@ + + + + + + , + + + + + + + + + + ISBN + + + + + -- cgit v1.2.3 From e259e1f748c7a6d67e307a90d6c27b8ab8b90df8 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Sun, 26 Mar 2017 15:03:27 -0400 Subject: doc: Fix oldhtml/old PDF build "xref to REFSECT1 unsupported" with the DSSSL stylesheets. Reported-by: Devrim Gündüz --- doc/src/sgml/ref/alter_collation.sgml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/alter_collation.sgml b/doc/src/sgml/ref/alter_collation.sgml index bf934ce75f..159f7e7472 100644 --- a/doc/src/sgml/ref/alter_collation.sgml +++ b/doc/src/sgml/ref/alter_collation.sgml @@ -93,7 +93,8 @@ ALTER COLLATION name SET SCHEMA new_sche Updated the collation version. - See below. + See below. -- cgit v1.2.3 From fc70a4b0df38bda6a13941f1581f25fbb643c7f3 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Sun, 26 Mar 2017 22:02:22 -0400 Subject: Show more processes in pg_stat_activity. Previously, auxiliary processes and background workers not connected to a database (such as the logical replication launcher) weren't shown. Include them, so that we can see the associated wait state information. Add a new column to identify the processes type, so that people can filter them out easily using SQL if they wish. Before this patch was written, there was discussion about whether we should expose this information in a separate view, so as to avoid contaminating pg_stat_activity with things people might not want to see. But putting everything in pg_stat_activity was a more popular choice, so that's what the patch does. Kuntal Ghosh, reviewed by Amit Langote and Michael Paquier. Some revisions and bug fixes by me. Discussion: http://postgr.es/m/CA+TgmoYES5nhkEGw9nZXU8_FhA8XEm8NTm3-SO+3ML1B81Hkww@mail.gmail.com --- doc/src/sgml/monitoring.sgml | 15 ++- src/backend/bootstrap/bootstrap.c | 4 + src/backend/catalog/system_views.sql | 3 +- src/backend/postmaster/pgstat.c | 194 ++++++++++++++++++++++++++++++----- src/backend/postmaster/startup.c | 1 + src/backend/replication/walsender.c | 4 +- src/backend/storage/lmgr/proc.c | 27 +++++ src/backend/utils/adt/pgstatfuncs.c | 60 ++++++++--- src/backend/utils/init/postinit.c | 8 ++ src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_proc.h | 2 +- src/include/pgstat.h | 26 +++++ src/include/storage/proc.h | 3 +- src/test/regress/expected/rules.out | 9 +- 14 files changed, 305 insertions(+), 53 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index e930731b16..9856968997 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -620,8 +620,8 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser backend_start timestamp with time zone - Time when this process was started, i.e., when the - client connected to the server + Time when this process was started. For client backends, + this is the time the client connected to the server. @@ -797,6 +797,17 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser . + + backend_type + text + Type of current backend. Possible types are + autovacuum launcher, autovacuum worker, + background worker, background writer, + client backend, checkpointer, + startup, walreceiver, + walsender and walwriter. + + diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c index 6cfce4f8dd..d8efdb5ed3 100644 --- a/src/backend/bootstrap/bootstrap.c +++ b/src/backend/bootstrap/bootstrap.c @@ -387,6 +387,10 @@ AuxiliaryProcessMain(int argc, char *argv[]) /* finish setting up bufmgr.c */ InitBufferPoolBackend(); + /* Initialize backend status information */ + pgstat_initialize(); + pgstat_bestart(); + /* register a before-shutdown callback for LWLock cleanup */ before_shmem_exit(ShutdownAuxiliaryProcess, 0); } diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index d8b762ee3f..d357c8b8fd 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -716,7 +716,8 @@ CREATE VIEW pg_stat_activity AS S.state, S.backend_xid, s.backend_xmin, - S.query + S.query, + S.backend_type FROM pg_stat_get_activity(NULL) AS S LEFT JOIN pg_database AS D ON (S.datid = D.oid) LEFT JOIN pg_authid AS U ON (S.usesysid = U.oid); diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index b704788eb5..869afd4c4a 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -50,6 +50,7 @@ #include "postmaster/autovacuum.h" #include "postmaster/fork_process.h" #include "postmaster/postmaster.h" +#include "replication/walsender.h" #include "storage/backendid.h" #include "storage/dsm.h" #include "storage/fd.h" @@ -102,6 +103,18 @@ #define PGSTAT_FUNCTION_HASH_SIZE 512 +/* ---------- + * Total number of backends including auxiliary + * + * We reserve a slot for each possible BackendId, plus one for each + * possible auxiliary process type. (This scheme assumes there is not + * more than one of any auxiliary process type at a time.) MaxBackends + * includes autovacuum workers and background workers as well. + * ---------- + */ +#define NumBackendStatSlots (MaxBackends + NUM_AUXPROCTYPES) + + /* ---------- * GUC parameters * ---------- @@ -212,7 +225,11 @@ typedef struct TwoPhasePgStatRecord */ static MemoryContext pgStatLocalContext = NULL; static HTAB *pgStatDBHash = NULL; + +/* Status for backends including auxiliary */ static LocalPgBackendStatus *localBackendStatusTable = NULL; + +/* Total number of backends including auxiliary */ static int localNumBackends = 0; /* @@ -2505,20 +2522,20 @@ BackendStatusShmemSize(void) Size size; /* BackendStatusArray: */ - size = mul_size(sizeof(PgBackendStatus), MaxBackends); + size = mul_size(sizeof(PgBackendStatus), NumBackendStatSlots); /* BackendAppnameBuffer: */ size = add_size(size, - mul_size(NAMEDATALEN, MaxBackends)); + mul_size(NAMEDATALEN, NumBackendStatSlots)); /* BackendClientHostnameBuffer: */ size = add_size(size, - mul_size(NAMEDATALEN, MaxBackends)); + mul_size(NAMEDATALEN, NumBackendStatSlots)); /* BackendActivityBuffer: */ size = add_size(size, - mul_size(pgstat_track_activity_query_size, MaxBackends)); + mul_size(pgstat_track_activity_query_size, NumBackendStatSlots)); #ifdef USE_SSL /* BackendSslStatusBuffer: */ size = add_size(size, - mul_size(sizeof(PgBackendSSLStatus), MaxBackends)); + mul_size(sizeof(PgBackendSSLStatus), NumBackendStatSlots)); #endif return size; } @@ -2536,7 +2553,7 @@ CreateSharedBackendStatus(void) char *buffer; /* Create or attach to the shared array */ - size = mul_size(sizeof(PgBackendStatus), MaxBackends); + size = mul_size(sizeof(PgBackendStatus), NumBackendStatSlots); BackendStatusArray = (PgBackendStatus *) ShmemInitStruct("Backend Status Array", size, &found); @@ -2559,7 +2576,7 @@ CreateSharedBackendStatus(void) /* Initialize st_appname pointers. */ buffer = BackendAppnameBuffer; - for (i = 0; i < MaxBackends; i++) + for (i = 0; i < NumBackendStatSlots; i++) { BackendStatusArray[i].st_appname = buffer; buffer += NAMEDATALEN; @@ -2577,7 +2594,7 @@ CreateSharedBackendStatus(void) /* Initialize st_clienthostname pointers. */ buffer = BackendClientHostnameBuffer; - for (i = 0; i < MaxBackends; i++) + for (i = 0; i < NumBackendStatSlots; i++) { BackendStatusArray[i].st_clienthostname = buffer; buffer += NAMEDATALEN; @@ -2586,7 +2603,7 @@ CreateSharedBackendStatus(void) /* Create or attach to the shared activity buffer */ BackendActivityBufferSize = mul_size(pgstat_track_activity_query_size, - MaxBackends); + NumBackendStatSlots); BackendActivityBuffer = (char *) ShmemInitStruct("Backend Activity Buffer", BackendActivityBufferSize, @@ -2598,7 +2615,7 @@ CreateSharedBackendStatus(void) /* Initialize st_activity pointers. */ buffer = BackendActivityBuffer; - for (i = 0; i < MaxBackends; i++) + for (i = 0; i < NumBackendStatSlots; i++) { BackendStatusArray[i].st_activity = buffer; buffer += pgstat_track_activity_query_size; @@ -2607,7 +2624,7 @@ CreateSharedBackendStatus(void) #ifdef USE_SSL /* Create or attach to the shared SSL status buffer */ - size = mul_size(sizeof(PgBackendSSLStatus), MaxBackends); + size = mul_size(sizeof(PgBackendSSLStatus), NumBackendStatSlots); BackendSslStatusBuffer = (PgBackendSSLStatus *) ShmemInitStruct("Backend SSL Status Buffer", size, &found); @@ -2619,7 +2636,7 @@ CreateSharedBackendStatus(void) /* Initialize st_sslstatus pointers. */ ptr = BackendSslStatusBuffer; - for (i = 0; i < MaxBackends; i++) + for (i = 0; i < NumBackendStatSlots; i++) { BackendStatusArray[i].st_sslstatus = ptr; ptr++; @@ -2633,7 +2650,8 @@ CreateSharedBackendStatus(void) * pgstat_initialize() - * * Initialize pgstats state, and set up our on-proc-exit hook. - * Called from InitPostgres. MyBackendId must be set, + * Called from InitPostgres and AuxiliaryProcessMain. For auxiliary process, + * MyBackendId is invalid. Otherwise, MyBackendId must be set, * but we must not have started any transaction yet (since the * exit hook must run after the last transaction exit). * NOTE: MyDatabaseId isn't set yet; so the shutdown hook has to be careful. @@ -2643,8 +2661,26 @@ void pgstat_initialize(void) { /* Initialize MyBEEntry */ - Assert(MyBackendId >= 1 && MyBackendId <= MaxBackends); - MyBEEntry = &BackendStatusArray[MyBackendId - 1]; + if (MyBackendId != InvalidBackendId) + { + Assert(MyBackendId >= 1 && MyBackendId <= MaxBackends); + MyBEEntry = &BackendStatusArray[MyBackendId - 1]; + } + else + { + /* Must be an auxiliary process */ + Assert(MyAuxProcType != NotAnAuxProcess); + + /* + * Assign the MyBEEntry for an auxiliary process. Since it doesn't + * have a BackendId, the slot is statically allocated based on the + * auxiliary process type (MyAuxProcType). Backends use slots indexed + * in the range from 1 to MaxBackends (inclusive), so we use + * MaxBackends + AuxBackendType + 1 as the index of the slot for an + * auxiliary process. + */ + MyBEEntry = &BackendStatusArray[MaxBackends + MyAuxProcType]; + } /* Set up a process-exit hook to clean up */ on_shmem_exit(pgstat_beshutdown_hook, 0); @@ -2655,15 +2691,16 @@ pgstat_initialize(void) * * Initialize this backend's entry in the PgBackendStatus array. * Called from InitPostgres. - * MyDatabaseId, session userid, and application_name must be set - * (hence, this cannot be combined with pgstat_initialize). + * + * Apart from auxiliary processes, MyBackendId, MyDatabaseId, + * session userid, and application_name must be set for a + * backend (hence, this cannot be combined with pgstat_initialize). * ---------- */ void pgstat_bestart(void) { TimestampTz proc_start_timestamp; - Oid userid; SockAddr clientaddr; volatile PgBackendStatus *beentry; @@ -2678,7 +2715,6 @@ pgstat_bestart(void) proc_start_timestamp = MyProcPort->SessionStartTime; else proc_start_timestamp = GetCurrentTimestamp(); - userid = GetSessionUserId(); /* * We may not have a MyProcPort (eg, if this is the autovacuum process). @@ -2697,6 +2733,66 @@ pgstat_bestart(void) * cute. */ beentry = MyBEEntry; + + /* pgstats state must be initialized from pgstat_initialize() */ + Assert(beentry != NULL); + + if (MyBackendId != InvalidBackendId) + { + if (IsAutoVacuumLauncherProcess()) + { + /* Autovacuum Launcher */ + beentry->st_backendType = B_AUTOVAC_LAUNCHER; + } + else if (IsAutoVacuumWorkerProcess()) + { + /* Autovacuum Worker */ + beentry->st_backendType = B_AUTOVAC_WORKER; + } + else if (am_walsender) + { + /* Wal sender */ + beentry->st_backendType = B_WAL_SENDER; + } + else if (IsBackgroundWorker) + { + /* bgworker */ + beentry->st_backendType = B_BG_WORKER; + } + else + { + /* client-backend */ + beentry->st_backendType = B_BACKEND; + } + } + else + { + /* Must be an auxiliary process */ + Assert(MyAuxProcType != NotAnAuxProcess); + switch (MyAuxProcType) + { + case StartupProcess: + beentry->st_backendType = B_STARTUP; + break; + case BgWriterProcess: + beentry->st_backendType = B_BG_WRITER; + break; + case CheckpointerProcess: + beentry->st_backendType = B_CHECKPOINTER; + break; + case WalWriterProcess: + beentry->st_backendType = B_WAL_WRITER; + break; + case WalReceiverProcess: + beentry->st_backendType = B_WAL_RECEIVER; + break; + default: + elog(FATAL, "unrecognized process type: %d", + (int) MyAuxProcType); + proc_exit(1); + } + } + do { pgstat_increment_changecount_before(beentry); @@ -2708,7 +2804,15 @@ pgstat_bestart(void) beentry->st_state_start_timestamp = 0; beentry->st_xact_start_timestamp = 0; beentry->st_databaseid = MyDatabaseId; - beentry->st_userid = userid; + + /* We have userid for client-backends, wal-sender and bgworker processes */ + if (beentry->st_backendType == B_BACKEND + || beentry->st_backendType == B_WAL_SENDER + || beentry->st_backendType == B_BG_WORKER) + beentry->st_userid = GetSessionUserId(); + else + beentry->st_userid = InvalidOid; + beentry->st_clientaddr = clientaddr; if (MyProcPort && MyProcPort->remote_hostname) strlcpy(beentry->st_clienthostname, MyProcPort->remote_hostname, @@ -3046,24 +3150,24 @@ pgstat_read_current_status(void) localtable = (LocalPgBackendStatus *) MemoryContextAlloc(pgStatLocalContext, - sizeof(LocalPgBackendStatus) * MaxBackends); + sizeof(LocalPgBackendStatus) * NumBackendStatSlots); localappname = (char *) MemoryContextAlloc(pgStatLocalContext, - NAMEDATALEN * MaxBackends); + NAMEDATALEN * NumBackendStatSlots); localactivity = (char *) MemoryContextAlloc(pgStatLocalContext, - pgstat_track_activity_query_size * MaxBackends); + pgstat_track_activity_query_size * NumBackendStatSlots); #ifdef USE_SSL localsslstatus = (PgBackendSSLStatus *) MemoryContextAlloc(pgStatLocalContext, - sizeof(PgBackendSSLStatus) * MaxBackends); + sizeof(PgBackendSSLStatus) * NumBackendStatSlots); #endif localNumBackends = 0; beentry = BackendStatusArray; localentry = localtable; - for (i = 1; i <= MaxBackends; i++) + for (i = 1; i <= NumBackendStatSlots; i++) { /* * Follow the protocol of retrying if st_changecount changes while we @@ -3829,7 +3933,47 @@ pgstat_get_crashed_backend_activity(int pid, char *buffer, int buflen) return NULL; } +const char * +pgstat_get_backend_desc(BackendType backendType) +{ + const char *backendDesc = "unknown process type"; + + switch (backendType) + { + case B_AUTOVAC_LAUNCHER: + backendDesc = "autovacuum launcher"; + break; + case B_AUTOVAC_WORKER: + backendDesc = "autovacuum worker"; + break; + case B_BACKEND: + backendDesc = "client backend"; + break; + case B_BG_WORKER: + backendDesc = "background worker"; + break; + case B_BG_WRITER: + backendDesc = "background writer"; + break; + case B_CHECKPOINTER: + backendDesc = "checkpointer"; + break; + case B_STARTUP: + backendDesc = "startup"; + break; + case B_WAL_RECEIVER: + backendDesc = "walreceiver"; + break; + case B_WAL_SENDER: + backendDesc = "walsender"; + break; + case B_WAL_WRITER: + backendDesc = "walwriter"; + break; + } + return backendDesc; +} /* ------------------------------------------------------------ * Local support functions follow * ------------------------------------------------------------ diff --git a/src/backend/postmaster/startup.c b/src/backend/postmaster/startup.c index b172b5e5d7..b623252475 100644 --- a/src/backend/postmaster/startup.c +++ b/src/backend/postmaster/startup.c @@ -25,6 +25,7 @@ #include "access/xlog.h" #include "libpq/pqsignal.h" #include "miscadmin.h" +#include "pgstat.h" #include "postmaster/startup.h" #include "storage/ipc.h" #include "storage/latch.h" diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 040ad7b737..cfc3fba1b7 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -2011,8 +2011,8 @@ WalSndLoop(WalSndSendDataCallback send_data) last_reply_timestamp = GetCurrentTimestamp(); waiting_for_ping_response = false; - /* Report to pgstat that this process is a WAL sender */ - pgstat_report_activity(STATE_RUNNING, "walsender"); + /* Report to pgstat that this process is running */ + pgstat_report_activity(STATE_RUNNING, NULL); /* * Loop until we reach the end of this timeline or the client requests to diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c index 8f467bef50..3e716b1c6c 100644 --- a/src/backend/storage/lmgr/proc.c +++ b/src/backend/storage/lmgr/proc.c @@ -941,6 +941,33 @@ AuxiliaryProcKill(int code, Datum arg) SpinLockRelease(ProcStructLock); } +/* + * AuxiliaryPidGetProc -- get PGPROC for an auxiliary process + * given its PID + * + * Returns NULL if not found. + */ +PGPROC * +AuxiliaryPidGetProc(int pid) +{ + PGPROC *result = NULL; + int index; + + if (pid == 0) /* never match dummy PGPROCs */ + return NULL; + + for (index = 0; index < NUM_AUXILIARY_PROCS; index++) + { + PGPROC *proc = &AuxiliaryProcs[index]; + + if (proc->pid == pid) + { + result = proc; + break; + } + } + return result; +} /* * ProcQueue package: routines for putting processes to sleep diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index a987d0d621..dd2b924d0a 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -20,6 +20,7 @@ #include "funcapi.h" #include "miscadmin.h" #include "pgstat.h" +#include "postmaster/postmaster.h" #include "storage/proc.h" #include "storage/procarray.h" #include "utils/acl.h" @@ -538,7 +539,7 @@ pg_stat_get_progress_info(PG_FUNCTION_ARGS) Datum pg_stat_get_activity(PG_FUNCTION_ARGS) { -#define PG_STAT_GET_ACTIVITY_COLS 23 +#define PG_STAT_GET_ACTIVITY_COLS 24 int num_backends = pgstat_fetch_stat_numbackends(); int curr_backend; int pid = PG_ARGISNULL(0) ? -1 : PG_GETARG_INT32(0); @@ -582,8 +583,8 @@ pg_stat_get_activity(PG_FUNCTION_ARGS) LocalPgBackendStatus *local_beentry; PgBackendStatus *beentry; PGPROC *proc; - const char *wait_event_type; - const char *wait_event; + const char *wait_event_type = NULL; + const char *wait_event = NULL; MemSet(values, 0, sizeof(values)); MemSet(nulls, 0, sizeof(nulls)); @@ -615,9 +616,18 @@ pg_stat_get_activity(PG_FUNCTION_ARGS) continue; /* Values available to all callers */ - values[0] = ObjectIdGetDatum(beentry->st_databaseid); + if (beentry->st_databaseid != InvalidOid) + values[0] = ObjectIdGetDatum(beentry->st_databaseid); + else + nulls[0] = true; + values[1] = Int32GetDatum(beentry->st_procpid); - values[2] = ObjectIdGetDatum(beentry->st_userid); + + if (beentry->st_userid != InvalidOid) + values[2] = ObjectIdGetDatum(beentry->st_userid); + else + nulls[2] = true; + if (beentry->st_appname) values[3] = CStringGetTextDatum(beentry->st_appname); else @@ -635,17 +645,17 @@ pg_stat_get_activity(PG_FUNCTION_ARGS) if (beentry->st_ssl) { - values[17] = BoolGetDatum(true); /* ssl */ - values[18] = CStringGetTextDatum(beentry->st_sslstatus->ssl_version); - values[19] = CStringGetTextDatum(beentry->st_sslstatus->ssl_cipher); - values[20] = Int32GetDatum(beentry->st_sslstatus->ssl_bits); - values[21] = BoolGetDatum(beentry->st_sslstatus->ssl_compression); - values[22] = CStringGetTextDatum(beentry->st_sslstatus->ssl_clientdn); + values[18] = BoolGetDatum(true); /* ssl */ + values[19] = CStringGetTextDatum(beentry->st_sslstatus->ssl_version); + values[20] = CStringGetTextDatum(beentry->st_sslstatus->ssl_cipher); + values[21] = Int32GetDatum(beentry->st_sslstatus->ssl_bits); + values[22] = BoolGetDatum(beentry->st_sslstatus->ssl_compression); + values[23] = CStringGetTextDatum(beentry->st_sslstatus->ssl_clientdn); } else { - values[17] = BoolGetDatum(false); /* ssl */ - nulls[18] = nulls[19] = nulls[20] = nulls[21] = nulls[22] = true; + values[18] = BoolGetDatum(false); /* ssl */ + nulls[19] = nulls[20] = nulls[21] = nulls[22] = nulls[23] = true; } /* Values only available to role member */ @@ -690,10 +700,24 @@ pg_stat_get_activity(PG_FUNCTION_ARGS) wait_event = pgstat_get_wait_event(raw_wait_event); } - else + else if (beentry->st_backendType != B_BACKEND) { - wait_event_type = NULL; - wait_event = NULL; + /* + * For an auxiliary process, retrieve process info from + * AuxiliaryProcs stored in shared-memory. + */ + proc = AuxiliaryPidGetProc(beentry->st_procpid); + + if (proc != NULL) + { + uint32 raw_wait_event; + + raw_wait_event = + UINT32_ACCESS_ONCE(proc->wait_event_info); + wait_event_type = + pgstat_get_wait_event_type(raw_wait_event); + wait_event = pgstat_get_wait_event(raw_wait_event); + } } if (wait_event_type) @@ -793,6 +817,9 @@ pg_stat_get_activity(PG_FUNCTION_ARGS) nulls[14] = true; } } + /* Add backend type */ + values[17] = + CStringGetTextDatum(pgstat_get_backend_desc(beentry->st_backendType)); } else { @@ -808,6 +835,7 @@ pg_stat_get_activity(PG_FUNCTION_ARGS) nulls[12] = true; nulls[13] = true; nulls[14] = true; + nulls[17] = true; } tuplestore_putvalues(tupstore, tupdesc, values, nulls); diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c index 9f938f2d27..b8b4a06350 100644 --- a/src/backend/utils/init/postinit.c +++ b/src/backend/utils/init/postinit.c @@ -665,7 +665,12 @@ InitPostgres(const char *in_dbname, Oid dboid, const char *username, /* The autovacuum launcher is done here */ if (IsAutoVacuumLauncherProcess()) + { + /* report this backend in the PgBackendStatus array */ + pgstat_bestart(); + return; + } /* * Start a new transaction here before first access to db, and get a @@ -874,7 +879,10 @@ InitPostgres(const char *in_dbname, Oid dboid, const char *username, * transaction we started before returning. */ if (!bootstrap) + { + pgstat_bestart(); CommitTransactionCommand(); + } return; } diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index b8fa18ae2e..fc374d79c8 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201703242 +#define CATALOG_VERSION_NO 201703261 #endif diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index ee67459c32..79f9b9012e 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -2811,7 +2811,7 @@ DATA(insert OID = 3057 ( pg_stat_get_autoanalyze_count PGNSP PGUID 12 1 0 0 0 f DESCR("statistics: number of auto analyzes for a table"); DATA(insert OID = 1936 ( pg_stat_get_backend_idset PGNSP PGUID 12 1 100 0 0 f f f f t t s r 0 0 23 "" _null_ _null_ _null_ _null_ _null_ pg_stat_get_backend_idset _null_ _null_ _null_ )); DESCR("statistics: currently active backend IDs"); -DATA(insert OID = 2022 ( pg_stat_get_activity PGNSP PGUID 12 1 100 0 0 f f f f f t s r 1 0 2249 "23" "{23,26,23,26,25,25,25,25,25,1184,1184,1184,1184,869,25,23,28,28,16,25,25,23,16,25}" "{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}" "{pid,datid,pid,usesysid,application_name,state,query,wait_event_type,wait_event,xact_start,query_start,backend_start,state_change,client_addr,client_hostname,client_port,backend_xid,backend_xmin,ssl,sslversion,sslcipher,sslbits,sslcompression,sslclientdn}" _null_ _null_ pg_stat_get_activity _null_ _null_ _null_ )); +DATA(insert OID = 2022 ( pg_stat_get_activity PGNSP PGUID 12 1 100 0 0 f f f f f t s r 1 0 2249 "23" "{23,26,23,26,25,25,25,25,25,1184,1184,1184,1184,869,25,23,28,28,25,16,25,25,23,16,25}" "{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}" "{pid,datid,pid,usesysid,application_name,state,query,wait_event_type,wait_event,xact_start,query_start,backend_start,state_change,client_addr,client_hostname,client_port,backend_xid,backend_xmin,backend_type,ssl,sslversion,sslcipher,sslbits,sslcompression,sslclientdn}" _null_ _null_ pg_stat_get_activity _null_ _null_ _null_ )); DESCR("statistics: information about currently active backends"); DATA(insert OID = 3318 ( pg_stat_get_progress_info PGNSP PGUID 12 1 100 0 0 f f f f t t s r 1 0 2249 "25" "{25,23,26,26,20,20,20,20,20,20,20,20,20,20}" "{i,o,o,o,o,o,o,o,o,o,o,o,o,o}" "{cmdtype,pid,datid,relid,param1,param2,param3,param4,param5,param6,param7,param8,param9,param10}" _null_ _null_ pg_stat_get_progress_info _null_ _null_ _null_ )); DESCR("statistics: information about progress of backends running maintenance command"); diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 201562521f..e29397f25b 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -695,6 +695,25 @@ typedef struct PgStat_GlobalStats } PgStat_GlobalStats; +/* ---------- + * Backend types + * ---------- + */ +typedef enum BackendType +{ + B_AUTOVAC_LAUNCHER, + B_AUTOVAC_WORKER, + B_BACKEND, + B_BG_WORKER, + B_BG_WRITER, + B_CHECKPOINTER, + B_STARTUP, + B_WAL_RECEIVER, + B_WAL_SENDER, + B_WAL_WRITER +} BackendType; + + /* ---------- * Backend states * ---------- @@ -927,6 +946,9 @@ typedef struct PgBackendSSLStatus * showing its current activity. (The structs are allocated according to * BackendId, but that is not critical.) Note that the collector process * has no involvement in, or even access to, these structs. + * + * Each auxiliary process also maintains a PgBackendStatus struct in shared + * memory. * ---------- */ typedef struct PgBackendStatus @@ -951,6 +973,9 @@ typedef struct PgBackendStatus /* The entry is valid iff st_procpid > 0, unused if st_procpid == 0 */ int st_procpid; + /* Type of backends */ + BackendType st_backendType; + /* Times when current backend, transaction, and activity started */ TimestampTz st_proc_start_timestamp; TimestampTz st_xact_start_timestamp; @@ -1149,6 +1174,7 @@ extern const char *pgstat_get_wait_event_type(uint32 wait_event_info); extern const char *pgstat_get_backend_current_activity(int pid, bool checkUser); extern const char *pgstat_get_crashed_backend_activity(int pid, char *buffer, int buflen); +extern const char *pgstat_get_backend_desc(BackendType backendType); extern void pgstat_progress_start_command(ProgressCommandType cmdtype, Oid relid); diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h index 1b345faa2d..1a125d83f4 100644 --- a/src/include/storage/proc.h +++ b/src/include/storage/proc.h @@ -272,7 +272,6 @@ extern PGPROC *PreparedXactProcs; */ #define NUM_AUXILIARY_PROCS 4 - /* configurable options */ extern int DeadlockTimeout; extern int StatementTimeout; @@ -309,6 +308,8 @@ extern void LockErrorCleanup(void); extern void ProcWaitForSignal(uint32 wait_event_info); extern void ProcSendSignal(int pid); +extern PGPROC *AuxiliaryPidGetProc(int pid); + extern void BecomeLockGroupLeader(void); extern bool BecomeLockGroupMember(PGPROC *leader, int pid); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index e8f8726c53..71121c8663 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1727,8 +1727,9 @@ pg_stat_activity| SELECT s.datid, s.state, s.backend_xid, s.backend_xmin, - s.query - FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, ssl, sslversion, sslcipher, sslbits, sslcompression, sslclientdn) + s.query, + s.backend_type + FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, sslcompression, sslclientdn) LEFT JOIN pg_database d ON ((s.datid = d.oid))) LEFT JOIN pg_authid u ON ((s.usesysid = u.oid))); pg_stat_all_indexes| SELECT c.oid AS relid, @@ -1859,7 +1860,7 @@ pg_stat_replication| SELECT s.pid, w.replay_lag, w.sync_priority, w.sync_state - FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, ssl, sslversion, sslcipher, sslbits, sslcompression, sslclientdn) + FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, sslcompression, sslclientdn) JOIN pg_stat_get_wal_senders() w(pid, state, sent_location, write_location, flush_location, replay_location, write_lag, flush_lag, replay_lag, sync_priority, sync_state) ON ((s.pid = w.pid))) LEFT JOIN pg_authid u ON ((s.usesysid = u.oid))); pg_stat_ssl| SELECT s.pid, @@ -1869,7 +1870,7 @@ pg_stat_ssl| SELECT s.pid, s.sslbits AS bits, s.sslcompression AS compression, s.sslclientdn AS clientdn - FROM pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, ssl, sslversion, sslcipher, sslbits, sslcompression, sslclientdn); + FROM pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, sslcompression, sslclientdn); pg_stat_subscription| SELECT su.oid AS subid, su.subname, st.pid, -- cgit v1.2.3 From facde2a98f0b5f7689b4e30a9e7376e926e733b8 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Sun, 26 Mar 2017 22:24:13 -0400 Subject: Clean up Perl code according to perlcritic Fix all perlcritic warnings of severity level 5, except in src/backend/utils/Gen_dummy_probes.pl, which is automatically generated. Reviewed-by: Dagfinn Ilmari Mannsåker Reviewed-by: Daniel Gustafsson --- contrib/intarray/bench/create_test.pl | 20 +-- doc/src/sgml/generate-errcodes-table.pl | 2 +- doc/src/sgml/mk_feature_tables.pl | 12 +- src/backend/catalog/Catalog.pm | 8 +- src/backend/catalog/genbki.pl | 64 ++++----- src/backend/parser/check_keywords.pl | 30 ++--- src/backend/storage/lmgr/generate-lwlocknames.pl | 30 ++--- src/backend/utils/Gen_fmgrtab.pl | 32 ++--- src/backend/utils/generate-errcodes.pl | 2 +- src/bin/pg_basebackup/t/010_pg_basebackup.pl | 26 ++-- src/bin/pg_ctl/t/001_start_stop.pl | 14 +- src/bin/psql/create_help.pl | 28 ++-- src/interfaces/ecpg/preproc/check_rules.pl | 12 +- src/interfaces/libpq/test/regress.pl | 14 +- src/pl/plperl/plc_perlboot.pl | 4 +- src/pl/plperl/plc_trusted.pl | 2 +- src/pl/plperl/text2macro.pl | 8 +- src/pl/plpgsql/src/generate-plerrcodes.pl | 2 +- src/pl/plpython/generate-spiexceptions.pl | 2 +- src/pl/tcl/generate-pltclerrcodes.pl | 2 +- src/test/locale/sort-test.pl | 6 +- src/test/perl/PostgresNode.pm | 8 +- src/test/perl/TestLib.pm | 16 +-- src/test/ssl/ServerSetup.pm | 48 +++---- src/tools/fix-old-flex-code.pl | 4 +- src/tools/msvc/Install.pm | 10 +- src/tools/msvc/Mkvcbuild.pm | 2 +- src/tools/msvc/Project.pm | 28 ++-- src/tools/msvc/Solution.pm | 162 +++++++++++------------ src/tools/msvc/build.pl | 8 +- src/tools/msvc/builddoc.pl | 2 +- src/tools/msvc/gendef.pl | 18 +-- src/tools/msvc/install.pl | 4 +- src/tools/msvc/mkvcbuild.pl | 4 +- src/tools/msvc/pgbison.pl | 4 +- src/tools/msvc/pgflex.pl | 12 +- src/tools/msvc/vcregress.pl | 19 +-- src/tools/pginclude/pgcheckdefines | 32 ++--- src/tools/pgindent/pgindent | 5 +- src/tools/version_stamp.pl | 6 +- src/tools/win32tzlist.pl | 6 +- 41 files changed, 360 insertions(+), 358 deletions(-) (limited to 'doc/src') diff --git a/contrib/intarray/bench/create_test.pl b/contrib/intarray/bench/create_test.pl index 1323b31e4d..f3262df05b 100755 --- a/contrib/intarray/bench/create_test.pl +++ b/contrib/intarray/bench/create_test.pl @@ -15,8 +15,8 @@ create table message_section_map ( EOT -open(MSG, ">message.tmp") || die; -open(MAP, ">message_section_map.tmp") || die; +open(my $msg, '>', "message.tmp") || die; +open(my $map, '>', "message_section_map.tmp") || die; srand(1); @@ -42,16 +42,16 @@ foreach my $i (1 .. 200000) } if ($#sect < 0 || rand() < 0.1) { - print MSG "$i\t\\N\n"; + print $msg "$i\t\\N\n"; } else { - print MSG "$i\t{" . join(',', @sect) . "}\n"; - map { print MAP "$i\t$_\n" } @sect; + print $msg "$i\t{" . join(',', @sect) . "}\n"; + map { print $map "$i\t$_\n" } @sect; } } -close MAP; -close MSG; +close $map; +close $msg; copytable('message'); copytable('message_section_map'); @@ -79,8 +79,8 @@ sub copytable my $t = shift; print "COPY $t from stdin;\n"; - open(FFF, "$t.tmp") || die; - while () { print; } - close FFF; + open(my $fff, '<', "$t.tmp") || die; + while (<$fff>) { print; } + close $fff; print "\\.\n"; } diff --git a/doc/src/sgml/generate-errcodes-table.pl b/doc/src/sgml/generate-errcodes-table.pl index 66be811adb..01fc6166bf 100644 --- a/doc/src/sgml/generate-errcodes-table.pl +++ b/doc/src/sgml/generate-errcodes-table.pl @@ -9,7 +9,7 @@ use strict; print "\n"; -open my $errcodes, $ARGV[0] or die; +open my $errcodes, '<', $ARGV[0] or die; while (<$errcodes>) { diff --git a/doc/src/sgml/mk_feature_tables.pl b/doc/src/sgml/mk_feature_tables.pl index 93dab2132e..9b111b8b40 100644 --- a/doc/src/sgml/mk_feature_tables.pl +++ b/doc/src/sgml/mk_feature_tables.pl @@ -6,11 +6,11 @@ use strict; my $yesno = $ARGV[0]; -open PACK, $ARGV[1] or die; +open my $pack, '<', $ARGV[1] or die; my %feature_packages; -while () +while (<$pack>) { chomp; my ($fid, $pname) = split /\t/; @@ -24,13 +24,13 @@ while () } } -close PACK; +close $pack; -open FEAT, $ARGV[2] or die; +open my $feat, '<', $ARGV[2] or die; print "\n"; -while () +while (<$feat>) { chomp; my ($feature_id, $feature_name, $subfeature_id, @@ -69,4 +69,4 @@ while () print "\n"; -close FEAT; +close $feat; diff --git a/src/backend/catalog/Catalog.pm b/src/backend/catalog/Catalog.pm index bccbc5118d..6ffd5f904a 100644 --- a/src/backend/catalog/Catalog.pm +++ b/src/backend/catalog/Catalog.pm @@ -44,13 +44,13 @@ sub Catalogs $catalog{columns} = []; $catalog{data} = []; - open(INPUT_FILE, '<', $input_file) || die "$input_file: $!"; + open(my $ifh, '<', $input_file) || die "$input_file: $!"; my ($filename) = ($input_file =~ m/(\w+)\.h$/); my $natts_pat = "Natts_$filename"; # Scan the input file. - while () + while (<$ifh>) { # Strip C-style comments. @@ -59,7 +59,7 @@ sub Catalogs { # handle multi-line comments properly. - my $next_line = ; + my $next_line = <$ifh>; die "$input_file: ends within C-style comment\n" if !defined $next_line; $_ .= $next_line; @@ -211,7 +211,7 @@ sub Catalogs } } $catalogs{$catname} = \%catalog; - close INPUT_FILE; + close $ifh; } return \%catalogs; } diff --git a/src/backend/catalog/genbki.pl b/src/backend/catalog/genbki.pl index 079516ca2f..f9ecb02548 100644 --- a/src/backend/catalog/genbki.pl +++ b/src/backend/catalog/genbki.pl @@ -66,16 +66,16 @@ if ($output_path ne '' && substr($output_path, -1) ne '/') # Open temp files my $tmpext = ".tmp$$"; my $bkifile = $output_path . 'postgres.bki'; -open BKI, '>', $bkifile . $tmpext +open my $bki, '>', $bkifile . $tmpext or die "can't open $bkifile$tmpext: $!"; my $schemafile = $output_path . 'schemapg.h'; -open SCHEMAPG, '>', $schemafile . $tmpext +open my $schemapg, '>', $schemafile . $tmpext or die "can't open $schemafile$tmpext: $!"; my $descrfile = $output_path . 'postgres.description'; -open DESCR, '>', $descrfile . $tmpext +open my $descr, '>', $descrfile . $tmpext or die "can't open $descrfile$tmpext: $!"; my $shdescrfile = $output_path . 'postgres.shdescription'; -open SHDESCR, '>', $shdescrfile . $tmpext +open my $shdescr, '>', $shdescrfile . $tmpext or die "can't open $shdescrfile$tmpext: $!"; # Fetch some special data that we will substitute into the output file. @@ -97,7 +97,7 @@ my $catalogs = Catalog::Catalogs(@input_files); # Generate postgres.bki, postgres.description, and postgres.shdescription # version marker for .bki file -print BKI "# PostgreSQL $major_version\n"; +print $bki "# PostgreSQL $major_version\n"; # vars to hold data needed for schemapg.h my %schemapg_entries; @@ -110,7 +110,7 @@ foreach my $catname (@{ $catalogs->{names} }) # .bki CREATE command for this catalog my $catalog = $catalogs->{$catname}; - print BKI "create $catname $catalog->{relation_oid}" + print $bki "create $catname $catalog->{relation_oid}" . $catalog->{shared_relation} . $catalog->{bootstrap} . $catalog->{without_oids} @@ -120,7 +120,7 @@ foreach my $catname (@{ $catalogs->{names} }) my @attnames; my $first = 1; - print BKI " (\n"; + print $bki " (\n"; foreach my $column (@{ $catalog->{columns} }) { my $attname = $column->{name}; @@ -130,27 +130,27 @@ foreach my $catname (@{ $catalogs->{names} }) if (!$first) { - print BKI " ,\n"; + print $bki " ,\n"; } $first = 0; - print BKI " $attname = $atttype"; + print $bki " $attname = $atttype"; if (defined $column->{forcenotnull}) { - print BKI " FORCE NOT NULL"; + print $bki " FORCE NOT NULL"; } elsif (defined $column->{forcenull}) { - print BKI " FORCE NULL"; + print $bki " FORCE NULL"; } } - print BKI "\n )\n"; + print $bki "\n )\n"; # open it, unless bootstrap case (create bootstrap does this automatically) if ($catalog->{bootstrap} eq '') { - print BKI "open $catname\n"; + print $bki "open $catname\n"; } if (defined $catalog->{data}) @@ -175,17 +175,17 @@ foreach my $catname (@{ $catalogs->{names} }) # Write to postgres.bki my $oid = $row->{oid} ? "OID = $row->{oid} " : ''; - printf BKI "insert %s( %s)\n", $oid, $row->{bki_values}; + printf $bki "insert %s( %s)\n", $oid, $row->{bki_values}; # Write comments to postgres.description and postgres.shdescription if (defined $row->{descr}) { - printf DESCR "%s\t%s\t0\t%s\n", $row->{oid}, $catname, + printf $descr "%s\t%s\t0\t%s\n", $row->{oid}, $catname, $row->{descr}; } if (defined $row->{shdescr}) { - printf SHDESCR "%s\t%s\t%s\n", $row->{oid}, $catname, + printf $shdescr "%s\t%s\t%s\n", $row->{oid}, $catname, $row->{shdescr}; } } @@ -267,7 +267,7 @@ foreach my $catname (@{ $catalogs->{names} }) } } - print BKI "close $catname\n"; + print $bki "close $catname\n"; } # Any information needed for the BKI that is not contained in a pg_*.h header @@ -276,19 +276,19 @@ foreach my $catname (@{ $catalogs->{names} }) # Write out declare toast/index statements foreach my $declaration (@{ $catalogs->{toasting}->{data} }) { - print BKI $declaration; + print $bki $declaration; } foreach my $declaration (@{ $catalogs->{indexing}->{data} }) { - print BKI $declaration; + print $bki $declaration; } # Now generate schemapg.h # Opening boilerplate for schemapg.h -print SCHEMAPG <{oid} ? "OID = $row->{oid} " : ''; my $bki_values = join ' ', map $row->{$_}, @attnames; - printf BKI "insert %s( %s)\n", $oid, $bki_values; + printf $bki "insert %s( %s)\n", $oid, $bki_values; } # The field values of a Schema_pg_xxx declaration are similar, but not @@ -472,15 +472,15 @@ sub find_defined_symbol } my $file = $path . $catalog_header; next if !-f $file; - open(FIND_DEFINED_SYMBOL, '<', $file) || die "$file: $!"; - while () + open(my $find_defined_symbol, '<', $file) || die "$file: $!"; + while (<$find_defined_symbol>) { if (/^#define\s+\Q$symbol\E\s+(\S+)/) { return $1; } } - close FIND_DEFINED_SYMBOL; + close $find_defined_symbol; die "$file: no definition found for $symbol\n"; } die "$catalog_header: not found in any include directory\n"; diff --git a/src/backend/parser/check_keywords.pl b/src/backend/parser/check_keywords.pl index 45862ce940..84fef1d95e 100644 --- a/src/backend/parser/check_keywords.pl +++ b/src/backend/parser/check_keywords.pl @@ -14,7 +14,7 @@ my $kwlist_filename = $ARGV[1]; my $errors = 0; -sub error(@) +sub error { print STDERR @_; $errors = 1; @@ -29,18 +29,18 @@ $keyword_categories{'col_name_keyword'} = 'COL_NAME_KEYWORD'; $keyword_categories{'type_func_name_keyword'} = 'TYPE_FUNC_NAME_KEYWORD'; $keyword_categories{'reserved_keyword'} = 'RESERVED_KEYWORD'; -open(GRAM, $gram_filename) || die("Could not open : $gram_filename"); +open(my $gram, '<', $gram_filename) || die("Could not open : $gram_filename"); -my ($S, $s, $k, $n, $kcat); +my $kcat; my $comment; my @arr; my %keywords; -line: while () +line: while (my $S = <$gram>) { - chomp; # strip record separator + chomp $S; # strip record separator - $S = $_; + my $s; # Make sure any braces are split $s = '{', $S =~ s/$s/ { /g; @@ -54,7 +54,7 @@ line: while () { # Is this the beginning of a keyword list? - foreach $k (keys %keyword_categories) + foreach my $k (keys %keyword_categories) { if ($S =~ m/^($k):/) { @@ -66,7 +66,7 @@ line: while () } # Now split the line into individual fields - $n = (@arr = split(' ', $S)); + my $n = (@arr = split(' ', $S)); # Ok, we're in a keyword list. Go through each field in turn for (my $fieldIndexer = 0; $fieldIndexer < $n; $fieldIndexer++) @@ -109,15 +109,15 @@ line: while () push @{ $keywords{$kcat} }, $arr[$fieldIndexer]; } } -close GRAM; +close $gram; # Check that each keyword list is in alphabetical order (just for neatnik-ism) -my ($prevkword, $kword, $bare_kword); -foreach $kcat (keys %keyword_categories) +my ($prevkword, $bare_kword); +foreach my $kcat (keys %keyword_categories) { $prevkword = ''; - foreach $kword (@{ $keywords{$kcat} }) + foreach my $kword (@{ $keywords{$kcat} }) { # Some keyword have a _P suffix. Remove it for the comparison. @@ -149,12 +149,12 @@ while (my ($kcat, $kcat_id) = each(%keyword_categories)) # Now read in kwlist.h -open(KWLIST, $kwlist_filename) || die("Could not open : $kwlist_filename"); +open(my $kwlist, '<', $kwlist_filename) || die("Could not open : $kwlist_filename"); my $prevkwstring = ''; my $bare_kwname; my %kwhash; -kwlist_line: while () +kwlist_line: while (<$kwlist>) { my ($line) = $_; @@ -219,7 +219,7 @@ kwlist_line: while () } } } -close KWLIST; +close $kwlist; # Check that we've paired up all keywords from gram.y with lines in kwlist.h while (my ($kwcat, $kwcat_id) = each(%keyword_categories)) diff --git a/src/backend/storage/lmgr/generate-lwlocknames.pl b/src/backend/storage/lmgr/generate-lwlocknames.pl index f80d2c8121..10d069896f 100644 --- a/src/backend/storage/lmgr/generate-lwlocknames.pl +++ b/src/backend/storage/lmgr/generate-lwlocknames.pl @@ -9,21 +9,21 @@ use strict; my $lastlockidx = -1; my $continue = "\n"; -open my $lwlocknames, $ARGV[0] or die; +open my $lwlocknames, '<', $ARGV[0] or die; # Include PID in suffix in case parallel make runs this multiple times. my $htmp = "lwlocknames.h.tmp$$"; my $ctmp = "lwlocknames.c.tmp$$"; -open H, '>', $htmp or die "Could not open $htmp: $!"; -open C, '>', $ctmp or die "Could not open $ctmp: $!"; +open my $h, '>', $htmp or die "Could not open $htmp: $!"; +open my $c, '>', $ctmp or die "Could not open $ctmp: $!"; my $autogen = "/* autogenerated from src/backend/storage/lmgr/lwlocknames.txt, do not edit */\n"; -print H $autogen; -print H "/* there is deliberately not an #ifndef LWLOCKNAMES_H here */\n\n"; -print C $autogen, "\n"; +print $h $autogen; +print $h "/* there is deliberately not an #ifndef LWLOCKNAMES_H here */\n\n"; +print $c $autogen, "\n"; -print C "char *MainLWLockNames[] = {"; +print $c "char *MainLWLockNames[] = {"; while (<$lwlocknames>) { @@ -44,22 +44,22 @@ while (<$lwlocknames>) while ($lastlockidx < $lockidx - 1) { ++$lastlockidx; - printf C "%s \"\"", $continue, $lastlockidx; + printf $c "%s \"\"", $continue, $lastlockidx; $continue = ",\n"; } - printf C "%s \"%s\"", $continue, $lockname; + printf $c "%s \"%s\"", $continue, $lockname; $lastlockidx = $lockidx; $continue = ",\n"; - print H "#define $lockname (&MainLWLockArray[$lockidx].lock)\n"; + print $h "#define $lockname (&MainLWLockArray[$lockidx].lock)\n"; } -printf C "\n};\n"; -print H "\n"; -printf H "#define NUM_INDIVIDUAL_LWLOCKS %s\n", $lastlockidx + 1; +printf $c "\n};\n"; +print $h "\n"; +printf $h "#define NUM_INDIVIDUAL_LWLOCKS %s\n", $lastlockidx + 1; -close H; -close C; +close $h; +close $c; rename($htmp, 'lwlocknames.h') || die "rename: $htmp: $!"; rename($ctmp, 'lwlocknames.c') || die "rename: $ctmp: $!"; diff --git a/src/backend/utils/Gen_fmgrtab.pl b/src/backend/utils/Gen_fmgrtab.pl index cdd603ab6f..2af9b355e7 100644 --- a/src/backend/utils/Gen_fmgrtab.pl +++ b/src/backend/utils/Gen_fmgrtab.pl @@ -90,11 +90,11 @@ my $oidsfile = $output_path . 'fmgroids.h'; my $protosfile = $output_path . 'fmgrprotos.h'; my $tabfile = $output_path . 'fmgrtab.c'; -open H, '>', $oidsfile . $tmpext or die "Could not open $oidsfile$tmpext: $!"; -open P, '>', $protosfile . $tmpext or die "Could not open $protosfile$tmpext: $!"; -open T, '>', $tabfile . $tmpext or die "Could not open $tabfile$tmpext: $!"; +open my $ofh, '>', $oidsfile . $tmpext or die "Could not open $oidsfile$tmpext: $!"; +open my $pfh, '>', $protosfile . $tmpext or die "Could not open $protosfile$tmpext: $!"; +open my $tfh, '>', $tabfile . $tmpext or die "Could not open $tabfile$tmpext: $!"; -print H +print $ofh qq|/*------------------------------------------------------------------------- * * fmgroids.h @@ -132,7 +132,7 @@ qq|/*------------------------------------------------------------------------- */ |; -print P +print $pfh qq|/*------------------------------------------------------------------------- * * fmgrprotos.h @@ -159,7 +159,7 @@ qq|/*------------------------------------------------------------------------- |; -print T +print $tfh qq|/*------------------------------------------------------------------------- * * fmgrtab.c @@ -193,26 +193,26 @@ foreach my $s (sort { $a->{oid} <=> $b->{oid} } @fmgr) { next if $seenit{ $s->{prosrc} }; $seenit{ $s->{prosrc} } = 1; - print H "#define F_" . uc $s->{prosrc} . " $s->{oid}\n"; - print P "extern Datum $s->{prosrc}(PG_FUNCTION_ARGS);\n"; + print $ofh "#define F_" . uc $s->{prosrc} . " $s->{oid}\n"; + print $pfh "extern Datum $s->{prosrc}(PG_FUNCTION_ARGS);\n"; } # Create the fmgr_builtins table -print T "\nconst FmgrBuiltin fmgr_builtins[] = {\n"; +print $tfh "\nconst FmgrBuiltin fmgr_builtins[] = {\n"; my %bmap; $bmap{'t'} = 'true'; $bmap{'f'} = 'false'; foreach my $s (sort { $a->{oid} <=> $b->{oid} } @fmgr) { - print T + print $tfh " { $s->{oid}, \"$s->{prosrc}\", $s->{nargs}, $bmap{$s->{strict}}, $bmap{$s->{retset}}, $s->{prosrc} },\n"; } # And add the file footers. -print H "\n#endif /* FMGROIDS_H */\n"; -print P "\n#endif /* FMGRPROTOS_H */\n"; +print $ofh "\n#endif /* FMGROIDS_H */\n"; +print $pfh "\n#endif /* FMGRPROTOS_H */\n"; -print T +print $tfh qq| /* dummy entry is easier than getting rid of comma after last real one */ /* (not that there has ever been anything wrong with *having* a comma after the last field in an array initializer) */ @@ -223,9 +223,9 @@ qq| /* dummy entry is easier than getting rid of comma after last real one */ const int fmgr_nbuiltins = (sizeof(fmgr_builtins) / sizeof(FmgrBuiltin)) - 1; |; -close(H); -close(P); -close(T); +close($ofh); +close($pfh); +close($tfh); # Finally, rename the completed files into place. Catalog::RenameTempFile($oidsfile, $tmpext); diff --git a/src/backend/utils/generate-errcodes.pl b/src/backend/utils/generate-errcodes.pl index b84c6b0d0f..6a577f657a 100644 --- a/src/backend/utils/generate-errcodes.pl +++ b/src/backend/utils/generate-errcodes.pl @@ -10,7 +10,7 @@ print "/* autogenerated from src/backend/utils/errcodes.txt, do not edit */\n"; print "/* there is deliberately not an #ifndef ERRCODES_H here */\n"; -open my $errcodes, $ARGV[0] or die; +open my $errcodes, '<', $ARGV[0] or die; while (<$errcodes>) { diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/010_pg_basebackup.pl index 14bd813896..1d3c498fb2 100644 --- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl +++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl @@ -24,10 +24,10 @@ $node->command_fails(['pg_basebackup'], # Some Windows ANSI code pages may reject this filename, in which case we # quietly proceed without this bit of test coverage. -if (open BADCHARS, ">>$tempdir/pgdata/FOO\xe0\xe0\xe0BAR") +if (open my $badchars, '>>', "$tempdir/pgdata/FOO\xe0\xe0\xe0BAR") { - print BADCHARS "test backup of file with non-UTF8 name\n"; - close BADCHARS; + print $badchars "test backup of file with non-UTF8 name\n"; + close $badchars; } $node->set_replication_conf(); @@ -45,19 +45,19 @@ $node->command_fails( ok(-d "$tempdir/backup", 'backup directory was created and left behind'); -open CONF, ">>$pgdata/postgresql.conf"; -print CONF "max_replication_slots = 10\n"; -print CONF "max_wal_senders = 10\n"; -print CONF "wal_level = replica\n"; -close CONF; +open my $conf, '>>', "$pgdata/postgresql.conf"; +print $conf "max_replication_slots = 10\n"; +print $conf "max_wal_senders = 10\n"; +print $conf "wal_level = replica\n"; +close $conf; $node->restart; # Write some files to test that they are not copied. foreach my $filename (qw(backup_label tablespace_map postgresql.auto.conf.tmp current_logfiles.tmp)) { - open FILE, ">>$pgdata/$filename"; - print FILE "DONOTCOPY"; - close FILE; + open my $file, '>>', "$pgdata/$filename"; + print $file "DONOTCOPY"; + close $file; } $node->command_ok([ 'pg_basebackup', '-D', "$tempdir/backup", '-X', 'none' ], @@ -124,8 +124,8 @@ $node->command_fails( my $superlongname = "superlongname_" . ("x" x 100); my $superlongpath = "$pgdata/$superlongname"; -open FILE, ">$superlongpath" or die "unable to create file $superlongpath"; -close FILE; +open my $file, '>', "$superlongpath" or die "unable to create file $superlongpath"; +close $file; $node->command_fails( [ 'pg_basebackup', '-D', "$tempdir/tarbackup_l1", '-Ft' ], 'pg_basebackup tar with long name fails'); diff --git a/src/bin/pg_ctl/t/001_start_stop.pl b/src/bin/pg_ctl/t/001_start_stop.pl index 8f16bf9795..918257441b 100644 --- a/src/bin/pg_ctl/t/001_start_stop.pl +++ b/src/bin/pg_ctl/t/001_start_stop.pl @@ -20,18 +20,18 @@ command_ok([ 'pg_ctl', 'initdb', '-D', "$tempdir/data", '-o', '-N' ], 'pg_ctl initdb'); command_ok([ $ENV{PG_REGRESS}, '--config-auth', "$tempdir/data" ], 'configure authentication'); -open CONF, ">>$tempdir/data/postgresql.conf"; -print CONF "fsync = off\n"; -if (!$windows_os) +open my $conf, '>>', "$tempdir/data/postgresql.conf"; +print $conf "fsync = off\n"; +if (! $windows_os) { - print CONF "listen_addresses = ''\n"; - print CONF "unix_socket_directories = '$tempdir_short'\n"; + print $conf "listen_addresses = ''\n"; + print $conf "unix_socket_directories = '$tempdir_short'\n"; } else { - print CONF "listen_addresses = '127.0.0.1'\n"; + print $conf "listen_addresses = '127.0.0.1'\n"; } -close CONF; +close $conf; command_ok([ 'pg_ctl', 'start', '-D', "$tempdir/data" ], 'pg_ctl start'); diff --git a/src/bin/psql/create_help.pl b/src/bin/psql/create_help.pl index 359670b6e9..cedb767b27 100644 --- a/src/bin/psql/create_help.pl +++ b/src/bin/psql/create_help.pl @@ -42,12 +42,12 @@ $define =~ s/\W/_/g; opendir(DIR, $docdir) or die "$0: could not open documentation source dir '$docdir': $!\n"; -open(HFILE, ">$hfile") +open(my $hfile_handle, '>', $hfile) or die "$0: could not open output file '$hfile': $!\n"; -open(CFILE, ">$cfile") +open(my $cfile_handle, '>', $cfile) or die "$0: could not open output file '$cfile': $!\n"; -print HFILE "/* +print $hfile_handle "/* * *** Do not change this file by hand. It is automatically * *** generated from the DocBook documentation. * @@ -72,7 +72,7 @@ struct _helpStruct extern const struct _helpStruct QL_HELP[]; "; -print CFILE "/* +print $cfile_handle "/* * *** Do not change this file by hand. It is automatically * *** generated from the DocBook documentation. * @@ -97,9 +97,9 @@ foreach my $file (sort readdir DIR) my (@cmdnames, $cmddesc, $cmdsynopsis); $file =~ /\.sgml$/ or next; - open(FILE, "$docdir/$file") or next; - my $filecontent = join('', ); - close FILE; + open(my $fh, '<', "$docdir/$file") or next; + my $filecontent = join('', <$fh>); + close $fh; # Ignore files that are not for SQL language statements $filecontent =~ @@ -171,7 +171,7 @@ foreach (sort keys %entries) $synopsis =~ s/\\n/\\n"\n$prefix"/g; my @args = ("buf", $synopsis, map("_(\"$_\")", @{ $entries{$_}{params} })); - print CFILE "static void + print $cfile_handle "static void sql_help_$id(PQExpBuffer buf) { \tappendPQExpBuffer(" . join(",\n$prefix", @args) . "); @@ -180,14 +180,14 @@ sql_help_$id(PQExpBuffer buf) "; } -print CFILE " +print $cfile_handle " const struct _helpStruct QL_HELP[] = { "; foreach (sort keys %entries) { my $id = $_; $id =~ s/ /_/g; - print CFILE " { \"$_\", + print $cfile_handle " { \"$_\", N_(\"$entries{$_}{cmddesc}\"), sql_help_$id, $entries{$_}{nl_count} }, @@ -195,12 +195,12 @@ foreach (sort keys %entries) "; } -print CFILE " +print $cfile_handle " { NULL, NULL, NULL } /* End of list marker */ }; "; -print HFILE " +print $hfile_handle " #define QL_HELP_COUNT " . scalar(keys %entries) . " /* number of help items */ #define QL_MAX_CMD_LEN $maxlen /* largest strlen(cmd) */ @@ -209,6 +209,6 @@ print HFILE " #endif /* $define */ "; -close CFILE; -close HFILE; +close $cfile_handle; +close $hfile_handle; closedir DIR; diff --git a/src/interfaces/ecpg/preproc/check_rules.pl b/src/interfaces/ecpg/preproc/check_rules.pl index dce4bc6a02..e681943856 100644 --- a/src/interfaces/ecpg/preproc/check_rules.pl +++ b/src/interfaces/ecpg/preproc/check_rules.pl @@ -53,8 +53,8 @@ my $comment = 0; my $non_term_id = ''; my $cc = 0; -open GRAM, $parser or die $!; -while () +open my $parser_fh, '<', $parser or die $!; +while (<$parser_fh>) { if (/^%%/) { @@ -145,7 +145,7 @@ while () } } -close GRAM; +close $parser_fh; if ($verbose) { print "$cc rules loaded\n"; @@ -154,8 +154,8 @@ if ($verbose) my $ret = 0; $cc = 0; -open ECPG, $filename or die $!; -while () +open my $ecpg_fh, '<', $filename or die $!; +while (<$ecpg_fh>) { if (!/^ECPG:/) { @@ -170,7 +170,7 @@ while () $ret = 1; } } -close ECPG; +close $ecpg_fh; if ($verbose) { diff --git a/src/interfaces/libpq/test/regress.pl b/src/interfaces/libpq/test/regress.pl index 1dab12282b..c403130c6a 100644 --- a/src/interfaces/libpq/test/regress.pl +++ b/src/interfaces/libpq/test/regress.pl @@ -14,19 +14,19 @@ my $expected_out = "$srcdir/$subdir/expected.out"; my $regress_out = "regress.out"; # open input file first, so possible error isn't sent to redirected STDERR -open(REGRESS_IN, "<", $regress_in) +open(my $regress_in_fh, "<", $regress_in) or die "can't open $regress_in for reading: $!"; # save STDOUT/ERR and redirect both to regress.out -open(OLDOUT, ">&", \*STDOUT) or die "can't dup STDOUT: $!"; -open(OLDERR, ">&", \*STDERR) or die "can't dup STDERR: $!"; +open(my $oldout_fh, ">&", \*STDOUT) or die "can't dup STDOUT: $!"; +open(my $olderr_fh, ">&", \*STDERR) or die "can't dup STDERR: $!"; open(STDOUT, ">", $regress_out) or die "can't open $regress_out for writing: $!"; open(STDERR, ">&", \*STDOUT) or die "can't dup STDOUT: $!"; # read lines from regress.in and run uri-regress on them -while () +while (<$regress_in_fh>) { chomp; print "trying $_\n"; @@ -35,11 +35,11 @@ while () } # restore STDOUT/ERR so we can print the outcome to the user -open(STDERR, ">&", \*OLDERR) or die; # can't complain as STDERR is still duped -open(STDOUT, ">&", \*OLDOUT) or die "can't restore STDOUT: $!"; +open(STDERR, ">&", $olderr_fh) or die; # can't complain as STDERR is still duped +open(STDOUT, ">&", $oldout_fh) or die "can't restore STDOUT: $!"; # just in case -close REGRESS_IN; +close $regress_in_fh; my $diff_status = system( "diff -c \"$srcdir/$subdir/expected.out\" regress.out >regress.diff"); diff --git a/src/pl/plperl/plc_perlboot.pl b/src/pl/plperl/plc_perlboot.pl index bb2d009be0..292c9101c9 100644 --- a/src/pl/plperl/plc_perlboot.pl +++ b/src/pl/plperl/plc_perlboot.pl @@ -52,7 +52,7 @@ sub ::encode_array_constructor { - package PostgreSQL::InServer; + package PostgreSQL::InServer; ## no critic (RequireFilenameMatchesPackage); use strict; use warnings; @@ -86,11 +86,13 @@ sub ::encode_array_constructor sub mkfunc { + ## no critic (ProhibitNoStrict, ProhibitStringyEval); no strict; # default to no strict for the eval no warnings; # default to no warnings for the eval my $ret = eval(mkfuncsrc(@_)); $@ =~ s/\(eval \d+\) //g if $@; return $ret; + ## use critic } 1; diff --git a/src/pl/plperl/plc_trusted.pl b/src/pl/plperl/plc_trusted.pl index cd61882eb6..38255b4afc 100644 --- a/src/pl/plperl/plc_trusted.pl +++ b/src/pl/plperl/plc_trusted.pl @@ -1,6 +1,6 @@ # src/pl/plperl/plc_trusted.pl -package PostgreSQL::InServer::safe; +package PostgreSQL::InServer::safe; ## no critic (RequireFilenameMatchesPackage); # Load widely useful pragmas into plperl to make them available. # diff --git a/src/pl/plperl/text2macro.pl b/src/pl/plperl/text2macro.pl index c88e5ec4be..e681fca21a 100644 --- a/src/pl/plperl/text2macro.pl +++ b/src/pl/plperl/text2macro.pl @@ -49,7 +49,7 @@ for my $src_file (@ARGV) (my $macro = $src_file) =~ s/ .*? (\w+) (?:\.\w+) $/$1/x; - open my $src_fh, $src_file # not 3-arg form + open my $src_fh, '<', $src_file or die "Can't open $src_file: $!"; printf qq{#define %s%s \\\n}, @@ -80,19 +80,19 @@ sub selftest my $tmp = "text2macro_tmp"; my $string = q{a '' '\\'' "" "\\"" "\\\\" "\\\\n" b}; - open my $fh, ">$tmp.pl" or die; + open my $fh, '>', "$tmp.pl" or die; print $fh $string; close $fh; system("perl $0 --name=X $tmp.pl > $tmp.c") == 0 or die; - open $fh, ">>$tmp.c"; + open $fh, '>>', "$tmp.c"; print $fh "#include \n"; print $fh "int main() { puts(X); return 0; }\n"; close $fh; system("cat -n $tmp.c"); system("make $tmp") == 0 or die; - open $fh, "./$tmp |" or die; + open $fh, '<', "./$tmp |" or die; my $result = <$fh>; unlink <$tmp.*>; diff --git a/src/pl/plpgsql/src/generate-plerrcodes.pl b/src/pl/plpgsql/src/generate-plerrcodes.pl index 6a676c0953..eb135bc25e 100644 --- a/src/pl/plpgsql/src/generate-plerrcodes.pl +++ b/src/pl/plpgsql/src/generate-plerrcodes.pl @@ -10,7 +10,7 @@ print "/* autogenerated from src/backend/utils/errcodes.txt, do not edit */\n"; print "/* there is deliberately not an #ifndef PLERRCODES_H here */\n"; -open my $errcodes, $ARGV[0] or die; +open my $errcodes, '<', $ARGV[0] or die; while (<$errcodes>) { diff --git a/src/pl/plpython/generate-spiexceptions.pl b/src/pl/plpython/generate-spiexceptions.pl index ab0fa4aeaa..a9ee9601b3 100644 --- a/src/pl/plpython/generate-spiexceptions.pl +++ b/src/pl/plpython/generate-spiexceptions.pl @@ -10,7 +10,7 @@ print "/* autogenerated from src/backend/utils/errcodes.txt, do not edit */\n"; print "/* there is deliberately not an #ifndef SPIEXCEPTIONS_H here */\n"; -open my $errcodes, $ARGV[0] or die; +open my $errcodes, '<', $ARGV[0] or die; while (<$errcodes>) { diff --git a/src/pl/tcl/generate-pltclerrcodes.pl b/src/pl/tcl/generate-pltclerrcodes.pl index e20a0aff4a..b4e429a4fb 100644 --- a/src/pl/tcl/generate-pltclerrcodes.pl +++ b/src/pl/tcl/generate-pltclerrcodes.pl @@ -10,7 +10,7 @@ print "/* autogenerated from src/backend/utils/errcodes.txt, do not edit */\n"; print "/* there is deliberately not an #ifndef PLTCLERRCODES_H here */\n"; -open my $errcodes, $ARGV[0] or die; +open my $errcodes, '<', $ARGV[0] or die; while (<$errcodes>) { diff --git a/src/test/locale/sort-test.pl b/src/test/locale/sort-test.pl index cb7e4934e4..b8fc93aab1 100755 --- a/src/test/locale/sort-test.pl +++ b/src/test/locale/sort-test.pl @@ -3,9 +3,9 @@ use strict; use locale; -open(INFILE, "<$ARGV[0]"); -chop(my (@words) = ); -close(INFILE); +open(my $in_fh, '<', $ARGV[0]) || die; +chop(my (@words) = <$in_fh>); +close($in_fh); $" = "\n"; my (@result) = sort @words; diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm index 5ef007f7d4..1ad8f7fc1c 100644 --- a/src/test/perl/PostgresNode.pm +++ b/src/test/perl/PostgresNode.pm @@ -347,7 +347,7 @@ sub set_replication_conf $self->host eq $test_pghost or die "set_replication_conf only works with the default host"; - open my $hba, ">>$pgdata/pg_hba.conf"; + open my $hba, '>>', "$pgdata/pg_hba.conf"; print $hba "\n# Allow replication (set up by PostgresNode.pm)\n"; if ($TestLib::windows_os) { @@ -399,7 +399,7 @@ sub init @{ $params{extra} }); TestLib::system_or_bail($ENV{PG_REGRESS}, '--config-auth', $pgdata); - open my $conf, ">>$pgdata/postgresql.conf"; + open my $conf, '>>', "$pgdata/postgresql.conf"; print $conf "\n# Added by PostgresNode.pm\n"; print $conf "fsync = off\n"; print $conf "log_line_prefix = '%m [%p] %q%a '\n"; @@ -820,7 +820,7 @@ sub _update_pid # If we can open the PID file, read its first line and that's the PID we # want. If the file cannot be opened, presumably the server is not # running; don't be noisy in that case. - if (open my $pidfile, $self->data_dir . "/postmaster.pid") + if (open my $pidfile, '<', $self->data_dir . "/postmaster.pid") { chomp($self->{_pid} = <$pidfile>); print "# Postmaster PID for node \"$name\" is $self->{_pid}\n"; @@ -1357,7 +1357,7 @@ sub lsn chomp($result); if ($result eq '') { - return undef; + return; } else { diff --git a/src/test/perl/TestLib.pm b/src/test/perl/TestLib.pm index d22957ceb0..ae8d1782da 100644 --- a/src/test/perl/TestLib.pm +++ b/src/test/perl/TestLib.pm @@ -84,14 +84,14 @@ INIT $test_logfile = basename($0); $test_logfile =~ s/\.[^.]+$//; $test_logfile = "$log_path/regress_log_$test_logfile"; - open TESTLOG, '>', $test_logfile + open my $testlog, '>', $test_logfile or die "could not open STDOUT to logfile \"$test_logfile\": $!"; # Hijack STDOUT and STDERR to the log file - open(ORIG_STDOUT, ">&STDOUT"); - open(ORIG_STDERR, ">&STDERR"); - open(STDOUT, ">&TESTLOG"); - open(STDERR, ">&TESTLOG"); + open(my $orig_stdout, '>&', \*STDOUT); + open(my $orig_stderr, '>&', \*STDERR); + open(STDOUT, '>&', $testlog); + open(STDERR, '>&', $testlog); # The test output (ok ...) needs to be printed to the original STDOUT so # that the 'prove' program can parse it, and display it to the user in @@ -99,16 +99,16 @@ INIT # in the log. my $builder = Test::More->builder; my $fh = $builder->output; - tie *$fh, "SimpleTee", *ORIG_STDOUT, *TESTLOG; + tie *$fh, "SimpleTee", $orig_stdout, $testlog; $fh = $builder->failure_output; - tie *$fh, "SimpleTee", *ORIG_STDERR, *TESTLOG; + tie *$fh, "SimpleTee", $orig_stderr, $testlog; # Enable auto-flushing for all the file handles. Stderr and stdout are # redirected to the same file, and buffering causes the lines to appear # in the log in confusing order. autoflush STDOUT 1; autoflush STDERR 1; - autoflush TESTLOG 1; + autoflush $testlog 1; } END diff --git a/src/test/ssl/ServerSetup.pm b/src/test/ssl/ServerSetup.pm index 9441249b3a..6d17d6d61a 100644 --- a/src/test/ssl/ServerSetup.pm +++ b/src/test/ssl/ServerSetup.pm @@ -58,21 +58,21 @@ sub configure_test_server_for_ssl $node->psql('postgres', "CREATE DATABASE certdb"); # enable logging etc. - open CONF, ">>$pgdata/postgresql.conf"; - print CONF "fsync=off\n"; - print CONF "log_connections=on\n"; - print CONF "log_hostname=on\n"; - print CONF "listen_addresses='$serverhost'\n"; - print CONF "log_statement=all\n"; + open my $conf, '>>', "$pgdata/postgresql.conf"; + print $conf "fsync=off\n"; + print $conf "log_connections=on\n"; + print $conf "log_hostname=on\n"; + print $conf "listen_addresses='$serverhost'\n"; + print $conf "log_statement=all\n"; # enable SSL and set up server key - print CONF "include 'sslconfig.conf'"; + print $conf "include 'sslconfig.conf'"; - close CONF; + close $conf; # ssl configuration will be placed here - open SSLCONF, ">$pgdata/sslconfig.conf"; - close SSLCONF; + open my $sslconf, '>', "$pgdata/sslconfig.conf"; + close $sslconf; # Copy all server certificates and keys, and client root cert, to the data dir copy_files("ssl/server-*.crt", $pgdata); @@ -100,13 +100,13 @@ sub switch_server_cert diag "Reloading server with certfile \"$certfile\" and cafile \"$cafile\"..."; - open SSLCONF, ">$pgdata/sslconfig.conf"; - print SSLCONF "ssl=on\n"; - print SSLCONF "ssl_ca_file='$cafile.crt'\n"; - print SSLCONF "ssl_cert_file='$certfile.crt'\n"; - print SSLCONF "ssl_key_file='$certfile.key'\n"; - print SSLCONF "ssl_crl_file='root+client.crl'\n"; - close SSLCONF; + open my $sslconf, '>', "$pgdata/sslconfig.conf"; + print $sslconf "ssl=on\n"; + print $sslconf "ssl_ca_file='root+client_ca.crt'\n"; + print $sslconf "ssl_cert_file='$certfile.crt'\n"; + print $sslconf "ssl_key_file='$certfile.key'\n"; + print $sslconf "ssl_crl_file='root+client.crl'\n"; + close $sslconf; $node->reload; } @@ -121,16 +121,16 @@ sub configure_hba_for_ssl # but seems best to keep it as narrow as possible for security reasons. # # When connecting to certdb, also check the client certificate. - open HBA, ">$pgdata/pg_hba.conf"; - print HBA + open my $hba, '>', "$pgdata/pg_hba.conf"; + print $hba "# TYPE DATABASE USER ADDRESS METHOD\n"; - print HBA + print $hba "hostssl trustdb ssltestuser $serverhost/32 trust\n"; - print HBA + print $hba "hostssl trustdb ssltestuser ::1/128 trust\n"; - print HBA + print $hba "hostssl certdb ssltestuser $serverhost/32 cert\n"; - print HBA + print $hba "hostssl certdb ssltestuser ::1/128 cert\n"; - close HBA; + close $hba; } diff --git a/src/tools/fix-old-flex-code.pl b/src/tools/fix-old-flex-code.pl index 8dafcae15e..bc868dfd7f 100644 --- a/src/tools/fix-old-flex-code.pl +++ b/src/tools/fix-old-flex-code.pl @@ -25,7 +25,7 @@ my $filename = shift; # Suck in the whole file. local $/ = undef; my $cfile; -open($cfile, $filename) || die "opening $filename for reading: $!"; +open($cfile, '<', $filename) || die "opening $filename for reading: $!"; my $ccode = <$cfile>; close($cfile); @@ -45,7 +45,7 @@ $ccode =~ s|(struct yyguts_t \* yyg = \(struct yyguts_t\*\)yyscanner; /\* This v |s; # Write the modified file back out. -open($cfile, ">$filename") || die "opening $filename for writing: $!"; +open($cfile, '>', $filename) || die "opening $filename for writing: $!"; print $cfile $ccode; close($cfile); diff --git a/src/tools/msvc/Install.pm b/src/tools/msvc/Install.pm index b81f4dd809..35ad5b8a44 100644 --- a/src/tools/msvc/Install.pm +++ b/src/tools/msvc/Install.pm @@ -58,8 +58,8 @@ sub Install # suppress warning about harmless redeclaration of $config no warnings 'misc'; - require "config_default.pl"; - require "config.pl" if (-f "config.pl"); + do "config_default.pl"; + do "config.pl" if (-f "config.pl"); } chdir("../../..") if (-f "../../../configure"); @@ -367,7 +367,7 @@ sub GenerateConversionScript $sql .= "COMMENT ON CONVERSION pg_catalog.$name IS 'conversion for $se to $de';\n\n"; } - open($F, ">$target/share/conversion_create.sql") + open($F, '>', "$target/share/conversion_create.sql") || die "Could not write to conversion_create.sql\n"; print $F $sql; close($F); @@ -409,7 +409,7 @@ sub GenerateTsearchFiles $mf =~ /^LANGUAGES\s*=\s*(.*)$/m || die "Could not find LANGUAGES line in snowball Makefile\n"; my @pieces = split /\s+/, $1; - open($F, ">$target/share/snowball_create.sql") + open($F, '>', "$target/share/snowball_create.sql") || die "Could not write snowball_create.sql"; print $F read_file('src/backend/snowball/snowball_func.sql.in'); @@ -735,7 +735,7 @@ sub read_file my $t = $/; undef $/; - open($F, $filename) || die "Could not open file $filename\n"; + open($F, '<', $filename) || die "Could not open file $filename\n"; my $txt = <$F>; close($F); $/ = $t; diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index 12f73f344c..ba1bf6d97a 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -825,7 +825,7 @@ sub GenerateContribSqlFiles $dn =~ s/\.sql$//; $cont =~ s/MODULE_PATHNAME/\$libdir\/$dn/g; my $o; - open($o, ">contrib/$n/$out") + open($o, '>', "contrib/$n/$out") || croak "Could not write to contrib/$n/$d"; print $o $cont; close($o); diff --git a/src/tools/msvc/Project.pm b/src/tools/msvc/Project.pm index faf1a683f6..9817b9439a 100644 --- a/src/tools/msvc/Project.pm +++ b/src/tools/msvc/Project.pm @@ -310,12 +310,12 @@ sub AddResourceFile if (Solution::IsNewer("$dir/win32ver.rc", 'src/port/win32ver.rc')) { print "Generating win32ver.rc for $dir\n"; - open(I, 'src/port/win32ver.rc') + open(my $i, '<', 'src/port/win32ver.rc') || confess "Could not open win32ver.rc"; - open(O, ">$dir/win32ver.rc") + open(my $o, '>', "$dir/win32ver.rc") || confess "Could not write win32ver.rc"; my $icostr = $ico ? "IDI_ICON ICON \"src/port/$ico.ico\"" : ""; - while () + while (<$i>) { s/FILEDESC/"$desc"/gm; s/_ICO_/$icostr/gm; @@ -324,11 +324,11 @@ sub AddResourceFile { s/VFT_APP/VFT_DLL/gm; } - print O; + print $o $_; } + close($o); + close($i); } - close(O); - close(I); $self->AddFile("$dir/win32ver.rc"); } @@ -357,13 +357,13 @@ sub Save $self->DisableLinkerWarnings('4197') if ($self->{platform} eq 'x64'); # Dump the project - open(F, ">$self->{name}$self->{filenameExtension}") + open(my $f, '>', "$self->{name}$self->{filenameExtension}") || croak( "Could not write to $self->{name}$self->{filenameExtension}\n"); - $self->WriteHeader(*F); - $self->WriteFiles(*F); - $self->Footer(*F); - close(F); + $self->WriteHeader($f); + $self->WriteFiles($f); + $self->Footer($f); + close($f); } sub GetAdditionalLinkerDependencies @@ -397,7 +397,7 @@ sub read_file my $t = $/; undef $/; - open($F, $filename) || croak "Could not open file $filename\n"; + open($F, '<', $filename) || croak "Could not open file $filename\n"; my $txt = <$F>; close($F); $/ = $t; @@ -412,8 +412,8 @@ sub read_makefile my $t = $/; undef $/; - open($F, "$reldir/GNUmakefile") - || open($F, "$reldir/Makefile") + open($F, '<', "$reldir/GNUmakefile") + || open($F, '<', "$reldir/Makefile") || confess "Could not open $reldir/Makefile\n"; my $txt = <$F>; close($F); diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index ff9064f923..abac2c7402 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -102,14 +102,14 @@ sub IsNewer sub copyFile { my ($src, $dest) = @_; - open(I, $src) || croak "Could not open $src"; - open(O, ">$dest") || croak "Could not open $dest"; - while () + open(my $i, '<', $src) || croak "Could not open $src"; + open(my $o, '>', $dest) || croak "Could not open $dest"; + while (<$i>) { - print O; + print $o $_; } - close(I); - close(O); + close($i); + close($o); } sub GenerateFiles @@ -118,9 +118,9 @@ sub GenerateFiles my $bits = $self->{platform} eq 'Win32' ? 32 : 64; # Parse configure.in to get version numbers - open(C, "configure.in") + open(my $c, '<', "configure.in") || confess("Could not open configure.in for reading\n"); - while () + while (<$c>) { if (/^AC_INIT\(\[PostgreSQL\], \[([^\]]+)\]/) { @@ -133,7 +133,7 @@ sub GenerateFiles $self->{majorver} = sprintf("%d", $1); } } - close(C); + close($c); confess "Unable to parse configure.in for all variables!" if ($self->{strver} eq '' || $self->{numver} eq ''); @@ -146,91 +146,91 @@ sub GenerateFiles if (IsNewer("src/include/pg_config.h", "src/include/pg_config.h.win32")) { print "Generating pg_config.h...\n"; - open(I, "src/include/pg_config.h.win32") + open(my $i, '<', "src/include/pg_config.h.win32") || confess "Could not open pg_config.h.win32\n"; - open(O, ">src/include/pg_config.h") + open(my $o, '>', "src/include/pg_config.h") || confess "Could not write to pg_config.h\n"; my $extraver = $self->{options}->{extraver}; $extraver = '' unless defined $extraver; - while () + while (<$i>) { s{PG_VERSION "[^"]+"}{PG_VERSION "$self->{strver}$extraver"}; s{PG_VERSION_NUM \d+}{PG_VERSION_NUM $self->{numver}}; s{PG_VERSION_STR "[^"]+"}{__STRINGIFY(x) #x\n#define __STRINGIFY2(z) __STRINGIFY(z)\n#define PG_VERSION_STR "PostgreSQL $self->{strver}$extraver, compiled by Visual C++ build " __STRINGIFY2(_MSC_VER) ", $bits-bit"}; - print O; + print $o $_; } - print O "#define PG_MAJORVERSION \"$self->{majorver}\"\n"; - print O "#define LOCALEDIR \"/share/locale\"\n" + print $o "#define PG_MAJORVERSION \"$self->{majorver}\"\n"; + print $o "#define LOCALEDIR \"/share/locale\"\n" if ($self->{options}->{nls}); - print O "/* defines added by config steps */\n"; - print O "#ifndef IGNORE_CONFIGURED_SETTINGS\n"; - print O "#define USE_ASSERT_CHECKING 1\n" + print $o "/* defines added by config steps */\n"; + print $o "#ifndef IGNORE_CONFIGURED_SETTINGS\n"; + print $o "#define USE_ASSERT_CHECKING 1\n" if ($self->{options}->{asserts}); - print O "#define USE_LDAP 1\n" if ($self->{options}->{ldap}); - print O "#define HAVE_LIBZ 1\n" if ($self->{options}->{zlib}); - print O "#define USE_OPENSSL 1\n" if ($self->{options}->{openssl}); - print O "#define ENABLE_NLS 1\n" if ($self->{options}->{nls}); + print $o "#define USE_LDAP 1\n" if ($self->{options}->{ldap}); + print $o "#define HAVE_LIBZ 1\n" if ($self->{options}->{zlib}); + print $o "#define USE_OPENSSL 1\n" if ($self->{options}->{openssl}); + print $o "#define ENABLE_NLS 1\n" if ($self->{options}->{nls}); - print O "#define BLCKSZ ", 1024 * $self->{options}->{blocksize}, "\n"; - print O "#define RELSEG_SIZE ", + print $o "#define BLCKSZ ", 1024 * $self->{options}->{blocksize}, "\n"; + print $o "#define RELSEG_SIZE ", (1024 / $self->{options}->{blocksize}) * $self->{options}->{segsize} * 1024, "\n"; - print O "#define XLOG_BLCKSZ ", + print $o "#define XLOG_BLCKSZ ", 1024 * $self->{options}->{wal_blocksize}, "\n"; - print O "#define XLOG_SEG_SIZE (", $self->{options}->{wal_segsize}, + print $o "#define XLOG_SEG_SIZE (", $self->{options}->{wal_segsize}, " * 1024 * 1024)\n"; if ($self->{options}->{float4byval}) { - print O "#define USE_FLOAT4_BYVAL 1\n"; - print O "#define FLOAT4PASSBYVAL true\n"; + print $o "#define USE_FLOAT4_BYVAL 1\n"; + print $o "#define FLOAT4PASSBYVAL true\n"; } else { - print O "#define FLOAT4PASSBYVAL false\n"; + print $o "#define FLOAT4PASSBYVAL false\n"; } if ($self->{options}->{float8byval}) { - print O "#define USE_FLOAT8_BYVAL 1\n"; - print O "#define FLOAT8PASSBYVAL true\n"; + print $o "#define USE_FLOAT8_BYVAL 1\n"; + print $o "#define FLOAT8PASSBYVAL true\n"; } else { - print O "#define FLOAT8PASSBYVAL false\n"; + print $o "#define FLOAT8PASSBYVAL false\n"; } if ($self->{options}->{uuid}) { - print O "#define HAVE_UUID_OSSP\n"; - print O "#define HAVE_UUID_H\n"; + print $o "#define HAVE_UUID_OSSP\n"; + print $o "#define HAVE_UUID_H\n"; } if ($self->{options}->{xml}) { - print O "#define HAVE_LIBXML2\n"; - print O "#define USE_LIBXML\n"; + print $o "#define HAVE_LIBXML2\n"; + print $o "#define USE_LIBXML\n"; } if ($self->{options}->{xslt}) { - print O "#define HAVE_LIBXSLT\n"; - print O "#define USE_LIBXSLT\n"; + print $o "#define HAVE_LIBXSLT\n"; + print $o "#define USE_LIBXSLT\n"; } if ($self->{options}->{gss}) { - print O "#define ENABLE_GSS 1\n"; + print $o "#define ENABLE_GSS 1\n"; } if (my $port = $self->{options}->{"--with-pgport"}) { - print O "#undef DEF_PGPORT\n"; - print O "#undef DEF_PGPORT_STR\n"; - print O "#define DEF_PGPORT $port\n"; - print O "#define DEF_PGPORT_STR \"$port\"\n"; + print $o "#undef DEF_PGPORT\n"; + print $o "#undef DEF_PGPORT_STR\n"; + print $o "#define DEF_PGPORT $port\n"; + print $o "#define DEF_PGPORT_STR \"$port\"\n"; } - print O "#define VAL_CONFIGURE \"" + print $o "#define VAL_CONFIGURE \"" . $self->GetFakeConfigure() . "\"\n"; - print O "#endif /* IGNORE_CONFIGURED_SETTINGS */\n"; - close(O); - close(I); + print $o "#endif /* IGNORE_CONFIGURED_SETTINGS */\n"; + close($o); + close($i); } if (IsNewer( @@ -379,17 +379,17 @@ s{PG_VERSION_STR "[^"]+"}{__STRINGIFY(x) #x\n#define __STRINGIFY2(z) __STRINGIFY my ($sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst) = localtime(time); my $d = ($year - 100) . "$yday"; - open(I, '<', 'src/interfaces/libpq/libpq.rc.in') + open(my $i, '<', 'src/interfaces/libpq/libpq.rc.in') || confess "Could not open libpq.rc.in"; - open(O, '>', 'src/interfaces/libpq/libpq.rc') + open(my $o, '>', 'src/interfaces/libpq/libpq.rc') || confess "Could not open libpq.rc"; - while () + while (<$i>) { s/(VERSION.*),0/$1,$d/; - print O; + print $o; } - close(I); - close(O); + close($i); + close($o); } if (IsNewer('src/bin/psql/sql_help.h', 'src/bin/psql/create_help.pl')) @@ -415,23 +415,23 @@ s{PG_VERSION_STR "[^"]+"}{__STRINGIFY(x) #x\n#define __STRINGIFY2(z) __STRINGIFY 'src/interfaces/ecpg/include/ecpg_config.h.in')) { print "Generating ecpg_config.h...\n"; - open(O, '>', 'src/interfaces/ecpg/include/ecpg_config.h') + open(my $o, '>', 'src/interfaces/ecpg/include/ecpg_config.h') || confess "Could not open ecpg_config.h"; - print O < 1200) #define HAVE_LONG_LONG_INT_64 #define ENABLE_THREAD_SAFETY 1 EOF - print O "#endif\n"; - close(O); + print $o "#endif\n"; + close($o); } unless (-f "src/port/pg_config_paths.h") { print "Generating pg_config_paths.h...\n"; - open(O, '>', 'src/port/pg_config_paths.h') + open(my $o, '>', 'src/port/pg_config_paths.h') || confess "Could not open pg_config_paths.h"; - print O <doc/src/sgml/version.sgml") + open(my $o, '>', "doc/src/sgml/version.sgml") || croak "Could not write to version.sgml\n"; - print O <{strver}"> {majorver}"> EOF - close(O); + close($o); } sub GenerateDefFile @@ -490,18 +490,18 @@ sub GenerateDefFile if (IsNewer($deffile, $txtfile)) { print "Generating $deffile...\n"; - open(I, $txtfile) || confess("Could not open $txtfile\n"); - open(O, ">$deffile") || confess("Could not open $deffile\n"); - print O "LIBRARY $libname\nEXPORTS\n"; - while () + open(my $if, '<', $txtfile) || confess("Could not open $txtfile\n"); + open(my $of, '>', $deffile) || confess("Could not open $deffile\n"); + print $of "LIBRARY $libname\nEXPORTS\n"; + while (<$if>) { next if (/^#/); next if (/^\s*$/); my ($f, $o) = split; - print O " $f @ $o\n"; + print $of " $f @ $o\n"; } - close(O); - close(I); + close($of); + close($if); } } @@ -575,19 +575,19 @@ sub Save } } - open(SLN, ">pgsql.sln") || croak "Could not write to pgsql.sln\n"; - print SLN <', "pgsql.sln") || croak "Could not write to pgsql.sln\n"; + print $sln <{solutionFileVersion} # $self->{visualStudioName} EOF - print SLN $self->GetAdditionalHeaders(); + print $sln $self->GetAdditionalHeaders(); foreach my $fld (keys %{ $self->{projects} }) { foreach my $proj (@{ $self->{projects}->{$fld} }) { - print SLN <{name}$proj->{filenameExtension}", "$proj->{guid}" EndProject EOF @@ -595,14 +595,14 @@ EOF if ($fld ne "") { $flduid{$fld} = Win32::GuidGen(); - print SLN <{platform}= Debug|$self->{platform} @@ -615,7 +615,7 @@ EOF { foreach my $proj (@{ $self->{projects}->{$fld} }) { - print SLN <{guid}.Debug|$self->{platform}.ActiveCfg = Debug|$self->{platform} $proj->{guid}.Debug|$self->{platform}.Build.0 = Debug|$self->{platform} $proj->{guid}.Release|$self->{platform}.ActiveCfg = Release|$self->{platform} @@ -624,7 +624,7 @@ EOF } } - print SLN <{projects}->{$fld} }) { - print SLN "\t\t$proj->{guid} = $flduid{$fld}\n"; + print $sln "\t\t$proj->{guid} = $flduid{$fld}\n"; } } - print SLN <) + open(my $f, '<', $symfile) || die "Could not open $symfile for $_\n"; + while (<$f>) { # Expected symbol lines look like: @@ -115,14 +115,14 @@ sub extract_syms # whatever came last. $def->{ $pieces[6] } = $pieces[3]; } - close(F); + close($f); } sub writedef { my ($deffile, $platform, $def) = @_; - open(DEF, ">$deffile") || die "Could not write to $deffile\n"; - print DEF "EXPORTS\n"; + open(my $fh, '>', $deffile) || die "Could not write to $deffile\n"; + print $fh "EXPORTS\n"; foreach my $f (sort keys %{$def}) { my $isdata = $def->{$f} eq 'data'; @@ -135,14 +135,14 @@ sub writedef # decorated with the DATA option for variables. if ($isdata) { - print DEF " $f DATA\n"; + print $fh " $f DATA\n"; } else { - print DEF " $f\n"; + print $fh " $f\n"; } } - close(DEF); + close($fh); } @@ -174,7 +174,7 @@ print "Generating $defname.DEF from directory $ARGV[0], platform $platform\n"; my %def = (); -while (<$ARGV[0]/*.obj>) +while (<$ARGV[0]/*.obj>) ## no critic (RequireGlobFunction); { my $objfile = $_; my $symfile = $objfile; diff --git a/src/tools/msvc/install.pl b/src/tools/msvc/install.pl index bde5b7c793..b2d7f9e040 100755 --- a/src/tools/msvc/install.pl +++ b/src/tools/msvc/install.pl @@ -14,11 +14,11 @@ use Install qw(Install); if (-e "src/tools/msvc/buildenv.pl") { - require "src/tools/msvc/buildenv.pl"; + do "src/tools/msvc/buildenv.pl"; } elsif (-e "./buildenv.pl") { - require "./buildenv.pl"; + do "./buildenv.pl"; } my $target = shift || Usage(); diff --git a/src/tools/msvc/mkvcbuild.pl b/src/tools/msvc/mkvcbuild.pl index 6f1c42e504..9255dff022 100644 --- a/src/tools/msvc/mkvcbuild.pl +++ b/src/tools/msvc/mkvcbuild.pl @@ -19,7 +19,7 @@ print "Warning: no config.pl found, using default.\n" unless (-f 'src/tools/msvc/config.pl'); our $config; -require 'src/tools/msvc/config_default.pl'; -require 'src/tools/msvc/config.pl' if (-f 'src/tools/msvc/config.pl'); +do 'src/tools/msvc/config_default.pl'; +do 'src/tools/msvc/config.pl' if (-f 'src/tools/msvc/config.pl'); Mkvcbuild::mkvcbuild($config); diff --git a/src/tools/msvc/pgbison.pl b/src/tools/msvc/pgbison.pl index 31e75403f5..e799d900fe 100644 --- a/src/tools/msvc/pgbison.pl +++ b/src/tools/msvc/pgbison.pl @@ -7,7 +7,7 @@ use File::Basename; # assume we are in the postgres source root -require 'src/tools/msvc/buildenv.pl' if -e 'src/tools/msvc/buildenv.pl'; +do 'src/tools/msvc/buildenv.pl' if -e 'src/tools/msvc/buildenv.pl'; my ($bisonver) = `bison -V`; # grab first line $bisonver = (split(/\s+/, $bisonver))[3]; # grab version number @@ -38,7 +38,7 @@ $output =~ s/gram\.c$/pl_gram.c/ if $input =~ /src.pl.plpgsql.src.gram\.y$/; my $makefile = dirname($input) . "/Makefile"; my ($mf, $make); -open($mf, $makefile); +open($mf, '<', $makefile); local $/ = undef; $make = <$mf>; close($mf); diff --git a/src/tools/msvc/pgflex.pl b/src/tools/msvc/pgflex.pl index fab0efa79f..67397ba644 100644 --- a/src/tools/msvc/pgflex.pl +++ b/src/tools/msvc/pgflex.pl @@ -10,7 +10,7 @@ $ENV{CYGWIN} = 'nodosfilewarning'; # assume we are in the postgres source root -require 'src/tools/msvc/buildenv.pl' if -e 'src/tools/msvc/buildenv.pl'; +do 'src/tools/msvc/buildenv.pl' if -e 'src/tools/msvc/buildenv.pl'; my ($flexver) = `flex -V`; # grab first line $flexver = (split(/\s+/, $flexver))[1]; @@ -41,7 +41,7 @@ elsif (!-e $input) # get flex flags from make file my $makefile = dirname($input) . "/Makefile"; my ($mf, $make); -open($mf, $makefile); +open($mf, '<', $makefile); local $/ = undef; $make = <$mf>; close($mf); @@ -53,7 +53,7 @@ if ($? == 0) { # Check for "%option reentrant" in .l file. my $lfile; - open($lfile, $input) || die "opening $input for reading: $!"; + open($lfile, '<', $input) || die "opening $input for reading: $!"; my $lcode = <$lfile>; close($lfile); if ($lcode =~ /\%option\sreentrant/) @@ -69,18 +69,18 @@ if ($? == 0) # For reentrant scanners (like the core scanner) we do not # need to (and must not) change the yywrap definition. my $cfile; - open($cfile, $output) || die "opening $output for reading: $!"; + open($cfile, '<', $output) || die "opening $output for reading: $!"; my $ccode = <$cfile>; close($cfile); $ccode =~ s/yywrap\(n\)/yywrap()/; - open($cfile, ">$output") || die "opening $output for writing: $!"; + open($cfile, '>', $output) || die "opening $output for writing: $!"; print $cfile $ccode; close($cfile); } if ($flexflags =~ /\s-b\s/) { my $lexback = "lex.backup"; - open($lfile, $lexback) || die "opening $lexback for reading: $!"; + open($lfile, '<', $lexback) || die "opening $lexback for reading: $!"; my $lexbacklines = <$lfile>; close($lfile); my $linecount = $lexbacklines =~ tr /\n/\n/; diff --git a/src/tools/msvc/vcregress.pl b/src/tools/msvc/vcregress.pl index f1b9819cd2..d9367f8fd5 100644 --- a/src/tools/msvc/vcregress.pl +++ b/src/tools/msvc/vcregress.pl @@ -20,8 +20,8 @@ chdir "../../.." if (-d "../../../src/tools/msvc"); my $topdir = getcwd(); my $tmp_installdir = "$topdir/tmp_install"; -require 'src/tools/msvc/config_default.pl'; -require 'src/tools/msvc/config.pl' if (-f 'src/tools/msvc/config.pl'); +do 'src/tools/msvc/config_default.pl'; +do 'src/tools/msvc/config.pl' if (-f 'src/tools/msvc/config.pl'); # buildenv.pl is for specifying the build environment settings # it should contain lines like: @@ -29,7 +29,7 @@ require 'src/tools/msvc/config.pl' if (-f 'src/tools/msvc/config.pl'); if (-e "src/tools/msvc/buildenv.pl") { - require "src/tools/msvc/buildenv.pl"; + do "src/tools/msvc/buildenv.pl"; } my $what = shift || ""; @@ -505,8 +505,8 @@ sub upgradecheck sub fetchRegressOpts { my $handle; - open($handle, "; @@ -521,8 +521,9 @@ sub fetchRegressOpts # an unhandled variable reference. Ignore anything that isn't an # option starting with "--". @opts = grep { - s/\Q$(top_builddir)\E/\"$topdir\"/; - $_ !~ /\$\(/ && $_ =~ /^--/ + my $x = $_; + $x =~ s/\Q$(top_builddir)\E/\"$topdir\"/; + $x !~ /\$\(/ && $x =~ /^--/ } split(/\s+/, $1); } if ($m =~ /^\s*ENCODING\s*=\s*(\S+)/m) @@ -540,8 +541,8 @@ sub fetchTests { my $handle; - open($handle, "; diff --git a/src/tools/pginclude/pgcheckdefines b/src/tools/pginclude/pgcheckdefines index e166efa08d..aa7c9c2fc1 100755 --- a/src/tools/pginclude/pgcheckdefines +++ b/src/tools/pginclude/pgcheckdefines @@ -42,25 +42,25 @@ my $MAKE = "make"; # my (@cfiles, @hfiles); -open PIPE, "$FIND * -type f -name '*.c' |" +open my $pipe, '-|', "$FIND * -type f -name '*.c'" or die "can't fork: $!"; -while () +while (<$pipe>) { chomp; push @cfiles, $_; } -close PIPE or die "$FIND failed: $!"; +close $pipe or die "$FIND failed: $!"; -open PIPE, "$FIND * -type f -name '*.h' |" +open $pipe, '-|', "$FIND * -type f -name '*.h'" or die "can't fork: $!"; -while () +while (<$pipe>) { chomp; push @hfiles, $_ unless m|^src/include/port/| || m|^src/backend/port/\w+/|; } -close PIPE or die "$FIND failed: $!"; +close $pipe or die "$FIND failed: $!"; # # For each .h file, extract all the symbols it #define's, and add them to @@ -71,16 +71,16 @@ my %defines; foreach my $hfile (@hfiles) { - open HFILE, $hfile + open my $fh, '<', $hfile or die "can't open $hfile: $!"; - while () + while (<$fh>) { if (m/^\s*#\s*define\s+(\w+)/) { $defines{$1}{$hfile} = 1; } } - close HFILE; + close $fh; } # @@ -124,9 +124,9 @@ foreach my $file (@hfiles, @cfiles) my ($CPPFLAGS, $CFLAGS, $CFLAGS_SL, $PTHREAD_CFLAGS, $CC); - open PIPE, "$MAKECMD |" + open $pipe, '-|', "$MAKECMD" or die "can't fork: $!"; - while () + while (<$pipe>) { if (m/^CPPFLAGS :?= (.*)/) { @@ -166,9 +166,9 @@ foreach my $file (@hfiles, @cfiles) # my @includes = (); my $COMPILE = "$CC $CPPFLAGS $CFLAGS -H -E $fname"; - open PIPE, "$COMPILE 2>&1 >/dev/null |" + open $pipe, '-|', "$COMPILE 2>&1 >/dev/null" or die "can't fork: $!"; - while () + while (<$pipe>) { if (m/^\.+ (.*)/) { @@ -211,10 +211,10 @@ foreach my $file (@hfiles, @cfiles) # We assume #ifdef isn't continued across lines, and that defined(foo) # isn't split across lines either # - open FILE, $fname + open my $fh, '<', $fname or die "can't open $file: $!"; my $inif = 0; - while () + while (<$fh>) { my $line = $_; if ($line =~ m/^\s*#\s*ifdef\s+(\w+)/) @@ -241,7 +241,7 @@ foreach my $file (@hfiles, @cfiles) } } } - close FILE; + close $fh; chdir $topdir or die "can't chdir to $topdir: $!"; } diff --git a/src/tools/pgindent/pgindent b/src/tools/pgindent/pgindent index 0d3859d029..0f3a1ba69a 100755 --- a/src/tools/pgindent/pgindent +++ b/src/tools/pgindent/pgindent @@ -159,8 +159,7 @@ sub process_exclude while (my $line = <$eh>) { chomp $line; - my $rgx; - eval " \$rgx = qr!$line!;"; + my $rgx = qr!$line!; @files = grep { $_ !~ /$rgx/ } @files if $rgx; } close($eh); @@ -435,7 +434,7 @@ sub diff sub run_build { - eval "use LWP::Simple;"; + eval "use LWP::Simple;"; ## no critic (ProhibitStringyEval); my $code_base = shift || '.'; my $save_dir = getcwd(); diff --git a/src/tools/version_stamp.pl b/src/tools/version_stamp.pl index dc9173f234..f973dd950c 100755 --- a/src/tools/version_stamp.pl +++ b/src/tools/version_stamp.pl @@ -80,8 +80,8 @@ my $padnumericversion = sprintf("%d%04d", $majorversion, $numericminor); # (this also ensures we're in the right directory) my $aconfver = ""; -open(FILE, "configure.in") || die "could not read configure.in: $!\n"; -while () +open(my $fh, '<', "configure.in") || die "could not read configure.in: $!\n"; +while (<$fh>) { if ( m/^m4_if\(m4_defn\(\[m4_PACKAGE_VERSION\]\), \[(.*)\], \[\], \[m4_fatal/) @@ -90,7 +90,7 @@ m/^m4_if\(m4_defn\(\[m4_PACKAGE_VERSION\]\), \[(.*)\], \[\], \[m4_fatal/) last; } } -close(FILE); +close($fh); $aconfver ne "" || die "could not find autoconf version number in configure.in\n"; diff --git a/src/tools/win32tzlist.pl b/src/tools/win32tzlist.pl index 6345465b19..0bdcc3610f 100755 --- a/src/tools/win32tzlist.pl +++ b/src/tools/win32tzlist.pl @@ -58,11 +58,11 @@ $basekey->Close(); # Fetch all timezones currently in the file # my @file_zones; -open(TZFILE, "<$tzfile") or die "Could not open $tzfile!\n"; +open(my $tzfh, '<', $tzfile) or die "Could not open $tzfile!\n"; my $t = $/; undef $/; -my $pgtz = ; -close(TZFILE); +my $pgtz = <$tzfh>; +close($tzfh); $/ = $t; # Attempt to locate and extract the complete win32_tzmap struct -- cgit v1.2.3 From 3371e4d9b12455fe1f8d1516d0bd915aab86be17 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Mon, 27 Mar 2017 10:34:33 -0400 Subject: Change default of log_directory to 'log' The previous default 'pg_log' might have indicated by its "pg_" prefix that it is an internal system directory. The new default is more in line with the typical naming of directories with user-facing log files. Together with the renaming of pg_clog and pg_xlog, this should clear up that difference. Author: Andreas Karlsson --- doc/src/sgml/config.sgml | 6 +++--- doc/src/sgml/file-fdw.sgml | 2 +- src/backend/utils/misc/guc.c | 2 +- src/backend/utils/misc/postgresql.conf.sample | 2 +- src/test/perl/PostgresNode.pm | 2 +- src/test/perl/RecursiveCopy.pm | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 2de3540def..ac339fb566 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -4363,8 +4363,8 @@ SELECT * FROM parent WHERE key = 2400; find the logs currently in use by the instance. Here is an example of this file's content: -stderr pg_log/postgresql.log -csvlog pg_log/postgresql.csv +stderr log/postgresql.log +csvlog log/postgresql.csv current_logfiles is recreated when a new log file @@ -4466,7 +4466,7 @@ local0.* /var/log/postgresql cluster data directory. This parameter can only be set in the postgresql.conf file or on the server command line. - The default is pg_log. + The default is log. diff --git a/doc/src/sgml/file-fdw.sgml b/doc/src/sgml/file-fdw.sgml index 309a303e03..74941a6f1e 100644 --- a/doc/src/sgml/file-fdw.sgml +++ b/doc/src/sgml/file-fdw.sgml @@ -262,7 +262,7 @@ CREATE FOREIGN TABLE pglog ( location text, application_name text ) SERVER pglog -OPTIONS ( filename '/home/josh/9.1/data/pg_log/pglog.csv', format 'csv' ); +OPTIONS ( filename '/home/josh/data/log/pglog.csv', format 'csv' ); diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 291bf7631d..e9d561b185 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -3320,7 +3320,7 @@ static struct config_string ConfigureNamesString[] = GUC_SUPERUSER_ONLY }, &Log_directory, - "pg_log", + "log", check_canonical_path, NULL, NULL }, { diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index a02b154863..8a93bdcb3f 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -344,7 +344,7 @@ # (change requires restart) # These are only used if logging_collector is on: -#log_directory = 'pg_log' # directory where log files are written, +#log_directory = 'log' # directory where log files are written, # can be absolute or relative to PGDATA #log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log' # log file name pattern, # can include strftime() escapes diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm index 1ad8f7fc1c..cb84f1f2c6 100644 --- a/src/test/perl/PostgresNode.pm +++ b/src/test/perl/PostgresNode.pm @@ -551,7 +551,7 @@ sub _backup_fs $backup_path, filterfn => sub { my $src = shift; - return ($src ne 'pg_log' and $src ne 'postmaster.pid'); + return ($src ne 'log' and $src ne 'postmaster.pid'); }); if ($hot) diff --git a/src/test/perl/RecursiveCopy.pm b/src/test/perl/RecursiveCopy.pm index 3e98813286..28ecaf6db2 100644 --- a/src/test/perl/RecursiveCopy.pm +++ b/src/test/perl/RecursiveCopy.pm @@ -48,9 +48,9 @@ attempted. RecursiveCopy::copypath('/some/path', '/empty/dir', filterfn => sub { - # omit pg_log and contents + # omit log/ and contents my $src = shift; - return $src ne 'pg_log'; + return $src ne 'log'; } ); -- cgit v1.2.3 From 8355a011a0124bdf7ccbada206a967d427039553 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Mon, 27 Mar 2017 10:37:29 -0400 Subject: Allow ON CONFLICT .. DO NOTHING on a partitioned table. ON CONFLICT .. DO UPDATE still doesn't work, for lack of a way of enforcing uniqueness across partitions, but we can still allow this case. Amit Langote, per discussion with Peter Geoghegan. Additional wordsmithing by me. Discussion: http://postgr.es/m/CAA-aLv7Z4uygtq-Q5CvDi9Y=VZxUyEnuWjL=EwCfOof=L04hgg@mail.gmail.com --- doc/src/sgml/ddl.sgml | 8 ++++++-- src/backend/parser/analyze.c | 8 -------- src/test/regress/expected/insert_conflict.out | 10 ++++++++++ src/test/regress/sql/insert_conflict.sql | 10 ++++++++++ 4 files changed, 26 insertions(+), 10 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ddl.sgml b/doc/src/sgml/ddl.sgml index 09b5b3ff70..d1e915c11a 100644 --- a/doc/src/sgml/ddl.sgml +++ b/doc/src/sgml/ddl.sgml @@ -3854,8 +3854,12 @@ ANALYZE measurement; - INSERT statements with ON CONFLICT - clause are currently not allowed on partitioned tables. + Using the ON CONFLICT clause with partitioned tables + will cause an error if DO UPDATE is specified as the + alternative action, because unique or exclusion constraints can only be + created on individual partitions. There is no support for enforcing + uniqueness (or an exclusion constraint) across an entire partitioning + hierarchy. diff --git a/src/backend/parser/analyze.c b/src/backend/parser/analyze.c index 3571e50aea..25699fbc4a 100644 --- a/src/backend/parser/analyze.c +++ b/src/backend/parser/analyze.c @@ -842,16 +842,8 @@ transformInsertStmt(ParseState *pstate, InsertStmt *stmt) /* Process ON CONFLICT, if any. */ if (stmt->onConflictClause) - { - /* Bail out if target relation is partitioned table */ - if (pstate->p_target_rangetblentry->relkind == RELKIND_PARTITIONED_TABLE) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("ON CONFLICT clause is not supported with partitioned tables"))); - qry->onConflict = transformOnConflictClause(pstate, stmt->onConflictClause); - } /* * If we have a RETURNING clause, we need to add the target relation to diff --git a/src/test/regress/expected/insert_conflict.out b/src/test/regress/expected/insert_conflict.out index 8d005fddd4..c90d381b34 100644 --- a/src/test/regress/expected/insert_conflict.out +++ b/src/test/regress/expected/insert_conflict.out @@ -786,3 +786,13 @@ select * from selfconflict; (3 rows) drop table selfconflict; +-- check that the following works: +-- insert into partitioned_table on conflict do nothing +create table parted_conflict_test (a int, b char) partition by list (a); +create table parted_conflict_test_1 partition of parted_conflict_test for values in (1); +insert into parted_conflict_test values (1, 'a') on conflict do nothing; +insert into parted_conflict_test values (1, 'a') on conflict do nothing; +-- however, on conflict do update not supported yet +insert into parted_conflict_test values (1) on conflict (a) do update set b = excluded.b where excluded.a = 1; +ERROR: there is no unique or exclusion constraint matching the ON CONFLICT specification +drop table parted_conflict_test, parted_conflict_test_1; diff --git a/src/test/regress/sql/insert_conflict.sql b/src/test/regress/sql/insert_conflict.sql index df3a9b59b5..78bffc783d 100644 --- a/src/test/regress/sql/insert_conflict.sql +++ b/src/test/regress/sql/insert_conflict.sql @@ -471,3 +471,13 @@ commit; select * from selfconflict; drop table selfconflict; + +-- check that the following works: +-- insert into partitioned_table on conflict do nothing +create table parted_conflict_test (a int, b char) partition by list (a); +create table parted_conflict_test_1 partition of parted_conflict_test for values in (1); +insert into parted_conflict_test values (1, 'a') on conflict do nothing; +insert into parted_conflict_test values (1, 'a') on conflict do nothing; +-- however, on conflict do update not supported yet +insert into parted_conflict_test values (1) on conflict (a) do update set b = excluded.b where excluded.a = 1; +drop table parted_conflict_test, parted_conflict_test_1; -- cgit v1.2.3 From 4785e377f9c3189c06051ece7ebb112220f365f6 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Mon, 27 Mar 2017 11:00:09 -0400 Subject: Tidy up the CREATE TABLE documentation for partitioning. Remove some tags that make this too "loud". Fix some typos. Amit Langote, with a few minor corrections by me Discussion: http://postgr.es/m/a6f99cdb-21e7-1d65-1381-91f2cfa156e2@lab.ntt.co.jp --- doc/src/sgml/ref/create_table.sgml | 74 +++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 37 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml index 9ed25c05da..283d53e203 100644 --- a/doc/src/sgml/ref/create_table.sgml +++ b/doc/src/sgml/ref/create_table.sgml @@ -261,43 +261,43 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI any existing partition of that parent. - - - Each of the values specified in the partition bound specification is - a literal, NULL, or UNBOUNDED. - A literal is either a numeric constant or a string constant that is - coercable to the corresponding partition key column's type. - - - - When creating a range partition, the lower bound specified with - FROM is an inclusive bound, whereas the upper bound - specified with TO is an exclusive bound. That is, - the values specified in the FROM list are accepted - values of the corresponding partition key columns in a given partition, - whereas those in the TO list are not. To be precise, - this applies only to the first of the partition key columns for which - the corresponding values in the FROM and - TO lists are not equal. All rows in a given - partition contain the same values for all preceding columns, equal to - those specified in FROM and TO - lists. On the other hand, any subsequent columns are insignificant - as far as implicit partition constraint is concerned. - - Specifying UNBOUNDED in FROM - signifies -infinity as the lower bound of the - corresponding column, whereas it signifies +infinity - as the upper bound when specified in TO. - - - - When creating a list partition, NULL can be specified - to signify that the partition allows the partition key column to be null. - However, there cannot be more than one such list partitions for a given - parent table. NULL cannot specified for range - partitions. - - + + Each of the values specified in the partition bound specification is + a literal, NULL, or UNBOUNDED. + A literal is either a numeric constant or a string constant that is + coercible to the corresponding partition key column's type. + + + + When creating a range partition, the lower bound specified with + FROM is an inclusive bound, whereas the upper + bound specified with TO is an exclusive bound. + That is, the values specified in the FROM list + are accepted values of the corresponding partition key columns in a + given partition, whereas those in the TO list are + not. To be precise, this applies only to the first of the partition + key columns for which the corresponding values in the FROM + and TO lists are not equal. All rows in a given + partition contain the same values for all preceding columns, equal to + those specified in FROM and TO + lists. On the other hand, any subsequent columns are insignificant + as far as implicit partition constraint is concerned. + + + + Specifying UNBOUNDED in FROM + signifies -infinity as the lower bound of the + corresponding column, whereas it signifies +infinity + as the upper bound when specified in TO. + + + + When creating a list partition, NULL can be + specified to signify that the partition allows the partition key + column to be null. However, there cannot be more than one such + list partition for a given parent table. NULL + cannot be specified for range partitions. + A partition must have the same column names and types as the partitioned -- cgit v1.2.3 From d65561464f45d325e82ad91918bcd4e2881ce567 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Mon, 27 Mar 2017 11:01:33 -0400 Subject: Improve documentation of how NOT NULL works with partitioning. Amit Langote Discussion: http://postgr.es/m/a6f99cdb-21e7-1d65-1381-91f2cfa156e2@lab.ntt.co.jp --- doc/src/sgml/ref/alter_table.sgml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/alter_table.sgml b/doc/src/sgml/ref/alter_table.sgml index 75de226253..7829f378bb 100644 --- a/doc/src/sgml/ref/alter_table.sgml +++ b/doc/src/sgml/ref/alter_table.sgml @@ -175,9 +175,14 @@ ALTER TABLE [ IF EXISTS ] name - If this table is a partition, one cannot perform DROP NOT NULL + If this table is a partition, one cannot perform DROP NOT NULL on a column if it is marked NOT NULL in the parent - table. + table. To drop the NOT NULL constraint from all the + partitions, perform DROP NOT NULL on the parent + table. Even if there is no NOT NULL constraint on the + parent, such a constraint can still be added to individual partitions, + if desired; that is, the children can disallow nulls even if the parent + allows them, but not the other way around. -- cgit v1.2.3 From 70ec3f1f8f0b753c38a1a582280a02930d7cac5f Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Sat, 25 Feb 2017 08:42:25 -0500 Subject: PL/Python: Add cursor and execute methods to plan object Instead of plan = plpy.prepare(...) res = plpy.execute(plan, ...) you can now write plan = plpy.prepare(...) res = plan.execute(...) or even res = plpy.prepare(...).execute(...) and similarly for the cursor() method. This is more in object oriented style, and makes the hybrid nature of the existing execute() function less confusing. Reviewed-by: Andrew Dunstan --- doc/src/sgml/plpython.sgml | 14 ++++++++++++-- src/pl/plpython/expected/plpython_spi.out | 19 ++++++++++++++++--- src/pl/plpython/plpy_cursorobject.c | 3 +-- src/pl/plpython/plpy_cursorobject.h | 1 + src/pl/plpython/plpy_planobject.c | 31 +++++++++++++++++++++++++++++++ src/pl/plpython/plpy_spi.c | 3 +-- src/pl/plpython/plpy_spi.h | 1 + src/pl/plpython/sql/plpython_spi.sql | 18 ++++++++++++++++-- 8 files changed, 79 insertions(+), 11 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/plpython.sgml b/doc/src/sgml/plpython.sgml index fb5d336efc..777a7ef780 100644 --- a/doc/src/sgml/plpython.sgml +++ b/doc/src/sgml/plpython.sgml @@ -1046,6 +1046,14 @@ rv = plpy.execute(plan, ["name"], 5) The third argument is the optional row limit as before. + + Alternatively, you can call the execute method on + the plan object: + +rv = plan.execute(["name"], 5) + + + Query parameters and result row fields are converted between PostgreSQL and Python data types as described in . @@ -1081,7 +1089,9 @@ $$ LANGUAGE plpythonu; as plpy.execute (except for the row limit) and returns a cursor object, which allows you to process large result sets in smaller chunks. As with plpy.execute, either a query string - or a plan object along with a list of arguments can be used. + or a plan object along with a list of arguments can be used, or + the cursor function can be called as a method of + the plan object. @@ -1125,7 +1135,7 @@ $$ LANGUAGE plpythonu; CREATE FUNCTION count_odd_prepared() RETURNS integer AS $$ odd = 0 plan = plpy.prepare("select num from largetable where num % $1 <> 0", ["integer"]) -rows = list(plpy.cursor(plan, [2])) +rows = list(plpy.cursor(plan, [2])) # or: = list(plan.cursor([2])) return len(rows) $$ LANGUAGE plpythonu; diff --git a/src/pl/plpython/expected/plpython_spi.out b/src/pl/plpython/expected/plpython_spi.out index 0d78ca1de4..e54dca9e2e 100644 --- a/src/pl/plpython/expected/plpython_spi.out +++ b/src/pl/plpython/expected/plpython_spi.out @@ -29,6 +29,19 @@ try: except Exception, ex: plpy.error(str(ex)) return None +' + LANGUAGE plpythonu; +CREATE FUNCTION spi_prepared_plan_test_two(a text) RETURNS text + AS +'if "myplan" not in SD: + q = "SELECT count(*) FROM users WHERE lname = $1" + SD["myplan"] = plpy.prepare(q, [ "text" ]) +try: + rv = SD["myplan"].execute([a]) + return "there are " + str(rv[0]["count"]) + " " + str(a) + "s" +except Exception, ex: + plpy.error(str(ex)) +return None ' LANGUAGE plpythonu; CREATE FUNCTION spi_prepared_plan_test_nested(a text) RETURNS text @@ -80,8 +93,8 @@ select spi_prepared_plan_test_one('doe'); there are 3 does (1 row) -select spi_prepared_plan_test_one('smith'); - spi_prepared_plan_test_one +select spi_prepared_plan_test_two('smith'); + spi_prepared_plan_test_two ---------------------------- there are 1 smiths (1 row) @@ -372,7 +385,7 @@ plan = plpy.prepare( ["text"]) for row in plpy.cursor(plan, ["w"]): yield row['fname'] -for row in plpy.cursor(plan, ["j"]): +for row in plan.cursor(["j"]): yield row['fname'] $$ LANGUAGE plpythonu; CREATE FUNCTION cursor_plan_wrong_args() RETURNS SETOF text AS $$ diff --git a/src/pl/plpython/plpy_cursorobject.c b/src/pl/plpython/plpy_cursorobject.c index 7bb8992148..18e689f141 100644 --- a/src/pl/plpython/plpy_cursorobject.c +++ b/src/pl/plpython/plpy_cursorobject.c @@ -25,7 +25,6 @@ static PyObject *PLy_cursor_query(const char *query); -static PyObject *PLy_cursor_plan(PyObject *ob, PyObject *args); static void PLy_cursor_dealloc(PyObject *arg); static PyObject *PLy_cursor_iternext(PyObject *self); static PyObject *PLy_cursor_fetch(PyObject *self, PyObject *args); @@ -160,7 +159,7 @@ PLy_cursor_query(const char *query) return (PyObject *) cursor; } -static PyObject * +PyObject * PLy_cursor_plan(PyObject *ob, PyObject *args) { PLyCursorObject *cursor; diff --git a/src/pl/plpython/plpy_cursorobject.h b/src/pl/plpython/plpy_cursorobject.h index c73033c486..ef23865dd2 100644 --- a/src/pl/plpython/plpy_cursorobject.h +++ b/src/pl/plpython/plpy_cursorobject.h @@ -19,5 +19,6 @@ typedef struct PLyCursorObject extern void PLy_cursor_init_type(void); extern PyObject *PLy_cursor(PyObject *self, PyObject *args); +extern PyObject *PLy_cursor_plan(PyObject *ob, PyObject *args); #endif /* PLPY_CURSOROBJECT_H */ diff --git a/src/pl/plpython/plpy_planobject.c b/src/pl/plpython/plpy_planobject.c index 16c39a05dd..390b4e90d4 100644 --- a/src/pl/plpython/plpy_planobject.c +++ b/src/pl/plpython/plpy_planobject.c @@ -10,11 +10,15 @@ #include "plpy_planobject.h" +#include "plpy_cursorobject.h" #include "plpy_elog.h" +#include "plpy_spi.h" #include "utils/memutils.h" static void PLy_plan_dealloc(PyObject *arg); +static PyObject *PLy_plan_cursor(PyObject *self, PyObject *args); +static PyObject *PLy_plan_execute(PyObject *self, PyObject *args); static PyObject *PLy_plan_status(PyObject *self, PyObject *args); static char PLy_plan_doc[] = { @@ -22,6 +26,8 @@ static char PLy_plan_doc[] = { }; static PyMethodDef PLy_plan_methods[] = { + {"cursor", PLy_plan_cursor, METH_VARARGS, NULL}, + {"execute", PLy_plan_execute, METH_VARARGS, NULL}, {"status", PLy_plan_status, METH_VARARGS, NULL}, {NULL, NULL, 0, NULL} }; @@ -111,6 +117,31 @@ PLy_plan_dealloc(PyObject *arg) } +static PyObject * +PLy_plan_cursor(PyObject *self, PyObject *args) +{ + PyObject *planargs = NULL; + + if (!PyArg_ParseTuple(args, "|O", &planargs)) + return NULL; + + return PLy_cursor_plan(self, planargs); +} + + +static PyObject * +PLy_plan_execute(PyObject *self, PyObject *args) +{ + PyObject *list = NULL; + long limit = 0; + + if (!PyArg_ParseTuple(args, "|Ol", &list, &limit)) + return NULL; + + return PLy_spi_execute_plan(self, list, limit); +} + + static PyObject * PLy_plan_status(PyObject *self, PyObject *args) { diff --git a/src/pl/plpython/plpy_spi.c b/src/pl/plpython/plpy_spi.c index 07ab6a087e..c6856ccbac 100644 --- a/src/pl/plpython/plpy_spi.c +++ b/src/pl/plpython/plpy_spi.c @@ -30,7 +30,6 @@ static PyObject *PLy_spi_execute_query(char *query, long limit); -static PyObject *PLy_spi_execute_plan(PyObject *ob, PyObject *list, long limit); static PyObject *PLy_spi_execute_fetch_result(SPITupleTable *tuptable, uint64 rows, int status); static void PLy_spi_exception_set(PyObject *excclass, ErrorData *edata); @@ -193,7 +192,7 @@ PLy_spi_execute(PyObject *self, PyObject *args) return NULL; } -static PyObject * +PyObject * PLy_spi_execute_plan(PyObject *ob, PyObject *list, long limit) { volatile int nargs; diff --git a/src/pl/plpython/plpy_spi.h b/src/pl/plpython/plpy_spi.h index b0427947ef..817a7584e7 100644 --- a/src/pl/plpython/plpy_spi.h +++ b/src/pl/plpython/plpy_spi.h @@ -10,6 +10,7 @@ extern PyObject *PLy_spi_prepare(PyObject *self, PyObject *args); extern PyObject *PLy_spi_execute(PyObject *self, PyObject *args); +extern PyObject *PLy_spi_execute_plan(PyObject *ob, PyObject *list, long limit); typedef struct PLyExceptionEntry { diff --git a/src/pl/plpython/sql/plpython_spi.sql b/src/pl/plpython/sql/plpython_spi.sql index 7427de824b..fcf049cb66 100644 --- a/src/pl/plpython/sql/plpython_spi.sql +++ b/src/pl/plpython/sql/plpython_spi.sql @@ -37,6 +37,20 @@ return None ' LANGUAGE plpythonu; +CREATE FUNCTION spi_prepared_plan_test_two(a text) RETURNS text + AS +'if "myplan" not in SD: + q = "SELECT count(*) FROM users WHERE lname = $1" + SD["myplan"] = plpy.prepare(q, [ "text" ]) +try: + rv = SD["myplan"].execute([a]) + return "there are " + str(rv[0]["count"]) + " " + str(a) + "s" +except Exception, ex: + plpy.error(str(ex)) +return None +' + LANGUAGE plpythonu; + CREATE FUNCTION spi_prepared_plan_test_nested(a text) RETURNS text AS 'if "myplan" not in SD: @@ -79,7 +93,7 @@ return a + r -- select nested_call_one('pass this along'); select spi_prepared_plan_test_one('doe'); -select spi_prepared_plan_test_one('smith'); +select spi_prepared_plan_test_two('smith'); select spi_prepared_plan_test_nested('smith'); SELECT join_sequences(sequences) FROM sequences; @@ -275,7 +289,7 @@ plan = plpy.prepare( ["text"]) for row in plpy.cursor(plan, ["w"]): yield row['fname'] -for row in plpy.cursor(plan, ["j"]): +for row in plan.cursor(["j"]): yield row['fname'] $$ LANGUAGE plpythonu; -- cgit v1.2.3 From 5196f13b2726eeabfe2c8b7a97fc05839766cdce Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Mon, 27 Mar 2017 11:57:10 -0400 Subject: doc: Fix oldhtml/old PDF build again Commit e259e1f748c7a6d67e307a90d6c27b8ab8b90df8 was faulty and created some broken output. This one fixes it better. --- doc/src/sgml/ref/alter_collation.sgml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/alter_collation.sgml b/doc/src/sgml/ref/alter_collation.sgml index 159f7e7472..71cf4de802 100644 --- a/doc/src/sgml/ref/alter_collation.sgml +++ b/doc/src/sgml/ref/alter_collation.sgml @@ -94,7 +94,7 @@ ALTER COLLATION name SET SCHEMA new_sche Updated the collation version. See below. + endterm="sql-altercollation-notes-title"> below. @@ -102,7 +102,7 @@ ALTER COLLATION name SET SCHEMA new_sche     - Notes + Notes When using collations provided by the ICU library, the ICU-specific version -- cgit v1.2.3 From 6e31c3e13514be4404f716f152ac4c434adad03a Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Mon, 27 Mar 2017 13:33:36 -0400 Subject: doc: Improve rendering of notes/cautions using XSL-FO Center title and put a border around it, like the output that the DSSSL version gave. --- doc/src/sgml/stylesheet-fo.xsl | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'doc/src') diff --git a/doc/src/sgml/stylesheet-fo.xsl b/doc/src/sgml/stylesheet-fo.xsl index 69536f92cd..8b555d12a7 100644 --- a/doc/src/sgml/stylesheet-fo.xsl +++ b/doc/src/sgml/stylesheet-fo.xsl @@ -18,6 +18,20 @@ wrap + + solid + 1pt + black + 12pt + 12pt + 6pt + 6pt + + + + center + + -- cgit v1.2.3 From a6f22e83562d8b78293229587cd3d9430d16d466 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 27 Mar 2017 20:14:36 -0400 Subject: Show ignored constants as "$N" rather than "?" in pg_stat_statements. The trouble with the original choice here is that "?" is a valid (and indeed used) operator name, so that you could end up with ambiguous statement texts like "SELECT ? ? ?". With this patch, you instead see "SELECT $1 ? $2", which seems significantly more readable. The numbers used for this purpose begin after the last actual $N parameter in the particular query. The conflict with external parameters has its own potential for confusion of course, but it was agreed to be an improvement over the previous behavior. Lukas Fittl Discussion: https://postgr.es/m/CAP53PkxeaCuwYmF-A4J5z2-qk5fYFo5_NH3gpXGJJBxv1DMwEw@mail.gmail.com --- .../expected/pg_stat_statements.out | 113 ++++++++++++++------- contrib/pg_stat_statements/pg_stat_statements.c | 40 ++++++-- .../pg_stat_statements/sql/pg_stat_statements.sql | 16 ++- doc/src/sgml/pgstatstatements.sgml | 29 ++++-- 4 files changed, 142 insertions(+), 56 deletions(-) (limited to 'doc/src') diff --git a/contrib/pg_stat_statements/expected/pg_stat_statements.out b/contrib/pg_stat_statements/expected/pg_stat_statements.out index fd53f15d8b..5318c3550c 100644 --- a/contrib/pg_stat_statements/expected/pg_stat_statements.out +++ b/contrib/pg_stat_statements/expected/pg_stat_statements.out @@ -68,6 +68,13 @@ SELECT 1 AS i UNION SELECT 2 ORDER BY i; 2 (2 rows) +-- ? operator +select '{"a":1, "b":2}'::jsonb ? 'b'; + ?column? +---------- + t +(1 row) + -- cte WITH t(f) AS ( VALUES (1.0), (2.0) @@ -79,24 +86,35 @@ WITH t(f) AS ( 2.0 (2 rows) +-- prepared statement with parameter +PREPARE pgss_test (int) AS SELECT $1, 'test' LIMIT 1; +EXECUTE pgss_test(1); + ?column? | ?column? +----------+---------- + 1 | test +(1 row) + +DEALLOCATE pgss_test; SELECT query, calls, rows FROM pg_stat_statements ORDER BY query COLLATE "C"; - query | calls | rows ------------------------------------------+-------+------ - SELECT ? +| 4 | 4 - +| | - AS "text" | | - SELECT ? + ? | 2 | 2 - SELECT ? + ? + ? AS "add" | 3 | 3 - SELECT ? AS "float" | 1 | 1 - SELECT ? AS "int" | 2 | 2 - SELECT ? AS i UNION SELECT ? ORDER BY i | 1 | 2 - SELECT ? || ? | 1 | 1 - SELECT pg_stat_statements_reset() | 1 | 1 - WITH t(f) AS ( +| 1 | 2 - VALUES (?), (?) +| | - ) +| | - SELECT f FROM t ORDER BY f | | -(9 rows) + query | calls | rows +---------------------------------------------------+-------+------ + PREPARE pgss_test (int) AS SELECT $1, $2 LIMIT $3 | 1 | 1 + SELECT $1 +| 4 | 4 + +| | + AS "text" | | + SELECT $1 + $2 | 2 | 2 + SELECT $1 + $2 + $3 AS "add" | 3 | 3 + SELECT $1 AS "float" | 1 | 1 + SELECT $1 AS "int" | 2 | 2 + SELECT $1 AS i UNION SELECT $2 ORDER BY i | 1 | 2 + SELECT $1 || $2 | 1 | 1 + SELECT pg_stat_statements_reset() | 1 | 1 + WITH t(f) AS ( +| 1 | 2 + VALUES ($1), ($2) +| | + ) +| | + SELECT f FROM t ORDER BY f | | + select $1::jsonb ? $2 | 1 | 1 +(11 rows) -- -- CRUD: INSERT SELECT UPDATE DELETE on test table @@ -108,7 +126,7 @@ SELECT pg_stat_statements_reset(); (1 row) -- utility "create table" should not be shown -CREATE TABLE test (a int, b char(20)); +CREATE TEMP TABLE test (a int, b char(20)); INSERT INTO test VALUES(generate_series(1, 10), 'aaa'); UPDATE test SET b = 'bbb' WHERE a > 7; DELETE FROM test WHERE a > 9; @@ -125,6 +143,8 @@ BEGIN \; UPDATE test SET b = '555' WHERE a = 5 \; UPDATE test SET b = '666' WHERE a = 6 \; COMMIT ; +-- many INSERT values +INSERT INTO test (a, b) VALUES (1, 'a'), (2, 'b'), (3, 'c'); -- SELECT with constants SELECT * FROM test WHERE a > 5 ORDER BY a ; a | b @@ -147,8 +167,11 @@ SELECT * SELECT * FROM test ORDER BY a; a | b ---+---------------------- + 1 | a 1 | 111 + 2 | b 2 | 222 + 3 | c 3 | 333 4 | 444 5 | 555 @@ -156,19 +179,35 @@ SELECT * FROM test ORDER BY a; 7 | aaa 8 | bbb 9 | bbb -(9 rows) +(12 rows) + +-- SELECT with IN clause +SELECT * FROM test WHERE a IN (1, 2, 3, 4, 5); + a | b +---+---------------------- + 1 | 111 + 2 | 222 + 3 | 333 + 4 | 444 + 5 | 555 + 1 | a + 2 | b + 3 | c +(8 rows) SELECT query, calls, rows FROM pg_stat_statements ORDER BY query COLLATE "C"; - query | calls | rows ----------------------------------------------------+-------+------ - DELETE FROM test WHERE a > ? | 1 | 1 - INSERT INTO test VALUES(generate_series(?, ?), ?) | 1 | 10 - SELECT * FROM test ORDER BY a | 1 | 9 - SELECT * FROM test WHERE a > ? ORDER BY a | 2 | 4 - SELECT pg_stat_statements_reset() | 1 | 1 - UPDATE test SET b = ? WHERE a = ? | 6 | 6 - UPDATE test SET b = ? WHERE a > ? | 1 | 3 -(7 rows) + query | calls | rows +-------------------------------------------------------------+-------+------ + DELETE FROM test WHERE a > $1 | 1 | 1 + INSERT INTO test (a, b) VALUES ($1, $2), ($3, $4), ($5, $6) | 1 | 3 + INSERT INTO test VALUES(generate_series($1, $2), $3) | 1 | 10 + SELECT * FROM test ORDER BY a | 1 | 12 + SELECT * FROM test WHERE a > $1 ORDER BY a | 2 | 4 + SELECT * FROM test WHERE a IN ($1, $2, $3, $4, $5) | 1 | 8 + SELECT pg_stat_statements_reset() | 1 | 1 + UPDATE test SET b = $1 WHERE a = $2 | 6 | 6 + UPDATE test SET b = $1 WHERE a > $2 | 1 | 3 +(9 rows) -- -- pg_stat_statements.track = none @@ -251,9 +290,9 @@ SELECT PLUS_ONE(10); SELECT query, calls, rows FROM pg_stat_statements ORDER BY query COLLATE "C"; query | calls | rows -----------------------------------+-------+------ - SELECT ?::TEXT | 1 | 1 - SELECT PLUS_ONE(?) | 2 | 2 - SELECT PLUS_TWO(?) | 2 | 2 + SELECT $1::TEXT | 1 | 1 + SELECT PLUS_ONE($1) | 2 | 2 + SELECT PLUS_TWO($1) | 2 | 2 SELECT pg_stat_statements_reset() | 1 | 1 (4 rows) @@ -308,10 +347,10 @@ SELECT PLUS_ONE(1); SELECT query, calls, rows FROM pg_stat_statements ORDER BY query COLLATE "C"; query | calls | rows -----------------------------------+-------+------ - SELECT (i + ? + ?)::INTEGER | 2 | 2 - SELECT (i + ?)::INTEGER LIMIT ? | 2 | 2 - SELECT PLUS_ONE(?) | 2 | 2 - SELECT PLUS_TWO(?) | 2 | 2 + SELECT (i + $2 + $3)::INTEGER | 2 | 2 + SELECT (i + $2)::INTEGER LIMIT $3 | 2 | 2 + SELECT PLUS_ONE($1) | 2 | 2 + SELECT PLUS_TWO($1) | 2 | 2 SELECT pg_stat_statements_reset() | 1 | 1 (5 rows) @@ -352,7 +391,7 @@ SELECT query, calls, rows FROM pg_stat_statements ORDER BY query COLLATE "C"; DROP FUNCTION PLUS_TWO(INTEGER) | 1 | 0 DROP TABLE IF EXISTS test | 3 | 0 DROP TABLE test | 1 | 0 - SELECT ? | 1 | 1 + SELECT $1 | 1 | 1 SELECT pg_stat_statements_reset() | 1 | 1 (8 rows) diff --git a/contrib/pg_stat_statements/pg_stat_statements.c b/contrib/pg_stat_statements/pg_stat_statements.c index 42f43233f8..cd4c16e9d2 100644 --- a/contrib/pg_stat_statements/pg_stat_statements.c +++ b/contrib/pg_stat_statements/pg_stat_statements.c @@ -27,10 +27,10 @@ * to blame query costs on the proper queryId. * * To facilitate presenting entries to users, we create "representative" query - * strings in which constants are replaced with '?' characters, to make it - * clearer what a normalized entry can represent. To save on shared memory, - * and to avoid having to truncate oversized query strings, we store these - * strings in a temporary external query-texts file. Offsets into this + * strings in which constants are replaced with parameter symbols ($n), to + * make it clearer what a normalized entry can represent. To save on shared + * memory, and to avoid having to truncate oversized query strings, we store + * these strings in a temporary external query-texts file. Offsets into this * file are kept in shared memory. * * Note about locking issues: to create or delete an entry in the shared @@ -219,6 +219,9 @@ typedef struct pgssJumbleState /* Current number of valid entries in clocations array */ int clocations_count; + + /* highest Param id we've seen, in order to start normalization correctly */ + int highest_extern_param_id; } pgssJumbleState; /*---- Local variables ----*/ @@ -803,6 +806,7 @@ pgss_post_parse_analyze(ParseState *pstate, Query *query) jstate.clocations = (pgssLocationLen *) palloc(jstate.clocations_buf_size * sizeof(pgssLocationLen)); jstate.clocations_count = 0; + jstate.highest_extern_param_id = 0; /* Compute query ID and mark the Query node with it */ JumbleQuery(&jstate, query); @@ -2482,6 +2486,10 @@ JumbleExpr(pgssJumbleState *jstate, Node *node) APP_JUMB(p->paramkind); APP_JUMB(p->paramid); APP_JUMB(p->paramtype); + /* Also, track the highest external Param id */ + if (p->paramkind == PARAM_EXTERN && + p->paramid > jstate->highest_extern_param_id) + jstate->highest_extern_param_id = p->paramid; } break; case T_Aggref: @@ -2874,7 +2882,7 @@ JumbleExpr(pgssJumbleState *jstate, Node *node) break; case T_TableFunc: { - TableFunc *tablefunc = (TableFunc *) node; + TableFunc *tablefunc = (TableFunc *) node; JumbleExpr(jstate, tablefunc->docexpr); JumbleExpr(jstate, tablefunc->rowexpr); @@ -2938,7 +2946,8 @@ RecordConstLocation(pgssJumbleState *jstate, int location) * of interest, so it's worth doing.) * * *query_len_p contains the input string length, and is updated with - * the result string length (which cannot be longer) on exit. + * the result string length on exit. The resulting string might be longer + * or shorter depending on what happens with replacement of constants. * * Returns a palloc'd string. */ @@ -2949,6 +2958,7 @@ generate_normalized_query(pgssJumbleState *jstate, const char *query, char *norm_query; int query_len = *query_len_p; int i, + norm_query_buflen, /* Space allowed for norm_query */ len_to_wrt, /* Length (in bytes) to write */ quer_loc = 0, /* Source query byte location */ n_quer_loc = 0, /* Normalized query byte location */ @@ -2961,8 +2971,17 @@ generate_normalized_query(pgssJumbleState *jstate, const char *query, */ fill_in_constant_lengths(jstate, query, query_loc); + /* + * Allow for $n symbols to be longer than the constants they replace. + * Constants must take at least one byte in text form, while a $n symbol + * certainly isn't more than 11 bytes, even if n reaches INT_MAX. We + * could refine that limit based on the max value of n for the current + * query, but it hardly seems worth any extra effort to do so. + */ + norm_query_buflen = query_len + jstate->clocations_count * 10; + /* Allocate result buffer */ - norm_query = palloc(query_len + 1); + norm_query = palloc(norm_query_buflen + 1); for (i = 0; i < jstate->clocations_count; i++) { @@ -2986,8 +3005,9 @@ generate_normalized_query(pgssJumbleState *jstate, const char *query, memcpy(norm_query + n_quer_loc, query + quer_loc, len_to_wrt); n_quer_loc += len_to_wrt; - /* And insert a '?' in place of the constant token */ - norm_query[n_quer_loc++] = '?'; + /* And insert a param symbol in place of the constant token */ + n_quer_loc += sprintf(norm_query + n_quer_loc, "$%d", + i + 1 + jstate->highest_extern_param_id); quer_loc = off + tok_len; last_off = off; @@ -3004,7 +3024,7 @@ generate_normalized_query(pgssJumbleState *jstate, const char *query, memcpy(norm_query + n_quer_loc, query + quer_loc, len_to_wrt); n_quer_loc += len_to_wrt; - Assert(n_quer_loc <= query_len); + Assert(n_quer_loc <= norm_query_buflen); norm_query[n_quer_loc] = '\0'; *query_len_p = n_quer_loc; diff --git a/contrib/pg_stat_statements/sql/pg_stat_statements.sql b/contrib/pg_stat_statements/sql/pg_stat_statements.sql index 385c8a8d99..a8361fd1bf 100644 --- a/contrib/pg_stat_statements/sql/pg_stat_statements.sql +++ b/contrib/pg_stat_statements/sql/pg_stat_statements.sql @@ -37,12 +37,20 @@ SELECT :add + 1 + 1 AS "add" \gset -- set operator SELECT 1 AS i UNION SELECT 2 ORDER BY i; +-- ? operator +select '{"a":1, "b":2}'::jsonb ? 'b'; + -- cte WITH t(f) AS ( VALUES (1.0), (2.0) ) SELECT f FROM t ORDER BY f; +-- prepared statement with parameter +PREPARE pgss_test (int) AS SELECT $1, 'test' LIMIT 1; +EXECUTE pgss_test(1); +DEALLOCATE pgss_test; + SELECT query, calls, rows FROM pg_stat_statements ORDER BY query COLLATE "C"; -- @@ -51,7 +59,7 @@ SELECT query, calls, rows FROM pg_stat_statements ORDER BY query COLLATE "C"; SELECT pg_stat_statements_reset(); -- utility "create table" should not be shown -CREATE TABLE test (a int, b char(20)); +CREATE TEMP TABLE test (a int, b char(20)); INSERT INTO test VALUES(generate_series(1, 10), 'aaa'); UPDATE test SET b = 'bbb' WHERE a > 7; @@ -74,6 +82,9 @@ UPDATE test SET b = '555' WHERE a = 5 \; UPDATE test SET b = '666' WHERE a = 6 \; COMMIT ; +-- many INSERT values +INSERT INTO test (a, b) VALUES (1, 'a'), (2, 'b'), (3, 'c'); + -- SELECT with constants SELECT * FROM test WHERE a > 5 ORDER BY a ; @@ -85,6 +96,9 @@ SELECT * -- SELECT without constants SELECT * FROM test ORDER BY a; +-- SELECT with IN clause +SELECT * FROM test WHERE a IN (1, 2, 3, 4, 5); + SELECT query, calls, rows FROM pg_stat_statements ORDER BY query COLLATE "C"; -- diff --git a/doc/src/sgml/pgstatstatements.sgml b/doc/src/sgml/pgstatstatements.sgml index d4f09fc2a3..082994cae0 100644 --- a/doc/src/sgml/pgstatstatements.sgml +++ b/doc/src/sgml/pgstatstatements.sgml @@ -244,11 +244,12 @@ - When a constant's value has been ignored for purposes of matching the - query to other queries, the constant is replaced by ? - in the pg_stat_statements display. The rest of the query - text is that of the first query that had the particular - queryid hash value associated with the + When a constant's value has been ignored for purposes of matching the query + to other queries, the constant is replaced by a parameter symbol, such + as $1, in the pg_stat_statements + display. + The rest of the query text is that of the first query that had the + particular queryid hash value associated with the pg_stat_statements entry. @@ -301,6 +302,18 @@ replicas. If in doubt, direct testing is recommended. + + The parameter symbols used to replace constants in + representative query texts start from the next number after the + highest $n parameter in the original query + text, or $1 if there was none. It's worth noting that in + some cases there may be hidden parameter symbols that affect this + numbering. For example, PL/pgSQL uses hidden parameter + symbols to insert values of function local variables into queries, so that + a PL/pgSQL statement like SELECT i + 1 INTO j + would have representative text like SELECT i + $2. + + The representative query texts are kept in an external disk file, and do not consume shared memory. Therefore, even very lengthy query texts can @@ -481,13 +494,13 @@ bench=# SELECT query, calls, total_time, rows, 100.0 * shared_blks_hit / nullif(shared_blks_hit + shared_blks_read, 0) AS hit_percent FROM pg_stat_statements ORDER BY total_time DESC LIMIT 5; -[ RECORD 1 ]--------------------------------------------------------------------- -query | UPDATE pgbench_branches SET bbalance = bbalance + ? WHERE bid = ?; +query | UPDATE pgbench_branches SET bbalance = bbalance + $1 WHERE bid = $2; calls | 3000 total_time | 9609.00100000002 rows | 2836 hit_percent | 99.9778970000200936 -[ RECORD 2 ]--------------------------------------------------------------------- -query | UPDATE pgbench_tellers SET tbalance = tbalance + ? WHERE tid = ?; +query | UPDATE pgbench_tellers SET tbalance = tbalance + $1 WHERE tid = $2; calls | 3000 total_time | 8015.156 rows | 2990 @@ -499,7 +512,7 @@ total_time | 310.624 rows | 100000 hit_percent | 0.30395136778115501520 -[ RECORD 4 ]--------------------------------------------------------------------- -query | UPDATE pgbench_accounts SET abalance = abalance + ? WHERE aid = ?; +query | UPDATE pgbench_accounts SET abalance = abalance + $1 WHERE aid = $2; calls | 3000 total_time | 271.741999999997 rows | 3000 -- cgit v1.2.3 From ff539da31691f2cd2694360250571c5c5fb7415e Mon Sep 17 00:00:00 2001 From: Simon Riggs Date: Tue, 28 Mar 2017 10:05:21 -0400 Subject: Cleanup slots during drop database Automatically drop all logical replication slots associated with a database when the database is dropped. Previously we threw an ERROR if a slot existed. Now we throw ERROR only if a slot is active in the database being dropped. Craig Ringer --- doc/src/sgml/func.sgml | 3 +- doc/src/sgml/protocol.sgml | 2 + src/backend/commands/dbcommands.c | 32 +++++--- src/backend/replication/slot.c | 88 ++++++++++++++++++++++ src/include/replication/slot.h | 1 + src/test/recovery/t/006_logical_decoding.pl | 40 +++++++++- .../recovery/t/010_logical_decoding_timelines.pl | 30 +++++++- 7 files changed, 182 insertions(+), 14 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index ba6f8dd8d2..78508d74ec 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -18876,7 +18876,8 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); Drops the physical or logical replication slot named slot_name. Same as replication protocol - command DROP_REPLICATION_SLOT. + command DROP_REPLICATION_SLOT. For logical slots, this must + be called when connected to the same database the slot was created on. diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index b3a50261c3..5f971412ae 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -2034,6 +2034,8 @@ The commands accepted in walsender mode are: Drops a replication slot, freeing any reserved server-side resources. If the slot is currently in use by an active connection, this command fails. + If the slot is a logical slot that was created in a database other than + the database the walsender is connected to, this command fails. diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c index 5a63b1abcb..c0ba2b451a 100644 --- a/src/backend/commands/dbcommands.c +++ b/src/backend/commands/dbcommands.c @@ -845,19 +845,22 @@ dropdb(const char *dbname, bool missing_ok) errmsg("cannot drop the currently open database"))); /* - * Check whether there are, possibly unconnected, logical slots that refer - * to the to-be-dropped database. The database lock we are holding - * prevents the creation of new slots using the database. + * Check whether there are active logical slots that refer to the + * to-be-dropped database. The database lock we are holding prevents the + * creation of new slots using the database or existing slots becoming + * active. */ - if (ReplicationSlotsCountDBSlots(db_id, &nslots, &nslots_active)) + (void) ReplicationSlotsCountDBSlots(db_id, &nslots, &nslots_active); + if (nslots_active) + { ereport(ERROR, (errcode(ERRCODE_OBJECT_IN_USE), - errmsg("database \"%s\" is used by a logical replication slot", + errmsg("database \"%s\" is used by an active logical replication slot", dbname), - errdetail_plural("There is %d slot, %d of them active.", - "There are %d slots, %d of them active.", - nslots, - nslots, nslots_active))); + errdetail_plural("There is %d active slot", + "There are %d active slots", + nslots_active, nslots_active))); + } /* * Check for other backends in the target database. (Because we hold the @@ -914,6 +917,11 @@ dropdb(const char *dbname, bool missing_ok) */ dropDatabaseDependencies(db_id); + /* + * Drop db-specific replication slots. + */ + ReplicationSlotsDropDBSlots(db_id); + /* * Drop pages for this database that are in the shared buffer cache. This * is important to ensure that no remaining backend tries to write out a @@ -2124,11 +2132,17 @@ dbase_redo(XLogReaderState *record) * InitPostgres() cannot fully re-execute concurrently. This * avoids backends re-connecting automatically to same database, * which can happen in some cases. + * + * This will lock out walsenders trying to connect to db-specific + * slots for logical decoding too, so it's safe for us to drop slots. */ LockSharedObjectForSession(DatabaseRelationId, xlrec->db_id, 0, AccessExclusiveLock); ResolveRecoveryConflictWithDatabase(xlrec->db_id); } + /* Drop any database-specific replication slots */ + ReplicationSlotsDropDBSlots(xlrec->db_id); + /* Drop pages for this database that are in the shared buffer cache */ DropDatabaseBuffers(xlrec->db_id); diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c index 5237a9fb07..6c5ec7a00e 100644 --- a/src/backend/replication/slot.c +++ b/src/backend/replication/slot.c @@ -796,6 +796,94 @@ ReplicationSlotsCountDBSlots(Oid dboid, int *nslots, int *nactive) return false; } +/* + * ReplicationSlotsDropDBSlots -- Drop all db-specific slots relating to the + * passed database oid. The caller should hold an exclusive lock on the + * pg_database oid for the database to prevent creation of new slots on the db + * or replay from existing slots. + * + * This routine isn't as efficient as it could be - but we don't drop databases + * often, especially databases with lots of slots. + * + * Another session that concurrently acquires an existing slot on the target DB + * (most likely to drop it) may cause this function to ERROR. If that happens + * it may have dropped some but not all slots. + */ +void +ReplicationSlotsDropDBSlots(Oid dboid) +{ + int i; + + if (max_replication_slots <= 0) + return; + +restart: + LWLockAcquire(ReplicationSlotControlLock, LW_SHARED); + for (i = 0; i < max_replication_slots; i++) + { + ReplicationSlot *s; + NameData slotname; + int active_pid; + + s = &ReplicationSlotCtl->replication_slots[i]; + + /* cannot change while ReplicationSlotCtlLock is held */ + if (!s->in_use) + continue; + + /* only logical slots are database specific, skip */ + if (!SlotIsLogical(s)) + continue; + + /* not our database, skip */ + if (s->data.database != dboid) + continue; + + /* Claim the slot, as if ReplicationSlotAcquire()ing. */ + SpinLockAcquire(&s->mutex); + strncpy(NameStr(slotname), NameStr(s->data.name), NAMEDATALEN); + NameStr(slotname)[NAMEDATALEN-1] = '\0'; + active_pid = s->active_pid; + if (active_pid == 0) + { + MyReplicationSlot = s; + s->active_pid = MyProcPid; + } + SpinLockRelease(&s->mutex); + + /* + * We might fail here if the slot was active. Even though we hold an + * exclusive lock on the database object a logical slot for that DB can + * still be active if it's being dropped by a backend connected to + * another DB or is otherwise acquired. + * + * It's an unlikely race that'll only arise from concurrent user action, + * so we'll just bail out. + */ + if (active_pid) + elog(ERROR, "replication slot %s is in use by pid %d", + NameStr(slotname), active_pid); + + /* + * To avoid largely duplicating ReplicationSlotDropAcquired() or + * complicating it with already_locked flags for ProcArrayLock, + * ReplicationSlotControlLock and ReplicationSlotAllocationLock, we + * just release our ReplicationSlotControlLock to drop the slot. + * + * For safety we'll restart our scan from the beginning each + * time we release the lock. + */ + LWLockRelease(ReplicationSlotControlLock); + ReplicationSlotDropAcquired(); + goto restart; + } + LWLockRelease(ReplicationSlotControlLock); + + /* recompute limits once after all slots are dropped */ + ReplicationSlotsComputeRequiredXmin(false); + ReplicationSlotsComputeRequiredLSN(); +} + /* * Check whether the server's configuration supports using replication diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h index 62cacdb384..9a2dbd7b61 100644 --- a/src/include/replication/slot.h +++ b/src/include/replication/slot.h @@ -177,6 +177,7 @@ extern void ReplicationSlotsComputeRequiredXmin(bool already_locked); extern void ReplicationSlotsComputeRequiredLSN(void); extern XLogRecPtr ReplicationSlotsComputeLogicalRestartLSN(void); extern bool ReplicationSlotsCountDBSlots(Oid dboid, int *nslots, int *nactive); +extern void ReplicationSlotsDropDBSlots(Oid dboid); extern void StartupReplicationSlots(void); extern void CheckPointReplicationSlots(void); diff --git a/src/test/recovery/t/006_logical_decoding.pl b/src/test/recovery/t/006_logical_decoding.pl index 66d5e4ad09..bf9b50a6a3 100644 --- a/src/test/recovery/t/006_logical_decoding.pl +++ b/src/test/recovery/t/006_logical_decoding.pl @@ -7,7 +7,7 @@ use strict; use warnings; use PostgresNode; use TestLib; -use Test::More tests => 5; +use Test::More tests => 16; # Initialize master node my $node_master = get_new_node('master'); @@ -54,7 +54,7 @@ my $stdout_sql = $node_master->safe_psql('postgres', qq[SELECT data FROM pg_logi is($stdout_sql, $expected, 'got expected output from SQL decoding session'); my $endpos = $node_master->safe_psql('postgres', "SELECT location FROM pg_logical_slot_peek_changes('test_slot', NULL, NULL) ORDER BY location DESC LIMIT 1;"); -diag "waiting to replay $endpos"; +print "waiting to replay $endpos\n"; my $stdout_recv = $node_master->pg_recvlogical_upto('postgres', 'test_slot', $endpos, 10, 'include-xids' => '0', 'skip-empty-xacts' => '1'); chomp($stdout_recv); @@ -64,5 +64,41 @@ $stdout_recv = $node_master->pg_recvlogical_upto('postgres', 'test_slot', $endpo chomp($stdout_recv); is($stdout_recv, '', 'pg_recvlogical acknowledged changes, nothing pending on slot'); +$node_master->safe_psql('postgres', 'CREATE DATABASE otherdb'); + +is($node_master->psql('otherdb', "SELECT location FROM pg_logical_slot_peek_changes('test_slot', NULL, NULL) ORDER BY location DESC LIMIT 1;"), 3, + 'replaying logical slot from another database fails'); + +$node_master->safe_psql('otherdb', qq[SELECT pg_create_logical_replication_slot('otherdb_slot', 'test_decoding');]); + +# make sure you can't drop a slot while active +my $pg_recvlogical = IPC::Run::start(['pg_recvlogical', '-d', $node_master->connstr('otherdb'), '-S', 'otherdb_slot', '-f', '-', '--start']); +$node_master->poll_query_until('otherdb', "SELECT EXISTS (SELECT 1 FROM pg_replication_slots WHERE slot_name = 'otherdb_slot' AND active_pid IS NOT NULL)"); +is($node_master->psql('postgres', 'DROP DATABASE otherdb'), 3, + 'dropping a DB with inactive logical slots fails'); +$pg_recvlogical->kill_kill; +is($node_master->slot('otherdb_slot')->{'slot_name'}, undef, + 'logical slot still exists'); + +$node_master->poll_query_until('otherdb', "SELECT EXISTS (SELECT 1 FROM pg_replication_slots WHERE slot_name = 'otherdb_slot' AND active_pid IS NULL)"); +is($node_master->psql('postgres', 'DROP DATABASE otherdb'), 0, + 'dropping a DB with inactive logical slots succeeds'); +is($node_master->slot('otherdb_slot')->{'slot_name'}, undef, + 'logical slot was actually dropped with DB'); + +# Restarting a node with wal_level = logical that has existing +# slots must succeed, but decoding from those slots must fail. +$node_master->safe_psql('postgres', 'ALTER SYSTEM SET wal_level = replica'); +is($node_master->safe_psql('postgres', 'SHOW wal_level'), 'logical', 'wal_level is still logical before restart'); +$node_master->restart; +is($node_master->safe_psql('postgres', 'SHOW wal_level'), 'replica', 'wal_level is replica'); +isnt($node_master->slot('test_slot')->{'catalog_xmin'}, '0', + 'restored slot catalog_xmin is nonzero'); +is($node_master->psql('postgres', qq[SELECT pg_logical_slot_get_changes('test_slot', NULL, NULL);]), 3, + 'reading from slot with wal_level < logical fails'); +is($node_master->psql('postgres', q[SELECT pg_drop_replication_slot('test_slot')]), 0, + 'can drop logical slot while wal_level = replica'); +is($node_master->slot('test_slot')->{'catalog_xmin'}, '', 'slot was dropped'); + # done with the node $node_master->stop; diff --git a/src/test/recovery/t/010_logical_decoding_timelines.pl b/src/test/recovery/t/010_logical_decoding_timelines.pl index 4561a06143..b618132e2b 100644 --- a/src/test/recovery/t/010_logical_decoding_timelines.pl +++ b/src/test/recovery/t/010_logical_decoding_timelines.pl @@ -15,12 +15,15 @@ # This module uses the first approach to show that timeline following # on a logical slot works. # +# (For convenience, it also tests some recovery-related operations +# on logical slots). +# use strict; use warnings; use PostgresNode; use TestLib; -use Test::More tests => 10; +use Test::More tests => 13; use RecursiveCopy; use File::Copy; use IPC::Run (); @@ -50,6 +53,16 @@ $node_master->safe_psql('postgres', $node_master->safe_psql('postgres', "CREATE TABLE decoding(blah text);"); $node_master->safe_psql('postgres', "INSERT INTO decoding(blah) VALUES ('beforebb');"); + +# We also want to verify that DROP DATABASE on a standby with a logical +# slot works. This isn't strictly related to timeline following, but +# the only way to get a logical slot on a standby right now is to use +# the same physical copy trick, so: +$node_master->safe_psql('postgres', 'CREATE DATABASE dropme;'); +$node_master->safe_psql('dropme', +"SELECT pg_create_logical_replication_slot('dropme_slot', 'test_decoding');" +); + $node_master->safe_psql('postgres', 'CHECKPOINT;'); my $backup_name = 'b1'; @@ -68,6 +81,17 @@ $node_replica->append_conf( $node_replica->start; +# If we drop 'dropme' on the master, the standby should drop the +# db and associated slot. +is($node_master->psql('postgres', 'DROP DATABASE dropme'), 0, + 'dropped DB with logical slot OK on master'); +$node_master->wait_for_catchup($node_replica, 'replay', $node_master->lsn('insert')); +is($node_replica->safe_psql('postgres', q[SELECT 1 FROM pg_database WHERE datname = 'dropme']), '', + 'dropped DB dropme on standby'); +is($node_master->slot('dropme_slot')->{'slot_name'}, undef, + 'logical slot was actually dropped on standby'); + +# Back to testing failover... $node_master->safe_psql('postgres', "SELECT pg_create_logical_replication_slot('after_basebackup', 'test_decoding');" ); @@ -99,10 +123,13 @@ isnt($phys_slot->{'catalog_xmin'}, '', cmp_ok($phys_slot->{'xmin'}, '>=', $phys_slot->{'catalog_xmin'}, 'xmin on physical slot must not be lower than catalog_xmin'); +$node_master->safe_psql('postgres', 'CHECKPOINT'); + # Boom, crash $node_master->stop('immediate'); $node_replica->promote; +print "waiting for replica to come up\n"; $node_replica->poll_query_until('postgres', "SELECT NOT pg_is_in_recovery();"); @@ -154,5 +181,4 @@ $stdout = $node_replica->pg_recvlogical_upto('postgres', 'before_basebackup', chomp($stdout); is($stdout, $final_expected_output_bb, 'got same output from walsender via pg_recvlogical on before_basebackup'); -# We don't need the standby anymore $node_replica->teardown_node(); -- cgit v1.2.3 From ab89e465cb2032017c4888399f47a76ac16eaf40 Mon Sep 17 00:00:00 2001 From: Teodor Sigaev Date: Tue, 28 Mar 2017 18:58:55 +0300 Subject: Altering default privileges on schemas Extend ALTER DEFAULT PRIVILEGES command to schemas. Author: Matheus Oliveira Reviewed-by: Petr Jelínek, Ashutosh Sharma https://commitfest.postgresql.org/13/887/ --- doc/src/sgml/ref/alter_default_privileges.sgml | 17 ++++++- src/backend/catalog/aclchk.c | 21 +++++++++ src/backend/catalog/objectaddress.c | 14 +++++- src/backend/catalog/pg_namespace.c | 21 ++++++--- src/backend/parser/gram.y | 4 +- src/bin/pg_dump/dumputils.c | 4 +- src/bin/pg_dump/pg_dump.c | 3 ++ src/bin/psql/describe.c | 4 +- src/bin/psql/tab-complete.c | 2 +- src/include/catalog/pg_default_acl.h | 1 + src/include/parser/kwlist.h | 1 + src/test/regress/expected/privileges.out | 62 ++++++++++++++++++++++++++ src/test/regress/sql/privileges.sql | 34 ++++++++++++++ 13 files changed, 176 insertions(+), 12 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/alter_default_privileges.sgml b/doc/src/sgml/ref/alter_default_privileges.sgml index 04064d399c..e3363f868a 100644 --- a/doc/src/sgml/ref/alter_default_privileges.sgml +++ b/doc/src/sgml/ref/alter_default_privileges.sgml @@ -46,6 +46,10 @@ GRANT { USAGE | ALL [ PRIVILEGES ] } ON TYPES TO { [ GROUP ] role_name | PUBLIC } [, ...] [ WITH GRANT OPTION ] +GRANT { USAGE | CREATE | ALL [ PRIVILEGES ] } + ON SCHEMAS + TO { [ GROUP ] role_name | PUBLIC } [, ...] [ WITH GRANT OPTION ] + REVOKE [ GRANT OPTION FOR ] { { SELECT | INSERT | UPDATE | DELETE | TRUNCATE | REFERENCES | TRIGGER } [, ...] | ALL [ PRIVILEGES ] } @@ -71,6 +75,12 @@ REVOKE [ GRANT OPTION FOR ] ON TYPES FROM { [ GROUP ] role_name | PUBLIC } [, ...] [ CASCADE | RESTRICT ] + +REVOKE [ GRANT OPTION FOR ] + { USAGE | CREATE | ALL [ PRIVILEGES ] } + ON SCHEMAS + FROM { [ GROUP ] role_name | PUBLIC } [, ...] + [ CASCADE | RESTRICT ] @@ -81,8 +91,9 @@ REVOKE [ GRANT OPTION FOR ] ALTER DEFAULT PRIVILEGES allows you to set the privileges that will be applied to objects created in the future. (It does not affect privileges assigned to already-existing objects.) Currently, - only the privileges for tables (including views and foreign tables), - sequences, functions, and types (including domains) can be altered. + only the privileges for schemas, tables (including views and foreign + tables), sequences, functions, and types (including domains) can be + altered. @@ -125,6 +136,8 @@ REVOKE [ GRANT OPTION FOR ] are altered for objects later created in that schema. If IN SCHEMA is omitted, the global default privileges are altered. + IN SCHEMA is not allowed when using ON SCHEMAS + as schemas can't be nested. diff --git a/src/backend/catalog/aclchk.c b/src/backend/catalog/aclchk.c index d01930f4a8..2d535c2aad 100644 --- a/src/backend/catalog/aclchk.c +++ b/src/backend/catalog/aclchk.c @@ -959,6 +959,10 @@ ExecAlterDefaultPrivilegesStmt(ParseState *pstate, AlterDefaultPrivilegesStmt *s all_privileges = ACL_ALL_RIGHTS_TYPE; errormsg = gettext_noop("invalid privilege type %s for type"); break; + case ACL_OBJECT_NAMESPACE: + all_privileges = ACL_ALL_RIGHTS_NAMESPACE; + errormsg = gettext_noop("invalid privilege type %s for schema"); + break; default: elog(ERROR, "unrecognized GrantStmt.objtype: %d", (int) action->objtype); @@ -1146,6 +1150,16 @@ SetDefaultACL(InternalDefaultACL *iacls) this_privileges = ACL_ALL_RIGHTS_TYPE; break; + case ACL_OBJECT_NAMESPACE: + if (OidIsValid(iacls->nspid)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_GRANT_OPERATION), + errmsg("cannot use IN SCHEMA clause when using GRANT/REVOKE ON SCHEMAS"))); + objtype = DEFACLOBJ_NAMESPACE; + if (iacls->all_privs && this_privileges == ACL_NO_RIGHTS) + this_privileges = ACL_ALL_RIGHTS_NAMESPACE; + break; + default: elog(ERROR, "unrecognized objtype: %d", (int) iacls->objtype); @@ -1369,6 +1383,9 @@ RemoveRoleFromObjectACL(Oid roleid, Oid classid, Oid objid) case DEFACLOBJ_TYPE: iacls.objtype = ACL_OBJECT_TYPE; break; + case DEFACLOBJ_NAMESPACE: + iacls.objtype = ACL_OBJECT_NAMESPACE; + break; default: /* Shouldn't get here */ elog(ERROR, "unexpected default ACL type: %d", @@ -5259,6 +5276,10 @@ get_user_default_acl(GrantObjectType objtype, Oid ownerId, Oid nsp_oid) defaclobjtype = DEFACLOBJ_TYPE; break; + case ACL_OBJECT_NAMESPACE: + defaclobjtype = DEFACLOBJ_NAMESPACE; + break; + default: return NULL; } diff --git a/src/backend/catalog/objectaddress.c b/src/backend/catalog/objectaddress.c index 2948d64fa7..1eb7930901 100644 --- a/src/backend/catalog/objectaddress.c +++ b/src/backend/catalog/objectaddress.c @@ -1843,11 +1843,14 @@ get_object_address_defacl(List *object, bool missing_ok) case DEFACLOBJ_TYPE: objtype_str = "types"; break; + case DEFACLOBJ_NAMESPACE: + objtype_str = "schemas"; + break; default: ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("unrecognized default ACL object type %c", objtype), - errhint("Valid object types are \"r\", \"S\", \"f\", and \"T\"."))); + errhint("Valid object types are \"r\", \"S\", \"f\", \"T\" and \"s\"."))); } /* @@ -3255,6 +3258,11 @@ getObjectDescription(const ObjectAddress *object) _("default privileges on new types belonging to role %s"), GetUserNameFromId(defacl->defaclrole, false)); break; + case DEFACLOBJ_NAMESPACE: + appendStringInfo(&buffer, + _("default privileges on new schemas belonging to role %s"), + GetUserNameFromId(defacl->defaclrole, false)); + break; default: /* shouldn't get here */ appendStringInfo(&buffer, @@ -4762,6 +4770,10 @@ getObjectIdentityParts(const ObjectAddress *object, appendStringInfoString(&buffer, " on types"); break; + case DEFACLOBJ_NAMESPACE: + appendStringInfoString(&buffer, + " on schemas"); + break; } if (objname) diff --git a/src/backend/catalog/pg_namespace.c b/src/backend/catalog/pg_namespace.c index 5672536d31..613b963683 100644 --- a/src/backend/catalog/pg_namespace.c +++ b/src/backend/catalog/pg_namespace.c @@ -31,10 +31,11 @@ * Create a namespace (schema) with the given name and owner OID. * * If isTemp is true, this schema is a per-backend schema for holding - * temporary tables. Currently, the only effect of that is to prevent it - * from being linked as a member of any active extension. (If someone - * does CREATE TEMP TABLE in an extension script, we don't want the temp - * schema to become part of the extension.) + * temporary tables. Currently, it is used to prevent it from being + * linked as a member of any active extension. (If someone does CREATE + * TEMP TABLE in an extension script, we don't want the temp schema to + * become part of the extension). And to avoid checking for default ACL + * for temp namespace (as it is not necessary). * --------------- */ Oid @@ -49,6 +50,7 @@ NamespaceCreate(const char *nspName, Oid ownerId, bool isTemp) TupleDesc tupDesc; ObjectAddress myself; int i; + Acl *nspacl; /* sanity checks */ if (!nspName) @@ -60,6 +62,12 @@ NamespaceCreate(const char *nspName, Oid ownerId, bool isTemp) (errcode(ERRCODE_DUPLICATE_SCHEMA), errmsg("schema \"%s\" already exists", nspName))); + if (!isTemp) + nspacl = get_user_default_acl(ACL_OBJECT_NAMESPACE, ownerId, + InvalidOid); + else + nspacl = NULL; + /* initialize nulls and values */ for (i = 0; i < Natts_pg_namespace; i++) { @@ -69,7 +77,10 @@ NamespaceCreate(const char *nspName, Oid ownerId, bool isTemp) namestrcpy(&nname, nspName); values[Anum_pg_namespace_nspname - 1] = NameGetDatum(&nname); values[Anum_pg_namespace_nspowner - 1] = ObjectIdGetDatum(ownerId); - nulls[Anum_pg_namespace_nspacl - 1] = true; + if (nspacl != NULL) + values[Anum_pg_namespace_nspacl - 1] = PointerGetDatum(nspacl); + else + nulls[Anum_pg_namespace_nspacl - 1] = true; nspdesc = heap_open(NamespaceRelationId, RowExclusiveLock); tupDesc = nspdesc->rd_att; diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 19dd77d787..20865c0ee0 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -668,7 +668,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); RESET RESTART RESTRICT RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP ROW ROWS RULE - SAVEPOINT SCHEMA SCROLL SEARCH SECOND_P SECURITY SELECT SEQUENCE SEQUENCES + SAVEPOINT SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SELECT SEQUENCE SEQUENCES SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF SHARE SHOW SIMILAR SIMPLE SKIP SLOT SMALLINT SNAPSHOT SOME SQL_P STABLE STANDALONE_P START STATEMENT STATISTICS STDIN STDOUT STORAGE STRICT_P STRIP_P @@ -7035,6 +7035,7 @@ defacl_privilege_target: | FUNCTIONS { $$ = ACL_OBJECT_FUNCTION; } | SEQUENCES { $$ = ACL_OBJECT_SEQUENCE; } | TYPES_P { $$ = ACL_OBJECT_TYPE; } + | SCHEMAS { $$ = ACL_OBJECT_NAMESPACE; } ; @@ -14713,6 +14714,7 @@ unreserved_keyword: | RULE | SAVEPOINT | SCHEMA + | SCHEMAS | SCROLL | SEARCH | SECOND_P diff --git a/src/bin/pg_dump/dumputils.c b/src/bin/pg_dump/dumputils.c index b41f2b9125..c74153acce 100644 --- a/src/bin/pg_dump/dumputils.c +++ b/src/bin/pg_dump/dumputils.c @@ -520,7 +520,9 @@ do { \ CONVERT_PRIV('X', "EXECUTE"); else if (strcmp(type, "LANGUAGE") == 0) CONVERT_PRIV('U', "USAGE"); - else if (strcmp(type, "SCHEMA") == 0) + else if (strcmp(type, "SCHEMA") == 0 || + strcmp(type, "SCHEMAS") == 0 + ) { CONVERT_PRIV('C', "CREATE"); CONVERT_PRIV('U', "USAGE"); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index ba34cc163e..262f5539bc 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -14295,6 +14295,9 @@ dumpDefaultACL(Archive *fout, DefaultACLInfo *daclinfo) case DEFACLOBJ_TYPE: type = "TYPES"; break; + case DEFACLOBJ_NAMESPACE: + type = "SCHEMAS"; + break; default: /* shouldn't get here */ exit_horribly(NULL, diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index bcf675208b..b0f3e5e347 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -1028,7 +1028,7 @@ listDefaultACLs(const char *pattern) printfPQExpBuffer(&buf, "SELECT pg_catalog.pg_get_userbyid(d.defaclrole) AS \"%s\",\n" " n.nspname AS \"%s\",\n" - " CASE d.defaclobjtype WHEN '%c' THEN '%s' WHEN '%c' THEN '%s' WHEN '%c' THEN '%s' WHEN '%c' THEN '%s' END AS \"%s\",\n" + " CASE d.defaclobjtype WHEN '%c' THEN '%s' WHEN '%c' THEN '%s' WHEN '%c' THEN '%s' WHEN '%c' THEN '%s' WHEN '%c' THEN '%s' END AS \"%s\",\n" " ", gettext_noop("Owner"), gettext_noop("Schema"), @@ -1040,6 +1040,8 @@ listDefaultACLs(const char *pattern) gettext_noop("function"), DEFACLOBJ_TYPE, gettext_noop("type"), + DEFACLOBJ_NAMESPACE, + gettext_noop("schema"), gettext_noop("Type")); printACLColumn(&buf, "d.defaclacl"); diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index f7494065de..dc2794d48a 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -2796,7 +2796,7 @@ psql_completion(const char *text, int start, int end) * to the kinds of objects supported. */ if (HeadMatches3("ALTER","DEFAULT","PRIVILEGES")) - COMPLETE_WITH_LIST4("TABLES", "SEQUENCES", "FUNCTIONS", "TYPES"); + COMPLETE_WITH_LIST5("TABLES", "SEQUENCES", "FUNCTIONS", "TYPES", "SCHEMAS"); else COMPLETE_WITH_SCHEMA_QUERY(Query_for_list_of_tsvmf, " UNION SELECT 'ALL FUNCTIONS IN SCHEMA'" diff --git a/src/include/catalog/pg_default_acl.h b/src/include/catalog/pg_default_acl.h index 42fb224f9d..78bbeb64fe 100644 --- a/src/include/catalog/pg_default_acl.h +++ b/src/include/catalog/pg_default_acl.h @@ -70,5 +70,6 @@ typedef FormData_pg_default_acl *Form_pg_default_acl; #define DEFACLOBJ_SEQUENCE 'S' /* sequence */ #define DEFACLOBJ_FUNCTION 'f' /* function */ #define DEFACLOBJ_TYPE 'T' /* type */ +#define DEFACLOBJ_NAMESPACE 'n' /* namespace */ #endif /* PG_DEFAULT_ACL_H */ diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h index 6cd36c7fe3..cd21a789d5 100644 --- a/src/include/parser/kwlist.h +++ b/src/include/parser/kwlist.h @@ -344,6 +344,7 @@ PG_KEYWORD("rows", ROWS, UNRESERVED_KEYWORD) PG_KEYWORD("rule", RULE, UNRESERVED_KEYWORD) PG_KEYWORD("savepoint", SAVEPOINT, UNRESERVED_KEYWORD) PG_KEYWORD("schema", SCHEMA, UNRESERVED_KEYWORD) +PG_KEYWORD("schemas", SCHEMAS, UNRESERVED_KEYWORD) PG_KEYWORD("scroll", SCROLL, UNRESERVED_KEYWORD) PG_KEYWORD("search", SEARCH, UNRESERVED_KEYWORD) PG_KEYWORD("second", SECOND_P, UNRESERVED_KEYWORD) diff --git a/src/test/regress/expected/privileges.out b/src/test/regress/expected/privileges.out index f349980759..c6e7031bef 100644 --- a/src/test/regress/expected/privileges.out +++ b/src/test/regress/expected/privileges.out @@ -1356,6 +1356,64 @@ SELECT has_table_privilege('regress_user1', 'testns.acltest1', 'INSERT'); -- no (1 row) ALTER DEFAULT PRIVILEGES FOR ROLE regress_user1 REVOKE EXECUTE ON FUNCTIONS FROM public; +ALTER DEFAULT PRIVILEGES IN SCHEMA testns GRANT USAGE ON SCHEMAS TO regress_user2; -- error +ERROR: cannot use IN SCHEMA clause when using GRANT/REVOKE ON SCHEMAS +ALTER DEFAULT PRIVILEGES GRANT USAGE ON SCHEMAS TO regress_user2; +CREATE SCHEMA testns2; +SELECT has_schema_privilege('regress_user2', 'testns2', 'USAGE'); -- yes + has_schema_privilege +---------------------- + t +(1 row) + +SELECT has_schema_privilege('regress_user2', 'testns2', 'CREATE'); -- no + has_schema_privilege +---------------------- + f +(1 row) + +ALTER DEFAULT PRIVILEGES REVOKE USAGE ON SCHEMAS FROM regress_user2; +CREATE SCHEMA testns3; +SELECT has_schema_privilege('regress_user2', 'testns3', 'USAGE'); -- no + has_schema_privilege +---------------------- + f +(1 row) + +SELECT has_schema_privilege('regress_user2', 'testns3', 'CREATE'); -- no + has_schema_privilege +---------------------- + f +(1 row) + +ALTER DEFAULT PRIVILEGES GRANT ALL ON SCHEMAS TO regress_user2; +CREATE SCHEMA testns4; +SELECT has_schema_privilege('regress_user2', 'testns4', 'USAGE'); -- yes + has_schema_privilege +---------------------- + t +(1 row) + +SELECT has_schema_privilege('regress_user2', 'testns4', 'CREATE'); -- yes + has_schema_privilege +---------------------- + t +(1 row) + +ALTER DEFAULT PRIVILEGES REVOKE ALL ON SCHEMAS FROM regress_user2; +CREATE SCHEMA testns5; +SELECT has_schema_privilege('regress_user2', 'testns5', 'USAGE'); -- no + has_schema_privilege +---------------------- + f +(1 row) + +SELECT has_schema_privilege('regress_user2', 'testns5', 'CREATE'); -- no + has_schema_privilege +---------------------- + f +(1 row) + SET ROLE regress_user1; CREATE FUNCTION testns.foo() RETURNS int AS 'select 1' LANGUAGE sql; SELECT has_function_privilege('regress_user2', 'testns.foo()', 'EXECUTE'); -- no @@ -1403,6 +1461,10 @@ SELECT count(*) DROP SCHEMA testns CASCADE; NOTICE: drop cascades to table testns.acltest1 +DROP SCHEMA testns2 CASCADE; +DROP SCHEMA testns3 CASCADE; +DROP SCHEMA testns4 CASCADE; +DROP SCHEMA testns5 CASCADE; SELECT d.* -- check that entries went away FROM pg_default_acl d LEFT JOIN pg_namespace n ON defaclnamespace = n.oid WHERE nspname IS NULL AND defaclnamespace != 0; diff --git a/src/test/regress/sql/privileges.sql b/src/test/regress/sql/privileges.sql index 166e903012..38215954da 100644 --- a/src/test/regress/sql/privileges.sql +++ b/src/test/regress/sql/privileges.sql @@ -816,6 +816,36 @@ SELECT has_table_privilege('regress_user1', 'testns.acltest1', 'INSERT'); -- no ALTER DEFAULT PRIVILEGES FOR ROLE regress_user1 REVOKE EXECUTE ON FUNCTIONS FROM public; +ALTER DEFAULT PRIVILEGES IN SCHEMA testns GRANT USAGE ON SCHEMAS TO regress_user2; -- error + +ALTER DEFAULT PRIVILEGES GRANT USAGE ON SCHEMAS TO regress_user2; + +CREATE SCHEMA testns2; + +SELECT has_schema_privilege('regress_user2', 'testns2', 'USAGE'); -- yes +SELECT has_schema_privilege('regress_user2', 'testns2', 'CREATE'); -- no + +ALTER DEFAULT PRIVILEGES REVOKE USAGE ON SCHEMAS FROM regress_user2; + +CREATE SCHEMA testns3; + +SELECT has_schema_privilege('regress_user2', 'testns3', 'USAGE'); -- no +SELECT has_schema_privilege('regress_user2', 'testns3', 'CREATE'); -- no + +ALTER DEFAULT PRIVILEGES GRANT ALL ON SCHEMAS TO regress_user2; + +CREATE SCHEMA testns4; + +SELECT has_schema_privilege('regress_user2', 'testns4', 'USAGE'); -- yes +SELECT has_schema_privilege('regress_user2', 'testns4', 'CREATE'); -- yes + +ALTER DEFAULT PRIVILEGES REVOKE ALL ON SCHEMAS FROM regress_user2; + +CREATE SCHEMA testns5; + +SELECT has_schema_privilege('regress_user2', 'testns5', 'USAGE'); -- no +SELECT has_schema_privilege('regress_user2', 'testns5', 'CREATE'); -- no + SET ROLE regress_user1; CREATE FUNCTION testns.foo() RETURNS int AS 'select 1' LANGUAGE sql; @@ -853,6 +883,10 @@ SELECT count(*) WHERE nspname = 'testns'; DROP SCHEMA testns CASCADE; +DROP SCHEMA testns2 CASCADE; +DROP SCHEMA testns3 CASCADE; +DROP SCHEMA testns4 CASCADE; +DROP SCHEMA testns5 CASCADE; SELECT d.* -- check that entries went away FROM pg_default_acl d LEFT JOIN pg_namespace n ON defaclnamespace = n.oid -- cgit v1.2.3 From 66b764341ba12206f01e2600713bdc3abdb070b3 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 28 Mar 2017 21:12:30 -0400 Subject: doc: Mention --enable-tap-tests in regression test chapter Reported-by: Jeff Janes --- doc/src/sgml/regress.sgml | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'doc/src') diff --git a/doc/src/sgml/regress.sgml b/doc/src/sgml/regress.sgml index b9d1af8707..8229528bd4 100644 --- a/doc/src/sgml/regress.sgml +++ b/doc/src/sgml/regress.sgml @@ -205,6 +205,12 @@ make installcheck-world ecpg1_regression, or ecpg2_regression, as well as regression. + + + The TAP tests are only run when PostgreSQL was configured with the + option . This is recommended for + development, but can be omitted if there is no suitable Perl installation. + -- cgit v1.2.3 From 5ded4bd21403e143dd3eb66b92d52732fdac1945 Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Wed, 29 Mar 2017 13:16:49 -0700 Subject: Remove support for version-0 calling conventions. The V0 convention is failure prone because we've so far assumed that a function is V0 if PG_FUNCTION_INFO_V1 is missing, leading to crashes if a function was coded against the V1 interface. V0 doesn't allow proper NULL, SRF and toast handling. V0 doesn't offer features that V1 doesn't. Thus remove V0 support and obsolete fmgr README contents relating to it. Author: Andres Freund, with contributions by Peter Eisentraut & Craig Ringer Reviewed-By: Peter Eisentraut, Craig Ringer Discussion: https://postgr.es/m/20161208213441.k3mbno4twhg2qf7g@alap3.anarazel.de --- contrib/earthdistance/earthdistance.c | 22 -- doc/src/sgml/xfunc.sgml | 269 ++++------------ src/backend/utils/fmgr/README | 254 +-------------- src/backend/utils/fmgr/fmgr.c | 377 +---------------------- src/include/fmgr.h | 8 +- src/test/regress/input/create_function_2.source | 5 - src/test/regress/input/misc.source | 13 - src/test/regress/output/create_function_2.source | 4 - src/test/regress/output/misc.source | 18 -- src/test/regress/regress.c | 47 ++- 10 files changed, 106 insertions(+), 911 deletions(-) (limited to 'doc/src') diff --git a/contrib/earthdistance/earthdistance.c b/contrib/earthdistance/earthdistance.c index 861b166373..6ad6d87ce8 100644 --- a/contrib/earthdistance/earthdistance.c +++ b/contrib/earthdistance/earthdistance.c @@ -88,16 +88,8 @@ geo_distance_internal(Point *pt1, Point *pt2) * * returns: float8 * distance between the points in miles on earth's surface - * - * If float8 is passed-by-value, the oldstyle version-0 calling convention - * is unportable, so we use version-1. However, if it's passed-by-reference, - * continue to use oldstyle. This is just because we'd like earthdistance - * to serve as a canary for any unintentional breakage of version-0 functions - * with float8 results. ******************************************************/ -#ifdef USE_FLOAT8_BYVAL - PG_FUNCTION_INFO_V1(geo_distance); Datum @@ -110,17 +102,3 @@ geo_distance(PG_FUNCTION_ARGS) result = geo_distance_internal(pt1, pt2); PG_RETURN_FLOAT8(result); } -#else /* !USE_FLOAT8_BYVAL */ - -double *geo_distance(Point *pt1, Point *pt2); - -double * -geo_distance(Point *pt1, Point *pt2) -{ - double *resultp = palloc(sizeof(double)); - - *resultp = geo_distance_internal(pt1, pt2); - return resultp; -} - -#endif /* USE_FLOAT8_BYVAL */ diff --git a/doc/src/sgml/xfunc.sgml b/doc/src/sgml/xfunc.sgml index 94a7ad747d..e6313ddd59 100644 --- a/doc/src/sgml/xfunc.sgml +++ b/doc/src/sgml/xfunc.sgml @@ -1610,14 +1610,10 @@ CREATE FUNCTION square_root(double precision) RETURNS double precision - Two different calling conventions are currently used for C functions. - The newer version 1 calling convention is indicated by writing - a PG_FUNCTION_INFO_V1() macro call for the function, - as illustrated below. Lack of such a macro indicates an old-style - (version 0) function. The language name specified in CREATE FUNCTION - is C in either case. Old-style functions are now deprecated - because of portability problems and lack of functionality, but they - are still supported for compatibility reasons. + Currently only one calling convention is used for C functions + (version 1). Support for that calling convention is + indicated by writing a PG_FUNCTION_INFO_V1() macro + call for the function, as illustrated below. @@ -2137,160 +2133,6 @@ memcpy(destination->data, buffer, 40); - - Version 0 Calling Conventions - - - We present the old style calling convention first — although - this approach is now deprecated, it's easier to get a handle on - initially. In the version-0 method, the arguments and result - of the C function are just declared in normal C style, but being - careful to use the C representation of each SQL data type as shown - above. - - - - Here are some examples: - - -#include "utils/geo_decls.h" - -#ifdef PG_MODULE_MAGIC -PG_MODULE_MAGIC; -#endif - -/* by value */ - -int -add_one(int arg) -{ - return arg + 1; -} - -/* by reference, fixed length */ - -float8 * -add_one_float8(float8 *arg) -{ - float8 *result = (float8 *) palloc(sizeof(float8)); - - *result = *arg + 1.0; - - return result; -} - -Point * -makepoint(Point *pointx, Point *pointy) -{ - Point *new_point = (Point *) palloc(sizeof(Point)); - - new_point->x = pointx->x; - new_point->y = pointy->y; - - return new_point; -} - -/* by reference, variable length */ - -text * -copytext(text *t) -{ - /* - * VARSIZE is the total size of the struct in bytes. - */ - text *new_t = (text *) palloc(VARSIZE(t)); - SET_VARSIZE(new_t, VARSIZE(t)); - /* - * VARDATA is a pointer to the data region of the struct. - */ - memcpy((void *) VARDATA(new_t), /* destination */ - (void *) VARDATA(t), /* source */ - VARSIZE(t) - VARHDRSZ); /* how many bytes */ - return new_t; -} - -text * -concat_text(text *arg1, text *arg2) -{ - int32 new_text_size = VARSIZE(arg1) + VARSIZE(arg2) - VARHDRSZ; - text *new_text = (text *) palloc(new_text_size); - - SET_VARSIZE(new_text, new_text_size); - memcpy(VARDATA(new_text), VARDATA(arg1), VARSIZE(arg1) - VARHDRSZ); - memcpy(VARDATA(new_text) + (VARSIZE(arg1) - VARHDRSZ), - VARDATA(arg2), VARSIZE(arg2) - VARHDRSZ); - return new_text; -} -]]> - - - - - Supposing that the above code has been prepared in file - funcs.c and compiled into a shared object, - we could define the functions to PostgreSQL - with commands like this: - - -CREATE FUNCTION add_one(integer) RETURNS integer - AS 'DIRECTORY/funcs', 'add_one' - LANGUAGE C STRICT; - --- note overloading of SQL function name "add_one" -CREATE FUNCTION add_one(double precision) RETURNS double precision - AS 'DIRECTORY/funcs', 'add_one_float8' - LANGUAGE C STRICT; - -CREATE FUNCTION makepoint(point, point) RETURNS point - AS 'DIRECTORY/funcs', 'makepoint' - LANGUAGE C STRICT; - -CREATE FUNCTION copytext(text) RETURNS text - AS 'DIRECTORY/funcs', 'copytext' - LANGUAGE C STRICT; - -CREATE FUNCTION concat_text(text, text) RETURNS text - AS 'DIRECTORY/funcs', 'concat_text' - LANGUAGE C STRICT; - - - - - Here, DIRECTORY stands for the - directory of the shared library file (for instance the - PostgreSQL tutorial directory, which - contains the code for the examples used in this section). - (Better style would be to use just 'funcs' in the - AS clause, after having added - DIRECTORY to the search path. In any - case, we can omit the system-specific extension for a shared - library, commonly .so or - .sl.) - - - - Notice that we have specified the functions as strict, - meaning that - the system should automatically assume a null result if any input - value is null. By doing this, we avoid having to check for null inputs - in the function code. Without this, we'd have to check for null values - explicitly, by checking for a null pointer for each - pass-by-reference argument. (For pass-by-value arguments, we don't - even have a way to check!) - - - - Although this calling convention is simple to use, - it is not very portable; on some architectures there are problems - with passing data types that are smaller than int this way. Also, there is - no simple way to return a null result, nor to cope with null arguments - in any way other than making the function strict. The version-1 - convention, presented next, overcomes these objections. - - - Version 1 Calling Conventions @@ -2316,8 +2158,10 @@ PG_FUNCTION_INFO_V1(funcname); In a version-1 function, each actual argument is fetched using a PG_GETARG_xxx() - macro that corresponds to the argument's data type, and the - result is returned using a + macro that corresponds to the argument's data type. In non-strict + functions there needs to be a previous check about argument null-ness + using PG_ARGNULL_xxx(). + The result is returned using a PG_RETURN_xxx() macro for the return type. PG_GETARG_xxx() @@ -2328,7 +2172,7 @@ PG_FUNCTION_INFO_V1(funcname); - Here we show the same functions as above, coded in version-1 style: + Here are some examples using the version-1 calling convention: + + + Supposing that the above code has been prepared in file + funcs.c and compiled into a shared object, + we could define the functions to PostgreSQL + with commands like this: + + +CREATE FUNCTION add_one(integer) RETURNS integer + AS 'DIRECTORY/funcs', 'add_one' + LANGUAGE C STRICT; + +-- note overloading of SQL function name "add_one" +CREATE FUNCTION add_one(double precision) RETURNS double precision + AS 'DIRECTORY/funcs', 'add_one_float8' + LANGUAGE C STRICT; + +CREATE FUNCTION makepoint(point, point) RETURNS point + AS 'DIRECTORY/funcs', 'makepoint' + LANGUAGE C STRICT; + +CREATE FUNCTION copytext(text) RETURNS text + AS 'DIRECTORY/funcs', 'copytext' + LANGUAGE C STRICT; + +CREATE FUNCTION concat_text(text, text) RETURNS text + AS 'DIRECTORY/funcs', 'concat_text' + LANGUAGE C STRICT; + + + + Here, DIRECTORY stands for the + directory of the shared library file (for instance the + PostgreSQL tutorial directory, which + contains the code for the examples used in this section). + (Better style would be to use just 'funcs' in the + AS clause, after having added + DIRECTORY to the search path. In any + case, we can omit the system-specific extension for a shared + library, commonly .so.) - The CREATE FUNCTION commands are the same as - for the version-0 equivalents. + Notice that we have specified the functions as strict, + meaning that + the system should automatically assume a null result if any input + value is null. By doing this, we avoid having to check for null inputs + in the function code. Without this, we'd have to check for null values + explicitly, using PG_ARGISNULL(). - At first glance, the version-1 coding conventions might appear to - be just pointless obscurantism. They do, however, offer a number - of improvements, because the macros can hide unnecessary detail. - An example is that in coding add_one_float8, we no longer need to - be aware that float8 is a pass-by-reference type. Another - example is that the GETARG macros for variable-length types allow - for more efficient fetching of toasted (compressed or + At first glance, the version-1 coding conventions might appear to be just + pointless obscurantism, over using plain C calling + conventions. They do however allow to deal with NULLable + arguments/return values, and toasted (compressed or out-of-line) values. - One big improvement in version-1 functions is better handling of null - inputs and results. The macro PG_ARGISNULL(n) + The macro PG_ARGISNULL(n) allows a function to test whether each input is null. (Of course, doing this is only necessary in functions not declared strict.) As with the @@ -2461,7 +2345,7 @@ concat_text(PG_FUNCTION_ARGS) - Other options provided in the new-style interface are two + Other options provided by the version-1 interface are two variants of the PG_GETARG_xxx() macros. The first of these, @@ -2493,9 +2377,7 @@ concat_text(PG_FUNCTION_ARGS) to return set results () and implement trigger functions () and procedural-language call handlers (). Version-1 code is also more - portable than version-0, because it does not break restrictions - on function call protocol in the C standard. For more details + linkend="plhandler">). For more details see src/backend/utils/fmgr/README in the source distribution. @@ -2630,7 +2512,7 @@ SELECT name, c_overpaid(emp, 1500) AS overpaid WHERE name = 'Bill' OR name = 'Sam'; - Using call conventions version 0, we can define + Using the version-1 calling conventions, we can define c_overpaid as: limit; -} -]]> - - - In version-1 coding, the above would look like this: - -arg[0]) + - DatumGetInt32(fcinfo->arg[1])); -} - -This is, of course, much uglier than the old-style code, but we can -improve matters with some well-chosen macros for the boilerplate parts. -I propose below macros that would make the code look like - -Datum -int4pl(PG_FUNCTION_ARGS) -{ - int32 arg1 = PG_GETARG_INT32(0); - int32 arg2 = PG_GETARG_INT32(1); - - PG_RETURN_INT32( arg1 + arg2 ); -} - -This is still more code than before, but it's fairly readable, and it's -also amenable to machine processing --- for example, we could probably -write a script that scans code like this and extracts argument and result -type info for comparison to the pg_proc table. - -For the standard data types float4, float8, and int8, these macros should hide -whether the types are pass-by-value or pass-by reference, by incorporating -indirection and space allocation if needed. This will offer a considerable -gain in readability, and it also opens up the opportunity to make these types -be pass-by-value on machines where it's feasible to do so. - Here are the proposed macros and coding conventions: The definition of an fmgr-callable function will always look like @@ -291,67 +204,6 @@ fields of FunctionCallInfo, it should just do it. I doubt that providing syntactic-sugar macros for these cases is useful. -Call-Site Coding Conventions ----------------------------- - -There are many places in the system that call either a specific function -(for example, the parser invokes "textin" by name in places) or a -particular group of functions that have a common argument list (for -example, the optimizer invokes selectivity estimation functions with -a fixed argument list). These places will need to change, but we should -try to avoid making them significantly uglier than before. - -Places that invoke an arbitrary function with an arbitrary argument list -can simply be changed to fill a FunctionCallInfoData structure directly; -that'll be no worse and possibly cleaner than what they do now. - -When invoking a specific built-in function by name, we have generally -just written something like - result = textin ( ... args ... ) -which will not work after textin() is converted to the new call style. -I suggest that code like this be converted to use "helper" functions -that will create and fill in a FunctionCallInfoData struct. For -example, if textin is being called with one argument, it'd look -something like - result = DirectFunctionCall1(textin, PointerGetDatum(argument)); -These helper routines will have declarations like - Datum DirectFunctionCall2(PGFunction func, Datum arg1, Datum arg2); -Note it will be the caller's responsibility to convert to and from -Datum; appropriate conversion macros should be used. - -The DirectFunctionCallN routines will not bother to fill in -fcinfo->flinfo (indeed cannot, since they have no idea about an OID for -the target function); they will just set it NULL. This is unlikely to -bother any built-in function that could be called this way. Note also -that this style of coding cannot pass a NULL input value nor cope with -a NULL result (it couldn't before, either!). We can make the helper -routines ereport an error if they see that the function returns a NULL. - -When invoking a function that has a known argument signature, we have -usually written either - result = fmgr(targetfuncOid, ... args ... ); -or - result = fmgr_ptr(FmgrInfo *finfo, ... args ... ); -depending on whether an FmgrInfo lookup has been done yet or not. -This kind of code can be recast using helper routines, in the same -style as above: - result = OidFunctionCall1(funcOid, PointerGetDatum(argument)); - result = FunctionCall2(funcCallInfo, - PointerGetDatum(argument), - Int32GetDatum(argument)); -Again, this style of coding does not allow for expressing NULL inputs -or receiving a NULL result. - -As with the callee-side situation, I propose adding argument conversion -macros that hide whether int8, float4, and float8 are pass-by-value or -pass-by-reference. - -The existing helper functions fmgr(), fmgr_c(), etc will be left in -place until all uses of them are gone. Of course their internals will -have to change in the first step of implementation, but they can -continue to support the same external appearance. - - Support for TOAST-Able Data Types --------------------------------- @@ -474,83 +326,3 @@ context. fn_mcxt normally points at the context that was CurrentMemoryContext at the time the FmgrInfo structure was created; in any case it is required to be a context at least as long-lived as the FmgrInfo itself. - - -Telling the Difference Between Old- and New-Style Functions ------------------------------------------------------------ - -During the conversion process, we carried two different pg_language -entries, "internal" and "newinternal", for internal functions. The -function manager used the language code to distinguish which calling -convention to use. (Old-style internal functions were supported via -a function handler.) As of Nov. 2000, no old-style internal functions -remain, so we can drop support for them. We will remove the old "internal" -pg_language entry and rename "newinternal" to "internal". - -The interim solution for dynamically-loaded compiled functions has been -similar: two pg_language entries "C" and "newC". This naming convention -is not desirable for the long run, and yet we cannot stop supporting -old-style user functions. Instead, it seems better to use just one -pg_language entry "C", and require the dynamically-loaded library to -provide additional information that identifies new-style functions. -This avoids compatibility problems --- for example, existing dump -scripts will identify PL language handlers as being in language "C", -which would be wrong under the "newC" convention. Also, this approach -should generalize more conveniently for future extensions to the function -interface specification. - -Given a dynamically loaded function named "foo" (note that the name being -considered here is the link-symbol name, not the SQL-level function name), -the function manager will look for another function in the same dynamically -loaded library named "pg_finfo_foo". If this second function does not -exist, then foo is assumed to be called old-style, thus ensuring backwards -compatibility with existing libraries. If the info function does exist, -it is expected to have the signature - - Pg_finfo_record * pg_finfo_foo (void); - -The info function will be called by the fmgr, and must return a pointer -to a Pg_finfo_record struct. (The returned struct will typically be a -statically allocated constant in the dynamic-link library.) The current -definition of the struct is just - - typedef struct { - int api_version; - } Pg_finfo_record; - -where api_version is 0 to indicate old-style or 1 to indicate new-style -calling convention. In future releases, additional fields may be defined -after api_version, but these additional fields will only be used if -api_version is greater than 1. - -These details will be hidden from the author of a dynamically loaded -function by using a macro. To define a new-style dynamically loaded -function named foo, write - - PG_FUNCTION_INFO_V1(foo); - - Datum - foo(PG_FUNCTION_ARGS) - { - ... - } - -The function itself is written using the same conventions as for new-style -internal functions; you just need to add the PG_FUNCTION_INFO_V1() macro. -Note that old-style and new-style functions can be intermixed in the same -library, depending on whether or not you write a PG_FUNCTION_INFO_V1() for -each one. - -The SQL declaration for a dynamically-loaded function is CREATE FUNCTION -foo ... LANGUAGE C regardless of whether it is old- or new-style. - -New-style dynamic functions will be invoked directly by fmgr, and will -therefore have the same performance as internal functions after the initial -pg_proc lookup overhead. Old-style dynamic functions will be invoked via -a handler, and will therefore have a small performance penalty. - -To allow old-style dynamic functions to work safely on toastable datatypes, -the handler for old-style functions will automatically detoast toastable -arguments before passing them to the old-style function. A new-style -function is expected to take care of toasted arguments by using the -standard argument access macros defined above. diff --git a/src/backend/utils/fmgr/fmgr.c b/src/backend/utils/fmgr/fmgr.c index 9fb695279b..68d2110890 100644 --- a/src/backend/utils/fmgr/fmgr.c +++ b/src/backend/utils/fmgr/fmgr.c @@ -36,37 +36,6 @@ PGDLLIMPORT needs_fmgr_hook_type needs_fmgr_hook = NULL; PGDLLIMPORT fmgr_hook_type fmgr_hook = NULL; -/* - * Declaration for old-style function pointer type. This is now used only - * in fmgr_oldstyle() and is no longer exported. - * - * The m68k SVR4 ABI defines that pointers are returned in %a0 instead of - * %d0. So if a function pointer is declared to return a pointer, the - * compiler may look only into %a0, but if the called function was declared - * to return an integer type, it puts its value only into %d0. So the - * caller doesn't pick up the correct return value. The solution is to - * declare the function pointer to return int, so the compiler picks up the - * return value from %d0. (Functions returning pointers put their value - * *additionally* into %d0 for compatibility.) The price is that there are - * some warnings about int->pointer conversions ... which we can suppress - * with suitably ugly casts in fmgr_oldstyle(). - */ -#if (defined(__mc68000__) || (defined(__m68k__))) && defined(__ELF__) -typedef int32 (*func_ptr) (); -#else -typedef char *(*func_ptr) (); -#endif - -/* - * For an oldstyle function, fn_extra points to a record like this: - */ -typedef struct -{ - func_ptr func; /* Address of the oldstyle function */ - bool arg_toastable[FUNC_MAX_ARGS]; /* is n'th arg of a toastable - * datatype? */ -} Oldstyle_fnextra; - /* * Hashtable for fast lookup of external C functions */ @@ -90,7 +59,6 @@ static void fmgr_info_other_lang(Oid functionId, FmgrInfo *finfo, HeapTuple proc static CFuncHashTabEntry *lookup_C_func(HeapTuple procedureTuple); static void record_C_func(HeapTuple procedureTuple, PGFunction user_fn, const Pg_finfo_record *inforec); -static Datum fmgr_oldstyle(PG_FUNCTION_ARGS); static Datum fmgr_security_definer(PG_FUNCTION_ARGS); @@ -304,13 +272,10 @@ fmgr_info_cxt_security(Oid functionId, FmgrInfo *finfo, MemoryContext mcxt, static void fmgr_info_C_lang(Oid functionId, FmgrInfo *finfo, HeapTuple procedureTuple) { - Form_pg_proc procedureStruct = (Form_pg_proc) GETSTRUCT(procedureTuple); CFuncHashTabEntry *hashentry; PGFunction user_fn; const Pg_finfo_record *inforec; - Oldstyle_fnextra *fnextra; bool isnull; - int i; /* * See if we have the function address cached already @@ -362,20 +327,6 @@ fmgr_info_C_lang(Oid functionId, FmgrInfo *finfo, HeapTuple procedureTuple) switch (inforec->api_version) { - case 0: - /* Old style: need to use a handler */ - finfo->fn_addr = fmgr_oldstyle; - fnextra = (Oldstyle_fnextra *) - MemoryContextAllocZero(finfo->fn_mcxt, - sizeof(Oldstyle_fnextra)); - finfo->fn_extra = (void *) fnextra; - fnextra->func = (func_ptr) user_fn; - for (i = 0; i < procedureStruct->pronargs; i++) - { - fnextra->arg_toastable[i] = - TypeIsToastable(procedureStruct->proargtypes.values[i]); - } - break; case 1: /* New style: call directly */ finfo->fn_addr = user_fn; @@ -415,14 +366,6 @@ fmgr_info_other_lang(Oid functionId, FmgrInfo *finfo, HeapTuple procedureTuple) CurrentMemoryContext, true); finfo->fn_addr = plfinfo.fn_addr; - /* - * If lookup of the PL handler function produced nonnull fn_extra, - * complain --- it must be an oldstyle function! We no longer support - * oldstyle PL handlers. - */ - if (plfinfo.fn_extra != NULL) - elog(ERROR, "language %u has old-style handler", language); - ReleaseSysCache(languageTuple); } @@ -431,10 +374,7 @@ fmgr_info_other_lang(Oid functionId, FmgrInfo *finfo, HeapTuple procedureTuple) * The function is specified by a handle for the containing library * (obtained from load_external_function) as well as the function name. * - * If no info function exists for the given name, it is not an error. - * Instead we return a default info record for a version-0 function. - * We want to raise an error here only if the info function returns - * something bogus. + * If no info function exists for the given name an error is raised. * * This function is broken out of fmgr_info_C_lang so that fmgr_c_validator * can validate the information record for a function not yet entered into @@ -446,7 +386,6 @@ fetch_finfo_record(void *filehandle, char *funcname) char *infofuncname; PGFInfoFunction infofunc; const Pg_finfo_record *inforec; - static Pg_finfo_record default_inforec = {0}; infofuncname = psprintf("pg_finfo_%s", funcname); @@ -455,9 +394,12 @@ fetch_finfo_record(void *filehandle, char *funcname) infofuncname); if (infofunc == NULL) { - /* Not found --- assume version 0 */ - pfree(infofuncname); - return &default_inforec; + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FUNCTION), + errmsg("could not find function information for function \"%s\"", + funcname), + errhint("SQL-callable functions need an accompanying PG_FUNCTION_INFO_V1(funcname)."))); + return NULL; /* silence compiler */ } /* Found, so call it */ @@ -468,7 +410,6 @@ fetch_finfo_record(void *filehandle, char *funcname) elog(ERROR, "null result from info function \"%s\"", infofuncname); switch (inforec->api_version) { - case 0: case 1: /* OK, no additional fields to validate */ break; @@ -585,18 +526,7 @@ fmgr_info_copy(FmgrInfo *dstinfo, FmgrInfo *srcinfo, { memcpy(dstinfo, srcinfo, sizeof(FmgrInfo)); dstinfo->fn_mcxt = destcxt; - if (dstinfo->fn_addr == fmgr_oldstyle) - { - /* For oldstyle functions we must copy fn_extra */ - Oldstyle_fnextra *fnextra; - - fnextra = (Oldstyle_fnextra *) - MemoryContextAlloc(destcxt, sizeof(Oldstyle_fnextra)); - memcpy(fnextra, srcinfo->fn_extra, sizeof(Oldstyle_fnextra)); - dstinfo->fn_extra = (void *) fnextra; - } - else - dstinfo->fn_extra = NULL; + dstinfo->fn_extra = NULL; } @@ -616,245 +546,6 @@ fmgr_internal_function(const char *proname) } -/* - * Handler for old-style "C" language functions - */ -static Datum -fmgr_oldstyle(PG_FUNCTION_ARGS) -{ - Oldstyle_fnextra *fnextra; - int n_arguments = fcinfo->nargs; - int i; - bool isnull; - func_ptr user_fn; - char *returnValue; - - if (fcinfo->flinfo == NULL || fcinfo->flinfo->fn_extra == NULL) - elog(ERROR, "fmgr_oldstyle received NULL pointer"); - fnextra = (Oldstyle_fnextra *) fcinfo->flinfo->fn_extra; - - /* - * Result is NULL if any argument is NULL, but we still call the function - * (peculiar, but that's the way it worked before, and after all this is a - * backwards-compatibility wrapper). Note, however, that we'll never get - * here with NULL arguments if the function is marked strict. - * - * We also need to detoast any TOAST-ed inputs, since it's unlikely that - * an old-style function knows about TOASTing. - */ - isnull = false; - for (i = 0; i < n_arguments; i++) - { - if (PG_ARGISNULL(i)) - isnull = true; - else if (fnextra->arg_toastable[i]) - fcinfo->arg[i] = PointerGetDatum(PG_DETOAST_DATUM(fcinfo->arg[i])); - } - fcinfo->isnull = isnull; - - user_fn = fnextra->func; - - switch (n_arguments) - { - case 0: - returnValue = (char *) (*user_fn) (); - break; - case 1: - - /* - * nullvalue() used to use isNull to check if arg is NULL; perhaps - * there are other functions still out there that also rely on - * this undocumented hack? - */ - returnValue = (char *) (*user_fn) (fcinfo->arg[0], - &fcinfo->isnull); - break; - case 2: - returnValue = (char *) (*user_fn) (fcinfo->arg[0], - fcinfo->arg[1]); - break; - case 3: - returnValue = (char *) (*user_fn) (fcinfo->arg[0], - fcinfo->arg[1], - fcinfo->arg[2]); - break; - case 4: - returnValue = (char *) (*user_fn) (fcinfo->arg[0], - fcinfo->arg[1], - fcinfo->arg[2], - fcinfo->arg[3]); - break; - case 5: - returnValue = (char *) (*user_fn) (fcinfo->arg[0], - fcinfo->arg[1], - fcinfo->arg[2], - fcinfo->arg[3], - fcinfo->arg[4]); - break; - case 6: - returnValue = (char *) (*user_fn) (fcinfo->arg[0], - fcinfo->arg[1], - fcinfo->arg[2], - fcinfo->arg[3], - fcinfo->arg[4], - fcinfo->arg[5]); - break; - case 7: - returnValue = (char *) (*user_fn) (fcinfo->arg[0], - fcinfo->arg[1], - fcinfo->arg[2], - fcinfo->arg[3], - fcinfo->arg[4], - fcinfo->arg[5], - fcinfo->arg[6]); - break; - case 8: - returnValue = (char *) (*user_fn) (fcinfo->arg[0], - fcinfo->arg[1], - fcinfo->arg[2], - fcinfo->arg[3], - fcinfo->arg[4], - fcinfo->arg[5], - fcinfo->arg[6], - fcinfo->arg[7]); - break; - case 9: - returnValue = (char *) (*user_fn) (fcinfo->arg[0], - fcinfo->arg[1], - fcinfo->arg[2], - fcinfo->arg[3], - fcinfo->arg[4], - fcinfo->arg[5], - fcinfo->arg[6], - fcinfo->arg[7], - fcinfo->arg[8]); - break; - case 10: - returnValue = (char *) (*user_fn) (fcinfo->arg[0], - fcinfo->arg[1], - fcinfo->arg[2], - fcinfo->arg[3], - fcinfo->arg[4], - fcinfo->arg[5], - fcinfo->arg[6], - fcinfo->arg[7], - fcinfo->arg[8], - fcinfo->arg[9]); - break; - case 11: - returnValue = (char *) (*user_fn) (fcinfo->arg[0], - fcinfo->arg[1], - fcinfo->arg[2], - fcinfo->arg[3], - fcinfo->arg[4], - fcinfo->arg[5], - fcinfo->arg[6], - fcinfo->arg[7], - fcinfo->arg[8], - fcinfo->arg[9], - fcinfo->arg[10]); - break; - case 12: - returnValue = (char *) (*user_fn) (fcinfo->arg[0], - fcinfo->arg[1], - fcinfo->arg[2], - fcinfo->arg[3], - fcinfo->arg[4], - fcinfo->arg[5], - fcinfo->arg[6], - fcinfo->arg[7], - fcinfo->arg[8], - fcinfo->arg[9], - fcinfo->arg[10], - fcinfo->arg[11]); - break; - case 13: - returnValue = (char *) (*user_fn) (fcinfo->arg[0], - fcinfo->arg[1], - fcinfo->arg[2], - fcinfo->arg[3], - fcinfo->arg[4], - fcinfo->arg[5], - fcinfo->arg[6], - fcinfo->arg[7], - fcinfo->arg[8], - fcinfo->arg[9], - fcinfo->arg[10], - fcinfo->arg[11], - fcinfo->arg[12]); - break; - case 14: - returnValue = (char *) (*user_fn) (fcinfo->arg[0], - fcinfo->arg[1], - fcinfo->arg[2], - fcinfo->arg[3], - fcinfo->arg[4], - fcinfo->arg[5], - fcinfo->arg[6], - fcinfo->arg[7], - fcinfo->arg[8], - fcinfo->arg[9], - fcinfo->arg[10], - fcinfo->arg[11], - fcinfo->arg[12], - fcinfo->arg[13]); - break; - case 15: - returnValue = (char *) (*user_fn) (fcinfo->arg[0], - fcinfo->arg[1], - fcinfo->arg[2], - fcinfo->arg[3], - fcinfo->arg[4], - fcinfo->arg[5], - fcinfo->arg[6], - fcinfo->arg[7], - fcinfo->arg[8], - fcinfo->arg[9], - fcinfo->arg[10], - fcinfo->arg[11], - fcinfo->arg[12], - fcinfo->arg[13], - fcinfo->arg[14]); - break; - case 16: - returnValue = (char *) (*user_fn) (fcinfo->arg[0], - fcinfo->arg[1], - fcinfo->arg[2], - fcinfo->arg[3], - fcinfo->arg[4], - fcinfo->arg[5], - fcinfo->arg[6], - fcinfo->arg[7], - fcinfo->arg[8], - fcinfo->arg[9], - fcinfo->arg[10], - fcinfo->arg[11], - fcinfo->arg[12], - fcinfo->arg[13], - fcinfo->arg[14], - fcinfo->arg[15]); - break; - default: - - /* - * Increasing FUNC_MAX_ARGS doesn't automatically add cases to the - * above code, so mention the actual value in this error not - * FUNC_MAX_ARGS. You could add cases to the above if you needed - * to support old-style functions with many arguments, but making - * 'em be new-style is probably a better idea. - */ - ereport(ERROR, - (errcode(ERRCODE_TOO_MANY_ARGUMENTS), - errmsg("function %u has too many arguments (%d, maximum is %d)", - fcinfo->flinfo->fn_oid, n_arguments, 16))); - returnValue = NULL; /* keep compiler quiet */ - break; - } - - return PointerGetDatum(returnValue); -} - - /* * Support for security-definer and proconfig-using functions. We support * both of these features using the same call handler, because they are @@ -2081,58 +1772,6 @@ OidSendFunctionCall(Oid functionId, Datum val) } -/* - * !!! OLD INTERFACE !!! - * - * fmgr() is the only remaining vestige of the old-style caller support - * functions. It's no longer used anywhere in the Postgres distribution, - * but we should leave it around for a release or two to ease the transition - * for user-supplied C functions. OidFunctionCallN() replaces it for new - * code. - * - * DEPRECATED, DO NOT USE IN NEW CODE - */ -char * -fmgr(Oid procedureId,...) -{ - FmgrInfo flinfo; - FunctionCallInfoData fcinfo; - int n_arguments; - Datum result; - - fmgr_info(procedureId, &flinfo); - - MemSet(&fcinfo, 0, sizeof(fcinfo)); - fcinfo.flinfo = &flinfo; - fcinfo.nargs = flinfo.fn_nargs; - n_arguments = fcinfo.nargs; - - if (n_arguments > 0) - { - va_list pvar; - int i; - - if (n_arguments > FUNC_MAX_ARGS) - ereport(ERROR, - (errcode(ERRCODE_TOO_MANY_ARGUMENTS), - errmsg("function %u has too many arguments (%d, maximum is %d)", - flinfo.fn_oid, n_arguments, FUNC_MAX_ARGS))); - va_start(pvar, procedureId); - for (i = 0; i < n_arguments; i++) - fcinfo.arg[i] = PointerGetDatum(va_arg(pvar, char *)); - va_end(pvar); - } - - result = FunctionCallInvoke(&fcinfo); - - /* Check for null result, since caller is clearly not expecting one */ - if (fcinfo.isnull) - elog(ERROR, "function %u returned NULL", flinfo.fn_oid); - - return DatumGetPointer(result); -} - - /*------------------------------------------------------------------------- * Support routines for standard maybe-pass-by-reference datatypes * diff --git a/src/include/fmgr.h b/src/include/fmgr.h index 6128752ab1..0c695e246a 100644 --- a/src/include/fmgr.h +++ b/src/include/fmgr.h @@ -336,10 +336,10 @@ extern struct varlena *pg_detoast_datum_packed(struct varlena * datum); /*------------------------------------------------------------------------- * Support for detecting call convention of dynamically-loaded functions * - * Dynamically loaded functions may use either the version-1 ("new style") - * or version-0 ("old style") calling convention. Version 1 is the call - * convention defined in this header file; version 0 is the old "plain C" - * convention. A version-1 function must be accompanied by the macro call + * Dynamically loaded functions currently can only use the version-1 ("new + * style") calling convention. Version-0 ("old style") is not supported + * anymore. Version 1 is the call convention defined in this header file, and + * must be accompanied by the macro call * * PG_FUNCTION_INFO_V1(function_name); * diff --git a/src/test/regress/input/create_function_2.source b/src/test/regress/input/create_function_2.source index 3c26b2fec6..b167c8ac6d 100644 --- a/src/test/regress/input/create_function_2.source +++ b/src/test/regress/input/create_function_2.source @@ -87,11 +87,6 @@ CREATE FUNCTION reverse_name(name) AS '@libdir@/regress@DLSUFFIX@' LANGUAGE C STRICT; -CREATE FUNCTION oldstyle_length(int4, text) - RETURNS int4 - AS '@libdir@/regress@DLSUFFIX@' - LANGUAGE C; -- intentionally not strict - -- -- Function dynamic loading -- diff --git a/src/test/regress/input/misc.source b/src/test/regress/input/misc.source index dd2d1b2033..b1dbc573c9 100644 --- a/src/test/regress/input/misc.source +++ b/src/test/regress/input/misc.source @@ -249,19 +249,6 @@ SELECT *, name(equipment(h.*)) FROM hobbies_r h; SELECT *, (equipment(CAST((h.*) AS hobbies_r))).name FROM hobbies_r h; --- --- check that old-style C functions work properly with TOASTed values --- -create table oldstyle_test(i int4, t text); -insert into oldstyle_test values(null,null); -insert into oldstyle_test values(0,'12'); -insert into oldstyle_test values(1000,'12'); -insert into oldstyle_test values(0, repeat('x', 50000)); - -select i, length(t), octet_length(t), oldstyle_length(i,t) from oldstyle_test; - -drop table oldstyle_test; - -- -- functional joins -- diff --git a/src/test/regress/output/create_function_2.source b/src/test/regress/output/create_function_2.source index bdd1b1bec5..8f28bff298 100644 --- a/src/test/regress/output/create_function_2.source +++ b/src/test/regress/output/create_function_2.source @@ -67,10 +67,6 @@ CREATE FUNCTION reverse_name(name) RETURNS name AS '@libdir@/regress@DLSUFFIX@' LANGUAGE C STRICT; -CREATE FUNCTION oldstyle_length(int4, text) - RETURNS int4 - AS '@libdir@/regress@DLSUFFIX@' - LANGUAGE C; -- intentionally not strict -- -- Function dynamic loading -- diff --git a/src/test/regress/output/misc.source b/src/test/regress/output/misc.source index 574ef0d2e3..b9595cc239 100644 --- a/src/test/regress/output/misc.source +++ b/src/test/regress/output/misc.source @@ -681,24 +681,6 @@ SELECT *, (equipment(CAST((h.*) AS hobbies_r))).name FROM hobbies_r h; skywalking | | guts (7 rows) --- --- check that old-style C functions work properly with TOASTed values --- -create table oldstyle_test(i int4, t text); -insert into oldstyle_test values(null,null); -insert into oldstyle_test values(0,'12'); -insert into oldstyle_test values(1000,'12'); -insert into oldstyle_test values(0, repeat('x', 50000)); -select i, length(t), octet_length(t), oldstyle_length(i,t) from oldstyle_test; - i | length | octet_length | oldstyle_length -------+--------+--------------+----------------- - | | | - 0 | 2 | 2 | 2 - 1000 | 2 | 2 | 1002 - 0 | 50000 | 50000 | 50000 -(4 rows) - -drop table oldstyle_test; -- -- functional joins -- diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 986d54ce2f..d7fb8498d8 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -45,8 +45,6 @@ extern PATH *poly2path(POLYGON *poly); extern void regress_lseg_construct(LSEG *lseg, Point *pt1, Point *pt2); -extern char *reverse_name(char *string); -extern int oldstyle_length(int n, text *t); #ifdef PG_MODULE_MAGIC PG_MODULE_MAGIC; @@ -240,14 +238,15 @@ typedef struct double radius; } WIDGET; -WIDGET *widget_in(char *str); -char *widget_out(WIDGET *widget); +PG_FUNCTION_INFO_V1(widget_in); +PG_FUNCTION_INFO_V1(widget_out); #define NARGS 3 -WIDGET * -widget_in(char *str) +Datum +widget_in(PG_FUNCTION_ARGS) { + char *str = PG_GETARG_CSTRING(0); char *p, *coord[NARGS]; int i; @@ -270,14 +269,16 @@ widget_in(char *str) result->center.y = atof(coord[1]); result->radius = atof(coord[2]); - return result; + PG_RETURN_POINTER(result); } -char * -widget_out(WIDGET *widget) +Datum +widget_out(PG_FUNCTION_ARGS) { - return psprintf("(%g,%g,%g)", - widget->center.x, widget->center.y, widget->radius); + WIDGET *widget = (WIDGET *) PG_GETARG_POINTER(0); + char *str = psprintf("(%g,%g,%g)", + widget->center.x, widget->center.y, widget->radius); + PG_RETURN_CSTRING(str); } PG_FUNCTION_INFO_V1(pt_in_widget); @@ -305,9 +306,12 @@ boxarea(PG_FUNCTION_ARGS) PG_RETURN_FLOAT8(width * height); } -char * -reverse_name(char *string) +PG_FUNCTION_INFO_V1(reverse_name); + +Datum +reverse_name(PG_FUNCTION_ARGS) { + char *string = PG_GETARG_CSTRING(0); int i; int len; char *new_string; @@ -320,22 +324,7 @@ reverse_name(char *string) len = i; for (; i >= 0; --i) new_string[len - i] = string[i]; - return new_string; -} - -/* - * This rather silly function is just to test that oldstyle functions - * work correctly on toast-able inputs. - */ -int -oldstyle_length(int n, text *t) -{ - int len = 0; - - if (t) - len = VARSIZE(t) - VARHDRSZ; - - return n + len; + PG_RETURN_CSTRING(new_string); } -- cgit v1.2.3 From ec19693014ed48fa1d8c7e0ce450795f752c4456 Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Fri, 31 Mar 2017 01:31:15 +0900 Subject: Simplify the example of VACUUM in documentation. Previously a detailed activity report by VACUUM VERBOSE ANALYZE was described as an example of VACUUM in docs. But it had been obsolete for a long time. For example, commit feb4f44d296b88b7f0723f4a4f3945a371276e0b updated the content of that activity report in 2003, but we had forgotten to update the example. So basically we need to update the example. But since no one cared about the details of VACUUM output and complained about that mistake for such long time, per discussion on hackers, we decided to get rid of the detailed activity report from the example and simplify it. Back-patch to all supported versions. Reported by Masahiko Sawada, patch by me. Discussion: https://postgr.es/m/CAD21AoAGA2pB3p-CWmTkxBsbkZS1bcDGBLcYVcvcDxspG_XAfA@mail.gmail.com --- doc/src/sgml/ref/vacuum.sgml | 34 +++------------------------------- 1 file changed, 3 insertions(+), 31 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/vacuum.sgml b/doc/src/sgml/ref/vacuum.sgml index 8ec5b40b71..421c18d117 100644 --- a/doc/src/sgml/ref/vacuum.sgml +++ b/doc/src/sgml/ref/vacuum.sgml @@ -246,39 +246,11 @@ VACUUM [ FULL ] [ FREEZE ] [ VERBOSE ] ANALYZE [ Examples - The following is an example from running VACUUM on a - table in the regression database: + To clean a single table onek, analyze it for + the optimizer and print a detailed vacuum activity report: -regression=# VACUUM (VERBOSE, ANALYZE) onek; -INFO: vacuuming "public.onek" -INFO: index "onek_unique1" now contains 1000 tuples in 14 pages -DETAIL: 3000 index tuples were removed. -0 index pages have been deleted, 0 are currently reusable. -CPU: user: 0.08 s, system: 0.01 s, elapsed: 0.18 s. -INFO: index "onek_unique2" now contains 1000 tuples in 16 pages -DETAIL: 3000 index tuples were removed. -0 index pages have been deleted, 0 are currently reusable. -CPU: user: 0.07 s, system: 0.00 s, elapsed: 0.23 s. -INFO: index "onek_hundred" now contains 1000 tuples in 13 pages -DETAIL: 3000 index tuples were removed. -0 index pages have been deleted, 0 are currently reusable. -CPU: user: 0.08 s, system: 0.01 s, elapsed: 0.17 s. -INFO: index "onek_stringu1" now contains 1000 tuples in 48 pages -DETAIL: 3000 index tuples were removed. -0 index pages have been deleted, 0 are currently reusable. -CPU: user: 0.09 s, system: 0.01 s, elapsed: 0.59 s. -INFO: "onek": removed 3000 tuples in 108 pages -DETAIL: CPU: user: 0.06 s, system: 0.01 s, elapsed: 0.07 s. -INFO: "onek": found 3000 removable, 1000 nonremovable tuples in 143 pages -DETAIL: 0 dead tuples cannot be removed yet. -There were 0 unused item pointers. -Skipped 0 pages due to buffer pins, 0 frozen pages. -0 pages are entirely empty. -CPU: user: 0.39 s, system: 0.07 s, elapsed: 1.56 s. -INFO: analyzing "public.onek" -INFO: "onek": 36 pages, 1000 rows sampled, 1000 estimated total rows -VACUUM +VACUUM (VERBOSE, ANALYZE) onek; -- cgit v1.2.3 From ffae6733db1f9d4a3a75d737a00ee2a4a3e01849 Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Thu, 30 Mar 2017 09:41:00 -0700 Subject: Try to fix xml docs build broken in 5ded4bd. Apparently the sgml to xml conversion treats non-closed s differently than jade does. --- doc/src/sgml/xfunc.sgml | 2 ++ 1 file changed, 2 insertions(+) (limited to 'doc/src') diff --git a/doc/src/sgml/xfunc.sgml b/doc/src/sgml/xfunc.sgml index e6313ddd59..0a12231396 100644 --- a/doc/src/sgml/xfunc.sgml +++ b/doc/src/sgml/xfunc.sgml @@ -2173,6 +2173,7 @@ PG_FUNCTION_INFO_V1(funcname); Here are some examples using the version-1 calling convention: + funcs.c and compiled into a shared object, we could define the functions to PostgreSQL with commands like this: + CREATE FUNCTION add_one(integer) RETURNS integer -- cgit v1.2.3 From e984ef5861df4bc9733b36271d05763e82de7c04 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 30 Mar 2017 12:59:11 -0400 Subject: Support \if ... \elif ... \else ... \endif in psql scripting. This patch adds nestable conditional blocks to psql. The control structure feature per se is complete, but the boolean expressions understood by \if and \elif are pretty primitive; basically, after variable substitution and backtick expansion, the result has to be "true" or "false" or one of the other standard spellings of a boolean value. But that's enough for many purposes, since you can always do the heavy lifting on the server side; and we can extend it later. Along the way, pay down some of the technical debt that had built up around psql/command.c: * Refactor exec_command() into a function per command, instead of being a 1500-line monstrosity. This makes the file noticeably longer because of repetitive function header/trailer overhead, but it seems much more readable. * Teach psql_get_variable() and psqlscanslash.l to suppress variable substitution and backtick expansion on the basis of the conditional stack state, thereby allowing removal of the OT_NO_EVAL kluge. * Fix the no-doubt-once-expedient hack of sometimes silently substituting mainloop.c's previous_buf for query_buf when calling HandleSlashCmds. (It's a bit remarkable that commands like \r worked at all with that.) Recall of a previous query is now done explicitly in the slash commands where that should happen. Corey Huinker, reviewed by Fabien Coelho, further hacking by me Discussion: https://postgr.es/m/CADkLM=c94OSRTnat=LX0ivNq4pxDNeoomFfYvBKM5N_xfmLtAA@mail.gmail.com --- doc/src/sgml/ref/psql-ref.sgml | 92 ++- src/bin/psql/Makefile | 8 +- src/bin/psql/command.c | 1578 +++++++++++++++++++++++++++++++----- src/bin/psql/command.h | 5 +- src/bin/psql/common.c | 7 +- src/bin/psql/conditional.c | 153 ++++ src/bin/psql/conditional.h | 83 ++ src/bin/psql/copy.c | 4 +- src/bin/psql/help.c | 9 +- src/bin/psql/mainloop.c | 119 ++- src/bin/psql/prompt.c | 6 +- src/bin/psql/prompt.h | 3 +- src/bin/psql/psqlscanslash.h | 7 +- src/bin/psql/psqlscanslash.l | 46 +- src/bin/psql/startup.c | 9 +- src/test/regress/expected/psql.out | 169 ++++ src/test/regress/sql/psql.sql | 144 ++++ 17 files changed, 2172 insertions(+), 270 deletions(-) create mode 100644 src/bin/psql/conditional.c create mode 100644 src/bin/psql/conditional.h (limited to 'doc/src') diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml index 2a9c412020..b51b11baa3 100644 --- a/doc/src/sgml/ref/psql-ref.sgml +++ b/doc/src/sgml/ref/psql-ref.sgml @@ -2063,6 +2063,95 @@ hello 10 + + \if expression + \elif expression + \else + \endif + + + This group of commands implements nestable conditional blocks. + A conditional block must begin with an \if and end + with an \endif. In between there may be any number + of \elif clauses, which may optionally be followed + by a single \else clause. Ordinary queries and + other types of backslash commands may (and usually do) appear between + the commands forming a conditional block. + + + The \if and \elif commands read + their argument(s) and evaluate them as a boolean expression. If the + expression yields true then processing continues + normally; otherwise, lines are skipped until a + matching \elif, \else, + or \endif is reached. Once + an \if or \elif test has + succeeded, the arguments of later \elif commands in + the same block are not evaluated but are treated as false. Lines + following an \else are processed only if no earlier + matching \if or \elif succeeded. + + + The expression argument + of an \if or \elif command + is subject to variable interpolation and backquote expansion, just + like any other backslash command argument. After that it is evaluated + like the value of an on/off option variable. So a valid value + is any unambiguous case-insensitive match for one of: + true, false, 1, + 0, on, off, + yes, no. For example, + t, T, and tR + will all be considered to be true. + + + Expressions that do not properly evaluate to true or false will + generate a warning and be treated as false. + + + Lines being skipped are parsed normally to identify queries and + backslash commands, but queries are not sent to the server, and + backslash commands other than conditionals + (\if, \elif, + \else, \endif) are + ignored. Conditional commands are checked only for valid nesting. + Variable references in skipped lines are not expanded, and backquote + expansion is not performed either. + + + All the backslash commands of a given conditional block must appear in + the same source file. If EOF is reached on the main input file or an + \include-ed file before all local + \if-blocks have been closed, + then psql will raise an error. + + + Here is an example: + + +-- check for the existence of two separate records in the database and store +-- the results in separate psql variables +SELECT + EXISTS(SELECT 1 FROM customer WHERE customer_id = 123) as is_customer, + EXISTS(SELECT 1 FROM employee WHERE employee_id = 456) as is_employee +\gset +\if :is_customer + SELECT * FROM customer WHERE customer_id = 123; +\elif :is_employee + \echo 'is not a customer but is an employee' + SELECT * FROM employee WHERE employee_id = 456; +\else + \if yes + \echo 'not a customer or employee' + \else + \echo 'this will never print' + \endif +\endif + + + + + \l[+] or \list[+] [ pattern ] @@ -3715,7 +3804,8 @@ testdb=> INSERT INTO my_table VALUES (:'content'); In prompt 1 normally =, - but ^ if in single-line mode, + but @ if the session is in an inactive branch of a + conditional block, or ^ if in single-line mode, or ! if the session is disconnected from the database (which can happen if \connect fails). In prompt 2 %R is replaced by a character that diff --git a/src/bin/psql/Makefile b/src/bin/psql/Makefile index f8e31eacbe..ab2cfa6353 100644 --- a/src/bin/psql/Makefile +++ b/src/bin/psql/Makefile @@ -21,10 +21,10 @@ REFDOCDIR= $(top_srcdir)/doc/src/sgml/ref override CPPFLAGS := -I. -I$(srcdir) -I$(libpq_srcdir) $(CPPFLAGS) LDFLAGS += -L$(top_builddir)/src/fe_utils -lpgfeutils -lpq -OBJS= command.o common.o help.o input.o stringutils.o mainloop.o copy.o \ - startup.o prompt.o variables.o large_obj.o describe.o \ - crosstabview.o tab-complete.o \ - sql_help.o psqlscanslash.o \ +OBJS= command.o common.o conditional.o copy.o crosstabview.o \ + describe.o help.o input.o large_obj.o mainloop.o \ + prompt.o psqlscanslash.o sql_help.o startup.o stringutils.o \ + tab-complete.o variables.o \ $(WIN32RES) diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c index 4f4a0aa9bd..94a3cfce90 100644 --- a/src/bin/psql/command.c +++ b/src/bin/psql/command.c @@ -56,14 +56,103 @@ typedef enum EditableObjectType EditableView } EditableObjectType; -/* functions for use in this file */ +/* local function declarations */ static backslashResult exec_command(const char *cmd, PsqlScanState scan_state, - PQExpBuffer query_buf); -static bool do_edit(const char *filename_arg, PQExpBuffer query_buf, - int lineno, bool *edited); + ConditionalStack cstack, + PQExpBuffer query_buf, + PQExpBuffer previous_buf); +static backslashResult exec_command_a(PsqlScanState scan_state, bool active_branch); +static backslashResult exec_command_C(PsqlScanState scan_state, bool active_branch); +static backslashResult exec_command_connect(PsqlScanState scan_state, bool active_branch); +static backslashResult exec_command_cd(PsqlScanState scan_state, bool active_branch, + const char *cmd); +static backslashResult exec_command_conninfo(PsqlScanState scan_state, bool active_branch); +static backslashResult exec_command_copy(PsqlScanState scan_state, bool active_branch); +static backslashResult exec_command_copyright(PsqlScanState scan_state, bool active_branch); +static backslashResult exec_command_crosstabview(PsqlScanState scan_state, bool active_branch); +static backslashResult exec_command_d(PsqlScanState scan_state, bool active_branch, + const char *cmd); +static backslashResult exec_command_edit(PsqlScanState scan_state, bool active_branch, + PQExpBuffer query_buf, PQExpBuffer previous_buf); +static backslashResult exec_command_ef(PsqlScanState scan_state, bool active_branch, + PQExpBuffer query_buf); +static backslashResult exec_command_ev(PsqlScanState scan_state, bool active_branch, + PQExpBuffer query_buf); +static backslashResult exec_command_echo(PsqlScanState scan_state, bool active_branch, + const char *cmd); +static backslashResult exec_command_elif(PsqlScanState scan_state, ConditionalStack cstack, + PQExpBuffer query_buf); +static backslashResult exec_command_else(PsqlScanState scan_state, ConditionalStack cstack, + PQExpBuffer query_buf); +static backslashResult exec_command_endif(PsqlScanState scan_state, ConditionalStack cstack, + PQExpBuffer query_buf); +static backslashResult exec_command_encoding(PsqlScanState scan_state, bool active_branch); +static backslashResult exec_command_errverbose(PsqlScanState scan_state, bool active_branch); +static backslashResult exec_command_f(PsqlScanState scan_state, bool active_branch); +static backslashResult exec_command_g(PsqlScanState scan_state, bool active_branch, + const char *cmd); +static backslashResult exec_command_gexec(PsqlScanState scan_state, bool active_branch); +static backslashResult exec_command_gset(PsqlScanState scan_state, bool active_branch); +static backslashResult exec_command_help(PsqlScanState scan_state, bool active_branch); +static backslashResult exec_command_html(PsqlScanState scan_state, bool active_branch); +static backslashResult exec_command_include(PsqlScanState scan_state, bool active_branch, + const char *cmd); +static backslashResult exec_command_if(PsqlScanState scan_state, ConditionalStack cstack, + PQExpBuffer query_buf); +static backslashResult exec_command_list(PsqlScanState scan_state, bool active_branch, + const char *cmd); +static backslashResult exec_command_lo(PsqlScanState scan_state, bool active_branch, + const char *cmd); +static backslashResult exec_command_out(PsqlScanState scan_state, bool active_branch); +static backslashResult exec_command_print(PsqlScanState scan_state, bool active_branch, + PQExpBuffer query_buf); +static backslashResult exec_command_password(PsqlScanState scan_state, bool active_branch); +static backslashResult exec_command_prompt(PsqlScanState scan_state, bool active_branch, + const char *cmd); +static backslashResult exec_command_pset(PsqlScanState scan_state, bool active_branch); +static backslashResult exec_command_quit(PsqlScanState scan_state, bool active_branch); +static backslashResult exec_command_reset(PsqlScanState scan_state, bool active_branch, + PQExpBuffer query_buf); +static backslashResult exec_command_s(PsqlScanState scan_state, bool active_branch); +static backslashResult exec_command_set(PsqlScanState scan_state, bool active_branch); +static backslashResult exec_command_setenv(PsqlScanState scan_state, bool active_branch, + const char *cmd); +static backslashResult exec_command_sf(PsqlScanState scan_state, bool active_branch, + const char *cmd); +static backslashResult exec_command_sv(PsqlScanState scan_state, bool active_branch, + const char *cmd); +static backslashResult exec_command_t(PsqlScanState scan_state, bool active_branch); +static backslashResult exec_command_T(PsqlScanState scan_state, bool active_branch); +static backslashResult exec_command_timing(PsqlScanState scan_state, bool active_branch); +static backslashResult exec_command_unset(PsqlScanState scan_state, bool active_branch, + const char *cmd); +static backslashResult exec_command_write(PsqlScanState scan_state, bool active_branch, + const char *cmd, + PQExpBuffer query_buf, PQExpBuffer previous_buf); +static backslashResult exec_command_watch(PsqlScanState scan_state, bool active_branch, + PQExpBuffer query_buf, PQExpBuffer previous_buf); +static backslashResult exec_command_x(PsqlScanState scan_state, bool active_branch); +static backslashResult exec_command_z(PsqlScanState scan_state, bool active_branch); +static backslashResult exec_command_shell_escape(PsqlScanState scan_state, bool active_branch); +static backslashResult exec_command_slash_command_help(PsqlScanState scan_state, bool active_branch); +static char *read_connect_arg(PsqlScanState scan_state); +static PQExpBuffer gather_boolean_expression(PsqlScanState scan_state); +static bool is_true_boolean_expression(PsqlScanState scan_state, const char *name); +static void ignore_boolean_expression(PsqlScanState scan_state); +static void ignore_slash_options(PsqlScanState scan_state); +static void ignore_slash_filepipe(PsqlScanState scan_state); +static void ignore_slash_whole_line(PsqlScanState scan_state); +static bool is_branching_command(const char *cmd); +static void save_query_text_state(PsqlScanState scan_state, ConditionalStack cstack, + PQExpBuffer query_buf); +static void discard_query_text(PsqlScanState scan_state, ConditionalStack cstack, + PQExpBuffer query_buf); +static void copy_previous_query(PQExpBuffer query_buf, PQExpBuffer previous_buf); static bool do_connect(enum trivalue reuse_previous_specification, char *dbname, char *user, char *host, char *port); +static bool do_edit(const char *filename_arg, PQExpBuffer query_buf, + int lineno, bool *edited); static bool do_shell(const char *command); static bool do_watch(PQExpBuffer query_buf, double sleep); static bool lookup_object_oid(EditableObjectType obj_type, const char *desc, @@ -96,9 +185,18 @@ static void checkWin32Codepage(void); * just after the '\'. The lexer is advanced past the command and all * arguments on return. * - * 'query_buf' contains the query-so-far, which may be modified by + * cstack is the current \if stack state. This will be examined, and + * possibly modified by conditional commands. + * + * query_buf contains the query-so-far, which may be modified by * execution of the backslash command (for example, \r clears it). - * query_buf can be NULL if there is no query so far. + * + * previous_buf contains the query most recently sent to the server + * (empty if none yet). This should not be modified here, but some + * commands copy its content into query_buf. + * + * query_buf and previous_buf will be NULL when executing a "-c" + * command-line option. * * Returns a status code indicating what action is desired, see command.h. *---------- @@ -106,19 +204,22 @@ static void checkWin32Codepage(void); backslashResult HandleSlashCmds(PsqlScanState scan_state, - PQExpBuffer query_buf) + ConditionalStack cstack, + PQExpBuffer query_buf, + PQExpBuffer previous_buf) { - backslashResult status = PSQL_CMD_SKIP_LINE; + backslashResult status; char *cmd; char *arg; Assert(scan_state != NULL); + Assert(cstack != NULL); /* Parse off the command name */ cmd = psql_scan_slash_command(scan_state); /* And try to execute it */ - status = exec_command(cmd, scan_state, query_buf); + status = exec_command(cmd, scan_state, cstack, query_buf, previous_buf); if (status == PSQL_CMD_UNKNOWN) { @@ -131,14 +232,22 @@ HandleSlashCmds(PsqlScanState scan_state, if (status != PSQL_CMD_ERROR) { - /* eat any remaining arguments after a valid command */ - /* note we suppress evaluation of backticks here */ + /* + * Eat any remaining arguments after a valid command. We want to + * suppress evaluation of backticks in this situation, so transiently + * push an inactive conditional-stack entry. + */ + bool active_branch = conditional_active(cstack); + + conditional_stack_push(cstack, IFSTATE_IGNORED); while ((arg = psql_scan_slash_option(scan_state, - OT_NO_EVAL, NULL, false))) + OT_NORMAL, NULL, false))) { - psql_error("\\%s: extra argument \"%s\" ignored\n", cmd, arg); + if (active_branch) + psql_error("\\%s: extra argument \"%s\" ignored\n", cmd, arg); free(arg); } + conditional_stack_pop(cstack); } else { @@ -159,56 +268,169 @@ HandleSlashCmds(PsqlScanState scan_state, return status; } + /* - * Read and interpret an argument to the \connect slash command. + * Subroutine to actually try to execute a backslash command. + * + * The typical "success" result code is PSQL_CMD_SKIP_LINE, although some + * commands return something else. Failure results are PSQL_CMD_ERROR, + * unless PSQL_CMD_UNKNOWN is more appropriate. */ -static char * -read_connect_arg(PsqlScanState scan_state) +static backslashResult +exec_command(const char *cmd, + PsqlScanState scan_state, + ConditionalStack cstack, + PQExpBuffer query_buf, + PQExpBuffer previous_buf) { - char *result; - char quote; + backslashResult status; + bool active_branch = conditional_active(cstack); /* - * Ideally we should treat the arguments as SQL identifiers. But for - * backwards compatibility with 7.2 and older pg_dump files, we have to - * take unquoted arguments verbatim (don't downcase them). For now, - * double-quoted arguments may be stripped of double quotes (as if SQL - * identifiers). By 7.4 or so, pg_dump files can be expected to - * double-quote all mixed-case \connect arguments, and then we can get rid - * of OT_SQLIDHACK. + * In interactive mode, warn when we're ignoring a command within a false + * \if-branch. But we continue on, so as to parse and discard the right + * amount of parameter text. Each individual backslash command subroutine + * is responsible for doing nothing after discarding appropriate + * arguments, if !active_branch. */ - result = psql_scan_slash_option(scan_state, OT_SQLIDHACK, "e, true); - - if (!result) - return NULL; + if (pset.cur_cmd_interactive && !active_branch && + !is_branching_command(cmd)) + { + psql_error("\\%s command ignored; use \\endif or Ctrl-C to exit current \\if block\n", + cmd); + } - if (quote) - return result; + if (strcmp(cmd, "a") == 0) + status = exec_command_a(scan_state, active_branch); + else if (strcmp(cmd, "C") == 0) + status = exec_command_C(scan_state, active_branch); + else if (strcmp(cmd, "c") == 0 || strcmp(cmd, "connect") == 0) + status = exec_command_connect(scan_state, active_branch); + else if (strcmp(cmd, "cd") == 0) + status = exec_command_cd(scan_state, active_branch, cmd); + else if (strcmp(cmd, "conninfo") == 0) + status = exec_command_conninfo(scan_state, active_branch); + else if (pg_strcasecmp(cmd, "copy") == 0) + status = exec_command_copy(scan_state, active_branch); + else if (strcmp(cmd, "copyright") == 0) + status = exec_command_copyright(scan_state, active_branch); + else if (strcmp(cmd, "crosstabview") == 0) + status = exec_command_crosstabview(scan_state, active_branch); + else if (cmd[0] == 'd') + status = exec_command_d(scan_state, active_branch, cmd); + else if (strcmp(cmd, "e") == 0 || strcmp(cmd, "edit") == 0) + status = exec_command_edit(scan_state, active_branch, + query_buf, previous_buf); + else if (strcmp(cmd, "ef") == 0) + status = exec_command_ef(scan_state, active_branch, query_buf); + else if (strcmp(cmd, "ev") == 0) + status = exec_command_ev(scan_state, active_branch, query_buf); + else if (strcmp(cmd, "echo") == 0 || strcmp(cmd, "qecho") == 0) + status = exec_command_echo(scan_state, active_branch, cmd); + else if (strcmp(cmd, "elif") == 0) + status = exec_command_elif(scan_state, cstack, query_buf); + else if (strcmp(cmd, "else") == 0) + status = exec_command_else(scan_state, cstack, query_buf); + else if (strcmp(cmd, "endif") == 0) + status = exec_command_endif(scan_state, cstack, query_buf); + else if (strcmp(cmd, "encoding") == 0) + status = exec_command_encoding(scan_state, active_branch); + else if (strcmp(cmd, "errverbose") == 0) + status = exec_command_errverbose(scan_state, active_branch); + else if (strcmp(cmd, "f") == 0) + status = exec_command_f(scan_state, active_branch); + else if (strcmp(cmd, "g") == 0 || strcmp(cmd, "gx") == 0) + status = exec_command_g(scan_state, active_branch, cmd); + else if (strcmp(cmd, "gexec") == 0) + status = exec_command_gexec(scan_state, active_branch); + else if (strcmp(cmd, "gset") == 0) + status = exec_command_gset(scan_state, active_branch); + else if (strcmp(cmd, "h") == 0 || strcmp(cmd, "help") == 0) + status = exec_command_help(scan_state, active_branch); + else if (strcmp(cmd, "H") == 0 || strcmp(cmd, "html") == 0) + status = exec_command_html(scan_state, active_branch); + else if (strcmp(cmd, "i") == 0 || strcmp(cmd, "include") == 0 || + strcmp(cmd, "ir") == 0 || strcmp(cmd, "include_relative") == 0) + status = exec_command_include(scan_state, active_branch, cmd); + else if (strcmp(cmd, "if") == 0) + status = exec_command_if(scan_state, cstack, query_buf); + else if (strcmp(cmd, "l") == 0 || strcmp(cmd, "list") == 0 || + strcmp(cmd, "l+") == 0 || strcmp(cmd, "list+") == 0) + status = exec_command_list(scan_state, active_branch, cmd); + else if (strncmp(cmd, "lo_", 3) == 0) + status = exec_command_lo(scan_state, active_branch, cmd); + else if (strcmp(cmd, "o") == 0 || strcmp(cmd, "out") == 0) + status = exec_command_out(scan_state, active_branch); + else if (strcmp(cmd, "p") == 0 || strcmp(cmd, "print") == 0) + status = exec_command_print(scan_state, active_branch, query_buf); + else if (strcmp(cmd, "password") == 0) + status = exec_command_password(scan_state, active_branch); + else if (strcmp(cmd, "prompt") == 0) + status = exec_command_prompt(scan_state, active_branch, cmd); + else if (strcmp(cmd, "pset") == 0) + status = exec_command_pset(scan_state, active_branch); + else if (strcmp(cmd, "q") == 0 || strcmp(cmd, "quit") == 0) + status = exec_command_quit(scan_state, active_branch); + else if (strcmp(cmd, "r") == 0 || strcmp(cmd, "reset") == 0) + status = exec_command_reset(scan_state, active_branch, query_buf); + else if (strcmp(cmd, "s") == 0) + status = exec_command_s(scan_state, active_branch); + else if (strcmp(cmd, "set") == 0) + status = exec_command_set(scan_state, active_branch); + else if (strcmp(cmd, "setenv") == 0) + status = exec_command_setenv(scan_state, active_branch, cmd); + else if (strcmp(cmd, "sf") == 0 || strcmp(cmd, "sf+") == 0) + status = exec_command_sf(scan_state, active_branch, cmd); + else if (strcmp(cmd, "sv") == 0 || strcmp(cmd, "sv+") == 0) + status = exec_command_sv(scan_state, active_branch, cmd); + else if (strcmp(cmd, "t") == 0) + status = exec_command_t(scan_state, active_branch); + else if (strcmp(cmd, "T") == 0) + status = exec_command_T(scan_state, active_branch); + else if (strcmp(cmd, "timing") == 0) + status = exec_command_timing(scan_state, active_branch); + else if (strcmp(cmd, "unset") == 0) + status = exec_command_unset(scan_state, active_branch, cmd); + else if (strcmp(cmd, "w") == 0 || strcmp(cmd, "write") == 0) + status = exec_command_write(scan_state, active_branch, cmd, + query_buf, previous_buf); + else if (strcmp(cmd, "watch") == 0) + status = exec_command_watch(scan_state, active_branch, + query_buf, previous_buf); + else if (strcmp(cmd, "x") == 0) + status = exec_command_x(scan_state, active_branch); + else if (strcmp(cmd, "z") == 0) + status = exec_command_z(scan_state, active_branch); + else if (strcmp(cmd, "!") == 0) + status = exec_command_shell_escape(scan_state, active_branch); + else if (strcmp(cmd, "?") == 0) + status = exec_command_slash_command_help(scan_state, active_branch); + else + status = PSQL_CMD_UNKNOWN; - if (*result == '\0' || strcmp(result, "-") == 0) - return NULL; + /* + * All the commands that return PSQL_CMD_SEND want to execute previous_buf + * if query_buf is empty. For convenience we implement that here, not in + * the individual command subroutines. + */ + if (status == PSQL_CMD_SEND) + copy_previous_query(query_buf, previous_buf); - return result; + return status; } /* - * Subroutine to actually try to execute a backslash command. + * \a -- toggle field alignment + * + * This makes little sense but we keep it around. */ static backslashResult -exec_command(const char *cmd, - PsqlScanState scan_state, - PQExpBuffer query_buf) +exec_command_a(PsqlScanState scan_state, bool active_branch) { - bool success = true; /* indicate here if the command ran ok or - * failed */ - backslashResult status = PSQL_CMD_SKIP_LINE; + bool success = true; - /* - * \a -- toggle field alignment This makes little sense but we keep it - * around. - */ - if (strcmp(cmd, "a") == 0) + if (active_branch) { if (pset.popt.topt.format != PRINT_ALIGNED) success = do_pset("format", "aligned", &pset.popt, pset.quiet); @@ -216,8 +438,18 @@ exec_command(const char *cmd, success = do_pset("format", "unaligned", &pset.popt, pset.quiet); } - /* \C -- override table title (formerly change HTML caption) */ - else if (strcmp(cmd, "C") == 0) + return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR; +} + +/* + * \C -- override table title (formerly change HTML caption) + */ +static backslashResult +exec_command_C(PsqlScanState scan_state, bool active_branch) +{ + bool success = true; + + if (active_branch) { char *opt = psql_scan_slash_option(scan_state, OT_NORMAL, NULL, true); @@ -225,20 +457,32 @@ exec_command(const char *cmd, success = do_pset("title", opt, &pset.popt, pset.quiet); free(opt); } + else + ignore_slash_options(scan_state); - /* - * \c or \connect -- connect to database using the specified parameters. - * - * \c [-reuse-previous=BOOL] dbname user host port - * - * Specifying a parameter as '-' is equivalent to omitting it. Examples: - * - * \c - - hst Connect to current database on current port of host - * "hst" as current user. \c - usr - prt Connect to current database on - * "prt" port of current host as user "usr". \c dbs Connect to - * "dbs" database on current port of current host as current user. - */ - else if (strcmp(cmd, "c") == 0 || strcmp(cmd, "connect") == 0) + return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR; +} + +/* + * \c or \connect -- connect to database using the specified parameters. + * + * \c [-reuse-previous=BOOL] dbname user host port + * + * Specifying a parameter as '-' is equivalent to omitting it. Examples: + * + * \c - - hst Connect to current database on current port of + * host "hst" as current user. + * \c - usr - prt Connect to current database on port "prt" of current host + * as user "usr". + * \c dbs Connect to database "dbs" on current port of current host + * as current user. + */ +static backslashResult +exec_command_connect(PsqlScanState scan_state, bool active_branch) +{ + bool success = true; + + if (active_branch) { static const char prefix[] = "-reuse-previous="; char *opt1, @@ -277,9 +521,21 @@ exec_command(const char *cmd, } free(opt1); } + else + ignore_slash_options(scan_state); - /* \cd */ - else if (strcmp(cmd, "cd") == 0) + return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR; +} + +/* + * \cd -- change directory + */ +static backslashResult +exec_command_cd(PsqlScanState scan_state, bool active_branch, const char *cmd) +{ + bool success = true; + + if (active_branch) { char *opt = psql_scan_slash_option(scan_state, OT_NORMAL, NULL, true); @@ -323,9 +579,19 @@ exec_command(const char *cmd, if (opt) free(opt); } + else + ignore_slash_options(scan_state); - /* \conninfo -- display information about the current connection */ - else if (strcmp(cmd, "conninfo") == 0) + return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR; +} + +/* + * \conninfo -- display information about the current connection + */ +static backslashResult +exec_command_conninfo(PsqlScanState scan_state, bool active_branch) +{ + if (active_branch) { char *db = PQdb(pset.db); @@ -366,8 +632,18 @@ exec_command(const char *cmd, } } - /* \copy */ - else if (pg_strcasecmp(cmd, "copy") == 0) + return PSQL_CMD_SKIP_LINE; +} + +/* + * \copy -- run a COPY command + */ +static backslashResult +exec_command_copy(PsqlScanState scan_state, bool active_branch) +{ + bool success = true; + + if (active_branch) { char *opt = psql_scan_slash_option(scan_state, OT_WHOLE_LINE, NULL, false); @@ -375,13 +651,33 @@ exec_command(const char *cmd, success = do_copy(opt); free(opt); } + else + ignore_slash_whole_line(scan_state); - /* \copyright */ - else if (strcmp(cmd, "copyright") == 0) + return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR; +} + +/* + * \copyright -- print copyright notice + */ +static backslashResult +exec_command_copyright(PsqlScanState scan_state, bool active_branch) +{ + if (active_branch) print_copyright(); - /* \crosstabview -- execute a query and display results in crosstab */ - else if (strcmp(cmd, "crosstabview") == 0) + return PSQL_CMD_SKIP_LINE; +} + +/* + * \crosstabview -- execute a query and display results in crosstab + */ +static backslashResult +exec_command_crosstabview(PsqlScanState scan_state, bool active_branch) +{ + backslashResult status = PSQL_CMD_SKIP_LINE; + + if (active_branch) { int i; @@ -391,9 +687,22 @@ exec_command(const char *cmd, pset.crosstab_flag = true; status = PSQL_CMD_SEND; } + else + ignore_slash_options(scan_state); - /* \d* commands */ - else if (cmd[0] == 'd') + return status; +} + +/* + * \d* commands + */ +static backslashResult +exec_command_d(PsqlScanState scan_state, bool active_branch, const char *cmd) +{ + backslashResult status = PSQL_CMD_SKIP_LINE; + bool success = true; + + if (active_branch) { char *pattern; bool show_verbose, @@ -502,7 +811,7 @@ exec_command(const char *cmd, success = listDbRoleSettings(pattern, pattern2); } else - success = PSQL_CMD_UNKNOWN; + status = PSQL_CMD_UNKNOWN; break; case 'R': switch (cmd[2]) @@ -580,13 +889,26 @@ exec_command(const char *cmd, if (pattern) free(pattern); } + else + ignore_slash_options(scan_state); + if (!success) + status = PSQL_CMD_ERROR; - /* - * \e or \edit -- edit the current query buffer, or edit a file and make - * it the query buffer - */ - else if (strcmp(cmd, "e") == 0 || strcmp(cmd, "edit") == 0) + return status; +} + +/* + * \e or \edit -- edit the current query buffer, or edit a file and + * make it the query buffer + */ +static backslashResult +exec_command_edit(PsqlScanState scan_state, bool active_branch, + PQExpBuffer query_buf, PQExpBuffer previous_buf) +{ + backslashResult status = PSQL_CMD_SKIP_LINE; + + if (active_branch) { if (!query_buf) { @@ -632,6 +954,10 @@ exec_command(const char *cmd, expand_tilde(&fname); if (fname) canonicalize_path(fname); + + /* Applies to previous query if current buffer is empty */ + copy_previous_query(query_buf, previous_buf); + if (do_edit(fname, query_buf, lineno, NULL)) status = PSQL_CMD_NEWEDIT; else @@ -643,13 +969,26 @@ exec_command(const char *cmd, free(ln); } } + else + ignore_slash_options(scan_state); - /* - * \ef -- edit the named function, or present a blank CREATE FUNCTION - * template if no argument is given - */ - else if (strcmp(cmd, "ef") == 0) + return status; +} + +/* + * \ef -- edit the named function, or present a blank CREATE FUNCTION + * template if no argument is given + */ +static backslashResult +exec_command_ef(PsqlScanState scan_state, bool active_branch, + PQExpBuffer query_buf) +{ + backslashResult status = PSQL_CMD_SKIP_LINE; + + if (active_branch) { + char *func = psql_scan_slash_option(scan_state, + OT_WHOLE_LINE, NULL, true); int lineno = -1; if (pset.sversion < 80400) @@ -668,11 +1007,8 @@ exec_command(const char *cmd, } else { - char *func; Oid foid = InvalidOid; - func = psql_scan_slash_option(scan_state, - OT_WHOLE_LINE, NULL, true); lineno = strip_lineno_from_objdesc(func); if (lineno == 0) { @@ -725,9 +1061,6 @@ exec_command(const char *cmd, lines++; } } - - if (func) - free(func); } if (status != PSQL_CMD_ERROR) @@ -741,14 +1074,30 @@ exec_command(const char *cmd, else status = PSQL_CMD_NEWEDIT; } + + if (func) + free(func); } + else + ignore_slash_whole_line(scan_state); - /* - * \ev -- edit the named view, or present a blank CREATE VIEW template if - * no argument is given - */ - else if (strcmp(cmd, "ev") == 0) + return status; +} + +/* + * \ev -- edit the named view, or present a blank CREATE VIEW + * template if no argument is given + */ +static backslashResult +exec_command_ev(PsqlScanState scan_state, bool active_branch, + PQExpBuffer query_buf) +{ + backslashResult status = PSQL_CMD_SKIP_LINE; + + if (active_branch) { + char *view = psql_scan_slash_option(scan_state, + OT_WHOLE_LINE, NULL, true); int lineno = -1; if (pset.sversion < 70400) @@ -767,11 +1116,8 @@ exec_command(const char *cmd, } else { - char *view; Oid view_oid = InvalidOid; - view = psql_scan_slash_option(scan_state, - OT_WHOLE_LINE, NULL, true); lineno = strip_lineno_from_objdesc(view); if (lineno == 0) { @@ -796,9 +1142,6 @@ exec_command(const char *cmd, /* error already reported */ status = PSQL_CMD_ERROR; } - - if (view) - free(view); } if (status != PSQL_CMD_ERROR) @@ -812,10 +1155,23 @@ exec_command(const char *cmd, else status = PSQL_CMD_NEWEDIT; } + + if (view) + free(view); } + else + ignore_slash_whole_line(scan_state); - /* \echo and \qecho */ - else if (strcmp(cmd, "echo") == 0 || strcmp(cmd, "qecho") == 0) + return status; +} + +/* + * \echo and \qecho -- echo arguments to stdout or query output + */ +static backslashResult +exec_command_echo(PsqlScanState scan_state, bool active_branch, const char *cmd) +{ + if (active_branch) { char *value; char quoted; @@ -846,9 +1202,19 @@ exec_command(const char *cmd, if (!no_newline) fputs("\n", fout); } + else + ignore_slash_options(scan_state); - /* \encoding -- set/show client side encoding */ - else if (strcmp(cmd, "encoding") == 0) + return PSQL_CMD_SKIP_LINE; +} + +/* + * \encoding -- set/show client side encoding + */ +static backslashResult +exec_command_encoding(PsqlScanState scan_state, bool active_branch) +{ + if (active_branch) { char *encoding = psql_scan_slash_option(scan_state, OT_NORMAL, NULL, false); @@ -874,9 +1240,19 @@ exec_command(const char *cmd, free(encoding); } } + else + ignore_slash_options(scan_state); - /* \errverbose -- display verbose message from last failed query */ - else if (strcmp(cmd, "errverbose") == 0) + return PSQL_CMD_SKIP_LINE; +} + +/* + * \errverbose -- display verbose message from last failed query + */ +static backslashResult +exec_command_errverbose(PsqlScanState scan_state, bool active_branch) +{ + if (active_branch) { if (pset.last_error_result) { @@ -897,8 +1273,18 @@ exec_command(const char *cmd, puts(_("There is no previous error.")); } - /* \f -- change field separator */ - else if (strcmp(cmd, "f") == 0) + return PSQL_CMD_SKIP_LINE; +} + +/* + * \f -- change field separator + */ +static backslashResult +exec_command_f(PsqlScanState scan_state, bool active_branch) +{ + bool success = true; + + if (active_branch) { char *fname = psql_scan_slash_option(scan_state, OT_NORMAL, NULL, false); @@ -906,12 +1292,22 @@ exec_command(const char *cmd, success = do_pset("fieldsep", fname, &pset.popt, pset.quiet); free(fname); } + else + ignore_slash_options(scan_state); - /* - * \g [filename] -- send query, optionally with output to file/pipe - * \gx [filename] -- same as \g, with expanded mode forced - */ - else if (strcmp(cmd, "g") == 0 || strcmp(cmd, "gx") == 0) + return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR; +} + +/* + * \g [filename] -- send query, optionally with output to file/pipe + * \gx [filename] -- same as \g, with expanded mode forced + */ +static backslashResult +exec_command_g(PsqlScanState scan_state, bool active_branch, const char *cmd) +{ + backslashResult status = PSQL_CMD_SKIP_LINE; + + if (active_branch) { char *fname = psql_scan_slash_option(scan_state, OT_FILEPIPE, NULL, false); @@ -928,16 +1324,38 @@ exec_command(const char *cmd, pset.g_expanded = true; status = PSQL_CMD_SEND; } + else + ignore_slash_filepipe(scan_state); - /* \gexec -- send query and execute each field of result */ - else if (strcmp(cmd, "gexec") == 0) + return status; +} + +/* + * \gexec -- send query and execute each field of result + */ +static backslashResult +exec_command_gexec(PsqlScanState scan_state, bool active_branch) +{ + backslashResult status = PSQL_CMD_SKIP_LINE; + + if (active_branch) { pset.gexec_flag = true; status = PSQL_CMD_SEND; } - /* \gset [prefix] -- send query and store result into variables */ - else if (strcmp(cmd, "gset") == 0) + return status; +} + +/* + * \gset [prefix] -- send query and store result into variables + */ +static backslashResult +exec_command_gset(PsqlScanState scan_state, bool active_branch) +{ + backslashResult status = PSQL_CMD_SKIP_LINE; + + if (active_branch) { char *prefix = psql_scan_slash_option(scan_state, OT_NORMAL, NULL, false); @@ -952,9 +1370,19 @@ exec_command(const char *cmd, /* gset_prefix is freed later */ status = PSQL_CMD_SEND; } + else + ignore_slash_options(scan_state); - /* help */ - else if (strcmp(cmd, "h") == 0 || strcmp(cmd, "help") == 0) + return status; +} + +/* + * \help [topic] -- print help about SQL commands + */ +static backslashResult +exec_command_help(PsqlScanState scan_state, bool active_branch) +{ + if (active_branch) { char *opt = psql_scan_slash_option(scan_state, OT_WHOLE_LINE, NULL, false); @@ -973,9 +1401,21 @@ exec_command(const char *cmd, helpSQL(opt, pset.popt.topt.pager); free(opt); } + else + ignore_slash_whole_line(scan_state); - /* HTML mode */ - else if (strcmp(cmd, "H") == 0 || strcmp(cmd, "html") == 0) + return PSQL_CMD_SKIP_LINE; +} + +/* + * \H and \html -- toggle HTML formatting + */ +static backslashResult +exec_command_html(PsqlScanState scan_state, bool active_branch) +{ + bool success = true; + + if (active_branch) { if (pset.popt.topt.format != PRINT_HTML) success = do_pset("format", "html", &pset.popt, pset.quiet); @@ -983,10 +1423,18 @@ exec_command(const char *cmd, success = do_pset("format", "aligned", &pset.popt, pset.quiet); } + return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR; +} - /* \i and \ir include files */ - else if (strcmp(cmd, "i") == 0 || strcmp(cmd, "include") == 0 - || strcmp(cmd, "ir") == 0 || strcmp(cmd, "include_relative") == 0) +/* + * \i and \ir -- include a file + */ +static backslashResult +exec_command_include(PsqlScanState scan_state, bool active_branch, const char *cmd) +{ + bool success = true; + + if (active_branch) { char *fname = psql_scan_slash_option(scan_state, OT_NORMAL, NULL, true); @@ -1007,10 +1455,254 @@ exec_command(const char *cmd, free(fname); } } + else + ignore_slash_options(scan_state); - /* \l is list databases */ - else if (strcmp(cmd, "l") == 0 || strcmp(cmd, "list") == 0 || - strcmp(cmd, "l+") == 0 || strcmp(cmd, "list+") == 0) + return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR; +} + +/* + * \if -- beginning of an \if..\endif block + * + * is parsed as a boolean expression. Invalid expressions will emit a + * warning and be treated as false. Statements that follow a false expression + * will be parsed but ignored. Note that in the case where an \if statement + * is itself within an inactive section of a block, then the entire inner + * \if..\endif block will be parsed but ignored. + */ +static backslashResult +exec_command_if(PsqlScanState scan_state, ConditionalStack cstack, + PQExpBuffer query_buf) +{ + if (conditional_active(cstack)) + { + /* + * First, push a new active stack entry; this ensures that the lexer + * will perform variable substitution and backtick evaluation while + * scanning the expression. (That should happen anyway, since we know + * we're in an active outer branch, but let's be sure.) + */ + conditional_stack_push(cstack, IFSTATE_TRUE); + + /* Remember current query state in case we need to restore later */ + save_query_text_state(scan_state, cstack, query_buf); + + /* + * Evaluate the expression; if it's false, change to inactive state. + */ + if (!is_true_boolean_expression(scan_state, "\\if expression")) + conditional_stack_poke(cstack, IFSTATE_FALSE); + } + else + { + /* + * We're within an inactive outer branch, so this entire \if block + * will be ignored. We don't want to evaluate the expression, so push + * the "ignored" stack state before scanning it. + */ + conditional_stack_push(cstack, IFSTATE_IGNORED); + + /* Remember current query state in case we need to restore later */ + save_query_text_state(scan_state, cstack, query_buf); + + ignore_boolean_expression(scan_state); + } + + return PSQL_CMD_SKIP_LINE; +} + +/* + * \elif -- alternative branch in an \if..\endif block + * + * is evaluated the same as in \if . + */ +static backslashResult +exec_command_elif(PsqlScanState scan_state, ConditionalStack cstack, + PQExpBuffer query_buf) +{ + bool success = true; + + switch (conditional_stack_peek(cstack)) + { + case IFSTATE_TRUE: + + /* + * Just finished active branch of this \if block. Update saved + * state so we will keep whatever data was put in query_buf by the + * active branch. + */ + save_query_text_state(scan_state, cstack, query_buf); + + /* + * Discard \elif expression and ignore the rest until \endif. + * Switch state before reading expression to ensure proper lexer + * behavior. + */ + conditional_stack_poke(cstack, IFSTATE_IGNORED); + ignore_boolean_expression(scan_state); + break; + case IFSTATE_FALSE: + + /* + * Discard any query text added by the just-skipped branch. + */ + discard_query_text(scan_state, cstack, query_buf); + + /* + * Have not yet found a true expression in this \if block, so this + * might be the first. We have to change state before examining + * the expression, or the lexer won't do the right thing. + */ + conditional_stack_poke(cstack, IFSTATE_TRUE); + if (!is_true_boolean_expression(scan_state, "\\elif expression")) + conditional_stack_poke(cstack, IFSTATE_FALSE); + break; + case IFSTATE_IGNORED: + + /* + * Discard any query text added by the just-skipped branch. + */ + discard_query_text(scan_state, cstack, query_buf); + + /* + * Skip expression and move on. Either the \if block already had + * an active section, or whole block is being skipped. + */ + ignore_boolean_expression(scan_state); + break; + case IFSTATE_ELSE_TRUE: + case IFSTATE_ELSE_FALSE: + psql_error("\\elif: cannot occur after \\else\n"); + success = false; + break; + case IFSTATE_NONE: + /* no \if to elif from */ + psql_error("\\elif: no matching \\if\n"); + success = false; + break; + } + + return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR; +} + +/* + * \else -- final alternative in an \if..\endif block + * + * Statements within an \else branch will only be executed if + * all previous \if and \elif expressions evaluated to false + * and the block was not itself being ignored. + */ +static backslashResult +exec_command_else(PsqlScanState scan_state, ConditionalStack cstack, + PQExpBuffer query_buf) +{ + bool success = true; + + switch (conditional_stack_peek(cstack)) + { + case IFSTATE_TRUE: + + /* + * Just finished active branch of this \if block. Update saved + * state so we will keep whatever data was put in query_buf by the + * active branch. + */ + save_query_text_state(scan_state, cstack, query_buf); + + /* Now skip the \else branch */ + conditional_stack_poke(cstack, IFSTATE_ELSE_FALSE); + break; + case IFSTATE_FALSE: + + /* + * Discard any query text added by the just-skipped branch. + */ + discard_query_text(scan_state, cstack, query_buf); + + /* + * We've not found any true \if or \elif expression, so execute + * the \else branch. + */ + conditional_stack_poke(cstack, IFSTATE_ELSE_TRUE); + break; + case IFSTATE_IGNORED: + + /* + * Discard any query text added by the just-skipped branch. + */ + discard_query_text(scan_state, cstack, query_buf); + + /* + * Either we previously processed the active branch of this \if, + * or the whole \if block is being skipped. Either way, skip the + * \else branch. + */ + conditional_stack_poke(cstack, IFSTATE_ELSE_FALSE); + break; + case IFSTATE_ELSE_TRUE: + case IFSTATE_ELSE_FALSE: + psql_error("\\else: cannot occur after \\else\n"); + success = false; + break; + case IFSTATE_NONE: + /* no \if to else from */ + psql_error("\\else: no matching \\if\n"); + success = false; + break; + } + + return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR; +} + +/* + * \endif -- ends an \if...\endif block + */ +static backslashResult +exec_command_endif(PsqlScanState scan_state, ConditionalStack cstack, + PQExpBuffer query_buf) +{ + bool success = true; + + switch (conditional_stack_peek(cstack)) + { + case IFSTATE_TRUE: + case IFSTATE_ELSE_TRUE: + /* Close the \if block, keeping the query text */ + success = conditional_stack_pop(cstack); + Assert(success); + break; + case IFSTATE_FALSE: + case IFSTATE_IGNORED: + case IFSTATE_ELSE_FALSE: + + /* + * Discard any query text added by the just-skipped branch. + */ + discard_query_text(scan_state, cstack, query_buf); + + /* Close the \if block */ + success = conditional_stack_pop(cstack); + Assert(success); + break; + case IFSTATE_NONE: + /* no \if to end */ + psql_error("\\endif: no matching \\if\n"); + success = false; + break; + } + + return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR; +} + +/* + * \l -- list databases + */ +static backslashResult +exec_command_list(PsqlScanState scan_state, bool active_branch, const char *cmd) +{ + bool success = true; + + if (active_branch) { char *pattern; bool show_verbose; @@ -1025,11 +1717,22 @@ exec_command(const char *cmd, if (pattern) free(pattern); } + else + ignore_slash_options(scan_state); - /* - * large object things - */ - else if (strncmp(cmd, "lo_", 3) == 0) + return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR; +} + +/* + * \lo_* -- large object operations + */ +static backslashResult +exec_command_lo(PsqlScanState scan_state, bool active_branch, const char *cmd) +{ + backslashResult status = PSQL_CMD_SKIP_LINE; + bool success = true; + + if (active_branch) { char *opt1, *opt2; @@ -1087,10 +1790,24 @@ exec_command(const char *cmd, free(opt1); free(opt2); } + else + ignore_slash_options(scan_state); + if (!success) + status = PSQL_CMD_ERROR; - /* \o -- set query output */ - else if (strcmp(cmd, "o") == 0 || strcmp(cmd, "out") == 0) + return status; +} + +/* + * \o -- set query output + */ +static backslashResult +exec_command_out(PsqlScanState scan_state, bool active_branch) +{ + bool success = true; + + if (active_branch) { char *fname = psql_scan_slash_option(scan_state, OT_FILEPIPE, NULL, true); @@ -1099,9 +1816,20 @@ exec_command(const char *cmd, success = setQFout(fname); free(fname); } + else + ignore_slash_filepipe(scan_state); - /* \p prints the current query buffer */ - else if (strcmp(cmd, "p") == 0 || strcmp(cmd, "print") == 0) + return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR; +} + +/* + * \p -- print the current query buffer + */ +static backslashResult +exec_command_print(PsqlScanState scan_state, bool active_branch, + PQExpBuffer query_buf) +{ + if (active_branch) { if (query_buf && query_buf->len > 0) puts(query_buf->data); @@ -1110,9 +1838,21 @@ exec_command(const char *cmd, fflush(stdout); } - /* \password -- set user password */ - else if (strcmp(cmd, "password") == 0) + return PSQL_CMD_SKIP_LINE; +} + +/* + * \password -- set user password + */ +static backslashResult +exec_command_password(PsqlScanState scan_state, bool active_branch) +{ + bool success = true; + + if (active_branch) { + char *opt0 = psql_scan_slash_option(scan_state, + OT_SQLID, NULL, true); char pw1[100]; char pw2[100]; @@ -1126,7 +1866,6 @@ exec_command(const char *cmd, } else { - char *opt0 = psql_scan_slash_option(scan_state, OT_SQLID, NULL, true); char *user; char *encrypted_password; @@ -1159,14 +1898,27 @@ exec_command(const char *cmd, PQclear(res); PQfreemem(encrypted_password); } - - if (opt0) - free(opt0); } + + if (opt0) + free(opt0); } + else + ignore_slash_options(scan_state); - /* \prompt -- prompt and set variable */ - else if (strcmp(cmd, "prompt") == 0) + return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR; +} + +/* + * \prompt -- prompt and set variable + */ +static backslashResult +exec_command_prompt(PsqlScanState scan_state, bool active_branch, + const char *cmd) +{ + bool success = true; + + if (active_branch) { char *opt, *prompt_text = NULL; @@ -1225,9 +1977,21 @@ exec_command(const char *cmd, free(opt); } } + else + ignore_slash_options(scan_state); - /* \pset -- set printing parameters */ - else if (strcmp(cmd, "pset") == 0) + return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR; +} + +/* + * \pset -- set printing parameters + */ +static backslashResult +exec_command_pset(PsqlScanState scan_state, bool active_branch) +{ + bool success = true; + + if (active_branch) { char *opt0 = psql_scan_slash_option(scan_state, OT_NORMAL, NULL, false); @@ -1267,13 +2031,34 @@ exec_command(const char *cmd, free(opt0); free(opt1); } + else + ignore_slash_options(scan_state); - /* \q or \quit */ - else if (strcmp(cmd, "q") == 0 || strcmp(cmd, "quit") == 0) + return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR; +} + +/* + * \q or \quit -- exit psql + */ +static backslashResult +exec_command_quit(PsqlScanState scan_state, bool active_branch) +{ + backslashResult status = PSQL_CMD_SKIP_LINE; + + if (active_branch) status = PSQL_CMD_TERMINATE; - /* reset(clear) the buffer */ - else if (strcmp(cmd, "r") == 0 || strcmp(cmd, "reset") == 0) + return status; +} + +/* + * \r -- reset (clear) the query buffer + */ +static backslashResult +exec_command_reset(PsqlScanState scan_state, bool active_branch, + PQExpBuffer query_buf) +{ + if (active_branch) { resetPQExpBuffer(query_buf); psql_scan_reset(scan_state); @@ -1281,8 +2066,18 @@ exec_command(const char *cmd, puts(_("Query buffer reset (cleared).")); } - /* \s save history in a file or show it on the screen */ - else if (strcmp(cmd, "s") == 0) + return PSQL_CMD_SKIP_LINE; +} + +/* + * \s -- save history in a file or show it on the screen + */ +static backslashResult +exec_command_s(PsqlScanState scan_state, bool active_branch) +{ + bool success = true; + + if (active_branch) { char *fname = psql_scan_slash_option(scan_state, OT_NORMAL, NULL, true); @@ -1295,9 +2090,21 @@ exec_command(const char *cmd, putchar('\n'); free(fname); } + else + ignore_slash_options(scan_state); - /* \set -- generalized set variable/option command */ - else if (strcmp(cmd, "set") == 0) + return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR; +} + +/* + * \set -- set variable + */ +static backslashResult +exec_command_set(PsqlScanState scan_state, bool active_branch) +{ + bool success = true; + + if (active_branch) { char *opt0 = psql_scan_slash_option(scan_state, OT_NORMAL, NULL, false); @@ -1336,10 +2143,22 @@ exec_command(const char *cmd, } free(opt0); } + else + ignore_slash_options(scan_state); + return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR; +} - /* \setenv -- set environment command */ - else if (strcmp(cmd, "setenv") == 0) +/* + * \setenv -- set environment variable + */ +static backslashResult +exec_command_setenv(PsqlScanState scan_state, bool active_branch, + const char *cmd) +{ + bool success = true; + + if (active_branch) { char *envvar = psql_scan_slash_option(scan_state, OT_NORMAL, NULL, false); @@ -1381,9 +2200,21 @@ exec_command(const char *cmd, free(envvar); free(envval); } + else + ignore_slash_options(scan_state); - /* \sf -- show a function's source code */ - else if (strcmp(cmd, "sf") == 0 || strcmp(cmd, "sf+") == 0) + return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR; +} + +/* + * \sf -- show a function's source code + */ +static backslashResult +exec_command_sf(PsqlScanState scan_state, bool active_branch, const char *cmd) +{ + backslashResult status = PSQL_CMD_SKIP_LINE; + + if (active_branch) { bool show_linenumbers = (strcmp(cmd, "sf+") == 0); PQExpBuffer func_buf; @@ -1463,9 +2294,21 @@ exec_command(const char *cmd, free(func); destroyPQExpBuffer(func_buf); } + else + ignore_slash_whole_line(scan_state); - /* \sv -- show a view's source code */ - else if (strcmp(cmd, "sv") == 0 || strcmp(cmd, "sv+") == 0) + return status; +} + +/* + * \sv -- show a view's source code + */ +static backslashResult +exec_command_sv(PsqlScanState scan_state, bool active_branch, const char *cmd) +{ + backslashResult status = PSQL_CMD_SKIP_LINE; + + if (active_branch) { bool show_linenumbers = (strcmp(cmd, "sv+") == 0); PQExpBuffer view_buf; @@ -1539,9 +2382,21 @@ exec_command(const char *cmd, free(view); destroyPQExpBuffer(view_buf); } + else + ignore_slash_whole_line(scan_state); - /* \t -- turn off headers and row count */ - else if (strcmp(cmd, "t") == 0) + return status; +} + +/* + * \t -- turn off table headers and row count + */ +static backslashResult +exec_command_t(PsqlScanState scan_state, bool active_branch) +{ + bool success = true; + + if (active_branch) { char *opt = psql_scan_slash_option(scan_state, OT_NORMAL, NULL, true); @@ -1549,9 +2404,21 @@ exec_command(const char *cmd, success = do_pset("tuples_only", opt, &pset.popt, pset.quiet); free(opt); } + else + ignore_slash_options(scan_state); - /* \T -- define html attributes */ - else if (strcmp(cmd, "T") == 0) + return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR; +} + +/* + * \T -- define html
    attributes + */ +static backslashResult +exec_command_T(PsqlScanState scan_state, bool active_branch) +{ + bool success = true; + + if (active_branch) { char *value = psql_scan_slash_option(scan_state, OT_NORMAL, NULL, false); @@ -1559,9 +2426,21 @@ exec_command(const char *cmd, success = do_pset("tableattr", value, &pset.popt, pset.quiet); free(value); } + else + ignore_slash_options(scan_state); - /* \timing -- toggle timing of queries */ - else if (strcmp(cmd, "timing") == 0) + return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR; +} + +/* + * \timing -- enable/disable timing of queries + */ +static backslashResult +exec_command_timing(PsqlScanState scan_state, bool active_branch) +{ + bool success = true; + + if (active_branch) { char *opt = psql_scan_slash_option(scan_state, OT_NORMAL, NULL, false); @@ -1579,9 +2458,22 @@ exec_command(const char *cmd, } free(opt); } + else + ignore_slash_options(scan_state); - /* \unset */ - else if (strcmp(cmd, "unset") == 0) + return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR; +} + +/* + * \unset -- unset variable + */ +static backslashResult +exec_command_unset(PsqlScanState scan_state, bool active_branch, + const char *cmd) +{ + bool success = true; + + if (active_branch) { char *opt = psql_scan_slash_option(scan_state, OT_NORMAL, NULL, false); @@ -1596,13 +2488,28 @@ exec_command(const char *cmd, free(opt); } + else + ignore_slash_options(scan_state); - /* \w -- write query buffer to file */ - else if (strcmp(cmd, "w") == 0 || strcmp(cmd, "write") == 0) + return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR; +} + +/* + * \w -- write query buffer to file + */ +static backslashResult +exec_command_write(PsqlScanState scan_state, bool active_branch, + const char *cmd, + PQExpBuffer query_buf, PQExpBuffer previous_buf) +{ + backslashResult status = PSQL_CMD_SKIP_LINE; + + if (active_branch) { + char *fname = psql_scan_slash_option(scan_state, + OT_FILEPIPE, NULL, true); FILE *fd = NULL; bool is_pipe = false; - char *fname = NULL; if (!query_buf) { @@ -1611,17 +2518,14 @@ exec_command(const char *cmd, } else { - fname = psql_scan_slash_option(scan_state, - OT_FILEPIPE, NULL, true); - expand_tilde(&fname); - if (!fname) { psql_error("\\%s: missing required argument\n", cmd); - success = false; + status = PSQL_CMD_ERROR; } else { + expand_tilde(&fname); if (fname[0] == '|') { is_pipe = true; @@ -1636,7 +2540,7 @@ exec_command(const char *cmd, if (!fd) { psql_error("%s: %s\n", fname, strerror(errno)); - success = false; + status = PSQL_CMD_ERROR; } } } @@ -1647,6 +2551,9 @@ exec_command(const char *cmd, if (query_buf && query_buf->len > 0) fprintf(fd, "%s\n", query_buf->data); + /* Applies to previous query if current buffer is empty */ + else if (previous_buf && previous_buf->len > 0) + fprintf(fd, "%s\n", previous_buf->data); if (is_pipe) result = pclose(fd); @@ -1656,7 +2563,7 @@ exec_command(const char *cmd, if (result == EOF) { psql_error("%s: %s\n", fname, strerror(errno)); - success = false; + status = PSQL_CMD_ERROR; } } @@ -1665,9 +2572,22 @@ exec_command(const char *cmd, free(fname); } + else + ignore_slash_filepipe(scan_state); - /* \watch -- execute a query every N seconds */ - else if (strcmp(cmd, "watch") == 0) + return status; +} + +/* + * \watch -- execute a query every N seconds + */ +static backslashResult +exec_command_watch(PsqlScanState scan_state, bool active_branch, + PQExpBuffer query_buf, PQExpBuffer previous_buf) +{ + bool success = true; + + if (active_branch) { char *opt = psql_scan_slash_option(scan_state, OT_NORMAL, NULL, true); @@ -1682,15 +2602,30 @@ exec_command(const char *cmd, free(opt); } + /* Applies to previous query if current buffer is empty */ + copy_previous_query(query_buf, previous_buf); + success = do_watch(query_buf, sleep); /* Reset the query buffer as though for \r */ resetPQExpBuffer(query_buf); psql_scan_reset(scan_state); } + else + ignore_slash_options(scan_state); - /* \x -- set or toggle expanded table representation */ - else if (strcmp(cmd, "x") == 0) + return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR; +} + +/* + * \x -- set or toggle expanded table representation + */ +static backslashResult +exec_command_x(PsqlScanState scan_state, bool active_branch) +{ + bool success = true; + + if (active_branch) { char *opt = psql_scan_slash_option(scan_state, OT_NORMAL, NULL, true); @@ -1698,9 +2633,21 @@ exec_command(const char *cmd, success = do_pset("expanded", opt, &pset.popt, pset.quiet); free(opt); } + else + ignore_slash_options(scan_state); - /* \z -- list table rights (equivalent to \dp) */ - else if (strcmp(cmd, "z") == 0) + return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR; +} + +/* + * \z -- list table privileges (equivalent to \dp) + */ +static backslashResult +exec_command_z(PsqlScanState scan_state, bool active_branch) +{ + bool success = true; + + if (active_branch) { char *pattern = psql_scan_slash_option(scan_state, OT_NORMAL, NULL, true); @@ -1709,9 +2656,21 @@ exec_command(const char *cmd, if (pattern) free(pattern); } + else + ignore_slash_options(scan_state); - /* \! -- shell escape */ - else if (strcmp(cmd, "!") == 0) + return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR; +} + +/* + * \! -- execute shell command + */ +static backslashResult +exec_command_shell_escape(PsqlScanState scan_state, bool active_branch) +{ + bool success = true; + + if (active_branch) { char *opt = psql_scan_slash_option(scan_state, OT_WHOLE_LINE, NULL, false); @@ -1719,9 +2678,19 @@ exec_command(const char *cmd, success = do_shell(opt); free(opt); } + else + ignore_slash_whole_line(scan_state); - /* \? -- slash command help */ - else if (strcmp(cmd, "?") == 0) + return success ? PSQL_CMD_SKIP_LINE : PSQL_CMD_ERROR; +} + +/* + * \? -- print help about backslash commands + */ +static backslashResult +exec_command_slash_command_help(PsqlScanState scan_state, bool active_branch) +{ + if (active_branch) { char *opt0 = psql_scan_slash_option(scan_state, OT_NORMAL, NULL, false); @@ -1734,34 +2703,231 @@ exec_command(const char *cmd, helpVariables(pset.popt.topt.pager); else slashUsage(pset.popt.topt.pager); + + if (opt0) + free(opt0); } + else + ignore_slash_options(scan_state); -#if 0 + return PSQL_CMD_SKIP_LINE; +} + + +/* + * Read and interpret an argument to the \connect slash command. + */ +static char * +read_connect_arg(PsqlScanState scan_state) +{ + char *result; + char quote; /* - * These commands don't do anything. I just use them to test the parser. + * Ideally we should treat the arguments as SQL identifiers. But for + * backwards compatibility with 7.2 and older pg_dump files, we have to + * take unquoted arguments verbatim (don't downcase them). For now, + * double-quoted arguments may be stripped of double quotes (as if SQL + * identifiers). By 7.4 or so, pg_dump files can be expected to + * double-quote all mixed-case \connect arguments, and then we can get rid + * of OT_SQLIDHACK. */ - else if (strcmp(cmd, "void") == 0 || strcmp(cmd, "#") == 0) - { - int i = 0; - char *value; + result = psql_scan_slash_option(scan_state, OT_SQLIDHACK, "e, true); - while ((value = psql_scan_slash_option(scan_state, - OT_NORMAL, NULL, true))) - { - psql_error("+ opt(%d) = |%s|\n", i++, value); - free(value); - } + if (!result) + return NULL; + + if (quote) + return result; + + if (*result == '\0' || strcmp(result, "-") == 0) + return NULL; + + return result; +} + +/* + * Read a boolean expression, return it as a PQExpBuffer string. + * + * Note: anything more or less than one token will certainly fail to be + * parsed by ParseVariableBool, so we don't worry about complaining here. + * This routine's return data structure will need to be rethought anyway + * to support likely future extensions such as "\if defined VARNAME". + */ +static PQExpBuffer +gather_boolean_expression(PsqlScanState scan_state) +{ + PQExpBuffer exp_buf = createPQExpBuffer(); + int num_options = 0; + char *value; + + /* collect all arguments for the conditional command into exp_buf */ + while ((value = psql_scan_slash_option(scan_state, + OT_NORMAL, NULL, false)) != NULL) + { + /* add spaces between tokens */ + if (num_options > 0) + appendPQExpBufferChar(exp_buf, ' '); + appendPQExpBufferStr(exp_buf, value); + num_options++; + free(value); } -#endif - else - status = PSQL_CMD_UNKNOWN; + return exp_buf; +} - if (!success) - status = PSQL_CMD_ERROR; +/* + * Read a boolean expression, return true if the expression + * was a valid boolean expression that evaluated to true. + * Otherwise return false. + * + * Note: conditional stack's top state must be active, else lexer will + * fail to expand variables and backticks. + */ +static bool +is_true_boolean_expression(PsqlScanState scan_state, const char *name) +{ + PQExpBuffer buf = gather_boolean_expression(scan_state); + bool value = false; + bool success = ParseVariableBool(buf->data, name, &value); - return status; + destroyPQExpBuffer(buf); + return success && value; +} + +/* + * Read a boolean expression, but do nothing with it. + * + * Note: conditional stack's top state must be INACTIVE, else lexer will + * expand variables and backticks, which we do not want here. + */ +static void +ignore_boolean_expression(PsqlScanState scan_state) +{ + PQExpBuffer buf = gather_boolean_expression(scan_state); + + destroyPQExpBuffer(buf); +} + +/* + * Read and discard "normal" slash command options. + * + * This should be used for inactive-branch processing of any slash command + * that eats one or more OT_NORMAL, OT_SQLID, or OT_SQLIDHACK parameters. + * We don't need to worry about exactly how many it would eat, since the + * cleanup logic in HandleSlashCmds would silently discard any extras anyway. + */ +static void +ignore_slash_options(PsqlScanState scan_state) +{ + char *arg; + + while ((arg = psql_scan_slash_option(scan_state, + OT_NORMAL, NULL, false)) != NULL) + free(arg); +} + +/* + * Read and discard FILEPIPE slash command argument. + * + * This *MUST* be used for inactive-branch processing of any slash command + * that takes an OT_FILEPIPE option. Otherwise we might consume a different + * amount of option text in active and inactive cases. + */ +static void +ignore_slash_filepipe(PsqlScanState scan_state) +{ + char *arg = psql_scan_slash_option(scan_state, + OT_FILEPIPE, NULL, false); + + if (arg) + free(arg); +} + +/* + * Read and discard whole-line slash command argument. + * + * This *MUST* be used for inactive-branch processing of any slash command + * that takes an OT_WHOLE_LINE option. Otherwise we might consume a different + * amount of option text in active and inactive cases. + */ +static void +ignore_slash_whole_line(PsqlScanState scan_state) +{ + char *arg = psql_scan_slash_option(scan_state, + OT_WHOLE_LINE, NULL, false); + + if (arg) + free(arg); +} + +/* + * Return true if the command given is a branching command. + */ +static bool +is_branching_command(const char *cmd) +{ + return (strcmp(cmd, "if") == 0 || + strcmp(cmd, "elif") == 0 || + strcmp(cmd, "else") == 0 || + strcmp(cmd, "endif") == 0); +} + +/* + * Prepare to possibly restore query buffer to its current state + * (cf. discard_query_text). + * + * We need to remember the length of the query buffer, and the lexer's + * notion of the parenthesis nesting depth. + */ +static void +save_query_text_state(PsqlScanState scan_state, ConditionalStack cstack, + PQExpBuffer query_buf) +{ + if (query_buf) + conditional_stack_set_query_len(cstack, query_buf->len); + conditional_stack_set_paren_depth(cstack, + psql_scan_get_paren_depth(scan_state)); +} + +/* + * Discard any query text absorbed during an inactive conditional branch. + * + * We must discard data that was appended to query_buf during an inactive + * \if branch. We don't have to do anything there if there's no query_buf. + * + * Also, reset the lexer state to the same paren depth there was before. + * (The rest of its state doesn't need attention, since we could not be + * inside a comment or literal or partial token.) + */ +static void +discard_query_text(PsqlScanState scan_state, ConditionalStack cstack, + PQExpBuffer query_buf) +{ + if (query_buf) + { + int new_len = conditional_stack_get_query_len(cstack); + + Assert(new_len >= 0 && new_len <= query_buf->len); + query_buf->len = new_len; + query_buf->data[new_len] = '\0'; + } + psql_scan_set_paren_depth(scan_state, + conditional_stack_get_paren_depth(cstack)); +} + +/* + * If query_buf is empty, copy previous_buf into it. + * + * This is used by various slash commands for which re-execution of a + * previous query is a common usage. For convenience, we allow the + * case of query_buf == NULL (and do nothing). + */ +static void +copy_previous_query(PQExpBuffer query_buf, PQExpBuffer previous_buf) +{ + if (query_buf && query_buf->len == 0) + appendPQExpBufferStr(query_buf, previous_buf->data); } /* diff --git a/src/bin/psql/command.h b/src/bin/psql/command.h index d0c32645f1..e8ea8473e8 100644 --- a/src/bin/psql/command.h +++ b/src/bin/psql/command.h @@ -10,6 +10,7 @@ #include "fe_utils/print.h" #include "fe_utils/psqlscan.h" +#include "conditional.h" typedef enum _backslashResult @@ -25,7 +26,9 @@ typedef enum _backslashResult extern backslashResult HandleSlashCmds(PsqlScanState scan_state, - PQExpBuffer query_buf); + ConditionalStack cstack, + PQExpBuffer query_buf, + PQExpBuffer previous_buf); extern int process_file(char *filename, bool use_relative_path); diff --git a/src/bin/psql/common.c b/src/bin/psql/common.c index e9d4fe6786..b06ae9779d 100644 --- a/src/bin/psql/common.c +++ b/src/bin/psql/common.c @@ -121,7 +121,8 @@ setQFout(const char *fname) * (Failure in escaping should lead to returning NULL.) * * "passthrough" is the pointer previously given to psql_scan_set_passthrough. - * psql currently doesn't use this. + * In psql, passthrough points to a ConditionalStack, which we check to + * determine whether variable expansion is allowed. */ char * psql_get_variable(const char *varname, bool escape, bool as_ident, @@ -130,6 +131,10 @@ psql_get_variable(const char *varname, bool escape, bool as_ident, char *result; const char *value; + /* In an inactive \if branch, suppress all variable substitutions */ + if (passthrough && !conditional_active((ConditionalStack) passthrough)) + return NULL; + value = GetVariable(pset.vars, varname); if (!value) return NULL; diff --git a/src/bin/psql/conditional.c b/src/bin/psql/conditional.c new file mode 100644 index 0000000000..63977ce5dd --- /dev/null +++ b/src/bin/psql/conditional.c @@ -0,0 +1,153 @@ +/* + * psql - the PostgreSQL interactive terminal + * + * Copyright (c) 2000-2017, PostgreSQL Global Development Group + * + * src/bin/psql/conditional.c + */ +#include "postgres_fe.h" + +#include "conditional.h" + +/* + * create stack + */ +ConditionalStack +conditional_stack_create(void) +{ + ConditionalStack cstack = pg_malloc(sizeof(ConditionalStackData)); + + cstack->head = NULL; + return cstack; +} + +/* + * destroy stack + */ +void +conditional_stack_destroy(ConditionalStack cstack) +{ + while (conditional_stack_pop(cstack)) + continue; + free(cstack); +} + +/* + * Create a new conditional branch. + */ +void +conditional_stack_push(ConditionalStack cstack, ifState new_state) +{ + IfStackElem *p = (IfStackElem *) pg_malloc(sizeof(IfStackElem)); + + p->if_state = new_state; + p->query_len = -1; + p->paren_depth = -1; + p->next = cstack->head; + cstack->head = p; +} + +/* + * Destroy the topmost conditional branch. + * Returns false if there was no branch to end. + */ +bool +conditional_stack_pop(ConditionalStack cstack) +{ + IfStackElem *p = cstack->head; + + if (!p) + return false; + cstack->head = cstack->head->next; + free(p); + return true; +} + +/* + * Fetch the current state of the top of the stack. + */ +ifState +conditional_stack_peek(ConditionalStack cstack) +{ + if (conditional_stack_empty(cstack)) + return IFSTATE_NONE; + return cstack->head->if_state; +} + +/* + * Change the state of the topmost branch. + * Returns false if there was no branch state to set. + */ +bool +conditional_stack_poke(ConditionalStack cstack, ifState new_state) +{ + if (conditional_stack_empty(cstack)) + return false; + cstack->head->if_state = new_state; + return true; +} + +/* + * True if there are no active \if-blocks. + */ +bool +conditional_stack_empty(ConditionalStack cstack) +{ + return cstack->head == NULL; +} + +/* + * True if we should execute commands normally; that is, the current + * conditional branch is active, or there is no open \if block. + */ +bool +conditional_active(ConditionalStack cstack) +{ + ifState s = conditional_stack_peek(cstack); + + return s == IFSTATE_NONE || s == IFSTATE_TRUE || s == IFSTATE_ELSE_TRUE; +} + +/* + * Save current query buffer length in topmost stack entry. + */ +void +conditional_stack_set_query_len(ConditionalStack cstack, int len) +{ + Assert(!conditional_stack_empty(cstack)); + cstack->head->query_len = len; +} + +/* + * Fetch last-recorded query buffer length from topmost stack entry. + * Will return -1 if no stack or it was never saved. + */ +int +conditional_stack_get_query_len(ConditionalStack cstack) +{ + if (conditional_stack_empty(cstack)) + return -1; + return cstack->head->query_len; +} + +/* + * Save current parenthesis nesting depth in topmost stack entry. + */ +void +conditional_stack_set_paren_depth(ConditionalStack cstack, int depth) +{ + Assert(!conditional_stack_empty(cstack)); + cstack->head->paren_depth = depth; +} + +/* + * Fetch last-recorded parenthesis nesting depth from topmost stack entry. + * Will return -1 if no stack or it was never saved. + */ +int +conditional_stack_get_paren_depth(ConditionalStack cstack) +{ + if (conditional_stack_empty(cstack)) + return -1; + return cstack->head->paren_depth; +} diff --git a/src/bin/psql/conditional.h b/src/bin/psql/conditional.h new file mode 100644 index 0000000000..90e4d93b40 --- /dev/null +++ b/src/bin/psql/conditional.h @@ -0,0 +1,83 @@ +/* + * psql - the PostgreSQL interactive terminal + * + * Copyright (c) 2000-2017, PostgreSQL Global Development Group + * + * src/bin/psql/conditional.h + */ +#ifndef CONDITIONAL_H +#define CONDITIONAL_H + +/* + * Possible states of a single level of \if block. + */ +typedef enum ifState +{ + IFSTATE_NONE = 0, /* not currently in an \if block */ + IFSTATE_TRUE, /* currently in an \if or \elif that is true + * and all parent branches (if any) are true */ + IFSTATE_FALSE, /* currently in an \if or \elif that is false + * but no true branch has yet been seen, and + * all parent branches (if any) are true */ + IFSTATE_IGNORED, /* currently in an \elif that follows a true + * branch, or the whole \if is a child of a + * false parent branch */ + IFSTATE_ELSE_TRUE, /* currently in an \else that is true and all + * parent branches (if any) are true */ + IFSTATE_ELSE_FALSE /* currently in an \else that is false or + * ignored */ +} ifState; + +/* + * The state of nested \ifs is stored in a stack. + * + * query_len is used to determine what accumulated text to throw away at the + * end of an inactive branch. (We could, perhaps, teach the lexer to not add + * stuff to the query buffer in the first place when inside an inactive branch; + * but that would be very invasive.) We also need to save and restore the + * lexer's parenthesis nesting depth when throwing away text. (We don't need + * to save and restore any of its other state, such as comment nesting depth, + * because a backslash command could never appear inside a comment or SQL + * literal.) + */ +typedef struct IfStackElem +{ + ifState if_state; /* current state, see enum above */ + int query_len; /* length of query_buf at last branch start */ + int paren_depth; /* parenthesis depth at last branch start */ + struct IfStackElem *next; /* next surrounding \if, if any */ +} IfStackElem; + +typedef struct ConditionalStackData +{ + IfStackElem *head; +} ConditionalStackData; + +typedef struct ConditionalStackData *ConditionalStack; + + +extern ConditionalStack conditional_stack_create(void); + +extern void conditional_stack_destroy(ConditionalStack cstack); + +extern void conditional_stack_push(ConditionalStack cstack, ifState new_state); + +extern bool conditional_stack_pop(ConditionalStack cstack); + +extern ifState conditional_stack_peek(ConditionalStack cstack); + +extern bool conditional_stack_poke(ConditionalStack cstack, ifState new_state); + +extern bool conditional_stack_empty(ConditionalStack cstack); + +extern bool conditional_active(ConditionalStack cstack); + +extern void conditional_stack_set_query_len(ConditionalStack cstack, int len); + +extern int conditional_stack_get_query_len(ConditionalStack cstack); + +extern void conditional_stack_set_paren_depth(ConditionalStack cstack, int depth); + +extern int conditional_stack_get_paren_depth(ConditionalStack cstack); + +#endif /* CONDITIONAL_H */ diff --git a/src/bin/psql/copy.c b/src/bin/psql/copy.c index 481031a229..2005b9a0bf 100644 --- a/src/bin/psql/copy.c +++ b/src/bin/psql/copy.c @@ -552,7 +552,7 @@ handleCopyIn(PGconn *conn, FILE *copystream, bool isbinary, PGresult **res) /* interactive input probably silly, but give one prompt anyway */ if (showprompt) { - const char *prompt = get_prompt(PROMPT_COPY); + const char *prompt = get_prompt(PROMPT_COPY, NULL); fputs(prompt, stdout); fflush(stdout); @@ -590,7 +590,7 @@ handleCopyIn(PGconn *conn, FILE *copystream, bool isbinary, PGresult **res) if (showprompt) { - const char *prompt = get_prompt(PROMPT_COPY); + const char *prompt = get_prompt(PROMPT_COPY, NULL); fputs(prompt, stdout); fflush(stdout); diff --git a/src/bin/psql/help.c b/src/bin/psql/help.c index ba14df0344..ac435220e6 100644 --- a/src/bin/psql/help.c +++ b/src/bin/psql/help.c @@ -167,7 +167,7 @@ slashUsage(unsigned short int pager) * Use "psql --help=commands | wc" to count correctly. It's okay to count * the USE_READLINE line even in builds without that. */ - output = PageOutput(113, pager ? &(pset.popt.topt) : NULL); + output = PageOutput(122, pager ? &(pset.popt.topt) : NULL); fprintf(output, _("General\n")); fprintf(output, _(" \\copyright show PostgreSQL usage and distribution terms\n")); @@ -210,6 +210,13 @@ slashUsage(unsigned short int pager) fprintf(output, _(" \\qecho [STRING] write string to query output stream (see \\o)\n")); fprintf(output, "\n"); + fprintf(output, _("Conditional\n")); + fprintf(output, _(" \\if EXPR begin conditional block\n")); + fprintf(output, _(" \\elif EXPR alternative within current conditional block\n")); + fprintf(output, _(" \\else final alternative within current conditional block\n")); + fprintf(output, _(" \\endif end conditional block\n")); + fprintf(output, "\n"); + fprintf(output, _("Informational\n")); fprintf(output, _(" (options: S = show system objects, + = additional detail)\n")); fprintf(output, _(" \\d[S+] list tables, views, and sequences\n")); diff --git a/src/bin/psql/mainloop.c b/src/bin/psql/mainloop.c index 6e358e2e1b..2bc2f43b4e 100644 --- a/src/bin/psql/mainloop.c +++ b/src/bin/psql/mainloop.c @@ -35,6 +35,7 @@ int MainLoop(FILE *source) { PsqlScanState scan_state; /* lexer working state */ + ConditionalStack cond_stack; /* \if status stack */ volatile PQExpBuffer query_buf; /* buffer for query being accumulated */ volatile PQExpBuffer previous_buf; /* if there isn't anything in the new * buffer yet, use this one for \e, @@ -50,16 +51,15 @@ MainLoop(FILE *source) volatile promptStatus_t prompt_status = PROMPT_READY; volatile int count_eof = 0; volatile bool die_on_error = false; - - /* Save the prior command source */ FILE *prev_cmd_source; bool prev_cmd_interactive; uint64 prev_lineno; - /* Save old settings */ + /* Save the prior command source */ prev_cmd_source = pset.cur_cmd_source; prev_cmd_interactive = pset.cur_cmd_interactive; prev_lineno = pset.lineno; + /* pset.stmt_lineno does not need to be saved and restored */ /* Establish new source */ pset.cur_cmd_source = source; @@ -69,6 +69,8 @@ MainLoop(FILE *source) /* Create working state */ scan_state = psql_scan_create(&psqlscan_callbacks); + cond_stack = conditional_stack_create(); + psql_scan_set_passthrough(scan_state, (void *) cond_stack); query_buf = createPQExpBuffer(); previous_buf = createPQExpBuffer(); @@ -122,7 +124,19 @@ MainLoop(FILE *source) cancel_pressed = false; if (pset.cur_cmd_interactive) + { putc('\n', stdout); + + /* + * if interactive user is in an \if block, then Ctrl-C will + * exit from the innermost \if. + */ + if (!conditional_stack_empty(cond_stack)) + { + psql_error("\\if: escaped\n"); + conditional_stack_pop(cond_stack); + } + } else { successResult = EXIT_USER; @@ -140,7 +154,8 @@ MainLoop(FILE *source) /* May need to reset prompt, eg after \r command */ if (query_buf->len == 0) prompt_status = PROMPT_READY; - line = gets_interactive(get_prompt(prompt_status), query_buf); + line = gets_interactive(get_prompt(prompt_status, cond_stack), + query_buf); } else { @@ -286,8 +301,10 @@ MainLoop(FILE *source) (scan_result == PSCAN_EOL && pset.singleline)) { /* - * Save query in history. We use history_buf to accumulate - * multi-line queries into a single history entry. + * Save line in history. We use history_buf to accumulate + * multi-line queries into a single history entry. Note that + * history accumulation works on input lines, so it doesn't + * matter whether the query will be ignored due to \if. */ if (pset.cur_cmd_interactive && !line_saved_in_history) { @@ -296,22 +313,36 @@ MainLoop(FILE *source) line_saved_in_history = true; } - /* execute query */ - success = SendQuery(query_buf->data); - slashCmdStatus = success ? PSQL_CMD_SEND : PSQL_CMD_ERROR; - pset.stmt_lineno = 1; - - /* transfer query to previous_buf by pointer-swapping */ + /* execute query unless we're in an inactive \if branch */ + if (conditional_active(cond_stack)) { - PQExpBuffer swap_buf = previous_buf; + success = SendQuery(query_buf->data); + slashCmdStatus = success ? PSQL_CMD_SEND : PSQL_CMD_ERROR; + pset.stmt_lineno = 1; - previous_buf = query_buf; - query_buf = swap_buf; - } - resetPQExpBuffer(query_buf); + /* transfer query to previous_buf by pointer-swapping */ + { + PQExpBuffer swap_buf = previous_buf; - added_nl_pos = -1; - /* we need not do psql_scan_reset() here */ + previous_buf = query_buf; + query_buf = swap_buf; + } + resetPQExpBuffer(query_buf); + + added_nl_pos = -1; + /* we need not do psql_scan_reset() here */ + } + else + { + /* if interactive, warn about non-executed query */ + if (pset.cur_cmd_interactive) + psql_error("query ignored; use \\endif or Ctrl-C to exit current \\if block\n"); + /* fake an OK result for purposes of loop checks */ + success = true; + slashCmdStatus = PSQL_CMD_SEND; + pset.stmt_lineno = 1; + /* note that query_buf doesn't change state */ + } } else if (scan_result == PSCAN_BACKSLASH) { @@ -343,21 +374,24 @@ MainLoop(FILE *source) /* execute backslash command */ slashCmdStatus = HandleSlashCmds(scan_state, - query_buf->len > 0 ? - query_buf : previous_buf); + cond_stack, + query_buf, + previous_buf); success = slashCmdStatus != PSQL_CMD_ERROR; - pset.stmt_lineno = 1; - if ((slashCmdStatus == PSQL_CMD_SEND || slashCmdStatus == PSQL_CMD_NEWEDIT) && - query_buf->len == 0) - { - /* copy previous buffer to current for handling */ - appendPQExpBufferStr(query_buf, previous_buf->data); - } + /* + * Resetting stmt_lineno after a backslash command isn't + * always appropriate, but it's what we've done historically + * and there have been few complaints. + */ + pset.stmt_lineno = 1; if (slashCmdStatus == PSQL_CMD_SEND) { + /* should not see this in inactive branch */ + Assert(conditional_active(cond_stack)); + success = SendQuery(query_buf->data); /* transfer query to previous_buf by pointer-swapping */ @@ -374,6 +408,8 @@ MainLoop(FILE *source) } else if (slashCmdStatus == PSQL_CMD_NEWEDIT) { + /* should not see this in inactive branch */ + Assert(conditional_active(cond_stack)); /* rescan query_buf as new input */ psql_scan_finish(scan_state); free(line); @@ -429,8 +465,17 @@ MainLoop(FILE *source) if (pset.cur_cmd_interactive) pg_send_history(history_buf); - /* execute query */ - success = SendQuery(query_buf->data); + /* execute query unless we're in an inactive \if branch */ + if (conditional_active(cond_stack)) + { + success = SendQuery(query_buf->data); + } + else + { + if (pset.cur_cmd_interactive) + psql_error("query ignored; use \\endif or Ctrl-C to exit current \\if block\n"); + success = true; + } if (!success && die_on_error) successResult = EXIT_USER; @@ -438,6 +483,19 @@ MainLoop(FILE *source) successResult = EXIT_BADCONN; } + /* + * Check for unbalanced \if-\endifs unless user explicitly quit, or the + * script is erroring out + */ + if (slashCmdStatus != PSQL_CMD_TERMINATE && + successResult != EXIT_USER && + !conditional_stack_empty(cond_stack)) + { + psql_error("reached EOF without finding closing \\endif(s)\n"); + if (die_on_error && !pset.cur_cmd_interactive) + successResult = EXIT_USER; + } + /* * Let's just make real sure the SIGINT handler won't try to use * sigint_interrupt_jmp after we exit this routine. If there is an outer @@ -452,6 +510,7 @@ MainLoop(FILE *source) destroyPQExpBuffer(history_buf); psql_scan_destroy(scan_state); + conditional_stack_destroy(cond_stack); pset.cur_cmd_source = prev_cmd_source; pset.cur_cmd_interactive = prev_cmd_interactive; diff --git a/src/bin/psql/prompt.c b/src/bin/psql/prompt.c index f7930c4a83..e502ff3f6e 100644 --- a/src/bin/psql/prompt.c +++ b/src/bin/psql/prompt.c @@ -66,7 +66,7 @@ */ char * -get_prompt(promptStatus_t status) +get_prompt(promptStatus_t status, ConditionalStack cstack) { #define MAX_PROMPT_SIZE 256 static char destination[MAX_PROMPT_SIZE + 1]; @@ -188,7 +188,9 @@ get_prompt(promptStatus_t status) switch (status) { case PROMPT_READY: - if (!pset.db) + if (cstack != NULL && !conditional_active(cstack)) + buf[0] = '@'; + else if (!pset.db) buf[0] = '!'; else if (!pset.singleline) buf[0] = '='; diff --git a/src/bin/psql/prompt.h b/src/bin/psql/prompt.h index 977e754293..b3d2d98fd7 100644 --- a/src/bin/psql/prompt.h +++ b/src/bin/psql/prompt.h @@ -10,7 +10,8 @@ /* enum promptStatus_t is now defined by psqlscan.h */ #include "fe_utils/psqlscan.h" +#include "conditional.h" -char *get_prompt(promptStatus_t status); +char *get_prompt(promptStatus_t status, ConditionalStack cstack); #endif /* PROMPT_H */ diff --git a/src/bin/psql/psqlscanslash.h b/src/bin/psql/psqlscanslash.h index 266e93af44..db76061332 100644 --- a/src/bin/psql/psqlscanslash.h +++ b/src/bin/psql/psqlscanslash.h @@ -18,8 +18,7 @@ enum slash_option_type OT_SQLID, /* treat as SQL identifier */ OT_SQLIDHACK, /* SQL identifier, but don't downcase */ OT_FILEPIPE, /* it's a filename or pipe */ - OT_WHOLE_LINE, /* just snarf the rest of the line */ - OT_NO_EVAL /* no expansion of backticks or variables */ + OT_WHOLE_LINE /* just snarf the rest of the line */ }; @@ -32,6 +31,10 @@ extern char *psql_scan_slash_option(PsqlScanState state, extern void psql_scan_slash_command_end(PsqlScanState state); +extern int psql_scan_get_paren_depth(PsqlScanState state); + +extern void psql_scan_set_paren_depth(PsqlScanState state, int depth); + extern void dequote_downcase_identifier(char *str, bool downcase, int encoding); #endif /* PSQLSCANSLASH_H */ diff --git a/src/bin/psql/psqlscanslash.l b/src/bin/psql/psqlscanslash.l index ba4a08d000..319afdc744 100644 --- a/src/bin/psql/psqlscanslash.l +++ b/src/bin/psql/psqlscanslash.l @@ -19,6 +19,7 @@ #include "postgres_fe.h" #include "psqlscanslash.h" +#include "conditional.h" #include "libpq-fe.h" } @@ -230,8 +231,7 @@ other . :{variable_char}+ { /* Possible psql variable substitution */ - if (option_type == OT_NO_EVAL || - cur_state->callbacks->get_variable == NULL) + if (cur_state->callbacks->get_variable == NULL) ECHO; else { @@ -268,25 +268,15 @@ other . } :'{variable_char}+' { - if (option_type == OT_NO_EVAL) - ECHO; - else - { - psqlscan_escape_variable(cur_state, yytext, yyleng, false); - *option_quote = ':'; - } + psqlscan_escape_variable(cur_state, yytext, yyleng, false); + *option_quote = ':'; unquoted_option_chars = 0; } :\"{variable_char}+\" { - if (option_type == OT_NO_EVAL) - ECHO; - else - { - psqlscan_escape_variable(cur_state, yytext, yyleng, true); - *option_quote = ':'; - } + psqlscan_escape_variable(cur_state, yytext, yyleng, true); + *option_quote = ':'; unquoted_option_chars = 0; } @@ -353,8 +343,9 @@ other . */ "`" { - /* In NO_EVAL mode, don't evaluate the command */ - if (option_type != OT_NO_EVAL) + /* In an inactive \if branch, don't evaluate the command */ + if (cur_state->cb_passthrough == NULL || + conditional_active((ConditionalStack) cur_state->cb_passthrough)) evaluate_backtick(cur_state); BEGIN(xslasharg); } @@ -641,6 +632,25 @@ psql_scan_slash_command_end(PsqlScanState state) psql_scan_reselect_sql_lexer(state); } +/* + * Fetch current paren nesting depth + */ +int +psql_scan_get_paren_depth(PsqlScanState state) +{ + return state->paren_depth; +} + +/* + * Set paren nesting depth + */ +void +psql_scan_set_paren_depth(PsqlScanState state, int depth) +{ + Assert(depth >= 0); + state->paren_depth = depth; +} + /* * De-quote and optionally downcase a SQL identifier. * diff --git a/src/bin/psql/startup.c b/src/bin/psql/startup.c index 694f0ef257..8068a28b4e 100644 --- a/src/bin/psql/startup.c +++ b/src/bin/psql/startup.c @@ -331,6 +331,7 @@ main(int argc, char *argv[]) else if (cell->action == ACT_SINGLE_SLASH) { PsqlScanState scan_state; + ConditionalStack cond_stack; if (pset.echo == PSQL_ECHO_ALL) puts(cell->val); @@ -339,11 +340,17 @@ main(int argc, char *argv[]) psql_scan_setup(scan_state, cell->val, strlen(cell->val), pset.encoding, standard_strings()); + cond_stack = conditional_stack_create(); + psql_scan_set_passthrough(scan_state, (void *) cond_stack); - successResult = HandleSlashCmds(scan_state, NULL) != PSQL_CMD_ERROR + successResult = HandleSlashCmds(scan_state, + cond_stack, + NULL, + NULL) != PSQL_CMD_ERROR ? EXIT_SUCCESS : EXIT_FAILURE; psql_scan_destroy(scan_state); + conditional_stack_destroy(cond_stack); } else if (cell->action == ACT_FILE) { diff --git a/src/test/regress/expected/psql.out b/src/test/regress/expected/psql.out index eb7f197b12..8aa914fa95 100644 --- a/src/test/regress/expected/psql.out +++ b/src/test/regress/expected/psql.out @@ -2735,6 +2735,175 @@ deallocate q; \pset format aligned \pset expanded off \pset border 1 +-- tests for \if ... \endif +\if true + select 'okay'; + ?column? +---------- + okay +(1 row) + + select 'still okay'; + ?column? +------------ + still okay +(1 row) + +\else + not okay; + still not okay +\endif +-- at this point query buffer should still have last valid line +\g + ?column? +------------ + still okay +(1 row) + +-- \if should work okay on part of a query +select + \if true + 42 + \else + (bogus + \endif + forty_two; + forty_two +----------- + 42 +(1 row) + +select \if false \\ (bogus \else \\ 42 \endif \\ forty_two; + forty_two +----------- + 42 +(1 row) + +-- test a large nested if using a variety of true-equivalents +\if true + \if 1 + \if yes + \if on + \echo 'all true' +all true + \else + \echo 'should not print #1-1' + \endif + \else + \echo 'should not print #1-2' + \endif + \else + \echo 'should not print #1-3' + \endif +\else + \echo 'should not print #1-4' +\endif +-- test a variety of false-equivalents in an if/elif/else structure +\if false + \echo 'should not print #2-1' +\elif 0 + \echo 'should not print #2-2' +\elif no + \echo 'should not print #2-3' +\elif off + \echo 'should not print #2-4' +\else + \echo 'all false' +all false +\endif +-- test simple true-then-else +\if true + \echo 'first thing true' +first thing true +\else + \echo 'should not print #3-1' +\endif +-- test simple false-true-else +\if false + \echo 'should not print #4-1' +\elif true + \echo 'second thing true' +second thing true +\else + \echo 'should not print #5-1' +\endif +-- invalid boolean expressions are false +\if invalid boolean expression +unrecognized value "invalid boolean expression" for "\if expression": boolean expected + \echo 'will not print #6-1' +\else + \echo 'will print anyway #6-2' +will print anyway #6-2 +\endif +-- test un-matched endif +\endif +\endif: no matching \if +-- test un-matched else +\else +\else: no matching \if +-- test un-matched elif +\elif +\elif: no matching \if +-- test double-else error +\if true +\else +\else +\else: cannot occur after \else +\endif +-- test elif out-of-order +\if false +\else +\elif +\elif: cannot occur after \else +\endif +-- test if-endif matching in a false branch +\if false + \if false + \echo 'should not print #7-1' + \else + \echo 'should not print #7-2' + \endif + \echo 'should not print #7-3' +\else + \echo 'should print #7-4' +should print #7-4 +\endif +-- show that vars and backticks are not expanded when ignoring extra args +\set foo bar +\echo :foo :'foo' :"foo" +bar 'bar' "bar" +\pset fieldsep | `nosuchcommand` :foo :'foo' :"foo" +\pset: extra argument "nosuchcommand" ignored +\pset: extra argument ":foo" ignored +\pset: extra argument ":'foo'" ignored +\pset: extra argument ":"foo"" ignored +-- show that vars and backticks are not expanded and commands are ignored +-- when in a false if-branch +\set try_to_quit '\\q' +\if false + :try_to_quit + \echo `nosuchcommand` :foo :'foo' :"foo" + \pset fieldsep | `nosuchcommand` :foo :'foo' :"foo" + \a \C arg1 \c arg1 arg2 arg3 arg4 \cd arg1 \conninfo + \copy arg1 arg2 arg3 arg4 arg5 arg6 + \copyright \dt arg1 \e arg1 arg2 + \ef whole_line + \ev whole_line + \echo arg1 arg2 arg3 arg4 arg5 \echo arg1 \encoding arg1 \errverbose + \g arg1 \gx arg1 \gexec \h \html \i arg1 \ir arg1 \l arg1 \lo arg1 arg2 + \o arg1 \p \password arg1 \prompt arg1 arg2 \pset arg1 arg2 \q + \reset \s arg1 \set arg1 arg2 arg3 arg4 arg5 arg6 arg7 \setenv arg1 arg2 + \sf whole_line + \sv whole_line + \t arg1 \T arg1 \timing arg1 \unset arg1 \w arg1 \watch arg1 \x arg1 + -- \else here is eaten as part of OT_FILEPIPE argument + \w |/no/such/file \else + -- \endif here is eaten as part of whole-line argument + \! whole_line \endif +\else + \echo 'should print #8-1' +should print #8-1 +\endif -- SHOW_CONTEXT \set SHOW_CONTEXT never do $$ diff --git a/src/test/regress/sql/psql.sql b/src/test/regress/sql/psql.sql index 8f8e17a87c..0ae4dd84ea 100644 --- a/src/test/regress/sql/psql.sql +++ b/src/test/regress/sql/psql.sql @@ -382,6 +382,150 @@ deallocate q; \pset expanded off \pset border 1 +-- tests for \if ... \endif + +\if true + select 'okay'; + select 'still okay'; +\else + not okay; + still not okay +\endif + +-- at this point query buffer should still have last valid line +\g + +-- \if should work okay on part of a query +select + \if true + 42 + \else + (bogus + \endif + forty_two; + +select \if false \\ (bogus \else \\ 42 \endif \\ forty_two; + +-- test a large nested if using a variety of true-equivalents +\if true + \if 1 + \if yes + \if on + \echo 'all true' + \else + \echo 'should not print #1-1' + \endif + \else + \echo 'should not print #1-2' + \endif + \else + \echo 'should not print #1-3' + \endif +\else + \echo 'should not print #1-4' +\endif + +-- test a variety of false-equivalents in an if/elif/else structure +\if false + \echo 'should not print #2-1' +\elif 0 + \echo 'should not print #2-2' +\elif no + \echo 'should not print #2-3' +\elif off + \echo 'should not print #2-4' +\else + \echo 'all false' +\endif + +-- test simple true-then-else +\if true + \echo 'first thing true' +\else + \echo 'should not print #3-1' +\endif + +-- test simple false-true-else +\if false + \echo 'should not print #4-1' +\elif true + \echo 'second thing true' +\else + \echo 'should not print #5-1' +\endif + +-- invalid boolean expressions are false +\if invalid boolean expression + \echo 'will not print #6-1' +\else + \echo 'will print anyway #6-2' +\endif + +-- test un-matched endif +\endif + +-- test un-matched else +\else + +-- test un-matched elif +\elif + +-- test double-else error +\if true +\else +\else +\endif + +-- test elif out-of-order +\if false +\else +\elif +\endif + +-- test if-endif matching in a false branch +\if false + \if false + \echo 'should not print #7-1' + \else + \echo 'should not print #7-2' + \endif + \echo 'should not print #7-3' +\else + \echo 'should print #7-4' +\endif + +-- show that vars and backticks are not expanded when ignoring extra args +\set foo bar +\echo :foo :'foo' :"foo" +\pset fieldsep | `nosuchcommand` :foo :'foo' :"foo" + +-- show that vars and backticks are not expanded and commands are ignored +-- when in a false if-branch +\set try_to_quit '\\q' +\if false + :try_to_quit + \echo `nosuchcommand` :foo :'foo' :"foo" + \pset fieldsep | `nosuchcommand` :foo :'foo' :"foo" + \a \C arg1 \c arg1 arg2 arg3 arg4 \cd arg1 \conninfo + \copy arg1 arg2 arg3 arg4 arg5 arg6 + \copyright \dt arg1 \e arg1 arg2 + \ef whole_line + \ev whole_line + \echo arg1 arg2 arg3 arg4 arg5 \echo arg1 \encoding arg1 \errverbose + \g arg1 \gx arg1 \gexec \h \html \i arg1 \ir arg1 \l arg1 \lo arg1 arg2 + \o arg1 \p \password arg1 \prompt arg1 arg2 \pset arg1 arg2 \q + \reset \s arg1 \set arg1 arg2 arg3 arg4 arg5 arg6 arg7 \setenv arg1 arg2 + \sf whole_line + \sv whole_line + \t arg1 \T arg1 \timing arg1 \unset arg1 \w arg1 \watch arg1 \x arg1 + -- \else here is eaten as part of OT_FILEPIPE argument + \w |/no/such/file \else + -- \endif here is eaten as part of whole-line argument + \! whole_line \endif +\else + \echo 'should print #8-1' +\endif + -- SHOW_CONTEXT \set SHOW_CONTEXT never -- cgit v1.2.3 From 25fff40798fc4ac11a241bfd9ab0c45c085e2212 Mon Sep 17 00:00:00 2001 From: Simon Riggs Date: Thu, 30 Mar 2017 14:18:53 -0400 Subject: Default monitoring roles Three nologin roles with non-overlapping privs are created by default * pg_read_all_settings - read all GUCs. * pg_read_all_stats - pg_stat_*, pg_database_size(), pg_tablespace_size() * pg_stat_scan_tables - may lock/scan tables Top level role - pg_monitor includes all of the above by default, plus others Author: Dave Page Reviewed-by: Stephen Frost, Robert Haas, Peter Eisentraut, Simon Riggs --- contrib/pg_buffercache/Makefile | 5 +-- .../pg_buffercache/pg_buffercache--1.2--1.3.sql | 7 +++++ contrib/pg_buffercache/pg_buffercache.control | 2 +- contrib/pg_freespacemap/Makefile | 4 +-- .../pg_freespacemap/pg_freespacemap--1.1--1.2.sql | 7 +++++ contrib/pg_freespacemap/pg_freespacemap.control | 2 +- contrib/pg_stat_statements/Makefile | 7 +++-- .../pg_stat_statements--1.4--1.5.sql | 6 ++++ contrib/pg_stat_statements/pg_stat_statements.c | 8 +++-- .../pg_stat_statements/pg_stat_statements.control | 2 +- contrib/pg_visibility/Makefile | 3 +- contrib/pg_visibility/pg_visibility--1.1--1.2.sql | 13 ++++++++ contrib/pg_visibility/pg_visibility.control | 2 +- contrib/pgrowlocks/pgrowlocks.c | 9 ++++-- contrib/pgstattuple/pgstattuple--1.4--1.5.sql | 9 ++++++ doc/src/sgml/catalogs.sgml | 8 +++-- doc/src/sgml/func.sgml | 23 ++++++++------ doc/src/sgml/pgbuffercache.sgml | 5 +-- doc/src/sgml/pgfreespacemap.sgml | 5 +-- doc/src/sgml/pgrowlocks.sgml | 7 +++++ doc/src/sgml/pgstatstatements.sgml | 9 +++--- doc/src/sgml/pgstattuple.sgml | 3 +- doc/src/sgml/pgvisibility.sgml | 5 ++- doc/src/sgml/user-manag.sgml | 36 ++++++++++++++++++++++ src/backend/catalog/system_views.sql | 6 ++++ src/backend/replication/walreceiver.c | 3 +- src/backend/utils/adt/dbsize.c | 20 ++++++++---- src/backend/utils/adt/pgstatfuncs.c | 6 ++-- src/backend/utils/misc/guc.c | 21 ++++++++----- src/include/catalog/pg_authid.h | 8 +++++ 30 files changed, 196 insertions(+), 55 deletions(-) create mode 100644 contrib/pg_buffercache/pg_buffercache--1.2--1.3.sql create mode 100644 contrib/pg_freespacemap/pg_freespacemap--1.1--1.2.sql create mode 100644 contrib/pg_stat_statements/pg_stat_statements--1.4--1.5.sql create mode 100644 contrib/pg_visibility/pg_visibility--1.1--1.2.sql (limited to 'doc/src') diff --git a/contrib/pg_buffercache/Makefile b/contrib/pg_buffercache/Makefile index 497dbeb229..18f7a87452 100644 --- a/contrib/pg_buffercache/Makefile +++ b/contrib/pg_buffercache/Makefile @@ -4,8 +4,9 @@ MODULE_big = pg_buffercache OBJS = pg_buffercache_pages.o $(WIN32RES) EXTENSION = pg_buffercache -DATA = pg_buffercache--1.2.sql pg_buffercache--1.1--1.2.sql \ - pg_buffercache--1.0--1.1.sql pg_buffercache--unpackaged--1.0.sql +DATA = pg_buffercache--1.2.sql pg_buffercache--1.2--1.3.sql \ + pg_buffercache--1.1--1.2.sql pg_buffercache--1.0--1.1.sql \ + pg_buffercache--unpackaged--1.0.sql PGFILEDESC = "pg_buffercache - monitoring of shared buffer cache in real-time" ifdef USE_PGXS diff --git a/contrib/pg_buffercache/pg_buffercache--1.2--1.3.sql b/contrib/pg_buffercache/pg_buffercache--1.2--1.3.sql new file mode 100644 index 0000000000..b37ef0112e --- /dev/null +++ b/contrib/pg_buffercache/pg_buffercache--1.2--1.3.sql @@ -0,0 +1,7 @@ +/* contrib/pg_buffercache/pg_buffercache--1.2--1.3.sql */ + +-- complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION pg_buffercache UPDATE TO '1.3'" to load this file. \quit + +GRANT EXECUTE ON FUNCTION pg_buffercache_pages() TO pg_monitor; +GRANT SELECT ON pg_buffercache TO pg_monitor; diff --git a/contrib/pg_buffercache/pg_buffercache.control b/contrib/pg_buffercache/pg_buffercache.control index a4d664f3fa..8c060ae9ab 100644 --- a/contrib/pg_buffercache/pg_buffercache.control +++ b/contrib/pg_buffercache/pg_buffercache.control @@ -1,5 +1,5 @@ # pg_buffercache extension comment = 'examine the shared buffer cache' -default_version = '1.2' +default_version = '1.3' module_pathname = '$libdir/pg_buffercache' relocatable = true diff --git a/contrib/pg_freespacemap/Makefile b/contrib/pg_freespacemap/Makefile index 7bc0e9555d..0a2f000ec6 100644 --- a/contrib/pg_freespacemap/Makefile +++ b/contrib/pg_freespacemap/Makefile @@ -4,8 +4,8 @@ MODULE_big = pg_freespacemap OBJS = pg_freespacemap.o $(WIN32RES) EXTENSION = pg_freespacemap -DATA = pg_freespacemap--1.1.sql pg_freespacemap--1.0--1.1.sql \ - pg_freespacemap--unpackaged--1.0.sql +DATA = pg_freespacemap--1.1.sql pg_freespacemap--1.1--1.2.sql \ + pg_freespacemap--1.0--1.1.sql pg_freespacemap--unpackaged--1.0.sql PGFILEDESC = "pg_freespacemap - monitoring of free space map" ifdef USE_PGXS diff --git a/contrib/pg_freespacemap/pg_freespacemap--1.1--1.2.sql b/contrib/pg_freespacemap/pg_freespacemap--1.1--1.2.sql new file mode 100644 index 0000000000..f558defadd --- /dev/null +++ b/contrib/pg_freespacemap/pg_freespacemap--1.1--1.2.sql @@ -0,0 +1,7 @@ +/* contrib/pg_freespacemap/pg_freespacemap--1.1--1.2.sql */ + +-- complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION pg_freespacemap UPDATE TO '1.2'" to load this file. \quit + +GRANT EXECUTE ON FUNCTION pg_freespace(regclass, bigint) TO pg_stat_scan_tables; +GRANT EXECUTE ON FUNCTION pg_freespace(regclass) TO pg_stat_scan_tables; diff --git a/contrib/pg_freespacemap/pg_freespacemap.control b/contrib/pg_freespacemap/pg_freespacemap.control index 764db30d18..ac8fc5050a 100644 --- a/contrib/pg_freespacemap/pg_freespacemap.control +++ b/contrib/pg_freespacemap/pg_freespacemap.control @@ -1,5 +1,5 @@ # pg_freespacemap extension comment = 'examine the free space map (FSM)' -default_version = '1.1' +default_version = '1.2' module_pathname = '$libdir/pg_freespacemap' relocatable = true diff --git a/contrib/pg_stat_statements/Makefile b/contrib/pg_stat_statements/Makefile index 298951a5f5..39b368b70e 100644 --- a/contrib/pg_stat_statements/Makefile +++ b/contrib/pg_stat_statements/Makefile @@ -4,9 +4,10 @@ MODULE_big = pg_stat_statements OBJS = pg_stat_statements.o $(WIN32RES) EXTENSION = pg_stat_statements -DATA = pg_stat_statements--1.4.sql pg_stat_statements--1.3--1.4.sql \ - pg_stat_statements--1.2--1.3.sql pg_stat_statements--1.1--1.2.sql \ - pg_stat_statements--1.0--1.1.sql pg_stat_statements--unpackaged--1.0.sql +DATA = pg_stat_statements--1.4.sql pg_stat_statements--1.4--1.5.sql \ + pg_stat_statements--1.3--1.4.sql pg_stat_statements--1.2--1.3.sql \ + pg_stat_statements--1.1--1.2.sql pg_stat_statements--1.0--1.1.sql \ + pg_stat_statements--unpackaged--1.0.sql PGFILEDESC = "pg_stat_statements - execution statistics of SQL statements" LDFLAGS_SL += $(filter -lm, $(LIBS)) diff --git a/contrib/pg_stat_statements/pg_stat_statements--1.4--1.5.sql b/contrib/pg_stat_statements/pg_stat_statements--1.4--1.5.sql new file mode 100644 index 0000000000..9c76122a2b --- /dev/null +++ b/contrib/pg_stat_statements/pg_stat_statements--1.4--1.5.sql @@ -0,0 +1,6 @@ +/* contrib/pg_stat_statements/pg_stat_statements--1.4--1.5.sql */ + +-- complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION pg_stat_statements UPDATE TO '1.5'" to load this file. \quit + +GRANT EXECUTE ON FUNCTION pg_stat_statements_reset() TO pg_read_all_stats; diff --git a/contrib/pg_stat_statements/pg_stat_statements.c b/contrib/pg_stat_statements/pg_stat_statements.c index cd4c16e9d2..c300261852 100644 --- a/contrib/pg_stat_statements/pg_stat_statements.c +++ b/contrib/pg_stat_statements/pg_stat_statements.c @@ -62,6 +62,7 @@ #include #include "access/hash.h" +#include "catalog/pg_authid.h" #include "executor/instrument.h" #include "funcapi.h" #include "mb/pg_wchar.h" @@ -1391,7 +1392,7 @@ pg_stat_statements_internal(FunctionCallInfo fcinfo, MemoryContext per_query_ctx; MemoryContext oldcontext; Oid userid = GetUserId(); - bool is_superuser = superuser(); + bool is_allowed_role = false; char *qbuffer = NULL; Size qbuffer_size = 0; Size extent = 0; @@ -1399,6 +1400,9 @@ pg_stat_statements_internal(FunctionCallInfo fcinfo, HASH_SEQ_STATUS hash_seq; pgssEntry *entry; + /* Superusers or members of pg_read_all_stats members are allowed */ + is_allowed_role = is_member_of_role(GetUserId(), DEFAULT_ROLE_READ_ALL_STATS); + /* hash table must exist already */ if (!pgss || !pgss_hash) ereport(ERROR, @@ -1541,7 +1545,7 @@ pg_stat_statements_internal(FunctionCallInfo fcinfo, values[i++] = ObjectIdGetDatum(entry->key.userid); values[i++] = ObjectIdGetDatum(entry->key.dbid); - if (is_superuser || entry->key.userid == userid) + if (is_allowed_role || entry->key.userid == userid) { if (api_version >= PGSS_V1_2) values[i++] = Int64GetDatumFast(queryid); diff --git a/contrib/pg_stat_statements/pg_stat_statements.control b/contrib/pg_stat_statements/pg_stat_statements.control index 24038f56b1..193fcdfafa 100644 --- a/contrib/pg_stat_statements/pg_stat_statements.control +++ b/contrib/pg_stat_statements/pg_stat_statements.control @@ -1,5 +1,5 @@ # pg_stat_statements extension comment = 'track execution statistics of all SQL statements executed' -default_version = '1.4' +default_version = '1.5' module_pathname = '$libdir/pg_stat_statements' relocatable = true diff --git a/contrib/pg_visibility/Makefile b/contrib/pg_visibility/Makefile index bc42944426..21d787ddf7 100644 --- a/contrib/pg_visibility/Makefile +++ b/contrib/pg_visibility/Makefile @@ -4,7 +4,8 @@ MODULE_big = pg_visibility OBJS = pg_visibility.o $(WIN32RES) EXTENSION = pg_visibility -DATA = pg_visibility--1.1.sql pg_visibility--1.0--1.1.sql +DATA = pg_visibility--1.1.sql pg_visibility--1.1--1.2.sql \ + pg_visibility--1.0--1.1.sql PGFILEDESC = "pg_visibility - page visibility information" REGRESS = pg_visibility diff --git a/contrib/pg_visibility/pg_visibility--1.1--1.2.sql b/contrib/pg_visibility/pg_visibility--1.1--1.2.sql new file mode 100644 index 0000000000..a5a4fe7ca8 --- /dev/null +++ b/contrib/pg_visibility/pg_visibility--1.1--1.2.sql @@ -0,0 +1,13 @@ +/* contrib/pg_visibility/pg_visibility--1.1--1.2.sql */ + +-- complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION pg_visibility UPDATE TO '1.2'" to load this file. \quit + +-- Allow use of monitoring functions by pg_monitor members +GRANT EXECUTE ON FUNCTION pg_visibility_map(regclass, bigint) TO pg_stat_scan_tables; +GRANT EXECUTE ON FUNCTION pg_visibility(regclass, bigint) TO pg_stat_scan_tables; +GRANT EXECUTE ON FUNCTION pg_visibility_map(regclass) TO pg_stat_scan_tables; +GRANT EXECUTE ON FUNCTION pg_visibility(regclass) TO pg_stat_scan_tables; +GRANT EXECUTE ON FUNCTION pg_visibility_map_summary(regclass) TO pg_stat_scan_tables; +GRANT EXECUTE ON FUNCTION pg_check_frozen(regclass) TO pg_stat_scan_tables; +GRANT EXECUTE ON FUNCTION pg_check_visible(regclass) TO pg_stat_scan_tables; diff --git a/contrib/pg_visibility/pg_visibility.control b/contrib/pg_visibility/pg_visibility.control index f93ed0176e..3cffa08b01 100644 --- a/contrib/pg_visibility/pg_visibility.control +++ b/contrib/pg_visibility/pg_visibility.control @@ -1,5 +1,5 @@ # pg_visibility extension comment = 'examine the visibility map (VM) and page-level visibility info' -default_version = '1.1' +default_version = '1.2' module_pathname = '$libdir/pg_visibility' relocatable = true diff --git a/contrib/pgrowlocks/pgrowlocks.c b/contrib/pgrowlocks/pgrowlocks.c index db9e0349a0..31b8626e3a 100644 --- a/contrib/pgrowlocks/pgrowlocks.c +++ b/contrib/pgrowlocks/pgrowlocks.c @@ -28,6 +28,7 @@ #include "access/relscan.h" #include "access/xact.h" #include "catalog/namespace.h" +#include "catalog/pg_authid.h" #include "funcapi.h" #include "miscadmin.h" #include "storage/bufmgr.h" @@ -98,9 +99,11 @@ pgrowlocks(PG_FUNCTION_ARGS) relrv = makeRangeVarFromNameList(textToQualifiedNameList(relname)); rel = heap_openrv(relrv, AccessShareLock); - /* check permissions: must have SELECT on table */ - aclresult = pg_class_aclcheck(RelationGetRelid(rel), GetUserId(), - ACL_SELECT); + /* check permissions: must have SELECT on table or be in pg_stat_scan_tables */ + aclresult = (pg_class_aclcheck(RelationGetRelid(rel), GetUserId(), + ACL_SELECT) || + is_member_of_role(GetUserId(), DEFAULT_ROLE_STAT_SCAN_TABLES); + if (aclresult != ACLCHECK_OK) aclcheck_error(aclresult, ACL_KIND_CLASS, RelationGetRelationName(rel)); diff --git a/contrib/pgstattuple/pgstattuple--1.4--1.5.sql b/contrib/pgstattuple/pgstattuple--1.4--1.5.sql index 84e112e1c2..05ae51fa4b 100644 --- a/contrib/pgstattuple/pgstattuple--1.4--1.5.sql +++ b/contrib/pgstattuple/pgstattuple--1.4--1.5.sql @@ -17,6 +17,7 @@ AS 'MODULE_PATHNAME', 'pgstattuple_v1_5' LANGUAGE C STRICT PARALLEL SAFE; REVOKE EXECUTE ON FUNCTION pgstattuple(text) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION pgstattuple(text) TO pg_stat_scan_tables; CREATE OR REPLACE FUNCTION pgstatindex(IN relname text, OUT version INT, @@ -33,6 +34,7 @@ AS 'MODULE_PATHNAME', 'pgstatindex_v1_5' LANGUAGE C STRICT PARALLEL SAFE; REVOKE EXECUTE ON FUNCTION pgstatindex(text) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION pgstatindex(text) TO pg_stat_scan_tables; CREATE OR REPLACE FUNCTION pg_relpages(IN relname text) RETURNS BIGINT @@ -40,6 +42,7 @@ AS 'MODULE_PATHNAME', 'pg_relpages_v1_5' LANGUAGE C STRICT PARALLEL SAFE; REVOKE EXECUTE ON FUNCTION pg_relpages(text) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION pg_relpages(text) TO pg_stat_scan_tables; /* New stuff in 1.1 begins here */ @@ -51,6 +54,7 @@ AS 'MODULE_PATHNAME', 'pgstatginindex_v1_5' LANGUAGE C STRICT PARALLEL SAFE; REVOKE EXECUTE ON FUNCTION pgstatginindex(regclass) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION pgstatginindex(regclass) TO pg_stat_scan_tables; /* New stuff in 1.2 begins here */ @@ -68,6 +72,7 @@ AS 'MODULE_PATHNAME', 'pgstattuplebyid_v1_5' LANGUAGE C STRICT PARALLEL SAFE; REVOKE EXECUTE ON FUNCTION pgstattuple(regclass) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION pgstattuple(regclass) TO pg_stat_scan_tables; CREATE OR REPLACE FUNCTION pgstatindex(IN relname regclass, OUT version INT, @@ -84,6 +89,7 @@ AS 'MODULE_PATHNAME', 'pgstatindexbyid_v1_5' LANGUAGE C STRICT PARALLEL SAFE; REVOKE EXECUTE ON FUNCTION pgstatindex(regclass) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION pgstatindex(regclass) TO pg_stat_scan_tables; CREATE OR REPLACE FUNCTION pg_relpages(IN relname regclass) RETURNS BIGINT @@ -91,6 +97,7 @@ AS 'MODULE_PATHNAME', 'pg_relpagesbyid_v1_5' LANGUAGE C STRICT PARALLEL SAFE; REVOKE EXECUTE ON FUNCTION pg_relpages(regclass) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION pg_relpages(regclass) TO pg_stat_scan_tables; /* New stuff in 1.3 begins here */ @@ -109,6 +116,7 @@ AS 'MODULE_PATHNAME', 'pgstattuple_approx_v1_5' LANGUAGE C STRICT PARALLEL SAFE; REVOKE EXECUTE ON FUNCTION pgstattuple_approx(regclass) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION pgstattuple_approx(regclass) TO pg_stat_scan_tables; /* New stuff in 1.5 begins here */ @@ -125,3 +133,4 @@ AS 'MODULE_PATHNAME', 'pgstathashindex' LANGUAGE C STRICT PARALLEL SAFE; REVOKE EXECUTE ON FUNCTION pgstathashindex(regclass) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION pgstathashindex(regclass) TO pg_stat_scan_tables; diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index ac39c639ed..65ba919e7b 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -10221,15 +10221,17 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx text Configuration file the current value was set in (null for values set from sources other than configuration files, or when - examined by a non-superuser); - helpful when using include directives in configuration files + examined by a user who is neither a superuser or a member of + pg_read_all_settings); helpful when using + include directives in configuration files sourceline integer Line number within the configuration file the current value was set at (null for values set from sources other than configuration files, - or when examined by a non-superuser) + or when examined by a user who is neither a superuser or a member of + pg_read_all_settings). diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 78508d74ec..076be587ea 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -19408,9 +19408,11 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); accept the OID or name of a database or tablespace, and return the total disk space used therein. To use pg_database_size, you must have CONNECT permission on the specified database - (which is granted by default). To use pg_tablespace_size, - you must have CREATE permission on the specified tablespace, - unless it is the default tablespace for the current database. + (which is granted by default), or be a member of the pg_read_all_stats + role. To use pg_tablespace_size, you must have + CREATE permission on the specified tablespace, or be a member + of the pg_read_all_stats role unless it is the default tablespace for + the current database. @@ -19736,7 +19738,8 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); setof record List the name, size, and last modification time of files in the log - directory. Access may be granted to non-superuser roles. + directory. Access is granted to members of the pg_monitor + role and may be granted to other non-superuser roles. @@ -19746,7 +19749,8 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); setof record List the name, size, and last modification time of files in the WAL - directory. Access may be granted to non-superuser roles. + directory. Access is granted to members of the pg_monitor + role and may be granted to other non-superuser roles. @@ -19807,8 +19811,8 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); pg_ls_logdir returns the name, size, and last modified time (mtime) of each file in the log directory. By default, only superusers - can use this function, but access may be granted to others using - GRANT. + and members of the pg_monitor role can use this function. + Access may be granted to others using GRANT. @@ -19817,8 +19821,9 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); pg_ls_waldir returns the name, size, and last modified time (mtime) of each file in the write ahead log (WAL) directory. By - default only superusers can use this function, but access may be granted - to others using GRANT. + default only superusers and members of the pg_monitor role + can use this function. Access may be granted to others using + GRANT. diff --git a/doc/src/sgml/pgbuffercache.sgml b/doc/src/sgml/pgbuffercache.sgml index b261a4dbe0..4e53009ae0 100644 --- a/doc/src/sgml/pgbuffercache.sgml +++ b/doc/src/sgml/pgbuffercache.sgml @@ -24,8 +24,9 @@ - By default public access is revoked from both of these, just in case there - are security issues lurking. + By default use is restricted to superusers and members of the + pg_read_all_stats role. Access may be granted to others + using GRANT. diff --git a/doc/src/sgml/pgfreespacemap.sgml b/doc/src/sgml/pgfreespacemap.sgml index f2f99d571e..43e154a2f3 100644 --- a/doc/src/sgml/pgfreespacemap.sgml +++ b/doc/src/sgml/pgfreespacemap.sgml @@ -16,8 +16,9 @@ - By default public access is revoked from the functions, just in case - there are security issues lurking. + By default use is restricted to superusers and members of the + pg_stat_scan_tables role. Access may be granted to others + using GRANT. diff --git a/doc/src/sgml/pgrowlocks.sgml b/doc/src/sgml/pgrowlocks.sgml index d73511579c..65d532e081 100644 --- a/doc/src/sgml/pgrowlocks.sgml +++ b/doc/src/sgml/pgrowlocks.sgml @@ -12,6 +12,13 @@ locking information for a specified table. + + By default use is restricted to superusers, members of the + pg_stat_scan_tables role, and users with + SELECT permissions on the table. + + + Overview diff --git a/doc/src/sgml/pgstatstatements.sgml b/doc/src/sgml/pgstatstatements.sgml index 082994cae0..2d55d3bdad 100644 --- a/doc/src/sgml/pgstatstatements.sgml +++ b/doc/src/sgml/pgstatstatements.sgml @@ -226,10 +226,11 @@
    - For security reasons, non-superusers are not allowed to see the SQL - text or queryid of queries executed by other users. - They can see the statistics, however, if the view has been installed in their - database. + For security reasons, only superusers and members of the + pg_read_all_stats role are allowed to see the SQL text and + queryid of queries executed by other users. + Other users can see the statistics, however, if the view has been installed + in their database. diff --git a/doc/src/sgml/pgstattuple.sgml b/doc/src/sgml/pgstattuple.sgml index 62b1a6f479..141d8e225f 100644 --- a/doc/src/sgml/pgstattuple.sgml +++ b/doc/src/sgml/pgstattuple.sgml @@ -16,7 +16,8 @@ As these functions return detailed page-level information, only the superuser has EXECUTE privileges on them upon installation. After the functions have been installed, users may issue GRANT commands to change - the privileges on the functions to allow non-superusers to execute them. See + the privileges on the functions to allow non-superusers to execute them. Members + of the pg_stat_scan_tables role are granted access by default. See the description of the command for specifics. diff --git a/doc/src/sgml/pgvisibility.sgml b/doc/src/sgml/pgvisibility.sgml index fd486696fc..d466a3bce8 100644 --- a/doc/src/sgml/pgvisibility.sgml +++ b/doc/src/sgml/pgvisibility.sgml @@ -140,7 +140,10 @@ - By default, these functions are executable only by superusers. + By default, these functions are executable only by superusers and members of the + pg_stat_scan_tables role, with the exception of + pg_truncate_visibility_map(relation regclass) which can only + be executed by superusers. diff --git a/doc/src/sgml/user-manag.sgml b/doc/src/sgml/user-manag.sgml index 7eaefe58c2..914f1505ab 100644 --- a/doc/src/sgml/user-manag.sgml +++ b/doc/src/sgml/user-manag.sgml @@ -515,14 +515,50 @@ DROP ROLE doomed_role; + + pg_read_all_settings + Read all configuration variables, even those normally visible only to + superusers. + + + pg_read_all_stats + Read all pg_stat_* views and use various statistics related extensions, + even those normally visible only to superusers. + + + pg_stat_scan_tables + Execute monitoring functions that may take AccessShareLocks on tables, + potentially for a long time. + pg_signal_backend Send signals to other backends (eg: cancel query, terminate). + + pg_monitor + Read/execute various monitoring views and functions. + This role is a member of pg_read_all_settings, + pg_read_all_stats and + pg_stat_scan_tables. + + + The pg_monitor, pg_read_all_settings, + pg_read_all_stats and pg_stat_scan_tables + roles are intended to allow administrators to easily configure a role for the + purpose of monitoring the database server. They grant a set of common privileges + allowing the role to read various useful configuration settings, statistics and + other system information normally restricted to superusers. + + + + Care should be taken when granting these roles to ensure they are only used where + needed to perform the desired monitoring. + + Administrators can grant access to these roles to users using the GRANT command: diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index d357c8b8fd..0217f3992f 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1148,3 +1148,9 @@ REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_function_counters(oid) FROM publ REVOKE EXECUTE ON FUNCTION pg_ls_logdir() FROM public; REVOKE EXECUTE ON FUNCTION pg_ls_waldir() FROM public; +GRANT EXECUTE ON FUNCTION pg_ls_logdir() TO pg_monitor; +GRANT EXECUTE ON FUNCTION pg_ls_waldir() TO pg_monitor; + +GRANT pg_read_all_settings TO pg_monitor; +GRANT pg_read_all_stats TO pg_monitor; +GRANT pg_stat_scan_tables TO pg_monitor; diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index 771ac305c3..df93265c20 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -50,6 +50,7 @@ #include "access/timeline.h" #include "access/transam.h" #include "access/xlog_internal.h" +#include "catalog/pg_authid.h" #include "catalog/pg_type.h" #include "funcapi.h" #include "libpq/pqformat.h" @@ -1421,7 +1422,7 @@ pg_stat_get_wal_receiver(PG_FUNCTION_ARGS) /* Fetch values */ values[0] = Int32GetDatum(walrcv->pid); - if (!superuser()) + if (!is_member_of_role(GetUserId(), DEFAULT_ROLE_READ_ALL_STATS)) { /* * Only superusers can see details. Other users only get the pid value diff --git a/src/backend/utils/adt/dbsize.c b/src/backend/utils/adt/dbsize.c index 58923912eb..6d56638208 100644 --- a/src/backend/utils/adt/dbsize.c +++ b/src/backend/utils/adt/dbsize.c @@ -17,6 +17,7 @@ #include "access/htup_details.h" #include "catalog/catalog.h" #include "catalog/namespace.h" +#include "catalog/pg_authid.h" #include "catalog/pg_tablespace.h" #include "commands/dbcommands.h" #include "commands/tablespace.h" @@ -88,11 +89,17 @@ calculate_database_size(Oid dbOid) char pathname[MAXPGPATH]; AclResult aclresult; - /* User must have connect privilege for target database */ + /* + * User must have connect privilege for target database + * or be a member of pg_read_all_stats + */ aclresult = pg_database_aclcheck(dbOid, GetUserId(), ACL_CONNECT); - if (aclresult != ACLCHECK_OK) + if (aclresult != ACLCHECK_OK && + !is_member_of_role(GetUserId(), DEFAULT_ROLE_READ_ALL_STATS)) + { aclcheck_error(aclresult, ACL_KIND_DATABASE, get_database_name(dbOid)); + } /* Shared storage in pg_global is not counted */ @@ -172,11 +179,12 @@ calculate_tablespace_size(Oid tblspcOid) AclResult aclresult; /* - * User must have CREATE privilege for target tablespace, either - * explicitly granted or implicitly because it is default for current - * database. + * User must be a member of pg_read_all_stats or have CREATE privilege for + * target tablespace, either explicitly granted or implicitly because + * it is default for current database. */ - if (tblspcOid != MyDatabaseTableSpace) + if (tblspcOid != MyDatabaseTableSpace && + !is_member_of_role(GetUserId(), DEFAULT_ROLE_READ_ALL_STATS)) { aclresult = pg_tablespace_aclcheck(tblspcOid, GetUserId(), ACL_CREATE); if (aclresult != ACLCHECK_OK) diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index dd2b924d0a..e0cae1ba1e 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -15,6 +15,7 @@ #include "postgres.h" #include "access/htup_details.h" +#include "catalog/pg_authid.h" #include "catalog/pg_type.h" #include "common/ip.h" #include "funcapi.h" @@ -658,8 +659,9 @@ pg_stat_get_activity(PG_FUNCTION_ARGS) nulls[19] = nulls[20] = nulls[21] = nulls[22] = nulls[23] = true; } - /* Values only available to role member */ - if (has_privs_of_role(GetUserId(), beentry->st_userid)) + /* Values only available to role member or pg_read_all_stats */ + if (has_privs_of_role(GetUserId(), beentry->st_userid) || + is_member_of_role(GetUserId(), DEFAULT_ROLE_READ_ALL_STATS)) { SockAddr zero_clientaddr; diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index e9d561b185..8b5f064d4e 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -34,6 +34,7 @@ #include "access/xact.h" #include "access/xlog_internal.h" #include "catalog/namespace.h" +#include "catalog/pg_authid.h" #include "commands/async.h" #include "commands/prepare.h" #include "commands/user.h" @@ -6689,10 +6690,11 @@ GetConfigOption(const char *name, bool missing_ok, bool restrict_superuser) } if (restrict_superuser && (record->flags & GUC_SUPERUSER_ONLY) && - !superuser()) + !is_member_of_role(GetUserId(), DEFAULT_ROLE_READ_ALL_SETTINGS)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("must be superuser to examine \"%s\"", name))); + errmsg("must be superuser or a member of pg_read_all_settings to examine \"%s\"", + name))); switch (record->vartype) { @@ -6737,10 +6739,12 @@ GetConfigOptionResetString(const char *name) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("unrecognized configuration parameter \"%s\"", name))); - if ((record->flags & GUC_SUPERUSER_ONLY) && !superuser()) + if ((record->flags & GUC_SUPERUSER_ONLY) && + !is_member_of_role(GetUserId(), DEFAULT_ROLE_READ_ALL_SETTINGS)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("must be superuser to examine \"%s\"", name))); + errmsg("must be superuser or a member of pg_read_all_settings to examine \"%s\"", + name))); switch (record->vartype) { @@ -8027,10 +8031,12 @@ GetConfigOptionByName(const char *name, const char **varname, bool missing_ok) errmsg("unrecognized configuration parameter \"%s\"", name))); } - if ((record->flags & GUC_SUPERUSER_ONLY) && !superuser()) + if ((record->flags & GUC_SUPERUSER_ONLY) && + !is_member_of_role(GetUserId(), DEFAULT_ROLE_READ_ALL_SETTINGS)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("must be superuser to examine \"%s\"", name))); + errmsg("must be superuser or a member of pg_read_all_settings to examine \"%s\"", + name))); if (varname) *varname = record->name; @@ -8056,7 +8062,8 @@ GetConfigOptionByNum(int varnum, const char **values, bool *noshow) if (noshow) { if ((conf->flags & GUC_NO_SHOW_ALL) || - ((conf->flags & GUC_SUPERUSER_ONLY) && !superuser())) + ((conf->flags & GUC_SUPERUSER_ONLY) && + !is_member_of_role(GetUserId(), DEFAULT_ROLE_READ_ALL_SETTINGS))) *noshow = true; else *noshow = false; diff --git a/src/include/catalog/pg_authid.h b/src/include/catalog/pg_authid.h index def71edaa8..a6c5c02ceb 100644 --- a/src/include/catalog/pg_authid.h +++ b/src/include/catalog/pg_authid.h @@ -99,10 +99,18 @@ typedef FormData_pg_authid *Form_pg_authid; * ---------------- */ DATA(insert OID = 10 ( "POSTGRES" t t t t t t t -1 _null_ _null_)); +DATA(insert OID = 3373 ( "pg_monitor" f t f f f f f -1 _null_ _null_)); +DATA(insert OID = 3374 ( "pg_read_all_settings" f t f f f f f -1 _null_ _null_)); +DATA(insert OID = 3375 ( "pg_read_all_stats" f t f f f f f -1 _null_ _null_)); +DATA(insert OID = 3377 ( "pg_stat_scan_tables" f t f f f f f -1 _null_ _null_)); DATA(insert OID = 4200 ( "pg_signal_backend" f t f f f f f -1 _null_ _null_)); #define BOOTSTRAP_SUPERUSERID 10 +#define DEFAULT_ROLE_MONITOR 3373 +#define DEFAULT_ROLE_READ_ALL_SETTINGS 3374 +#define DEFAULT_ROLE_READ_ALL_STATS 3375 +#define DEFAULT_ROLE_STAT_SCAN_TABLES 3377 #define DEFAULT_ROLE_SIGNAL_BACKENDID 4200 #endif /* PG_AUTHID_H */ -- cgit v1.2.3 From ab1e644005b6ef77dada51188d7b92905e2444d7 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 30 Mar 2017 17:13:44 -0400 Subject: Fix broken markup. Per buildfarm. --- doc/src/sgml/pgstatstatements.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/pgstatstatements.sgml b/doc/src/sgml/pgstatstatements.sgml index 2d55d3bdad..f9dd43e891 100644 --- a/doc/src/sgml/pgstatstatements.sgml +++ b/doc/src/sgml/pgstatstatements.sgml @@ -227,7 +227,7 @@ For security reasons, only superusers and members of the - pg_read_all_stats role are allowed to see the SQL text and + pg_read_all_stats role are allowed to see the SQL text and queryid of queries executed by other users. Other users can see the statistics, however, if the view has been installed in their database. -- cgit v1.2.3 From e306df7f9cd6b4433273e006df11bdc966b7079e Mon Sep 17 00:00:00 2001 From: Andrew Dunstan Date: Fri, 31 Mar 2017 14:26:03 -0400 Subject: Full Text Search support for json and jsonb The new functions are ts_headline() and to_tsvector. Dmitry Dolgov, edited and documented by me. --- doc/src/sgml/func.sgml | 18 ++++ src/backend/tsearch/to_tsany.c | 138 ++++++++++++++++++++++++++ src/backend/tsearch/wparser.c | 190 ++++++++++++++++++++++++++++++++++++ src/include/catalog/pg_proc.h | 26 +++++ src/include/tsearch/ts_type.h | 9 ++ src/test/regress/expected/json.out | 90 +++++++++++++++++ src/test/regress/expected/jsonb.out | 90 +++++++++++++++++ src/test/regress/sql/json.sql | 26 +++++ src/test/regress/sql/jsonb.sql | 26 +++++ 9 files changed, 613 insertions(+) (limited to 'doc/src') diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 076be587ea..6887eabd0e 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -9564,6 +9564,15 @@ CREATE TYPE rainbow AS ENUM ('red', 'orange', 'yellow', 'green', 'blue', 'purple to_tsvector('english', 'The Fat Rats') 'fat':2 'rat':3 + + + to_tsvector( config regconfig , document json(b)) + + tsvector + reduce document text to tsvector + to_tsvector('english', '{"a": "The Fat Rats"}'::json) + 'fat':2 'rat':3 + @@ -9610,6 +9619,15 @@ CREATE TYPE rainbow AS ENUM ('red', 'orange', 'yellow', 'green', 'blue', 'purple ts_headline('x y z', 'z'::tsquery) x y <b>z</b> + + + ts_headline( config regconfig, document json(b), query tsquery , options text ) + + text + display a query match + ts_headline('{"a":"x y z"}'::json, 'z'::tsquery) + {"a":"x y <b>z</b>"} + diff --git a/src/backend/tsearch/to_tsany.c b/src/backend/tsearch/to_tsany.c index 398a781c03..93c08bcf85 100644 --- a/src/backend/tsearch/to_tsany.c +++ b/src/backend/tsearch/to_tsany.c @@ -16,6 +16,7 @@ #include "tsearch/ts_cache.h" #include "tsearch/ts_utils.h" #include "utils/builtins.h" +#include "utils/jsonapi.h" typedef struct MorphOpaque @@ -24,6 +25,14 @@ typedef struct MorphOpaque int qoperator; /* query operator */ } MorphOpaque; +typedef struct TSVectorBuildState +{ + ParsedText *prs; + TSVector result; + Oid cfgId; +} TSVectorBuildState; + +static void add_to_tsvector(void *state, char *elem_value, int elem_len); Datum get_current_ts_config(PG_FUNCTION_ARGS) @@ -256,6 +265,135 @@ to_tsvector(PG_FUNCTION_ARGS) PointerGetDatum(in))); } +Datum +jsonb_to_tsvector_byid(PG_FUNCTION_ARGS) +{ + Oid cfgId = PG_GETARG_OID(0); + Jsonb *jb = PG_GETARG_JSONB(1); + TSVectorBuildState state; + ParsedText *prs = (ParsedText *) palloc(sizeof(ParsedText)); + + prs->words = NULL; + state.result = NULL; + state.cfgId = cfgId; + state.prs = prs; + + iterate_jsonb_string_values(jb, &state, (JsonIterateStringValuesAction) add_to_tsvector); + + PG_FREE_IF_COPY(jb, 1); + + if (state.result == NULL) + { + /* There weren't any string elements in jsonb, + * so wee need to return an empty vector */ + + if (prs->words != NULL) + pfree(prs->words); + + state.result = palloc(CALCDATASIZE(0, 0)); + SET_VARSIZE(state.result, CALCDATASIZE(0, 0)); + state.result->size = 0; + } + + PG_RETURN_TSVECTOR(state.result); +} + +Datum +jsonb_to_tsvector(PG_FUNCTION_ARGS) +{ + Jsonb *jb = PG_GETARG_JSONB(0); + Oid cfgId; + + cfgId = getTSCurrentConfig(true); + PG_RETURN_DATUM(DirectFunctionCall2(jsonb_to_tsvector_byid, + ObjectIdGetDatum(cfgId), + JsonbGetDatum(jb))); +} + +Datum +json_to_tsvector_byid(PG_FUNCTION_ARGS) +{ + Oid cfgId = PG_GETARG_OID(0); + text *json = PG_GETARG_TEXT_P(1); + TSVectorBuildState state; + ParsedText *prs = (ParsedText *) palloc(sizeof(ParsedText)); + + prs->words = NULL; + state.result = NULL; + state.cfgId = cfgId; + state.prs = prs; + + iterate_json_string_values(json, &state, (JsonIterateStringValuesAction) add_to_tsvector); + + PG_FREE_IF_COPY(json, 1); + if (state.result == NULL) + { + /* There weren't any string elements in json, + * so wee need to return an empty vector */ + + if (prs->words != NULL) + pfree(prs->words); + + state.result = palloc(CALCDATASIZE(0, 0)); + SET_VARSIZE(state.result, CALCDATASIZE(0, 0)); + state.result->size = 0; + } + + PG_RETURN_TSVECTOR(state.result); +} + +Datum +json_to_tsvector(PG_FUNCTION_ARGS) +{ + text *json = PG_GETARG_TEXT_P(0); + Oid cfgId; + + cfgId = getTSCurrentConfig(true); + PG_RETURN_DATUM(DirectFunctionCall2(json_to_tsvector_byid, + ObjectIdGetDatum(cfgId), + PointerGetDatum(json))); +} + +/* + * Extend current TSVector from _state with a new one, + * build over a json(b) element. + */ +static void +add_to_tsvector(void *_state, char *elem_value, int elem_len) +{ + TSVectorBuildState *state = (TSVectorBuildState *) _state; + ParsedText *prs = state->prs; + TSVector item_vector; + int i; + + prs->lenwords = elem_len / 6; + if (prs->lenwords == 0) + prs->lenwords = 2; + + prs->words = (ParsedWord *) palloc(sizeof(ParsedWord) * prs->lenwords); + prs->curwords = 0; + prs->pos = 0; + + parsetext(state->cfgId, prs, elem_value, elem_len); + + if (prs->curwords) + { + if (state->result != NULL) + { + for (i = 0; i < prs->curwords; i++) + prs->words[i].pos.pos = prs->words[i].pos.pos + TS_JUMP; + + item_vector = make_tsvector(prs); + + state->result = (TSVector) DirectFunctionCall2(tsvector_concat, + TSVectorGetDatum(state->result), + PointerGetDatum(item_vector)); + } + else + state->result = make_tsvector(prs); + } +} + /* * to_tsquery */ diff --git a/src/backend/tsearch/wparser.c b/src/backend/tsearch/wparser.c index d8f2f65542..c19937d644 100644 --- a/src/backend/tsearch/wparser.c +++ b/src/backend/tsearch/wparser.c @@ -20,6 +20,7 @@ #include "tsearch/ts_cache.h" #include "tsearch/ts_utils.h" #include "utils/builtins.h" +#include "utils/jsonapi.h" #include "utils/varlena.h" @@ -31,6 +32,19 @@ typedef struct LexDescr *list; } TSTokenTypeStorage; +/* state for ts_headline_json_* */ +typedef struct HeadlineJsonState +{ + HeadlineParsedText *prs; + TSConfigCacheEntry *cfg; + TSParserCacheEntry *prsobj; + TSQuery query; + List *prsoptions; + bool transformed; +} HeadlineJsonState; + +static text * headline_json_value(void *_state, char *elem_value, int elem_len); + static void tt_setup_firstcall(FuncCallContext *funcctx, Oid prsid) { @@ -363,3 +377,179 @@ ts_headline_opt(PG_FUNCTION_ARGS) PG_GETARG_DATUM(1), PG_GETARG_DATUM(2))); } + +Datum +ts_headline_jsonb_byid_opt(PG_FUNCTION_ARGS) +{ + Jsonb *out, *jb = PG_GETARG_JSONB(1); + TSQuery query = PG_GETARG_TSQUERY(2); + text *opt = (PG_NARGS() > 3 && PG_GETARG_POINTER(3)) ? PG_GETARG_TEXT_P(3) : NULL; + JsonTransformStringValuesAction action = (JsonTransformStringValuesAction) headline_json_value; + + HeadlineParsedText prs; + HeadlineJsonState *state = palloc0(sizeof(HeadlineJsonState)); + + memset(&prs, 0, sizeof(HeadlineParsedText)); + prs.lenwords = 32; + prs.words = (HeadlineWordEntry *) palloc(sizeof(HeadlineWordEntry) * prs.lenwords); + + state->prs = &prs; + state->cfg = lookup_ts_config_cache(PG_GETARG_OID(0)); + state->prsobj = lookup_ts_parser_cache(state->cfg->prsId); + state->query = query; + if (opt) + state->prsoptions = deserialize_deflist(PointerGetDatum(opt)); + else + state->prsoptions = NIL; + + if (!OidIsValid(state->prsobj->headlineOid)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("text search parser does not support headline creation"))); + + out = transform_jsonb_string_values(jb, state, action); + + PG_FREE_IF_COPY(jb, 1); + PG_FREE_IF_COPY(query, 2); + if (opt) + PG_FREE_IF_COPY(opt, 3); + + pfree(prs.words); + + if (state->transformed) + { + pfree(prs.startsel); + pfree(prs.stopsel); + } + + PG_RETURN_JSONB(out); +} + +Datum +ts_headline_jsonb(PG_FUNCTION_ARGS) +{ + PG_RETURN_DATUM(DirectFunctionCall3(ts_headline_jsonb_byid_opt, + ObjectIdGetDatum(getTSCurrentConfig(true)), + PG_GETARG_DATUM(0), + PG_GETARG_DATUM(1))); +} + +Datum +ts_headline_jsonb_byid(PG_FUNCTION_ARGS) +{ + PG_RETURN_DATUM(DirectFunctionCall3(ts_headline_jsonb_byid_opt, + PG_GETARG_DATUM(0), + PG_GETARG_DATUM(1), + PG_GETARG_DATUM(2))); +} + +Datum +ts_headline_jsonb_opt(PG_FUNCTION_ARGS) +{ + PG_RETURN_DATUM(DirectFunctionCall4(ts_headline_jsonb_byid_opt, + ObjectIdGetDatum(getTSCurrentConfig(true)), + PG_GETARG_DATUM(0), + PG_GETARG_DATUM(1), + PG_GETARG_DATUM(2))); +} + +Datum +ts_headline_json_byid_opt(PG_FUNCTION_ARGS) +{ + text *json = PG_GETARG_TEXT_P(1); + TSQuery query = PG_GETARG_TSQUERY(2); + text *opt = (PG_NARGS() > 3 && PG_GETARG_POINTER(3)) ? PG_GETARG_TEXT_P(3) : NULL; + text *out; + JsonTransformStringValuesAction action = (JsonTransformStringValuesAction) headline_json_value; + + HeadlineParsedText prs; + HeadlineJsonState *state = palloc0(sizeof(HeadlineJsonState)); + + memset(&prs, 0, sizeof(HeadlineParsedText)); + prs.lenwords = 32; + prs.words = (HeadlineWordEntry *) palloc(sizeof(HeadlineWordEntry) * prs.lenwords); + + state->prs = &prs; + state->cfg = lookup_ts_config_cache(PG_GETARG_OID(0)); + state->prsobj = lookup_ts_parser_cache(state->cfg->prsId); + state->query = query; + if (opt) + state->prsoptions = deserialize_deflist(PointerGetDatum(opt)); + else + state->prsoptions = NIL; + + if (!OidIsValid(state->prsobj->headlineOid)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("text search parser does not support headline creation"))); + + out = transform_json_string_values(json, state, action); + + PG_FREE_IF_COPY(json, 1); + PG_FREE_IF_COPY(query, 2); + if (opt) + PG_FREE_IF_COPY(opt, 3); + pfree(prs.words); + + if (state->transformed) + { + pfree(prs.startsel); + pfree(prs.stopsel); + } + + PG_RETURN_TEXT_P(out); +} + +Datum +ts_headline_json(PG_FUNCTION_ARGS) +{ + PG_RETURN_DATUM(DirectFunctionCall3(ts_headline_json_byid_opt, + ObjectIdGetDatum(getTSCurrentConfig(true)), + PG_GETARG_DATUM(0), + PG_GETARG_DATUM(1))); +} + +Datum +ts_headline_json_byid(PG_FUNCTION_ARGS) +{ + PG_RETURN_DATUM(DirectFunctionCall3(ts_headline_json_byid_opt, + PG_GETARG_DATUM(0), + PG_GETARG_DATUM(1), + PG_GETARG_DATUM(2))); +} + +Datum +ts_headline_json_opt(PG_FUNCTION_ARGS) +{ + PG_RETURN_DATUM(DirectFunctionCall4(ts_headline_json_byid_opt, + ObjectIdGetDatum(getTSCurrentConfig(true)), + PG_GETARG_DATUM(0), + PG_GETARG_DATUM(1), + PG_GETARG_DATUM(2))); +} + + +/* + * Return headline in text from, generated from a json(b) element + */ +static text * +headline_json_value(void *_state, char *elem_value, int elem_len) +{ + HeadlineJsonState *state = (HeadlineJsonState *) _state; + + HeadlineParsedText *prs = state->prs; + TSConfigCacheEntry *cfg = state->cfg; + TSParserCacheEntry *prsobj = state->prsobj; + TSQuery query = state->query; + List *prsoptions = state->prsoptions; + + prs->curwords = 0; + hlparsetext(cfg->cfgId, prs, query, elem_value, elem_len); + FunctionCall3(&(prsobj->prsheadline), + PointerGetDatum(prs), + PointerGetDatum(prsoptions), + PointerGetDatum(query)); + + state->transformed = true; + return generateHeadline(prs); +} diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index 220ba7be60..1132a6052e 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -4812,6 +4812,24 @@ DESCR("generate headline"); DATA(insert OID = 3755 ( ts_headline PGNSP PGUID 12 100 0 0 0 f f f f t f s s 2 0 25 "25 3615" _null_ _null_ _null_ _null_ _null_ ts_headline _null_ _null_ _null_ )); DESCR("generate headline"); +DATA(insert OID = 4201 ( ts_headline PGNSP PGUID 12 100 0 0 0 f f f f t f i s 4 0 3802 "3734 3802 3615 25" _null_ _null_ _null_ _null_ _null_ ts_headline_jsonb_byid_opt _null_ _null_ _null_ )); +DESCR("generate headline from jsonb"); +DATA(insert OID = 4202 ( ts_headline PGNSP PGUID 12 100 0 0 0 f f f f t f i s 3 0 3802 "3734 3802 3615" _null_ _null_ _null_ _null_ _null_ ts_headline_jsonb_byid _null_ _null_ _null_ )); +DESCR("generate headline from jsonb"); +DATA(insert OID = 4203 ( ts_headline PGNSP PGUID 12 100 0 0 0 f f f f t f s s 3 0 3802 "3802 3615 25" _null_ _null_ _null_ _null_ _null_ ts_headline_jsonb_opt _null_ _null_ _null_ )); +DESCR("generate headline from jsonb"); +DATA(insert OID = 4204 ( ts_headline PGNSP PGUID 12 100 0 0 0 f f f f t f s s 2 0 3802 "3802 3615" _null_ _null_ _null_ _null_ _null_ ts_headline_jsonb _null_ _null_ _null_ )); +DESCR("generate headline from jsonb"); + +DATA(insert OID = 4205 ( ts_headline PGNSP PGUID 12 100 0 0 0 f f f f t f i s 4 0 114 "3734 114 3615 25" _null_ _null_ _null_ _null_ _null_ ts_headline_json_byid_opt _null_ _null_ _null_ )); +DESCR("generate headline from json"); +DATA(insert OID = 4206 ( ts_headline PGNSP PGUID 12 100 0 0 0 f f f f t f i s 3 0 114 "3734 114 3615" _null_ _null_ _null_ _null_ _null_ ts_headline_json_byid _null_ _null_ _null_ )); +DESCR("generate headline from json"); +DATA(insert OID = 4207 ( ts_headline PGNSP PGUID 12 100 0 0 0 f f f f t f s s 3 0 114 "114 3615 25" _null_ _null_ _null_ _null_ _null_ ts_headline_json_opt _null_ _null_ _null_ )); +DESCR("generate headline from json"); +DATA(insert OID = 4208 ( ts_headline PGNSP PGUID 12 100 0 0 0 f f f f t f s s 2 0 114 "114 3615" _null_ _null_ _null_ _null_ _null_ ts_headline_json _null_ _null_ _null_ )); +DESCR("generate headline from json"); + DATA(insert OID = 3745 ( to_tsvector PGNSP PGUID 12 100 0 0 0 f f f f t f i s 2 0 3614 "3734 25" _null_ _null_ _null_ _null_ _null_ to_tsvector_byid _null_ _null_ _null_ )); DESCR("transform to tsvector"); DATA(insert OID = 3746 ( to_tsquery PGNSP PGUID 12 100 0 0 0 f f f f t f i s 2 0 3615 "3734 25" _null_ _null_ _null_ _null_ _null_ to_tsquery_byid _null_ _null_ _null_ )); @@ -4828,6 +4846,14 @@ DATA(insert OID = 3751 ( plainto_tsquery PGNSP PGUID 12 100 0 0 0 f f f f t f s DESCR("transform to tsquery"); DATA(insert OID = 5001 ( phraseto_tsquery PGNSP PGUID 12 100 0 0 0 f f f f t f s s 1 0 3615 "25" _null_ _null_ _null_ _null_ _null_ phraseto_tsquery _null_ _null_ _null_ )); DESCR("transform to tsquery"); +DATA(insert OID = 4209 ( to_tsvector PGNSP PGUID 12 100 0 0 0 f f f f t f s s 1 0 3614 "3802" _null_ _null_ _null_ _null_ _null_ jsonb_to_tsvector _null_ _null_ _null_ )); +DESCR("transform jsonb to tsvector"); +DATA(insert OID = 4210 ( to_tsvector PGNSP PGUID 12 100 0 0 0 f f f f t f s s 1 0 3614 "114" _null_ _null_ _null_ _null_ _null_ json_to_tsvector _null_ _null_ _null_ )); +DESCR("transform json to tsvector"); +DATA(insert OID = 4211 ( to_tsvector PGNSP PGUID 12 100 0 0 0 f f f f t f s s 2 0 3614 "3734 3802" _null_ _null_ _null_ _null_ _null_ jsonb_to_tsvector_byid _null_ _null_ _null_ )); +DESCR("transform jsonb to tsvector"); +DATA(insert OID = 4212 ( to_tsvector PGNSP PGUID 12 100 0 0 0 f f f f t f s s 2 0 3614 "3734 114" _null_ _null_ _null_ _null_ _null_ json_to_tsvector_byid _null_ _null_ _null_ )); +DESCR("transform json to tsvector"); DATA(insert OID = 3752 ( tsvector_update_trigger PGNSP PGUID 12 1 0 0 0 f f f f f f v s 0 0 2279 "" _null_ _null_ _null_ _null_ _null_ tsvector_update_trigger_byid _null_ _null_ _null_ )); DESCR("trigger for automatic update of tsvector column"); diff --git a/src/include/tsearch/ts_type.h b/src/include/tsearch/ts_type.h index 155650c6f3..873e2e1856 100644 --- a/src/include/tsearch/ts_type.h +++ b/src/include/tsearch/ts_type.h @@ -86,6 +86,15 @@ typedef struct #define MAXNUMPOS (256) #define LIMITPOS(x) ( ( (x) >= MAXENTRYPOS ) ? (MAXENTRYPOS-1) : (x) ) +/* + * In case if a TSVector contains several parts and we want to treat them as + * separate, it's necessary to add an artificial increment to position of each + * lexeme from every next part. It's required to avoid the situation when + * tsquery can find a phrase consisting of lexemes from two of such parts. + * TS_JUMP defined a value of this increment. + */ +#define TS_JUMP 1 + /* This struct represents a complete tsvector datum */ typedef struct { diff --git a/src/test/regress/expected/json.out b/src/test/regress/expected/json.out index 1bb87689fb..47b2b6e6a5 100644 --- a/src/test/regress/expected/json.out +++ b/src/test/regress/expected/json.out @@ -1674,3 +1674,93 @@ select json_strip_nulls('{"a": {"b": null, "c": null}, "d": {} }'); {"a":{},"d":{}} (1 row) +-- json to tsvector +select to_tsvector('{"a": "aaa bbb ddd ccc", "b": ["eee fff ggg"], "c": {"d": "hhh iii"}}'::json); + to_tsvector +--------------------------------------------------------------------------- + 'aaa':1 'bbb':2 'ccc':4 'ddd':3 'eee':6 'fff':7 'ggg':8 'hhh':10 'iii':11 +(1 row) + +-- json to tsvector with config +select to_tsvector('simple', '{"a": "aaa bbb ddd ccc", "b": ["eee fff ggg"], "c": {"d": "hhh iii"}}'::json); + to_tsvector +--------------------------------------------------------------------------- + 'aaa':1 'bbb':2 'ccc':4 'ddd':3 'eee':6 'fff':7 'ggg':8 'hhh':10 'iii':11 +(1 row) + +-- json to tsvector with stop words +select to_tsvector('{"a": "aaa in bbb ddd ccc", "b": ["the eee fff ggg"], "c": {"d": "hhh. iii"}}'::json); + to_tsvector +---------------------------------------------------------------------------- + 'aaa':1 'bbb':3 'ccc':5 'ddd':4 'eee':8 'fff':9 'ggg':10 'hhh':12 'iii':13 +(1 row) + +-- ts_vector corner cases +select to_tsvector('""'::json); + to_tsvector +------------- + +(1 row) + +select to_tsvector('{}'::json); + to_tsvector +------------- + +(1 row) + +select to_tsvector('[]'::json); + to_tsvector +------------- + +(1 row) + +select to_tsvector('null'::json); + to_tsvector +------------- + +(1 row) + +-- ts_headline for json +select ts_headline('{"a": "aaa bbb", "b": {"c": "ccc ddd fff", "c1": "ccc1 ddd1"}, "d": ["ggg hhh", "iii jjj"]}'::json, tsquery('bbb & ddd & hhh')); + ts_headline +--------------------------------------------------------------------------------------------------------- + {"a":"aaa bbb","b":{"c":"ccc ddd fff","c1":"ccc1 ddd1"},"d":["ggg hhh","iii jjj"]} +(1 row) + +select ts_headline('english', '{"a": "aaa bbb", "b": {"c": "ccc ddd fff"}, "d": ["ggg hhh", "iii jjj"]}'::json, tsquery('bbb & ddd & hhh')); + ts_headline +---------------------------------------------------------------------------------------- + {"a":"aaa bbb","b":{"c":"ccc ddd fff"},"d":["ggg hhh","iii jjj"]} +(1 row) + +select ts_headline('{"a": "aaa bbb", "b": {"c": "ccc ddd fff", "c1": "ccc1 ddd1"}, "d": ["ggg hhh", "iii jjj"]}'::json, tsquery('bbb & ddd & hhh'), 'StartSel = <, StopSel = >'); + ts_headline +------------------------------------------------------------------------------------------ + {"a":"aaa ","b":{"c":"ccc fff","c1":"ccc1 ddd1"},"d":["ggg ","iii jjj"]} +(1 row) + +select ts_headline('english', '{"a": "aaa bbb", "b": {"c": "ccc ddd fff", "c1": "ccc1 ddd1"}, "d": ["ggg hhh", "iii jjj"]}'::json, tsquery('bbb & ddd & hhh'), 'StartSel = <, StopSel = >'); + ts_headline +------------------------------------------------------------------------------------------ + {"a":"aaa ","b":{"c":"ccc fff","c1":"ccc1 ddd1"},"d":["ggg ","iii jjj"]} +(1 row) + +-- corner cases for ts_headline with json +select ts_headline('null'::json, tsquery('aaa & bbb')); + ts_headline +------------- + null +(1 row) + +select ts_headline('{}'::json, tsquery('aaa & bbb')); + ts_headline +------------- + {} +(1 row) + +select ts_headline('[]'::json, tsquery('aaa & bbb')); + ts_headline +------------- + [] +(1 row) + diff --git a/src/test/regress/expected/jsonb.out b/src/test/regress/expected/jsonb.out index 8ec4150bc2..e72a950599 100644 --- a/src/test/regress/expected/jsonb.out +++ b/src/test/regress/expected/jsonb.out @@ -3474,3 +3474,93 @@ HINT: Try using the function jsonb_set to replace key value. select jsonb_insert('{"a": {"b": "value"}}', '{a, b}', '"new_value"', true); ERROR: cannot replace existing key HINT: Try using the function jsonb_set to replace key value. +-- jsonb to tsvector +select to_tsvector('{"a": "aaa bbb ddd ccc", "b": ["eee fff ggg"], "c": {"d": "hhh iii"}}'::jsonb); + to_tsvector +--------------------------------------------------------------------------- + 'aaa':1 'bbb':2 'ccc':4 'ddd':3 'eee':6 'fff':7 'ggg':8 'hhh':10 'iii':11 +(1 row) + +-- jsonb to tsvector with config +select to_tsvector('simple', '{"a": "aaa bbb ddd ccc", "b": ["eee fff ggg"], "c": {"d": "hhh iii"}}'::jsonb); + to_tsvector +--------------------------------------------------------------------------- + 'aaa':1 'bbb':2 'ccc':4 'ddd':3 'eee':6 'fff':7 'ggg':8 'hhh':10 'iii':11 +(1 row) + +-- jsonb to tsvector with stop words +select to_tsvector('{"a": "aaa in bbb ddd ccc", "b": ["the eee fff ggg"], "c": {"d": "hhh. iii"}}'::jsonb); + to_tsvector +---------------------------------------------------------------------------- + 'aaa':1 'bbb':3 'ccc':5 'ddd':4 'eee':8 'fff':9 'ggg':10 'hhh':12 'iii':13 +(1 row) + +-- ts_vector corner cases +select to_tsvector('""'::jsonb); + to_tsvector +------------- + +(1 row) + +select to_tsvector('{}'::jsonb); + to_tsvector +------------- + +(1 row) + +select to_tsvector('[]'::jsonb); + to_tsvector +------------- + +(1 row) + +select to_tsvector('null'::jsonb); + to_tsvector +------------- + +(1 row) + +-- ts_headline for jsonb +select ts_headline('{"a": "aaa bbb", "b": {"c": "ccc ddd fff", "c1": "ccc1 ddd1"}, "d": ["ggg hhh", "iii jjj"]}'::jsonb, tsquery('bbb & ddd & hhh')); + ts_headline +------------------------------------------------------------------------------------------------------------------ + {"a": "aaa bbb", "b": {"c": "ccc ddd fff", "c1": "ccc1 ddd1"}, "d": ["ggg hhh", "iii jjj"]} +(1 row) + +select ts_headline('english', '{"a": "aaa bbb", "b": {"c": "ccc ddd fff"}, "d": ["ggg hhh", "iii jjj"]}'::jsonb, tsquery('bbb & ddd & hhh')); + ts_headline +----------------------------------------------------------------------------------------------- + {"a": "aaa bbb", "b": {"c": "ccc ddd fff"}, "d": ["ggg hhh", "iii jjj"]} +(1 row) + +select ts_headline('{"a": "aaa bbb", "b": {"c": "ccc ddd fff", "c1": "ccc1 ddd1"}, "d": ["ggg hhh", "iii jjj"]}'::jsonb, tsquery('bbb & ddd & hhh'), 'StartSel = <, StopSel = >'); + ts_headline +--------------------------------------------------------------------------------------------------- + {"a": "aaa ", "b": {"c": "ccc fff", "c1": "ccc1 ddd1"}, "d": ["ggg ", "iii jjj"]} +(1 row) + +select ts_headline('english', '{"a": "aaa bbb", "b": {"c": "ccc ddd fff", "c1": "ccc1 ddd1"}, "d": ["ggg hhh", "iii jjj"]}'::jsonb, tsquery('bbb & ddd & hhh'), 'StartSel = <, StopSel = >'); + ts_headline +--------------------------------------------------------------------------------------------------- + {"a": "aaa ", "b": {"c": "ccc fff", "c1": "ccc1 ddd1"}, "d": ["ggg ", "iii jjj"]} +(1 row) + +-- corner cases for ts_headline with jsonb +select ts_headline('null'::jsonb, tsquery('aaa & bbb')); + ts_headline +------------- + null +(1 row) + +select ts_headline('{}'::jsonb, tsquery('aaa & bbb')); + ts_headline +------------- + {} +(1 row) + +select ts_headline('[]'::jsonb, tsquery('aaa & bbb')); + ts_headline +------------- + [] +(1 row) + diff --git a/src/test/regress/sql/json.sql b/src/test/regress/sql/json.sql index 5e61922fbf..1acf4decd6 100644 --- a/src/test/regress/sql/json.sql +++ b/src/test/regress/sql/json.sql @@ -551,3 +551,29 @@ select json_strip_nulls('[1,{"a":1,"b":null,"c":2},3]'); -- an empty object is not null and should not be stripped select json_strip_nulls('{"a": {"b": null, "c": null}, "d": {} }'); + +-- json to tsvector +select to_tsvector('{"a": "aaa bbb ddd ccc", "b": ["eee fff ggg"], "c": {"d": "hhh iii"}}'::json); + +-- json to tsvector with config +select to_tsvector('simple', '{"a": "aaa bbb ddd ccc", "b": ["eee fff ggg"], "c": {"d": "hhh iii"}}'::json); + +-- json to tsvector with stop words +select to_tsvector('{"a": "aaa in bbb ddd ccc", "b": ["the eee fff ggg"], "c": {"d": "hhh. iii"}}'::json); + +-- ts_vector corner cases +select to_tsvector('""'::json); +select to_tsvector('{}'::json); +select to_tsvector('[]'::json); +select to_tsvector('null'::json); + +-- ts_headline for json +select ts_headline('{"a": "aaa bbb", "b": {"c": "ccc ddd fff", "c1": "ccc1 ddd1"}, "d": ["ggg hhh", "iii jjj"]}'::json, tsquery('bbb & ddd & hhh')); +select ts_headline('english', '{"a": "aaa bbb", "b": {"c": "ccc ddd fff"}, "d": ["ggg hhh", "iii jjj"]}'::json, tsquery('bbb & ddd & hhh')); +select ts_headline('{"a": "aaa bbb", "b": {"c": "ccc ddd fff", "c1": "ccc1 ddd1"}, "d": ["ggg hhh", "iii jjj"]}'::json, tsquery('bbb & ddd & hhh'), 'StartSel = <, StopSel = >'); +select ts_headline('english', '{"a": "aaa bbb", "b": {"c": "ccc ddd fff", "c1": "ccc1 ddd1"}, "d": ["ggg hhh", "iii jjj"]}'::json, tsquery('bbb & ddd & hhh'), 'StartSel = <, StopSel = >'); + +-- corner cases for ts_headline with json +select ts_headline('null'::json, tsquery('aaa & bbb')); +select ts_headline('{}'::json, tsquery('aaa & bbb')); +select ts_headline('[]'::json, tsquery('aaa & bbb')); diff --git a/src/test/regress/sql/jsonb.sql b/src/test/regress/sql/jsonb.sql index e2eaca0e27..c9fa1fc393 100644 --- a/src/test/regress/sql/jsonb.sql +++ b/src/test/regress/sql/jsonb.sql @@ -878,3 +878,29 @@ select jsonb_insert('{"a": {"b": "value"}}', '{a, c}', '"new_value"', true); select jsonb_insert('{"a": {"b": "value"}}', '{a, b}', '"new_value"'); select jsonb_insert('{"a": {"b": "value"}}', '{a, b}', '"new_value"', true); + +-- jsonb to tsvector +select to_tsvector('{"a": "aaa bbb ddd ccc", "b": ["eee fff ggg"], "c": {"d": "hhh iii"}}'::jsonb); + +-- jsonb to tsvector with config +select to_tsvector('simple', '{"a": "aaa bbb ddd ccc", "b": ["eee fff ggg"], "c": {"d": "hhh iii"}}'::jsonb); + +-- jsonb to tsvector with stop words +select to_tsvector('{"a": "aaa in bbb ddd ccc", "b": ["the eee fff ggg"], "c": {"d": "hhh. iii"}}'::jsonb); + +-- ts_vector corner cases +select to_tsvector('""'::jsonb); +select to_tsvector('{}'::jsonb); +select to_tsvector('[]'::jsonb); +select to_tsvector('null'::jsonb); + +-- ts_headline for jsonb +select ts_headline('{"a": "aaa bbb", "b": {"c": "ccc ddd fff", "c1": "ccc1 ddd1"}, "d": ["ggg hhh", "iii jjj"]}'::jsonb, tsquery('bbb & ddd & hhh')); +select ts_headline('english', '{"a": "aaa bbb", "b": {"c": "ccc ddd fff"}, "d": ["ggg hhh", "iii jjj"]}'::jsonb, tsquery('bbb & ddd & hhh')); +select ts_headline('{"a": "aaa bbb", "b": {"c": "ccc ddd fff", "c1": "ccc1 ddd1"}, "d": ["ggg hhh", "iii jjj"]}'::jsonb, tsquery('bbb & ddd & hhh'), 'StartSel = <, StopSel = >'); +select ts_headline('english', '{"a": "aaa bbb", "b": {"c": "ccc ddd fff", "c1": "ccc1 ddd1"}, "d": ["ggg hhh", "iii jjj"]}'::jsonb, tsquery('bbb & ddd & hhh'), 'StartSel = <, StopSel = >'); + +-- corner cases for ts_headline with jsonb +select ts_headline('null'::jsonb, tsquery('aaa & bbb')); +select ts_headline('{}'::jsonb, tsquery('aaa & bbb')); +select ts_headline('[]'::jsonb, tsquery('aaa & bbb')); -- cgit v1.2.3 From f49bcd4ef3e9a75de210357a4d9bbe3e004db956 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Fri, 31 Mar 2017 15:01:35 -0400 Subject: postgres_fdw: Teach IMPORT FOREIGN SCHEMA about partitioning. Don't import partitions. Do import partitioned tables which are not themselves partitions. Report by Stephen Frost. Design and patch by Michael Paquier, reviewed by Amit Langote. Documentation revised by me. Discussion: http://postgr.es/m/20170309141531.GD9812@tamriel.snowman.net --- contrib/postgres_fdw/expected/postgres_fdw.out | 36 +++++++++++++++++++++++--- contrib/postgres_fdw/postgres_fdw.c | 11 +++++++- contrib/postgres_fdw/sql/postgres_fdw.sql | 3 +++ doc/src/sgml/postgres-fdw.sgml | 10 +++++++ 4 files changed, 55 insertions(+), 5 deletions(-) (limited to 'doc/src') diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index a466bf2079..1a9e6c87f6 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -6907,6 +6907,9 @@ CREATE TABLE import_source.t3 (c1 timestamptz default now(), c2 typ1); CREATE TABLE import_source."x 4" (c1 float8, "C 2" text, c3 varchar(42)); CREATE TABLE import_source."x 5" (c1 float8); ALTER TABLE import_source."x 5" DROP COLUMN c1; +CREATE TABLE import_source.t4 (c1 int) PARTITION BY RANGE (c1); +CREATE TABLE import_source.t4_part PARTITION OF import_source.t4 + FOR VALUES FROM (1) TO (100); CREATE SCHEMA import_dest1; IMPORT FOREIGN SCHEMA import_source FROM SERVER loopback INTO import_dest1; \det+ import_dest1.* @@ -6916,9 +6919,10 @@ IMPORT FOREIGN SCHEMA import_source FROM SERVER loopback INTO import_dest1; import_dest1 | t1 | loopback | (schema_name 'import_source', table_name 't1') | import_dest1 | t2 | loopback | (schema_name 'import_source', table_name 't2') | import_dest1 | t3 | loopback | (schema_name 'import_source', table_name 't3') | + import_dest1 | t4 | loopback | (schema_name 'import_source', table_name 't4') | import_dest1 | x 4 | loopback | (schema_name 'import_source', table_name 'x 4') | import_dest1 | x 5 | loopback | (schema_name 'import_source', table_name 'x 5') | -(5 rows) +(6 rows) \d import_dest1.* Foreign table "import_dest1.t1" @@ -6946,6 +6950,13 @@ FDW Options: (schema_name 'import_source', table_name 't2') Server: loopback FDW Options: (schema_name 'import_source', table_name 't3') + Foreign table "import_dest1.t4" + Column | Type | Collation | Nullable | Default | FDW Options +--------+---------+-----------+----------+---------+-------------------- + c1 | integer | | not null | | (column_name 'c1') +Server: loopback +FDW Options: (schema_name 'import_source', table_name 't4') + Foreign table "import_dest1.x 4" Column | Type | Collation | Nullable | Default | FDW Options --------+-----------------------+-----------+----------+---------+--------------------- @@ -6972,9 +6983,10 @@ IMPORT FOREIGN SCHEMA import_source FROM SERVER loopback INTO import_dest2 import_dest2 | t1 | loopback | (schema_name 'import_source', table_name 't1') | import_dest2 | t2 | loopback | (schema_name 'import_source', table_name 't2') | import_dest2 | t3 | loopback | (schema_name 'import_source', table_name 't3') | + import_dest2 | t4 | loopback | (schema_name 'import_source', table_name 't4') | import_dest2 | x 4 | loopback | (schema_name 'import_source', table_name 'x 4') | import_dest2 | x 5 | loopback | (schema_name 'import_source', table_name 'x 5') | -(5 rows) +(6 rows) \d import_dest2.* Foreign table "import_dest2.t1" @@ -7002,6 +7014,13 @@ FDW Options: (schema_name 'import_source', table_name 't2') Server: loopback FDW Options: (schema_name 'import_source', table_name 't3') + Foreign table "import_dest2.t4" + Column | Type | Collation | Nullable | Default | FDW Options +--------+---------+-----------+----------+---------+-------------------- + c1 | integer | | not null | | (column_name 'c1') +Server: loopback +FDW Options: (schema_name 'import_source', table_name 't4') + Foreign table "import_dest2.x 4" Column | Type | Collation | Nullable | Default | FDW Options --------+-----------------------+-----------+----------+---------+--------------------- @@ -7027,9 +7046,10 @@ IMPORT FOREIGN SCHEMA import_source FROM SERVER loopback INTO import_dest3 import_dest3 | t1 | loopback | (schema_name 'import_source', table_name 't1') | import_dest3 | t2 | loopback | (schema_name 'import_source', table_name 't2') | import_dest3 | t3 | loopback | (schema_name 'import_source', table_name 't3') | + import_dest3 | t4 | loopback | (schema_name 'import_source', table_name 't4') | import_dest3 | x 4 | loopback | (schema_name 'import_source', table_name 'x 4') | import_dest3 | x 5 | loopback | (schema_name 'import_source', table_name 'x 5') | -(5 rows) +(6 rows) \d import_dest3.* Foreign table "import_dest3.t1" @@ -7057,6 +7077,13 @@ FDW Options: (schema_name 'import_source', table_name 't2') Server: loopback FDW Options: (schema_name 'import_source', table_name 't3') + Foreign table "import_dest3.t4" + Column | Type | Collation | Nullable | Default | FDW Options +--------+---------+-----------+----------+---------+-------------------- + c1 | integer | | | | (column_name 'c1') +Server: loopback +FDW Options: (schema_name 'import_source', table_name 't4') + Foreign table "import_dest3.x 4" Column | Type | Collation | Nullable | Default | FDW Options --------+-----------------------+-----------+----------+---------+--------------------- @@ -7092,8 +7119,9 @@ IMPORT FOREIGN SCHEMA import_source EXCEPT (t1, "x 4", nonesuch) import_dest4 | t1 | loopback | (schema_name 'import_source', table_name 't1') | import_dest4 | t2 | loopback | (schema_name 'import_source', table_name 't2') | import_dest4 | t3 | loopback | (schema_name 'import_source', table_name 't3') | + import_dest4 | t4 | loopback | (schema_name 'import_source', table_name 't4') | import_dest4 | x 5 | loopback | (schema_name 'import_source', table_name 'x 5') | -(4 rows) +(5 rows) -- Assorted error cases IMPORT FOREIGN SCHEMA import_source FROM SERVER loopback INTO import_dest4; diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c index 03f14800b0..54b938734a 100644 --- a/contrib/postgres_fdw/postgres_fdw.c +++ b/contrib/postgres_fdw/postgres_fdw.c @@ -3849,6 +3849,10 @@ postgresImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid) * should save a few cycles to not process excluded tables in the * first place.) * + * Ignore table data for partitions and only include the definitions + * of the root partitioned tables to allow access to the complete + * remote data set locally in the schema imported. + * * Note: because we run the connection with search_path restricted to * pg_catalog, the format_type() and pg_get_expr() outputs will always * include a schema name for types/functions in other schemas, which @@ -3897,10 +3901,15 @@ postgresImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid) CppAsString2(RELKIND_RELATION) "," CppAsString2(RELKIND_VIEW) "," CppAsString2(RELKIND_FOREIGN_TABLE) "," - CppAsString2(RELKIND_MATVIEW) ") " + CppAsString2(RELKIND_MATVIEW) "," + CppAsString2(RELKIND_PARTITIONED_TABLE) ") " " AND n.nspname = "); deparseStringLiteral(&buf, stmt->remote_schema); + /* Partitions are supported since Postgres 10 */ + if (PQserverVersion(conn) >= 100000) + appendStringInfoString(&buf, " AND NOT c.relispartition "); + /* Apply restrictions for LIMIT TO and EXCEPT */ if (stmt->list_type == FDW_IMPORT_SCHEMA_LIMIT_TO || stmt->list_type == FDW_IMPORT_SCHEMA_EXCEPT) diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index 8f3edc13e1..cf70ca2c01 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -1618,6 +1618,9 @@ CREATE TABLE import_source.t3 (c1 timestamptz default now(), c2 typ1); CREATE TABLE import_source."x 4" (c1 float8, "C 2" text, c3 varchar(42)); CREATE TABLE import_source."x 5" (c1 float8); ALTER TABLE import_source."x 5" DROP COLUMN c1; +CREATE TABLE import_source.t4 (c1 int) PARTITION BY RANGE (c1); +CREATE TABLE import_source.t4_part PARTITION OF import_source.t4 + FOR VALUES FROM (1) TO (100); CREATE SCHEMA import_dest1; IMPORT FOREIGN SCHEMA import_source FROM SERVER loopback INTO import_dest1; diff --git a/doc/src/sgml/postgres-fdw.sgml b/doc/src/sgml/postgres-fdw.sgml index 7a9b655d36..3dfc0f84ed 100644 --- a/doc/src/sgml/postgres-fdw.sgml +++ b/doc/src/sgml/postgres-fdw.sgml @@ -425,6 +425,16 @@ For more detail about the treatment of CHECK constraints on foreign tables, see . + + + Tables or foreign tables which are partitions of some other table are + automatically excluded. Partitioned tables are imported, unless they + are a partition of some other table. Since all data can be accessed + through the partitioned table which is the root of the partitioning + hierarchy, this approach should allow access to all the data without + creating extra objects. + + -- cgit v1.2.3 From c94e6942cefe7d20c5feed856e27f672734b1e2b Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Fri, 31 Mar 2017 16:28:30 -0400 Subject: Don't allocate storage for partitioned tables. Also, don't allow setting reloptions on them, since that would have no effect given the lack of storage. The patch does this by introducing a new reloption kind for which there are currently no reloptions -- we might have some in the future -- so it adjusts parseRelOptions to handle that case correctly. Bumped catversion. System catalogs that contained reloptions for partitioned tables are no longer valid; plus, there are now fewer physical files on disk, which is not technically a catalog change but still a good reason to re-initdb. Amit Langote, reviewed by Maksim Milyutin and Kyotaro Horiguchi and revised a bit by me. Discussion: http://postgr.es/m/20170331.173326.212311140.horiguchi.kyotaro@lab.ntt.co.jp --- doc/src/sgml/ref/create_table.sgml | 2 ++ src/backend/access/common/reloptions.c | 30 +++++++++++++++--------------- src/backend/catalog/heap.c | 20 ++++++++++++-------- src/include/access/reloptions.h | 3 ++- src/include/catalog/catversion.h | 2 +- 5 files changed, 32 insertions(+), 25 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml index 283d53e203..e1ec14e1c1 100644 --- a/doc/src/sgml/ref/create_table.sgml +++ b/doc/src/sgml/ref/create_table.sgml @@ -1038,6 +1038,8 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI If a table parameter value is set and the equivalent toast. parameter is not, the TOAST table will use the table's parameter value. + Specifying these parameters for partitioned tables is not supported, + but you may specify them for individual leaf partitions. diff --git a/src/backend/access/common/reloptions.c b/src/backend/access/common/reloptions.c index 72e12532ab..de7507aa68 100644 --- a/src/backend/access/common/reloptions.c +++ b/src/backend/access/common/reloptions.c @@ -1000,7 +1000,8 @@ extractRelOptions(HeapTuple tuple, TupleDesc tupdesc, * array; this is so that the caller can easily locate the default values. * * If there are no options of the given kind, numrelopts is set to 0 and NULL - * is returned. + * is returned (unless options are illegally supplied despite none being + * defined, in which case an error occurs). * * Note: values of type int, bool and real are allocated as part of the * returned array. Values of type string are allocated separately and must @@ -1010,7 +1011,7 @@ relopt_value * parseRelOptions(Datum options, bool validate, relopt_kind kind, int *numrelopts) { - relopt_value *reloptions; + relopt_value *reloptions = NULL; int numoptions = 0; int i; int j; @@ -1024,21 +1025,18 @@ parseRelOptions(Datum options, bool validate, relopt_kind kind, if (relOpts[i]->kinds & kind) numoptions++; - if (numoptions == 0) + if (numoptions > 0) { - *numrelopts = 0; - return NULL; - } + reloptions = palloc(numoptions * sizeof(relopt_value)); - reloptions = palloc(numoptions * sizeof(relopt_value)); - - for (i = 0, j = 0; relOpts[i]; i++) - { - if (relOpts[i]->kinds & kind) + for (i = 0, j = 0; relOpts[i]; i++) { - reloptions[j].gen = relOpts[i]; - reloptions[j].isset = false; - j++; + if (relOpts[i]->kinds & kind) + { + reloptions[j].gen = relOpts[i]; + reloptions[j].isset = false; + j++; + } } } @@ -1418,8 +1416,10 @@ heap_reloptions(char relkind, Datum reloptions, bool validate) return (bytea *) rdopts; case RELKIND_RELATION: case RELKIND_MATVIEW: - case RELKIND_PARTITIONED_TABLE: return default_reloptions(reloptions, validate, RELOPT_KIND_HEAP); + case RELKIND_PARTITIONED_TABLE: + return default_reloptions(reloptions, validate, + RELOPT_KIND_PARTITIONED); default: /* other relkinds are not supported */ return NULL; diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index eee5e2f6ca..1cbe7f907f 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -293,6 +293,7 @@ heap_create(const char *relname, case RELKIND_VIEW: case RELKIND_COMPOSITE_TYPE: case RELKIND_FOREIGN_TABLE: + case RELKIND_PARTITIONED_TABLE: create_storage = false; /* @@ -1347,14 +1348,13 @@ heap_create_with_catalog(const char *relname, if (oncommit != ONCOMMIT_NOOP) register_on_commit_action(relid, oncommit); - if (relpersistence == RELPERSISTENCE_UNLOGGED) - { - Assert(relkind == RELKIND_RELATION || relkind == RELKIND_MATVIEW || - relkind == RELKIND_TOASTVALUE || - relkind == RELKIND_PARTITIONED_TABLE); - + /* + * Unlogged objects need an init fork, except for partitioned tables which + * have no storage at all. + */ + if (relpersistence == RELPERSISTENCE_UNLOGGED && + relkind != RELKIND_PARTITIONED_TABLE) heap_create_init_fork(new_rel_desc); - } /* * ok, the relation has been cataloged, so close our relations and return @@ -1378,6 +1378,9 @@ heap_create_with_catalog(const char *relname, void heap_create_init_fork(Relation rel) { + Assert(rel->rd_rel->relkind == RELKIND_RELATION || + rel->rd_rel->relkind == RELKIND_MATVIEW || + rel->rd_rel->relkind == RELKIND_TOASTVALUE); RelationOpenSmgr(rel); smgrcreate(rel->rd_smgr, INIT_FORKNUM, false); log_smgrcreate(&rel->rd_smgr->smgr_rnode.node, INIT_FORKNUM); @@ -1824,7 +1827,8 @@ heap_drop_with_catalog(Oid relid) */ if (rel->rd_rel->relkind != RELKIND_VIEW && rel->rd_rel->relkind != RELKIND_COMPOSITE_TYPE && - rel->rd_rel->relkind != RELKIND_FOREIGN_TABLE) + rel->rd_rel->relkind != RELKIND_FOREIGN_TABLE && + rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE) { RelationDropStorage(rel); } diff --git a/src/include/access/reloptions.h b/src/include/access/reloptions.h index 861977a608..91b2cd7bb2 100644 --- a/src/include/access/reloptions.h +++ b/src/include/access/reloptions.h @@ -48,8 +48,9 @@ typedef enum relopt_kind RELOPT_KIND_SPGIST = (1 << 8), RELOPT_KIND_VIEW = (1 << 9), RELOPT_KIND_BRIN = (1 << 10), + RELOPT_KIND_PARTITIONED = (1 << 11), /* if you add a new kind, make sure you update "last_default" too */ - RELOPT_KIND_LAST_DEFAULT = RELOPT_KIND_BRIN, + RELOPT_KIND_LAST_DEFAULT = RELOPT_KIND_PARTITIONED, /* some compilers treat enums as signed ints, so we can't use 1 << 31 */ RELOPT_KIND_MAX = (1 << 30) } relopt_kind; diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index bee5711da8..d067b757b0 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201703292 +#define CATALOG_VERSION_NO 201703311 #endif -- cgit v1.2.3 From f05230752d53c4aa74cffa9b699983bbb6bcb118 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Fri, 31 Mar 2017 16:47:38 -0400 Subject: Revert "Allow ON CONFLICT .. DO NOTHING on a partitioned table." This reverts commit 8355a011a0124bdf7ccbada206a967d427039553, which turns out to have been a misguided effort. We can't really support this in a partitioning hierarchy after all for exactly the reasons stated in the documentation removed by that commit. It's still possible to use ON CONFLICT .. DO NOTHING (or for that matter ON CONFLICT .. DO UPDATE) on individual partitions if desired, but but to allow this on a partitioned table implies that we have some way of evaluating uniqueness across the whole partitioning hierarchy, which is false. Shinoda Noriyoshi noticed that the old code was crashing (which we could fix, though not in a nice way) and Amit Langote realized that this was indicative of a fundamental problem with the commit being reverted here. Discussion: http://postgr.es/m/ff3dc21d-7204-c09c-50ac-cf11a8c45c81@lab.ntt.co.jp --- doc/src/sgml/ddl.sgml | 8 ++------ src/backend/parser/analyze.c | 8 ++++++++ src/test/regress/expected/insert_conflict.out | 10 ---------- src/test/regress/sql/insert_conflict.sql | 10 ---------- 4 files changed, 10 insertions(+), 26 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ddl.sgml b/doc/src/sgml/ddl.sgml index d1e915c11a..09b5b3ff70 100644 --- a/doc/src/sgml/ddl.sgml +++ b/doc/src/sgml/ddl.sgml @@ -3854,12 +3854,8 @@ ANALYZE measurement; - Using the ON CONFLICT clause with partitioned tables - will cause an error if DO UPDATE is specified as the - alternative action, because unique or exclusion constraints can only be - created on individual partitions. There is no support for enforcing - uniqueness (or an exclusion constraint) across an entire partitioning - hierarchy. + INSERT statements with ON CONFLICT + clause are currently not allowed on partitioned tables. diff --git a/src/backend/parser/analyze.c b/src/backend/parser/analyze.c index f6025225be..8f11c46621 100644 --- a/src/backend/parser/analyze.c +++ b/src/backend/parser/analyze.c @@ -842,8 +842,16 @@ transformInsertStmt(ParseState *pstate, InsertStmt *stmt) /* Process ON CONFLICT, if any. */ if (stmt->onConflictClause) + { + /* Bail out if target relation is partitioned table */ + if (pstate->p_target_rangetblentry->relkind == RELKIND_PARTITIONED_TABLE) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("ON CONFLICT clause is not supported with partitioned tables"))); + qry->onConflict = transformOnConflictClause(pstate, stmt->onConflictClause); + } /* * If we have a RETURNING clause, we need to add the target relation to diff --git a/src/test/regress/expected/insert_conflict.out b/src/test/regress/expected/insert_conflict.out index c90d381b34..8d005fddd4 100644 --- a/src/test/regress/expected/insert_conflict.out +++ b/src/test/regress/expected/insert_conflict.out @@ -786,13 +786,3 @@ select * from selfconflict; (3 rows) drop table selfconflict; --- check that the following works: --- insert into partitioned_table on conflict do nothing -create table parted_conflict_test (a int, b char) partition by list (a); -create table parted_conflict_test_1 partition of parted_conflict_test for values in (1); -insert into parted_conflict_test values (1, 'a') on conflict do nothing; -insert into parted_conflict_test values (1, 'a') on conflict do nothing; --- however, on conflict do update not supported yet -insert into parted_conflict_test values (1) on conflict (a) do update set b = excluded.b where excluded.a = 1; -ERROR: there is no unique or exclusion constraint matching the ON CONFLICT specification -drop table parted_conflict_test, parted_conflict_test_1; diff --git a/src/test/regress/sql/insert_conflict.sql b/src/test/regress/sql/insert_conflict.sql index 78bffc783d..df3a9b59b5 100644 --- a/src/test/regress/sql/insert_conflict.sql +++ b/src/test/regress/sql/insert_conflict.sql @@ -471,13 +471,3 @@ commit; select * from selfconflict; drop table selfconflict; - --- check that the following works: --- insert into partitioned_table on conflict do nothing -create table parted_conflict_test (a int, b char) partition by list (a); -create table parted_conflict_test_1 partition of parted_conflict_test for values in (1); -insert into parted_conflict_test values (1, 'a') on conflict do nothing; -insert into parted_conflict_test values (1, 'a') on conflict do nothing; --- however, on conflict do update not supported yet -insert into parted_conflict_test values (1) on conflict (a) do update set b = excluded.b where excluded.a = 1; -drop table parted_conflict_test, parted_conflict_test_1; -- cgit v1.2.3 From 8f18a880a5f138d4da94173d15514142331f8de6 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Fri, 31 Mar 2017 17:33:34 -0400 Subject: Improve documentation for table partitioning. Emphasize the new declarative partitioning more, and compare and contrast it more clearly with inheritance-based partitioning. Amit Langote, reviewed and somewhat revised by me Discussion: http://postgr.es/m/a6f99cdb-21e7-1d65-1381-91f2cfa156e2@lab.ntt.co.jp --- doc/src/sgml/ddl.sgml | 1433 +++++++++++++++++++++++-------------------------- 1 file changed, 684 insertions(+), 749 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ddl.sgml b/doc/src/sgml/ddl.sgml index 09b5b3ff70..5109778196 100644 --- a/doc/src/sgml/ddl.sgml +++ b/doc/src/sgml/ddl.sgml @@ -2772,586 +2772,691 @@ VALUES ('Albany', NULL, NULL, 'NY'); - - Partitioned Tables + + Table Partitioning - partitioned table + partitioning - - PostgreSQL offers a way to specify how to divide a table into pieces - called partitions. The table that is divided is referred to as a - partitioned table. The specification consists - of the partitioning method and a list of columns - or expressions to be used as the partition key. - - - - All rows inserted into a partitioned table will be routed to one of the - partitions based on the value of the partition - key. Each partition has a subset defined by its partition - bounds. Currently supported partitioning methods include - range and list, wherein each partition is assigned a range of keys or - a list of keys, respectively. - + + table + partitioning + - - Partitions may have their own indexes, constraints and default values, - distinct from other partitions. Partitions do not inherit indexes from - the partitioned table. - + + partitioned table + - Partitions may themselves be defined as partitioned tables, referred to as - sub-partitioning. See - for more details creating partitioned tables and partitions. It is not - currently possible to alter a regular table into a partitioned table or - vice versa. However, it is possible to add a regular table containing - data into a partition of a partitioned table, or remove a partition; see - to learn more about the - ATTACH PARTITION and DETACH PARTITION sub-commands. + PostgreSQL supports basic table + partitioning. This section describes why and how to implement + partitioning as part of your database design. - - Individual partitions are linked to the partitioned table with inheritance - behind-the-scenes, however it is not possible to use some of the inheritance - features discussed in the previous section with partitioned tables and - partitions. For example, partitions cannot have any other parents than - the partitioned table it is a partition of, nor can a regular table inherit - from a partitioned table making the latter its parent. That means - partitioned table and partitions do not participate in inheritance with - regular tables. Since a partition hierarchy consisting of the - partitioned table and its partitions is still an inheritance hierarchy, - all the normal rules of inheritance apply as described in the previous - section () with some exceptions, most notably: + + Overview + + Partitioning refers to splitting what is logically one large table into + smaller physical pieces. Partitioning can provide several benefits: - Both CHECK and NOT NULL - constraints of a partitioned table are always inherited by all its - partitions. There cannot be any CHECK constraints - that are marked NO INHERIT. + Query performance can be improved dramatically in certain situations, + particularly when most of the heavily accessed rows of the table are in a + single partition or a small number of partitions. The partitioning + substitutes for leading columns of indexes, reducing index size and + making it more likely that the heavily-used parts of the indexes + fit in memory. - The ONLY notation used to exclude child tables - would either cause error or will be ignored in some cases for - partitioned tables. For example, specifying ONLY - when querying data from a partitioned table would not make much sense, - because all the data is contained in partitions, so this raises an - error. Specifying ONLY when modifying schema is - not desirable in certain cases with partitioned tables where it may be - fine for regular inheritance parents (for example, dropping a column - from only the parent); an error will be thrown in that case. + When queries or updates access a large percentage of a single + partition, performance can be improved by taking advantage + of sequential scan of that partition instead of using an + index and random access reads scattered across the whole table. - Partitions cannot have columns that are not present in the parent. - It is neither possible to specify columns when creating partitions - with CREATE TABLE nor is it possible to add columns to - partitions using ALTER TABLE. Tables may be added with - ALTER TABLE ... ATTACH PARTITION if their columns exactly - match the parent, including oids. + Bulk loads and deletes can be accomplished by adding or removing + partitions, if that requirement is planned into the partitioning design. + Doing ALTER TABLE DETACH PARTITION or dropping an individual + partition using DROP TABLE is far faster than a bulk + operation. These commands also entirely avoid the + VACUUM overhead caused by a bulk DELETE. - One cannot drop a NOT NULL constraint on a - partition's column, if the constraint is present in the parent table. + Seldom-used data can be migrated to cheaper and slower storage media. + + The benefits will normally be worthwhile only when a table would + otherwise be very large. The exact point at which a table will + benefit from partitioning depends on the application, although a + rule of thumb is that the size of the table should exceed the physical + memory of the database server. + + + + PostgreSQL offers built-in support for the + following forms of partitioning: + + + + Range Partitioning + + + + The table is partitioned into ranges defined + by a key column or set of columns, with no overlap between + the ranges of values assigned to different partitions. For + example, one might partition by date ranges, or by ranges of + identifiers for particular business objects. + + + + + + List Partitioning + + + + The table is partitioned by explicitly listing which key values + appear in each partition. + + + + + + If your application needs to use other forms of partitioning not listed + above, alternative methods such as inheritance and + UNION ALL views can be used instead. Such methods + offer flexibility but do not have some of the performance benefits + of built-in declarative partitioning. + + + + + Declarative Partitioning + + + PostgreSQL offers a way to specify how to + divide a table into pieces called partitions. The table that is divided + is referred to as a partitioned table. The + specification consists of the partitioning method + and a list of columns or expressions to be used as the + partition key. + + + + All rows inserted into a partitioned table will be routed to one of the + partitions based on the value of the partition + key. Each partition has a subset of the data defined by its + partition bounds. Currently supported + partitioning methods include range and list, where each partition is + assigned a range of keys and a list of keys, respectively. + + + + Partitions may themselves be defined as partitioned tables, using what is + called sub-partitioning. Partitions may have their + own indexes, constraints and default values, distinct from those of other + partitions. Indexes must be created separately for each partition. See + for more details on creating partitioned + tables and partitions. - Partitions can also be foreign tables (see ), - although certain limitations exist currently in their usage. For example, - data inserted into the partitioned table cannot be routed to foreign table - partitions. + It is not possible to turn a regular table into a partitioned table or + vice versa. However, it is possible to add a regular or partitioned table + containing data as a partition of a partitioned table, or remove a + partition from a partitioned table turning it into a standalone table; + see to learn more about the + ATTACH PARTITION and DETACH PARTITION + sub-commands. - There are currently the following limitations of using partitioned tables: + Individual partitions are linked to the partitioned table with inheritance + behind-the-scenes; however, it is not possible to use some of the + inheritance features discussed in the previous section with partitioned + tables and partitions. For example, a partition cannot have any parents + other than the partitioned table it is a partition of, nor can a regular + table inherit from a partitioned table making the latter its parent. + That means partitioned table and partitions do not participate in + inheritance with regular tables. Since a partition hierarchy consisting + of the partitioned table and its partitions is still an inheritance + hierarchy, all the normal rules of inheritance apply as described in + with some exceptions, most notably: + - It is currently not possible to add same set of indexes on all partitions - automatically. Indexes must be added to each partition with separate - commands. + Both CHECK and NOT NULL + constraints of a partitioned table are always inherited by all its + partitions. CHECK constraints that are marked + NO INHERIT are not allowed. - It is currently not possible to define indexes on partitioned tables - that include all rows from all partitions in one global index. - Consequently, it is not possible to create constraints that are realized - using an index such as UNIQUE. + The ONLY notation used to exclude child tables + will cause an error for partitioned tables in the case of + schema-modifying commands such as most ALTER TABLE + commands. For example, dropping a column from only the parent does + not make sense for partitioned tables. - Since primary keys are not supported on partitioned tables, - foreign keys referencing partitioned tables are not supported, nor - are foreign key references from a partitioned table to some other table. + Partitions cannot have columns that are not present in the parent. It + is neither possible to specify columns when creating partitions with + CREATE TABLE nor is it possible to add columns to + partitions after-the-fact using ALTER TABLE. Tables may be + added as a partition with ALTER TABLE ... ATTACH PARTITION + only if their columns exactly match the parent, including oids. - Row triggers, if necessary, must be defined on individual partitions, not - the partitioned table as it is currently not supported. + You cannot drop the NOT NULL constraint on a + partition's column if the constraint is present in the parent table. - A detailed example that shows how to use partitioned tables is discussed in - the next chapter. + Partitions can also be foreign tables + (see ), + although these have some limitations that normal tables do not. For + example, data inserted into the partitioned table is not routed to + foreign table partitions. - - - - Partitioning + + Example - - partitioning - + + Suppose we are constructing a database for a large ice cream company. + The company measures peak temperatures every day as well as ice cream + sales in each region. Conceptually, we want a table like: - - table - partitioning - + +CREATE TABLE measurement ( + city_id int not null, + logdate date not null, + peaktemp int, + unitsales int +); + - - PostgreSQL supports basic table - partitioning. This section describes why and how to implement - partitioning as part of your database design. + We know that most queries will access just the last week's, month's or + quarter's data, since the main use of this table will be to prepare + online reports for management. To reduce the amount of old data that + needs to be stored, we decide to only keep the most recent 3 years + worth of data. At the beginning of each month we will remove the oldest + month's data. In this situation we can use partitioning to help us meet + all of our different requirements for the measurements table. - - Overview - - Partitioning refers to splitting what is logically one large table - into smaller physical pieces. - Partitioning can provide several benefits: - - - - Query performance can be improved dramatically in certain situations, - particularly when most of the heavily accessed rows of the table are in a - single partition or a small number of partitions. The partitioning - substitutes for leading columns of indexes, reducing index size and - making it more likely that the heavily-used parts of the indexes - fit in memory. - - + To use declarative partitioning in this case, use the following steps: - - - When queries or updates access a large percentage of a single - partition, performance can be improved by taking advantage - of sequential scan of that partition instead of using an - index and random access reads scattered across the whole table. - - + + + + Create measurement table as a partitioned + table by specifying the PARTITION BY clause, which + includes the partitioning method (RANGE in this + case) and the list of column(s) to use as the partition key. - - - Bulk loads and deletes can be accomplished by adding or removing - partitions, if that requirement is planned into the partitioning design. - ALTER TABLE NO INHERIT or ALTER TABLE DETACH PARTITION - and DROP TABLE are both far faster than a bulk operation. - These commands also entirely avoid the VACUUM - overhead caused by a bulk DELETE. - - + +CREATE TABLE measurement ( + city_id int not null, + logdate date not null, + peaktemp int, + unitsales int +) PARTITION BY RANGE (logdate); + + - - - Seldom-used data can be migrated to cheaper and slower storage media. - - - + + You may decide to use multiple columns in the partition key for range + partitioning, if desired. Of course, this will often result in a larger + number of partitions, each of which is individually smaller. + criteria. Using fewer columns may lead to coarser-grained + A query accessing the partitioned table will have + to scan fewer partitions if the conditions involve some or all of these + columns. For example, consider a table range partitioned using columns + lastname and firstname (in that order) + as the partition key. + +     - The benefits will normally be worthwhile only when a table would - otherwise be very large. The exact point at which a table will - benefit from partitioning depends on the application, although a - rule of thumb is that the size of the table should exceed the physical - memory of the database server. - + + + Create partitions. Each partition's definition must specify the bounds + that correspond to the partitioning method and partition key of the + parent. Note that specifying bounds such that the new partition's + values will overlap with those in one or more existing partitions will + cause an error. Inserting data into the parent table that does not map + to one of the existing partitions will cause an error; appropriate + partition must be added manually. + - - Currently, PostgreSQL supports partitioning - using two methods: + + Partitions thus created are in every way normal + PostgreSQL + tables (or, possibly, foreign tables). It is possible to specify a + tablespace and storage parameters for each partition separately. + - - - Using Table Inheritance + + It is not necessary to create table constraints describing partition + boundary condition for partitions. Instead, partition constraints are + generated implicitly from the partition bound specification whenever + there is need to refer to them. - - - Each partition must be created as a child table of a single parent - table. The parent table itself is normally empty; it exists just to - represent the entire data set. You should be familiar with - inheritance (see ) before attempting to - set up partitioning with it. This was the only method to implement - partitioning in older versions. - - - + +CREATE TABLE measurement_y2006m02 PARTITION OF measurement + FOR VALUES FROM ('2006-02-01') TO ('2006-03-01') - - Using Partitioned Tables +CREATE TABLE measurement_y2006m03 PARTITION OF measurement + FOR VALUES FROM ('2006-03-01') TO ('2006-04-01') - - - See last section for some general information: - - - - - - +... +CREATE TABLE measurement_y2007m11 PARTITION OF measurement + FOR VALUES FROM ('2007-11-01') TO ('2007-12-01') - - The following forms of partitioning can be implemented in - PostgreSQL using either of the above mentioned - methods, although the latter provides dedicated syntax for each: +CREATE TABLE measurement_y2007m12 PARTITION OF measurement + FOR VALUES FROM ('2007-12-01') TO ('2008-01-01') + TABLESPACE fasttablespace; - - - Range Partitioning +CREATE TABLE measurement_y2008m01 PARTITION OF measurement + FOR VALUES FROM ('2008-01-01') TO ('2008-02-01') + TABLESPACE fasttablespace + WITH (parallel_workers = 4); +     + - - - The table is partitioned into ranges defined - by a key column or set of columns, with no overlap between - the ranges of values assigned to different partitions. For - example one might partition by date ranges, or by ranges of - identifiers for particular business objects. - - - + + To implement sub-partitioning, specify the + PARTITION BY clause in the commands used to create + individual partitions, for example: - - List Partitioning + +CREATE TABLE measurement_y2006m02 PARTITION OF measurement + FOR VALUES FROM ('2006-02-01') TO ('2006-03-01') + PARTITION BY RANGE (peaktemp); + + + After creating partitions of measurement_y2006m02, + any data inserted into measurement that is mapped to + measurement_y2006m02 (or data that is directly inserted + into measurement_y2006m02, provided it satisfies its + partition constraint) will be further redirected to one of its + partitions based on the peaktemp column. The partition + key specified may overlap with the parent's partition key, although + care should be taken when specifying the bounds of a sub-partition + such that the set of data it accepts constitutes a subset of what + the partition's own bounds allows; the system does not try to check + whether that's really the case. + + + + + + Create an index on the key column(s), as well as any other indexes you + might want for every partition. (The key index is not strictly + necessary, but in most scenarios it is helpful. If you intend the key + values to be unique then you should always create a unique or + primary-key constraint for each partition.) + + +CREATE INDEX ON measurement_y2006m02 (logdate); +CREATE INDEX ON measurement_y2006m03 (logdate); +... +CREATE INDEX ON measurement_y2007m11 (logdate); +CREATE INDEX ON measurement_y2007m12 (logdate); +CREATE INDEX ON measurement_y2008m01 (logdate); + + + - The table is partitioned by explicitly listing which key values - appear in each partition. + Ensure that the + configuration parameter is not disabled in postgresql.conf. + If it is, queries will not be optimized as desired. - - + - - - Implementing Partitioning + + In the above example we would be creating a new partition each month, so + it might be wise to write a script that generates the required DDL + automatically. + + + + + Partition Maintenance - To set up a partitioned table using inheritance, do the following: - - - - Create the master table, from which all of the - partitions will inherit. - - - This table will contain no data. Do not define any check - constraints on this table, unless you intend them to - be applied equally to all partitions. There is no point - in defining any indexes or unique constraints on it, either. - - + Normally the set of partitions established when initially defining the + the table are not intended to remain static. It is common to want to + remove old partitions of data and periodically add new partitions for + new data. One of the most important advantages of partitioning is + precisely that it allows this otherwise painful task to be executed + nearly instantaneously by manipulating the partition structure, rather + than physically moving large amounts of data around. + - - - Create several child tables that each inherit from - the master table. Normally, these tables will not add any columns - to the set inherited from the master. - + + The simplest option for removing old data is simply to drop the partition + that is no longer necessary: + +DROP TABLE measurement_y2006m02; + + This can very quickly delete millions of records because it doesn't have + to individually delete every record. Note however that the above command + requires taking an ACCESS EXCLUSIVE lock on the parent + table. + - - We will refer to the child tables as partitions, though they - are in every way normal PostgreSQL tables - (or, possibly, foreign tables). - - + + Another option that is often preferable is to remove the partition from + the partitioned table but retain access to it as a table in its own + right: - - - Add table constraints to the partition tables to define the - allowed key values in each partition. - + +ALTER TABLE measurement DETACH PARTITION measurement_y2006m02; + + + This allows further operations to be performed on the data before + it is dropped. For example, this is often a useful time to back up + the data using COPY, pg_dump, or + similar tools. It might also be a useful time to aggregate data + into smaller formats, perform other data manipulations, or run + reports. + + + + Similarly we can add a new partition to handle new data. We can create an + empty partition in the partitioned table just as the original partitions + were created above: - - Typical examples would be: -CHECK ( x = 1 ) -CHECK ( county IN ( 'Oxfordshire', 'Buckinghamshire', 'Warwickshire' )) -CHECK ( outletID >= 100 AND outletID < 200 ) +CREATE TABLE measurement_y2008m02 PARTITION OF measurement + FOR VALUES FROM ('2008-02-01') TO ('2008-03-01') + TABLESPACE fasttablespace; - Ensure that the constraints guarantee that there is no overlap - between the key values permitted in different partitions. A common - mistake is to set up range constraints like: + + As an alternative, it is sometimes more convenient to create the + new table outside the partition structure, and make it a proper + partition later. This allows the data to be loaded, checked, and + transformed prior to it appearing in the partitioned table: + -CHECK ( outletID BETWEEN 100 AND 200 ) -CHECK ( outletID BETWEEN 200 AND 300 ) +CREATE TABLE measurement_y2008m02 + (LIKE measurement INCLUDING DEFAULTS INCLUDING CONSTRAINTS) + TABLESPACE fasttablespace; + +ALTER TABLE measurement_y2008m02 ADD CONSTRAINT y2008m02 + CHECK ( logdate >= DATE '2008-02-01' AND logdate < DATE '2008-03-01' ); + +\copy measurement_y2008m02 from 'measurement_y2008m02' +-- possibly some other data preparation work + +ALTER TABLE measurement ATTACH PARTITION measurement_y2008m02 + FOR VALUES FROM ('2008-02-01') TO ('2008-03-01' ); - This is wrong since it is not clear which partition the key value - 200 belongs in. - + - - Note that there is no difference in - syntax between range and list partitioning; those terms are - descriptive only. - - + + Before running the ATTACH PARTITION command, it is + recommended to create a CHECK constraint on the table to + be attached describing the desired partition constraint. That way, + the system will be able to skip the scan to validate the implicit + partition constraint. Without such a constraint, the table will be + scanned to validate the partition constraint while holding an + ACCESS EXCLUSIVE lock on the parent table. + One may then drop the constraint after ATTACH PARTITION + is finished, because it is no longer necessary. + + - - - For each partition, create an index on the key column(s), - as well as any other indexes you might want. (The key index is - not strictly necessary, but in most scenarios it is helpful. - If you intend the key values to be unique then you should - always create a unique or primary-key constraint for each - partition.) - - + + Limitations - - - Optionally, define a trigger or rule to redirect data inserted into - the master table to the appropriate partition. - - + + The following limitations apply to partitioned tables: + + + + There is no facility available to create the matching indexes on all + partitions automatically. Indexes must be added to each partition with + separate commands. This also means that there is no way to create a + primary key, unique constraint, or exclusion constraint spanning all + partitions; it is only possible to constrain each leaf partition + individually. + + - - - Ensure that the - configuration parameter is not disabled in - postgresql.conf. - If it is, queries will not be optimized as desired. - - + + + Since primary keys are not supported on partitioned tables, foreign + keys referencing partitioned tables are not supported, nor are foreign + key references from a partitioned table to some other table. + + + + + + Using the ON CONFLICT clause with partitioned tables + will cause an error, because unique or exclusion constraints can only be + created on individual partitions. There is no support for enforcing + uniqueness (or an exclusion constraint) across an entire partitioning + hierarchy. + + - + + + An UPDATE that causes a row to move from one partition to + another fails, because the new value of the row fails to satisfy the + implicit partition constraint of the original partition. + + + + + + Row triggers, if necessary, must be defined on individual partitions, + not the partitioned table. + + + + + + + Implementation Using Inheritance - To use partitioned tables, do the following: - + While the built-in declarative partitioning is suitable for most + common use cases, there are some circumstances where a more flexible + approach may be useful. Partitioning can be implemented using table + inheritance, which allows for several features which are not supported + by declarative partitioning, such as: + + - Create master table as a partitioned table by - specifying the PARTITION BY clause, which includes - the partitioning method (RANGE or - LIST) and the list of column(s) to use as the - partition key. To be able to insert data into the table, one must - create partitions, as described below. + Partitioning enforces a rule that all partitions must have exactly + the same set of columns as the parent, but table inheritance allows + children to have extra columns not present in the parent. - - - - To decide when to use multiple columns in the partition key for range - partitioning, consider whether queries accessing the partitioned - in question will include conditions that involve multiple columns, - especially the columns being considered to be the partition key. - If so, the optimizer can create a plan that will scan fewer partitions - if a query's conditions are such that there is equality constraint on - leading partition key columns, because they limit the number of - partitions of interest. The first partition key column with - inequality constraint also further eliminates some partitions of - those chosen by equality constraints on earlier columns. - - - Create partitions of the master partitioned table, with the partition - bounds specified for each partition matching the partitioning method - and partition key of the master table. Note that specifying partition - bounds such that the new partition's values will overlap with one or - more existing partitions will cause an error. It is only after - creating partitions that one is able to insert data into the master - partitioned table, provided it maps to one of the existing partitions. - If a data row does not map to any of the existing partitions, it will - cause an error. - - - - Partitions thus created are also in every way normal - PostgreSQL tables (or, possibly, foreign tables), - whereas partitioned tables differ in a number of ways. - - - - It is not necessary to create table constraints for partitions. - Instead, partition constraints are generated implicitly whenever - there is a need to refer to them. Also, since any data inserted into - the master partitioned table is automatically inserted into the - appropriate partition, it is not necessary to create triggers for the - same. + Table inheritance allows for multiple inheritance. - Just like with inheritance, create an index on the key column(s), - as well as any other indexes you might want for every partition. - Note that it is currently not supported to propagate index definition - from the master partitioned table to its partitions; in fact, it is - not possible to define indexes on partitioned tables in the first - place. This might change in future releases. + Declarative partitioning only supports list and range partitioning, + whereas table inheritance allows data to be divided in a manner of + the user's choosing. (Note, however, that if constraint exclusion is + unable to prune partitions effectively, query performance will be very + poor.) - Currently, partitioned tables also depend on constraint exclusion - for query optimization, so ensure that the - configuration parameter is - not disabled in postgresql.conf. This might change in - future releases. + Some operations require a stronger lock when using declarative + partitioning than when using table inheritance. For example, adding + or removing a partition to or from a partitioned table requires taking + an ACCESS EXCLUSIVE lock on the parent table, + whereas a SHARE UPDATE EXCLUSIVE lock is enough + in the case of regular inheritance. - - + - - For example, suppose we are constructing a database for a large - ice cream company. The company measures peak temperatures every - day as well as ice cream sales in each region. Conceptually, - we want a table like: - - -CREATE TABLE measurement ( - city_id int not null, - logdate date not null, - peaktemp int, - unitsales int -); - - - We know that most queries will access just the last week's, month's or - quarter's data, since the main use of this table will be to prepare - online reports for management. - To reduce the amount of old data that needs to be stored, we - decide to only keep the most recent 3 years worth of data. At the - beginning of each month we will remove the oldest month's data. - + + Example - - In this situation we can use partitioning to help us meet all of our - different requirements for the measurements table. Following the - steps outlined above for both methods, partitioning can be set up as - follows: - + + We use the same measurement table we used + above. To implement it as a partitioned table using inheritance, use + the following steps: - - - - - The master table is the measurement table, declared - exactly as above. - - + + + + Create the master table, from which all of the + partitions will inherit. This table will contain no data. Do not + define any check constraints on this table, unless you intend them + to be applied equally to all partitions. There is no point in + defining any indexes or unique constraints on it, either. For our + example, master table is the measurement + table as originally defined. + + - - - Next we create one partition for each active month: + + + Create several child tables that each inherit from + the master table. Normally, these tables will not add any columns + to the set inherited from the master. Just as with declarative + partitioning, these partitions are in every way normal + PostgreSQL tables (or foreign tables). + + -CREATE TABLE measurement_y2006m02 ( ) INHERITS (measurement); -CREATE TABLE measurement_y2006m03 ( ) INHERITS (measurement); +CREATE TABLE measurement_y2006m02 () INHERITS (measurement); +CREATE TABLE measurement_y2006m03 () INHERITS (measurement); ... -CREATE TABLE measurement_y2007m11 ( ) INHERITS (measurement); -CREATE TABLE measurement_y2007m12 ( ) INHERITS (measurement); -CREATE TABLE measurement_y2008m01 ( ) INHERITS (measurement); +CREATE TABLE measurement_y2007m11 () INHERITS (measurement); +CREATE TABLE measurement_y2007m12 () INHERITS (measurement); +CREATE TABLE measurement_y2008m01 () INHERITS (measurement); + + - Each of the partitions are complete tables in their own right, - but they inherit their definitions from the - measurement table. - + + + Add non-overlapping table constraints to the partition tables to + define the allowed key values in each partition. + - - This solves one of our problems: deleting old data. Each - month, all we will need to do is perform a DROP - TABLE on the oldest child table and create a new - child table for the new month's data. - - + + Typical examples would be: + +CHECK ( x = 1 ) +CHECK ( county IN ( 'Oxfordshire', 'Buckinghamshire', 'Warwickshire' )) +CHECK ( outletID >= 100 AND outletID < 200 ) + + Ensure that the constraints guarantee that there is no overlap + between the key values permitted in different partitions. A common + mistake is to set up range constraints like: + +CHECK ( outletID BETWEEN 100 AND 200 ) +CHECK ( outletID BETWEEN 200 AND 300 ) + + This is wrong since it is not clear which partition the key value + 200 belongs in. + - - - We must provide non-overlapping table constraints. Rather than - just creating the partition tables as above, the table creation - script should really be: + + It would be better to instead create partitions as follows: CREATE TABLE measurement_y2006m02 ( CHECK ( logdate >= DATE '2006-02-01' AND logdate < DATE '2006-03-01' ) ) INHERITS (measurement); + CREATE TABLE measurement_y2006m03 ( CHECK ( logdate >= DATE '2006-03-01' AND logdate < DATE '2006-04-01' ) ) INHERITS (measurement); + ... CREATE TABLE measurement_y2007m11 ( CHECK ( logdate >= DATE '2007-11-01' AND logdate < DATE '2007-12-01' ) ) INHERITS (measurement); + CREATE TABLE measurement_y2007m12 ( CHECK ( logdate >= DATE '2007-12-01' AND logdate < DATE '2008-01-01' ) ) INHERITS (measurement); -CREATE TABLE measurement_y2008m01 ( - CHECK ( logdate >= DATE '2008-01-01' AND logdate < DATE '2008-02-01' ) -) INHERITS (measurement); - - - - - - We probably need indexes on the key columns too: +CREATE TABLE measurement_y2008m01 ( + CHECK ( logdate >= DATE '2008-01-01' AND logdate < DATE '2008-02-01' ) +) INHERITS (measurement); + + + + + + For each partition, create an index on the key column(s), + as well as any other indexes you might want. CREATE INDEX measurement_y2006m02_logdate ON measurement_y2006m02 (logdate); CREATE INDEX measurement_y2006m03_logdate ON measurement_y2006m03 (logdate); -... CREATE INDEX measurement_y2007m11_logdate ON measurement_y2007m11 (logdate); CREATE INDEX measurement_y2007m12_logdate ON measurement_y2007m12 (logdate); CREATE INDEX measurement_y2008m01_logdate ON measurement_y2008m01 (logdate); + + - We choose not to add further indexes at this time. - - - - - - We want our application to be able to say INSERT INTO - measurement ... and have the data be redirected into the - appropriate partition table. We can arrange that by attaching - a suitable trigger function to the master table. - If data will be added only to the latest partition, we can - use a very simple trigger function: + + + We want our application to be able to say INSERT INTO + measurement ... and have the data be redirected into the + appropriate partition table. We can arrange that by attaching + a suitable trigger function to the master table. + If data will be added only to the latest partition, we can + use a very simple trigger function: CREATE OR REPLACE FUNCTION measurement_insert_trigger() @@ -3363,9 +3468,11 @@ END; $$ LANGUAGE plpgsql; + - After creating the function, we create a trigger which - calls the trigger function: + + After creating the function, we create a trigger which + calls the trigger function: CREATE TRIGGER insert_measurement_trigger @@ -3373,15 +3480,15 @@ CREATE TRIGGER insert_measurement_trigger FOR EACH ROW EXECUTE PROCEDURE measurement_insert_trigger(); - We must redefine the trigger function each month so that it always - points to the current partition. The trigger definition does - not need to be updated, however. - + We must redefine the trigger function each month so that it always + points to the current partition. The trigger definition does + not need to be updated, however. + - - We might want to insert data and have the server automatically - locate the partition into which the row should be added. We - could do this with a more complex trigger function, for example: + + We might want to insert data and have the server automatically + locate the partition into which the row should be added. We + could do this with a more complex trigger function, for example: CREATE OR REPLACE FUNCTION measurement_insert_trigger() @@ -3406,170 +3513,107 @@ $$ LANGUAGE plpgsql; - The trigger definition is the same as before. - Note that each IF test must exactly match the - CHECK constraint for its partition. - - - - While this function is more complex than the single-month case, - it doesn't need to be updated as often, since branches can be - added in advance of being needed. - + The trigger definition is the same as before. + Note that each IF test must exactly match the + CHECK constraint for its partition. + - - In practice it might be best to check the newest partition first, - if most inserts go into that partition. For simplicity we have - shown the trigger's tests in the same order as in other parts - of this example. + While this function is more complex than the single-month case, + it doesn't need to be updated as often, since branches can be + added in advance of being needed. - - - - - - - Steps when using a partitioned table are as follows: - - - - - - - Create the measurement table as a partitioned table: - -CREATE TABLE measurement ( - city_id int not null, - logdate date not null, - peaktemp int, - unitsales int -) PARTITION BY RANGE (logdate); - - - + + + In practice it might be best to check the newest partition first, + if most inserts go into that partition. For simplicity we have + shown the trigger's tests in the same order as in other parts + of this example. + + - - - Then create partitions as follows: + + A different approach to redirecting inserts into the appropriate + partition table is to set up rules, instead of a trigger, on the + master table. For example: -CREATE TABLE measurement_y2006m02 PARTITION OF measurement - FOR VALUES FROM ('2006-02-01') TO ('2006-03-01'); -CREATE TABLE measurement_y2006m03 PARTITION OF measurement - FOR VALUES FROM ('2006-03-01') TO ('2006-04-01'); +CREATE RULE measurement_insert_y2006m02 AS +ON INSERT TO measurement WHERE + ( logdate >= DATE '2006-02-01' AND logdate < DATE '2006-03-01' ) +DO INSTEAD + INSERT INTO measurement_y2006m02 VALUES (NEW.*); ... -CREATE TABLE measurement_y2007m11 PARTITION OF measurement - FOR VALUES FROM ('2007-11-01') TO ('2007-12-01'); -CREATE TABLE measurement_y2007m12 PARTITION OF measurement - FOR VALUES FROM ('2007-12-01') TO ('2008-01-01'); -CREATE TABLE measurement_y2008m01 PARTITION OF measurement - FOR VALUES FROM ('2008-01-01') TO ('2008-02-01'); - - - - - - - Create indexes on the key columns just like in case of inheritance - partitions. - - - - - - - To implement sub-partitioning, specify the - PARTITION BY clause in the commands used to create - individual partitions, for example: - - -CREATE TABLE measurement_y2006m02 PARTITION OF measurement - FOR VALUES FROM ('2006-02-01') TO ('2006-03-01') - PARTITION BY RANGE (peaktemp); +CREATE RULE measurement_insert_y2008m01 AS +ON INSERT TO measurement WHERE + ( logdate >= DATE '2008-01-01' AND logdate < DATE '2008-02-01' ) +DO INSTEAD + INSERT INTO measurement_y2008m01 VALUES (NEW.*); - After creating partitions of measurement_y2006m02, any - data inserted into measurement that is mapped to - measurement_y2006m02 will be further redirected to one - of its partitions based on the peaktemp column. - Partition key specified may overlap with the parent's partition key, - although care must be taken when specifying the bounds of sub-partitions - such that the accepted set of data constitutes a subset of what a - partition's own bounds allows; the system does not try to check if - that's really the case. - - - - - - As we can see, a complex partitioning scheme could require a - substantial amount of DDL, although significantly less when using - partitioned tables. In the above example we would be creating a new - partition each month, so it might be wise to write a script that - generates the required DDL automatically. - + A rule has significantly more overhead than a trigger, but the + overhead is paid once per query rather than once per row, so this + method might be advantageous for bulk-insert situations. In most + cases, however, the trigger method will offer better performance. + - + + Be aware that COPY ignores rules. If you want to + use COPY to insert data, you'll need to copy into the + correct partition table rather than into the master. COPY + does fire triggers, so you can use it normally if you use the trigger + approach. + - - Managing Partitions + + Another disadvantage of the rule approach is that there is no simple + way to force an error if the set of rules doesn't cover the insertion + date; the data will silently go into the master table instead. + + - - Normally the set of partitions established when initially - defining the table are not intended to remain static. It is - common to want to remove old partitions of data and periodically - add new partitions for new data. One of the most important - advantages of partitioning is precisely that it allows this - otherwise painful task to be executed nearly instantaneously by - manipulating the partition structure, rather than physically moving large - amounts of data around. - + + + Ensure that the + configuration parameter is not disabled in + postgresql.conf. + If it is, queries will not be optimized as desired. + + + + - - Both the inheritance-based and partitioned table methods allow this to - be done, although the latter requires taking an ACCESS EXCLUSIVE - lock on the master table for various commands mentioned below. - + + As we can see, a complex partitioning scheme could require a + substantial amount of DDL. In the above example we would be creating + a new partition each month, so it might be wise to write a script that + generates the required DDL automatically. + + - - The simplest option for removing old data is simply to drop the partition - that is no longer necessary, which works using both methods of - partitioning: + + Partition Maintenance + + To remove old data quickly, simply to drop the partition that is no + longer necessary: DROP TABLE measurement_y2006m02; - This can very quickly delete millions of records because it doesn't have - to individually delete every record. - - - - Another option that is often preferable is to remove the partition from - the partitioned table but retain access to it as a table in its own - right: - -ALTER TABLE measurement_y2006m02 NO INHERIT measurement; - + - When using a partitioned table: + + To remove the partition from the partitioned table but retain access to + it as a table in its own right: -ALTER TABLE measurement DETACH PARTITION measurement_y2006m02; +ALTER TABLE measurement_y2006m02 NO INHERIT measurement; + - This allows further operations to be performed on the data before - it is dropped. For example, this is often a useful time to back up - the data using COPY, pg_dump, or - similar tools. It might also be a useful time to aggregate data - into smaller formats, perform other data manipulations, or run - reports. - - - - Similarly we can add a new partition to handle new data. We can create an - empty partition in the partitioned table just as the original partitions - were created above: + + To add a new partition to handle new data, create an empty partition + just as the original partitions were created above: CREATE TABLE measurement_y2008m02 ( @@ -3577,17 +3621,9 @@ CREATE TABLE measurement_y2008m02 ( ) INHERITS (measurement); - When using a partitioned table: - - -CREATE TABLE measurement_y2008m02 PARTITION OF measurement - FOR VALUES FROM ('2008-02-01') TO ('2008-03-01'); - - - As an alternative, it is sometimes more convenient to create the - new table outside the partition structure, and make it a proper - partition later. This allows the data to be loaded, checked, and - transformed prior to it appearing in the partitioned table: + Alternatively, one may want to create the new table outside the partition + structure, and make it a partition after the data is loaded, checked, + and transformed. CREATE TABLE measurement_y2008m02 @@ -3598,31 +3634,74 @@ ALTER TABLE measurement_y2008m02 ADD CONSTRAINT y2008m02 -- possibly some other data preparation work ALTER TABLE measurement_y2008m02 INHERIT measurement; + + + + + Caveats + + + The following caveats apply to partitioned tables implemented using + inheritance: + + + + There is no automatic way to verify that all of the + CHECK constraints are mutually + exclusive. It is safer to create code that generates + partitions and creates and/or modifies associated objects than + to write each by hand. + + - The last of the above commands when using a partitioned table would be: + + + The schemes shown here assume that the partition key column(s) + of a row never change, or at least do not change enough to require + it to move to another partition. An UPDATE that attempts + to do that will fail because of the CHECK constraints. + If you need to handle such cases, you can put suitable update triggers + on the partition tables, but it makes management of the structure + much more complicated. + + + + + If you are using manual VACUUM or + ANALYZE commands, don't forget that + you need to run them on each partition individually. A command like: -ALTER TABLE measurement ATTACH PARTITION measurement_y2008m02 - FOR VALUES FROM ('2008-02-01') TO ('2008-03-01' ); +ANALYZE measurement; - + will only process the master table. + + - - - Before running the ATTACH PARTITION command, it is - recommended to create a CHECK constraint on the table to - be attached describing the desired partition constraint. Using the - same, system is able to skip the scan to validate the implicit - partition constraint. Without such a constraint, the table will be - scanned to validate the partition constraint, while holding an - ACCESS EXCLUSIVE lock on the parent table. - One may want to drop the constraint after ATTACH PARTITION - is finished, because it is no longer necessary. - - - + + + INSERT statements with ON CONFLICT + clauses are unlikely to work as expected, as the ON CONFLICT + action is only taken in case of unique violations on the specified + target relation, not its child relations. + + + + + + Triggers or rules will be needed to route rows to the desired + partition, unless the application is explicitly aware of the + partitioning scheme. Triggers may be complicated to write, and will + be much slower than the tuple routing performed interally by + declarative partitioning. + + + + + + - + Partitioning and Constraint Exclusion @@ -3632,7 +3711,8 @@ ALTER TABLE measurement ATTACH PARTITION measurement_y2008m02 Constraint exclusion is a query optimization technique that improves performance for partitioned tables defined in the - fashion described above. As an example: + fashion described above (both declaratively partitioned tables and those + implemented using inheritance). As an example: SET constraint_exclusion = on; @@ -3715,156 +3795,9 @@ EXPLAIN SELECT count(*) FROM measurement WHERE logdate >= DATE '2008-01-01'; are unlikely to benefit. - - - Currently, constraint exclusion is also used for partitioned tables. - However, we did not create any CHECK constraints - for individual partitions as seen above. In this case, the optimizer - uses internally generated constraint for every partition. - - - - - - - Alternative Partitioning Methods - - - A different approach to redirecting inserts into the appropriate - partition table is to set up rules, instead of a trigger, on the - master table (unless it is a partitioned table). For example: - - -CREATE RULE measurement_insert_y2006m02 AS -ON INSERT TO measurement WHERE - ( logdate >= DATE '2006-02-01' AND logdate < DATE '2006-03-01' ) -DO INSTEAD - INSERT INTO measurement_y2006m02 VALUES (NEW.*); -... -CREATE RULE measurement_insert_y2008m01 AS -ON INSERT TO measurement WHERE - ( logdate >= DATE '2008-01-01' AND logdate < DATE '2008-02-01' ) -DO INSTEAD - INSERT INTO measurement_y2008m01 VALUES (NEW.*); - - - A rule has significantly more overhead than a trigger, but the overhead - is paid once per query rather than once per row, so this method might be - advantageous for bulk-insert situations. In most cases, however, the - trigger method will offer better performance. - - - - Be aware that COPY ignores rules. If you want to - use COPY to insert data, you'll need to copy into the correct - partition table rather than into the master. COPY does fire - triggers, so you can use it normally if you use the trigger approach. - - - - Another disadvantage of the rule approach is that there is no simple - way to force an error if the set of rules doesn't cover the insertion - date; the data will silently go into the master table instead. - - - - Partitioning can also be arranged using a UNION ALL - view, instead of table inheritance. For example, - - -CREATE VIEW measurement AS - SELECT * FROM measurement_y2006m02 -UNION ALL SELECT * FROM measurement_y2006m03 -... -UNION ALL SELECT * FROM measurement_y2007m11 -UNION ALL SELECT * FROM measurement_y2007m12 -UNION ALL SELECT * FROM measurement_y2008m01; - - - However, the need to recreate the view adds an extra step to adding and - dropping individual partitions of the data set. In practice this - method has little to recommend it compared to using inheritance. - - - - - - Caveats - - - The following caveats apply to using inheritance to implement partitioning: - - - - There is no automatic way to verify that all of the - CHECK constraints are mutually - exclusive. It is safer to create code that generates - partitions and creates and/or modifies associated objects than - to write each by hand. - - - - - - The schemes shown here assume that the partition key column(s) - of a row never change, or at least do not change enough to require - it to move to another partition. An UPDATE that attempts - to do that will fail because of the CHECK constraints. - If you need to handle such cases, you can put suitable update triggers - on the partition tables, but it makes management of the structure - much more complicated. - - - - - - If you are using manual VACUUM or - ANALYZE commands, don't forget that - you need to run them on each partition individually. A command like: - -ANALYZE measurement; - - will only process the master table. - - - - - - INSERT statements with ON CONFLICT - clauses are unlikely to work as expected, as the ON CONFLICT - action is only taken in case of unique violations on the specified - target relation, not its child relations. - - - - - - - The following caveats apply to partitioned tables created with the - explicit syntax: - - - - An UPDATE that causes a row to move from one partition to - another fails, because the new value of the row fails to satisfy the - implicit partition constraint of the original partition. This might - change in future releases. - - - - - - INSERT statements with ON CONFLICT - clause are currently not allowed on partitioned tables. - - - - - - - The following caveats apply to constraint exclusion, which is currently - used by both inheritance and partitioned tables: + The following caveats apply to constraint exclusion, which is used by + both inheritance and partitioned tables: @@ -3888,7 +3821,9 @@ ANALYZE measurement; contain only comparisons of the partitioning column(s) to constants using B-tree-indexable operators, which applies even to partitioned tables, because only B-tree-indexable column(s) are allowed in the - partition key. + partition key. (This is not a problem when using declarative + partitioning, since the automatically generated constraints are simple + enough to be understood by the planner.) -- cgit v1.2.3 From 64d4da511c012faff8ac309595620938a43c6817 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 31 Mar 2017 18:11:25 -0400 Subject: For foreign keys, check REFERENCES privilege only on the referenced table. We were requiring that the user have REFERENCES permission on both the referenced and referencing tables --- but this doesn't seem to have any support in the SQL standard, which says only that you need REFERENCES permission on the referenced table. And ALTER TABLE ADD FOREIGN KEY has already checked that you own the referencing table, so the check could only fail if a table owner has revoked his own REFERENCES permission. Moreover, the symmetric interpretation of this permission is unintuitive and confusing, as per complaint from Paul Jungwirth. So let's drop the referencing-side check. In passing, do a bit of wordsmithing on the GRANT reference page so that all the privilege types are described in similar fashion. Discussion: https://postgr.es/m/8940.1490906755@sss.pgh.pa.us --- doc/src/sgml/ref/create_table.sgml | 8 +++++--- doc/src/sgml/ref/grant.sgml | 19 +++++++++---------- src/backend/commands/tablecmds.c | 8 ++++++-- 3 files changed, 20 insertions(+), 15 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml index e1ec14e1c1..121418b6ca 100644 --- a/doc/src/sgml/ref/create_table.sgml +++ b/doc/src/sgml/ref/create_table.sgml @@ -750,9 +750,11 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI class="parameter">refcolumn     list is omitted, the primary key of the reftable is used. The referenced columns must be the columns of a non-deferrable - unique or primary key constraint in the referenced table. Note that - foreign key constraints cannot be defined between temporary tables and - permanent tables. + unique or primary key constraint in the referenced table. The user + must have REFERENCES permission on the referenced table + (either the whole table, or the specific referenced columns). + Note that foreign key constraints cannot be defined between temporary + tables and permanent tables. diff --git a/doc/src/sgml/ref/grant.sgml b/doc/src/sgml/ref/grant.sgml index 9fb4c2fd7e..d7b6d5fb30 100644 --- a/doc/src/sgml/ref/grant.sgml +++ b/doc/src/sgml/ref/grant.sgml @@ -257,10 +257,9 @@ GRANT role_name [, ...] TO REFERENCES - To create a foreign key constraint, it is - necessary to have this privilege on both the referencing and - referenced columns. The privilege may be granted for all columns - of a table, or just specific columns. + Allows creation of a foreign key constraint referencing the specified + table, or specified column(s) of the table. (See the + statement.) @@ -351,7 +350,7 @@ GRANT role_name [, ...] TO currval and nextval functions. - For types and domains, this privilege allow the use of the type or + For types and domains, this privilege allows the use of the type or domain in the creation of tables, functions, and other schema objects. (Note that it does not control general usage of the type, such as values of the type appearing in queries. It only prevents @@ -360,13 +359,13 @@ GRANT role_name [, ...] TO - For foreign-data wrappers, this privilege enables the grantee - to create new servers using that foreign-data wrapper. + For foreign-data wrappers, this privilege allows creation of + new servers using the foreign-data wrapper. - For servers, this privilege enables the grantee to create foreign - tables using the server, and also to create, alter, or drop their own - user's user mappings associated with that server. + For servers, this privilege allows creation of foreign tables using + the server. Grantees may also create, alter, or drop their own + user mappings associated with that server. diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 4cf2efb2ad..d418d56b54 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -6817,7 +6817,6 @@ ATAddForeignKeyConstraint(AlteredTableInfo *tab, Relation rel, * Now we can check permissions. */ checkFkeyPermissions(pkrel, pkattnum, numpks); - checkFkeyPermissions(rel, fkattnum, numfks); /* * Look up the equality operators to use in the constraint. @@ -7745,7 +7744,12 @@ findFkeyCast(Oid targetTypeId, Oid sourceTypeId, Oid *funcid) return ret; } -/* Permissions checks for ADD FOREIGN KEY */ +/* + * Permissions checks on the referenced table for ADD FOREIGN KEY + * + * Note: we have already checked that the user owns the referencing table, + * else we'd have failed much earlier; no additional checks are needed for it. + */ static void checkFkeyPermissions(Relation rel, int16 *attnums, int natts) { -- cgit v1.2.3 From 2113ac4cbb12b815804e8873d761cade9ddf49b9 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Fri, 31 Mar 2017 20:35:51 -0400 Subject: Don't use bgw_main even to specify in-core bgworker entrypoints. On EXEC_BACKEND builds, this can fail if ASLR is in use. Backpatch to 9.5. On master, completely remove the bgw_main field completely, since there is no situation in which it is safe for an EXEC_BACKEND build. On 9.6 and 9.5, leave the field intact to avoid breaking things for third-party code that doesn't care about working under EXEC_BACKEND. Prior to 9.5, there are no in-core bgworker entrypoints. Petr Jelinek, reviewed by me. Discussion: http://postgr.es/m/09d8ad33-4287-a09b-a77f-77f8761adb5e@2ndquadrant.com --- doc/src/sgml/bgworker.sgml | 36 ++++--------- src/backend/access/transam/parallel.c | 6 +-- src/backend/postmaster/bgworker.c | 81 +++++++++++++++++++----------- src/backend/replication/logical/launcher.c | 6 ++- src/include/access/parallel.h | 2 + src/include/postmaster/bgworker.h | 5 +- src/test/modules/test_shm_mq/setup.c | 1 - src/test/modules/worker_spi/worker_spi.c | 4 +- 8 files changed, 75 insertions(+), 66 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/bgworker.sgml b/doc/src/sgml/bgworker.sgml index 07f9f1081c..b422323081 100644 --- a/doc/src/sgml/bgworker.sgml +++ b/doc/src/sgml/bgworker.sgml @@ -54,9 +54,8 @@ typedef struct BackgroundWorker int bgw_flags; BgWorkerStartTime bgw_start_time; int bgw_restart_time; /* in seconds, or BGW_NEVER_RESTART */ - bgworker_main_type bgw_main; - char bgw_library_name[BGW_MAXLEN]; /* only if bgw_main is NULL */ - char bgw_function_name[BGW_MAXLEN]; /* only if bgw_main is NULL */ + char bgw_library_name[BGW_MAXLEN]; + char bgw_function_name[BGW_MAXLEN]; Datum bgw_main_arg; char bgw_extra[BGW_EXTRALEN]; int bgw_notify_pid; @@ -130,27 +129,13 @@ typedef struct BackgroundWorker process in case of a crash. - - bgw_main is a pointer to the function to run when - the process is started. This field can only safely be used to launch - functions within the core server, because shared libraries may be loaded - at different starting addresses in different backend processes. This will - happen on all platforms when the library is loaded using any mechanism - other than . Even when that - mechanism is used, address space layout variations will still occur on - Windows, and when EXEC_BACKEND is used. Therefore, most users - of this API should set this field to NULL. If it is non-NULL, it takes - precedence over bgw_library_name and - bgw_function_name. - - bgw_library_name is the name of a library in which the initial entry point for the background worker should be sought. The named library will be dynamically loaded by the worker process and bgw_function_name will be used to identify the - function to be called. If loading a function from the core code, - bgw_main should be set instead. + function to be called. If loading a function from the core code, this must + be set to "postgres". @@ -161,13 +146,10 @@ typedef struct BackgroundWorker bgw_main_arg is the Datum argument - to the background worker main function. Regardless of whether that - function is specified via bgw_main or via the combination - of bgw_library_name and bgw_function_name, - this main function should take a single argument of type Datum - and return void. bgw_main_arg will be - passed as the argument. In addition, the global variable - MyBgworkerEntry + to the background worker main function. This main function should take a + single argument of type Datum and return void. + bgw_main_arg will be passed as the argument. + In addition, the global variable MyBgworkerEntry points to a copy of the BackgroundWorker structure passed at registration time; the worker may find it helpful to examine this structure. @@ -215,7 +197,7 @@ typedef struct BackgroundWorker Signals are initially blocked when control reaches the - bgw_main function, and must be unblocked by it; this is to + background worker's main function, and must be unblocked by it; this is to allow the process to customize its signal handlers, if necessary. Signals can be unblocked in the new process by calling BackgroundWorkerUnblockSignals and blocked by calling diff --git a/src/backend/access/transam/parallel.c b/src/backend/access/transam/parallel.c index 3e0ee87e20..b3d3853fbc 100644 --- a/src/backend/access/transam/parallel.c +++ b/src/backend/access/transam/parallel.c @@ -110,7 +110,6 @@ static dlist_head pcxt_list = DLIST_STATIC_INIT(pcxt_list); /* Private functions. */ static void HandleParallelMessage(ParallelContext *pcxt, int i, StringInfo msg); static void ParallelExtensionTrampoline(dsm_segment *seg, shm_toc *toc); -static void ParallelWorkerMain(Datum main_arg); static void WaitForParallelWorkersToExit(ParallelContext *pcxt); @@ -458,7 +457,8 @@ LaunchParallelWorkers(ParallelContext *pcxt) | BGWORKER_CLASS_PARALLEL; worker.bgw_start_time = BgWorkerStart_ConsistentState; worker.bgw_restart_time = BGW_NEVER_RESTART; - worker.bgw_main = ParallelWorkerMain; + sprintf(worker.bgw_library_name, "postgres"); + sprintf(worker.bgw_function_name, "ParallelWorkerMain"); worker.bgw_main_arg = UInt32GetDatum(dsm_segment_handle(pcxt->seg)); worker.bgw_notify_pid = MyProcPid; memset(&worker.bgw_extra, 0, BGW_EXTRALEN); @@ -931,7 +931,7 @@ AtEOXact_Parallel(bool isCommit) /* * Main entrypoint for parallel workers. */ -static void +void ParallelWorkerMain(Datum main_arg) { dsm_segment *seg; diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c index 10e0f88b0d..0823317b78 100644 --- a/src/backend/postmaster/bgworker.c +++ b/src/backend/postmaster/bgworker.c @@ -15,12 +15,14 @@ #include #include "libpq/pqsignal.h" +#include "access/parallel.h" #include "miscadmin.h" #include "pgstat.h" #include "port/atomics.h" #include "postmaster/bgworker_internals.h" #include "postmaster/postmaster.h" #include "replication/logicallauncher.h" +#include "replication/logicalworker.h" #include "storage/dsm.h" #include "storage/ipc.h" #include "storage/latch.h" @@ -109,14 +111,26 @@ struct BackgroundWorkerHandle static BackgroundWorkerArray *BackgroundWorkerData; /* - * List of workers that are allowed to be started outside of - * shared_preload_libraries. + * List of internal background workers. These are used for mapping the + * function name to actual function when building with EXEC_BACKEND and also + * to allow these to be loaded outside of shared_preload_libraries. */ -static const bgworker_main_type InternalBGWorkers[] = { - ApplyLauncherMain, - NULL +typedef struct InternalBGWorkerMain +{ + char *bgw_function_name; + bgworker_main_type bgw_main; +} InternalBGWorkerMain; + +static const InternalBGWorkerMain InternalBGWorkers[] = { + {"ParallelWorkerMain", ParallelWorkerMain}, + {"ApplyLauncherMain", ApplyLauncherMain}, + {"ApplyWorkerMain", ApplyWorkerMain}, + /* Dummy entry marking end of the array. */ + {NULL, NULL} }; +static bgworker_main_type GetInternalBgWorkerMain(BackgroundWorker *worker); + /* * Calculate shared memory needed. */ @@ -341,7 +355,6 @@ BackgroundWorkerStateChange(void) rw->rw_worker.bgw_flags = slot->worker.bgw_flags; rw->rw_worker.bgw_start_time = slot->worker.bgw_start_time; rw->rw_worker.bgw_restart_time = slot->worker.bgw_restart_time; - rw->rw_worker.bgw_main = slot->worker.bgw_main; rw->rw_worker.bgw_main_arg = slot->worker.bgw_main_arg; memcpy(rw->rw_worker.bgw_extra, slot->worker.bgw_extra, BGW_EXTRALEN); @@ -763,17 +776,14 @@ StartBackgroundWorker(void) } /* - * If bgw_main is set, we use that value as the initial entrypoint. - * However, if the library containing the entrypoint wasn't loaded at - * postmaster startup time, passing it as a direct function pointer is not - * possible. To work around that, we allow callers for whom a function - * pointer is not available to pass a library name (which will be loaded, - * if necessary) and a function name (which will be looked up in the named - * library). + * For internal workers set the entry point to known function address. + * Otherwise use the entry point specified by library name (which will + * be loaded, if necessary) and a function name (which will be looked up + * in the named library). */ - if (worker->bgw_main != NULL) - entrypt = worker->bgw_main; - else + entrypt = GetInternalBgWorkerMain(worker); + + if (entrypt == NULL) entrypt = (bgworker_main_type) load_external_function(worker->bgw_library_name, worker->bgw_function_name, @@ -806,23 +816,13 @@ RegisterBackgroundWorker(BackgroundWorker *worker) { RegisteredBgWorker *rw; static int numworkers = 0; - bool internal = false; - int i; if (!IsUnderPostmaster) ereport(DEBUG1, (errmsg("registering background worker \"%s\"", worker->bgw_name))); - for (i = 0; InternalBGWorkers[i]; i++) - { - if (worker->bgw_main == InternalBGWorkers[i]) - { - internal = true; - break; - } - } - - if (!process_shared_preload_libraries_in_progress && !internal) + if (!process_shared_preload_libraries_in_progress && + GetInternalBgWorkerMain(worker) == NULL) { if (!IsUnderPostmaster) ereport(LOG, @@ -1152,3 +1152,28 @@ TerminateBackgroundWorker(BackgroundWorkerHandle *handle) if (signal_postmaster) SendPostmasterSignal(PMSIGNAL_BACKGROUND_WORKER_CHANGE); } + +/* + * Search the known internal worker array and return its main function + * pointer if found. + * + * Returns NULL if not known internal worker. + */ +static bgworker_main_type +GetInternalBgWorkerMain(BackgroundWorker *worker) +{ + int i; + + /* Internal workers always have to use postgres as library name. */ + if (strncmp(worker->bgw_library_name, "postgres", BGW_MAXLEN) != 0) + return NULL; + + for (i = 0; InternalBGWorkers[i].bgw_function_name; i++) + { + if (strncmp(InternalBGWorkers[i].bgw_function_name, + worker->bgw_function_name, BGW_MAXLEN) == 0) + return InternalBGWorkers[i].bgw_main; + } + + return NULL; +} diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c index 255b22597b..fecff936c0 100644 --- a/src/backend/replication/logical/launcher.c +++ b/src/backend/replication/logical/launcher.c @@ -295,7 +295,8 @@ logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid, bgw.bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION; bgw.bgw_start_time = BgWorkerStart_RecoveryFinished; - bgw.bgw_main = ApplyWorkerMain; + snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres"); + snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyWorkerMain"); if (OidIsValid(relid)) snprintf(bgw.bgw_name, BGW_MAXLEN, "logical replication worker for subscription %u sync %u", subid, relid); @@ -553,7 +554,8 @@ ApplyLauncherRegister(void) bgw.bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION; bgw.bgw_start_time = BgWorkerStart_RecoveryFinished; - bgw.bgw_main = ApplyLauncherMain; + snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres"); + snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyLauncherMain"); snprintf(bgw.bgw_name, BGW_MAXLEN, "logical replication launcher"); bgw.bgw_restart_time = 5; diff --git a/src/include/access/parallel.h b/src/include/access/parallel.h index 96ce4803da..5065a3830c 100644 --- a/src/include/access/parallel.h +++ b/src/include/access/parallel.h @@ -67,4 +67,6 @@ extern void AtEOXact_Parallel(bool isCommit); extern void AtEOSubXact_Parallel(bool isCommit, SubTransactionId mySubId); extern void ParallelWorkerReportLastRecEnd(XLogRecPtr last_xlog_end); +extern void ParallelWorkerMain(Datum main_arg); + #endif /* PARALLEL_H */ diff --git a/src/include/postmaster/bgworker.h b/src/include/postmaster/bgworker.h index 128e92cea1..51a5978ea8 100644 --- a/src/include/postmaster/bgworker.h +++ b/src/include/postmaster/bgworker.h @@ -91,9 +91,8 @@ typedef struct BackgroundWorker int bgw_flags; BgWorkerStartTime bgw_start_time; int bgw_restart_time; /* in seconds, or BGW_NEVER_RESTART */ - bgworker_main_type bgw_main; - char bgw_library_name[BGW_MAXLEN]; /* only if bgw_main is NULL */ - char bgw_function_name[BGW_MAXLEN]; /* only if bgw_main is NULL */ + char bgw_library_name[BGW_MAXLEN]; + char bgw_function_name[BGW_MAXLEN]; Datum bgw_main_arg; char bgw_extra[BGW_EXTRALEN]; pid_t bgw_notify_pid; /* SIGUSR1 this backend on start/stop */ diff --git a/src/test/modules/test_shm_mq/setup.c b/src/test/modules/test_shm_mq/setup.c index e3c024ecb4..319a67f49a 100644 --- a/src/test/modules/test_shm_mq/setup.c +++ b/src/test/modules/test_shm_mq/setup.c @@ -216,7 +216,6 @@ setup_background_workers(int nworkers, dsm_segment *seg) worker.bgw_flags = BGWORKER_SHMEM_ACCESS; worker.bgw_start_time = BgWorkerStart_ConsistentState; worker.bgw_restart_time = BGW_NEVER_RESTART; - worker.bgw_main = NULL; /* new worker might not have library loaded */ sprintf(worker.bgw_library_name, "test_shm_mq"); sprintf(worker.bgw_function_name, "test_shm_mq_main"); snprintf(worker.bgw_name, BGW_MAXLEN, "test_shm_mq"); diff --git a/src/test/modules/worker_spi/worker_spi.c b/src/test/modules/worker_spi/worker_spi.c index 72ab8464e1..421ec76ba3 100644 --- a/src/test/modules/worker_spi/worker_spi.c +++ b/src/test/modules/worker_spi/worker_spi.c @@ -347,7 +347,8 @@ _PG_init(void) BGWORKER_BACKEND_DATABASE_CONNECTION; worker.bgw_start_time = BgWorkerStart_RecoveryFinished; worker.bgw_restart_time = BGW_NEVER_RESTART; - worker.bgw_main = worker_spi_main; + sprintf(worker.bgw_library_name, "worker_spi"); + sprintf(worker.bgw_function_name, "worker_spi_main"); worker.bgw_notify_pid = 0; /* @@ -378,7 +379,6 @@ worker_spi_launch(PG_FUNCTION_ARGS) BGWORKER_BACKEND_DATABASE_CONNECTION; worker.bgw_start_time = BgWorkerStart_RecoveryFinished; worker.bgw_restart_time = BGW_NEVER_RESTART; - worker.bgw_main = NULL; /* new worker might not have library loaded */ sprintf(worker.bgw_library_name, "worker_spi"); sprintf(worker.bgw_function_name, "worker_spi_main"); snprintf(worker.bgw_name, BGW_MAXLEN, "worker %d", i); -- cgit v1.2.3 From 18ce3a4ab22d2984f8540ab480979c851dae5338 Mon Sep 17 00:00:00 2001 From: Kevin Grittner Date: Fri, 31 Mar 2017 23:17:18 -0500 Subject: Add infrastructure to support EphemeralNamedRelation references. A QueryEnvironment concept is added, which allows new types of objects to be passed into queries from parsing on through execution. At this point, the only thing implemented is a collection of EphemeralNamedRelation objects -- relations which can be referenced by name in queries, but do not exist in the catalogs. The only type of ENR implemented is NamedTuplestore, but provision is made to add more types fairly easily. An ENR can carry its own TupleDesc or reference a relation in the catalogs by relid. Although these features can be used without SPI, convenience functions are added to SPI so that ENRs can easily be used by code run through SPI. The initial use of all this is going to be transition tables in AFTER triggers, but that will be added to each PL as a separate commit. An incidental effect of this patch is to produce a more informative error message if an attempt is made to modify the contents of a CTE from a referencing DML statement. No tests previously covered that possibility, so one is added. Kevin Grittner and Thomas Munro Reviewed by Heikki Linnakangas, David Fetter, and Thomas Munro with valuable comments and suggestions from many others --- contrib/pg_stat_statements/pg_stat_statements.c | 15 +- doc/src/sgml/spi.sgml | 204 ++++++++++++++++++++++++ src/backend/catalog/pg_proc.c | 3 +- src/backend/commands/copy.c | 5 +- src/backend/commands/createas.c | 5 +- src/backend/commands/explain.c | 40 +++-- src/backend/commands/extension.c | 6 +- src/backend/commands/foreigncmds.c | 2 +- src/backend/commands/matview.c | 2 +- src/backend/commands/prepare.c | 17 +- src/backend/commands/schemacmds.c | 1 + src/backend/commands/trigger.c | 9 +- src/backend/commands/view.c | 2 +- src/backend/executor/Makefile | 3 +- src/backend/executor/execAmi.c | 6 + src/backend/executor/execMain.c | 5 + src/backend/executor/execParallel.c | 2 +- src/backend/executor/execProcnode.c | 14 ++ src/backend/executor/execUtils.c | 2 + src/backend/executor/functions.c | 8 +- src/backend/executor/nodeNamedtuplestorescan.c | 198 +++++++++++++++++++++++ src/backend/executor/spi.c | 116 ++++++++++++-- src/backend/nodes/copyfuncs.c | 25 +++ src/backend/nodes/nodeFuncs.c | 2 + src/backend/nodes/outfuncs.c | 20 +++ src/backend/nodes/print.c | 4 + src/backend/nodes/readfuncs.c | 7 + src/backend/optimizer/path/allpaths.c | 45 ++++++ src/backend/optimizer/path/costsize.c | 70 ++++++++ src/backend/optimizer/plan/createplan.c | 71 +++++++++ src/backend/optimizer/plan/setrefs.c | 16 ++ src/backend/optimizer/plan/subselect.c | 4 + src/backend/optimizer/prep/prepjointree.c | 2 + src/backend/optimizer/util/clauses.c | 2 +- src/backend/optimizer/util/pathnode.c | 26 +++ src/backend/optimizer/util/plancat.c | 7 +- src/backend/optimizer/util/relnode.c | 1 + src/backend/parser/Makefile | 5 +- src/backend/parser/analyze.c | 14 +- src/backend/parser/parse_clause.c | 59 +++++-- src/backend/parser/parse_enr.c | 29 ++++ src/backend/parser/parse_node.c | 2 + src/backend/parser/parse_relation.c | 143 ++++++++++++++++- src/backend/parser/parse_target.c | 2 + src/backend/tcop/postgres.c | 22 ++- src/backend/tcop/pquery.c | 10 +- src/backend/tcop/utility.c | 34 ++-- src/backend/utils/adt/ruleutils.c | 1 + src/backend/utils/cache/plancache.c | 34 ++-- src/backend/utils/misc/Makefile | 4 +- src/backend/utils/misc/queryenvironment.c | 144 +++++++++++++++++ src/backend/utils/sort/tuplestore.c | 17 ++ src/include/catalog/catversion.h | 2 +- src/include/commands/createas.h | 3 +- src/include/commands/explain.h | 9 +- src/include/commands/prepare.h | 4 +- src/include/executor/execdesc.h | 2 + src/include/executor/nodeNamedtuplestorescan.h | 24 +++ src/include/executor/spi.h | 7 + src/include/executor/spi_priv.h | 2 + src/include/nodes/execnodes.h | 21 +++ src/include/nodes/nodes.h | 2 + src/include/nodes/parsenodes.h | 6 +- src/include/nodes/plannodes.h | 10 ++ src/include/optimizer/cost.h | 3 + src/include/optimizer/pathnode.h | 2 + src/include/parser/analyze.h | 2 +- src/include/parser/parse_enr.h | 22 +++ src/include/parser/parse_node.h | 3 + src/include/parser/parse_relation.h | 4 + src/include/tcop/tcopprot.h | 7 +- src/include/tcop/utility.h | 5 +- src/include/utils/plancache.h | 10 +- src/include/utils/portal.h | 1 + src/include/utils/queryenvironment.h | 74 +++++++++ src/include/utils/tuplestore.h | 2 + src/test/regress/expected/with.out | 3 + src/test/regress/sql/with.sql | 3 + 78 files changed, 1598 insertions(+), 122 deletions(-) create mode 100644 src/backend/executor/nodeNamedtuplestorescan.c create mode 100644 src/backend/parser/parse_enr.c create mode 100644 src/backend/utils/misc/queryenvironment.c create mode 100644 src/include/executor/nodeNamedtuplestorescan.h create mode 100644 src/include/parser/parse_enr.h create mode 100644 src/include/utils/queryenvironment.h (limited to 'doc/src') diff --git a/contrib/pg_stat_statements/pg_stat_statements.c b/contrib/pg_stat_statements/pg_stat_statements.c index c300261852..6b7503df42 100644 --- a/contrib/pg_stat_statements/pg_stat_statements.c +++ b/contrib/pg_stat_statements/pg_stat_statements.c @@ -299,6 +299,7 @@ static void pgss_ExecutorFinish(QueryDesc *queryDesc); static void pgss_ExecutorEnd(QueryDesc *queryDesc); static void pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, ProcessUtilityContext context, ParamListInfo params, + QueryEnvironment *queryEnv, DestReceiver *dest, char *completionTag); static uint32 pgss_hash_fn(const void *key, Size keysize); static int pgss_match_fn(const void *key1, const void *key2, Size keysize); @@ -956,7 +957,8 @@ pgss_ExecutorEnd(QueryDesc *queryDesc) */ static void pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, - ProcessUtilityContext context, ParamListInfo params, + ProcessUtilityContext context, + ParamListInfo params, QueryEnvironment *queryEnv, DestReceiver *dest, char *completionTag) { Node *parsetree = pstmt->utilityStmt; @@ -994,11 +996,11 @@ pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, { if (prev_ProcessUtility) prev_ProcessUtility(pstmt, queryString, - context, params, + context, params, queryEnv, dest, completionTag); else standard_ProcessUtility(pstmt, queryString, - context, params, + context, params, queryEnv, dest, completionTag); nested_level--; } @@ -1058,11 +1060,11 @@ pgss_ProcessUtility(PlannedStmt *pstmt, const char *queryString, { if (prev_ProcessUtility) prev_ProcessUtility(pstmt, queryString, - context, params, + context, params, queryEnv, dest, completionTag); else standard_ProcessUtility(pstmt, queryString, - context, params, + context, params, queryEnv, dest, completionTag); } } @@ -2424,6 +2426,9 @@ JumbleRangeTable(pgssJumbleState *jstate, List *rtable) APP_JUMB_STRING(rte->ctename); APP_JUMB(rte->ctelevelsup); break; + case RTE_NAMEDTUPLESTORE: + APP_JUMB_STRING(rte->enrname); + break; default: elog(ERROR, "unrecognized RTE kind: %d", (int) rte->rtekind); break; diff --git a/doc/src/sgml/spi.sgml b/doc/src/sgml/spi.sgml index 836ce0822f..af7eada2e1 100644 --- a/doc/src/sgml/spi.sgml +++ b/doc/src/sgml/spi.sgml @@ -2639,6 +2639,210 @@ SPIPlanPtr SPI_saveplan(SPIPlanPtr plan) + + + + SPI_register_relation + + + SPI_register_relation + 3 + + + + SPI_register_relation + make a ephemeral named relation available by name in SPI queries + + + + +int SPI_register_relation(EphemeralNamedRelation enr) + + + + + Description + + + SPI_register_relation makes an ephemeral named + relation, with associated information, available to queries planned and + executed through the current SPI connection. + + + + + Arguments + + + + EphemeralNamedRelation enr + + + the ephemeral named relation registry entry + + + + + + + + Return Value + + + If the execution of the command was successful then the following + (nonnegative) value will be returned: + + + + SPI_OK_REL_REGISTER + + + if the relation has been successfully registered by name + + + + + + + + On error, one of the following negative values is returned: + + + + SPI_ERROR_ARGUMENT + + + if enr is NULL or its + name field is NULL + + + + + + SPI_ERROR_UNCONNECTED + + + if called from an unconnected procedure + + + + + + SPI_ERROR_REL_DUPLICATE + + + if the name specified in the name field of + enr is already registered for this connection + + + + + + + + + + + + SPI_unregister_relation + + + SPI_unregister_relation + 3 + + + + SPI_unregister_relation + remove an ephemeral named relation from the registry + + + + +int SPI_unregister_relation(const char * name) + + + + + Description + + + SPI_unregister_relation removes an ephemeral named + relation from the registry for the current connection. + + + + + Arguments + + + + const char * name + + + the relation registry entry name + + + + + + + + Return Value + + + If the execution of the command was successful then the following + (nonnegative) value will be returned: + + + + SPI_OK_REL_UNREGISTER + + + if the tuplestore has been successfully removed from the registry + + + + + + + + On error, one of the following negative values is returned: + + + + SPI_ERROR_ARGUMENT + + + if name is NULL + + + + + + SPI_ERROR_UNCONNECTED + + + if called from an unconnected procedure + + + + + + SPI_ERROR_REL_NOT_FOUND + + + if name is not found in the registry for the + current connection + + + + + + + + + + diff --git a/src/backend/catalog/pg_proc.c b/src/backend/catalog/pg_proc.c index ae27848116..f058d1274f 100644 --- a/src/backend/catalog/pg_proc.c +++ b/src/backend/catalog/pg_proc.c @@ -934,7 +934,8 @@ fmgr_sql_validator(PG_FUNCTION_ARGS) querytree_sublist = pg_analyze_and_rewrite_params(parsetree, prosrc, (ParserSetupHook) sql_fn_parser_setup, - pinfo); + pinfo, + NULL); querytree_list = list_concat(querytree_list, querytree_sublist); } diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 0158eda591..8c58808686 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -1471,7 +1471,8 @@ BeginCopy(ParseState *pstate, * DECLARE CURSOR and PREPARE.) XXX FIXME someday. */ rewritten = pg_analyze_and_rewrite(copyObject(raw_query), - pstate->p_sourcetext, NULL, 0); + pstate->p_sourcetext, NULL, 0, + NULL); /* check that we got back something we can work with */ if (rewritten == NIL) @@ -1574,7 +1575,7 @@ BeginCopy(ParseState *pstate, cstate->queryDesc = CreateQueryDesc(plan, pstate->p_sourcetext, GetActiveSnapshot(), InvalidSnapshot, - dest, NULL, 0); + dest, NULL, NULL, 0); /* * Call ExecutorStart to prepare the plan for execution. diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 20cb64661a..f49b391505 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -222,7 +222,8 @@ create_ctas_nodata(List *tlist, IntoClause *into) */ ObjectAddress ExecCreateTableAs(CreateTableAsStmt *stmt, const char *queryString, - ParamListInfo params, char *completionTag) + ParamListInfo params, QueryEnvironment *queryEnv, + char *completionTag) { Query *query = castNode(Query, stmt->query); IntoClause *into = stmt->into; @@ -341,7 +342,7 @@ ExecCreateTableAs(CreateTableAsStmt *stmt, const char *queryString, /* Create a QueryDesc, redirecting output to our tuple receiver */ queryDesc = CreateQueryDesc(plan, queryString, GetActiveSnapshot(), InvalidSnapshot, - dest, params, 0); + dest, params, queryEnv, 0); /* call ExecutorStart to prepare the plan for execution */ ExecutorStart(queryDesc, GetIntoRelEFlags(into)); diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index ea19ba60c5..a18ab43616 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -55,7 +55,8 @@ explain_get_index_name_hook_type explain_get_index_name_hook = NULL; static void ExplainOneQuery(Query *query, int cursorOptions, IntoClause *into, ExplainState *es, - const char *queryString, ParamListInfo params); + const char *queryString, ParamListInfo params, + QueryEnvironment *queryEnv); static void report_triggers(ResultRelInfo *rInfo, bool show_relname, ExplainState *es); static double elapsed_time(instr_time *starttime); @@ -142,7 +143,8 @@ static void escape_yaml(StringInfo buf, const char *str); */ void ExplainQuery(ParseState *pstate, ExplainStmt *stmt, const char *queryString, - ParamListInfo params, DestReceiver *dest) + ParamListInfo params, QueryEnvironment *queryEnv, + DestReceiver *dest) { ExplainState *es = NewExplainState(); TupOutputState *tstate; @@ -253,7 +255,7 @@ ExplainQuery(ParseState *pstate, ExplainStmt *stmt, const char *queryString, { ExplainOneQuery(castNode(Query, lfirst(l)), CURSOR_OPT_PARALLEL_OK, NULL, es, - queryString, params); + queryString, params, queryEnv); /* Separate plans with an appropriate separator */ if (lnext(l) != NULL) @@ -338,12 +340,14 @@ ExplainResultDesc(ExplainStmt *stmt) static void ExplainOneQuery(Query *query, int cursorOptions, IntoClause *into, ExplainState *es, - const char *queryString, ParamListInfo params) + const char *queryString, ParamListInfo params, + QueryEnvironment *queryEnv) { /* planner will not cope with utility statements */ if (query->commandType == CMD_UTILITY) { - ExplainOneUtility(query->utilityStmt, into, es, queryString, params); + ExplainOneUtility(query->utilityStmt, into, es, queryString, params, + queryEnv); return; } @@ -366,7 +370,8 @@ ExplainOneQuery(Query *query, int cursorOptions, INSTR_TIME_SUBTRACT(planduration, planstart); /* run it (if needed) and produce output */ - ExplainOnePlan(plan, into, es, queryString, params, &planduration); + ExplainOnePlan(plan, into, es, queryString, params, queryEnv, + &planduration); } } @@ -383,7 +388,8 @@ ExplainOneQuery(Query *query, int cursorOptions, */ void ExplainOneUtility(Node *utilityStmt, IntoClause *into, ExplainState *es, - const char *queryString, ParamListInfo params) + const char *queryString, ParamListInfo params, + QueryEnvironment *queryEnv) { if (utilityStmt == NULL) return; @@ -404,7 +410,7 @@ ExplainOneUtility(Node *utilityStmt, IntoClause *into, ExplainState *es, Assert(list_length(rewritten) == 1); ExplainOneQuery(castNode(Query, linitial(rewritten)), 0, ctas->into, es, - queryString, params); + queryString, params, queryEnv); } else if (IsA(utilityStmt, DeclareCursorStmt)) { @@ -423,11 +429,11 @@ ExplainOneUtility(Node *utilityStmt, IntoClause *into, ExplainState *es, Assert(list_length(rewritten) == 1); ExplainOneQuery(castNode(Query, linitial(rewritten)), dcs->options, NULL, es, - queryString, params); + queryString, params, queryEnv); } else if (IsA(utilityStmt, ExecuteStmt)) ExplainExecuteQuery((ExecuteStmt *) utilityStmt, into, es, - queryString, params); + queryString, params, queryEnv); else if (IsA(utilityStmt, NotifyStmt)) { if (es->format == EXPLAIN_FORMAT_TEXT) @@ -460,7 +466,7 @@ ExplainOneUtility(Node *utilityStmt, IntoClause *into, ExplainState *es, void ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es, const char *queryString, ParamListInfo params, - const instr_time *planduration) + QueryEnvironment *queryEnv, const instr_time *planduration) { DestReceiver *dest; QueryDesc *queryDesc; @@ -505,7 +511,7 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es, /* Create a QueryDesc for the query */ queryDesc = CreateQueryDesc(plannedstmt, queryString, GetActiveSnapshot(), InvalidSnapshot, - dest, params, instrument_option); + dest, params, queryEnv, instrument_option); /* Select execution options */ if (es->analyze) @@ -796,6 +802,7 @@ ExplainPreScanNode(PlanState *planstate, Bitmapset **rels_used) case T_TableFuncScan: case T_ValuesScan: case T_CteScan: + case T_NamedTuplestoreScan: case T_WorkTableScan: *rels_used = bms_add_member(*rels_used, ((Scan *) plan)->scanrelid); @@ -951,6 +958,9 @@ ExplainNode(PlanState *planstate, List *ancestors, case T_CteScan: pname = sname = "CTE Scan"; break; + case T_NamedTuplestoreScan: + pname = sname = "Named Tuplestore Scan"; + break; case T_WorkTableScan: pname = sname = "WorkTable Scan"; break; @@ -1389,6 +1399,7 @@ ExplainNode(PlanState *planstate, List *ancestors, case T_SeqScan: case T_ValuesScan: case T_CteScan: + case T_NamedTuplestoreScan: case T_WorkTableScan: case T_SubqueryScan: show_scan_qual(plan->qual, "Filter", planstate, ancestors, es); @@ -2679,6 +2690,11 @@ ExplainTargetRel(Plan *plan, Index rti, ExplainState *es) objectname = rte->ctename; objecttag = "CTE Name"; break; + case T_NamedTuplestoreScan: + Assert(rte->rtekind == RTE_NAMEDTUPLESTORE); + objectname = rte->enrname; + objecttag = "Tuplestore Name"; + break; case T_WorkTableScan: /* Assert it's on a self-reference CTE */ Assert(rte->rtekind == RTE_CTE); diff --git a/src/backend/commands/extension.c b/src/backend/commands/extension.c index 5a84bedf46..6be9bc457c 100644 --- a/src/backend/commands/extension.c +++ b/src/backend/commands/extension.c @@ -721,7 +721,8 @@ execute_sql_string(const char *sql, const char *filename) stmt_list = pg_analyze_and_rewrite(parsetree, sql, NULL, - 0); + 0, + NULL); stmt_list = pg_plan_queries(stmt_list, CURSOR_OPT_PARALLEL_OK, NULL); foreach(lc2, stmt_list) @@ -739,7 +740,7 @@ execute_sql_string(const char *sql, const char *filename) qdesc = CreateQueryDesc(stmt, sql, GetActiveSnapshot(), NULL, - dest, NULL, 0); + dest, NULL, NULL, 0); ExecutorStart(qdesc, 0); ExecutorRun(qdesc, ForwardScanDirection, 0, true); @@ -759,6 +760,7 @@ execute_sql_string(const char *sql, const char *filename) sql, PROCESS_UTILITY_QUERY, NULL, + NULL, dest, NULL); } diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c index 68100df083..4ffe1bca75 100644 --- a/src/backend/commands/foreigncmds.c +++ b/src/backend/commands/foreigncmds.c @@ -1623,7 +1623,7 @@ ImportForeignSchema(ImportForeignSchemaStmt *stmt) /* Execute statement */ ProcessUtility(pstmt, cmd, - PROCESS_UTILITY_SUBCOMMAND, NULL, + PROCESS_UTILITY_SUBCOMMAND, NULL, NULL, None_Receiver, NULL); /* Be sure to advance the command counter between subcommands */ diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 9d41ad8fad..2f93328318 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -418,7 +418,7 @@ refresh_matview_datafill(DestReceiver *dest, Query *query, /* Create a QueryDesc, redirecting output to our tuple receiver */ queryDesc = CreateQueryDesc(plan, queryString, GetActiveSnapshot(), InvalidSnapshot, - dest, NULL, 0); + dest, NULL, NULL, 0); /* call ExecutorStart to prepare the plan for execution */ ExecutorStart(queryDesc, EXEC_FLAG_WITHOUT_OIDS); diff --git a/src/backend/commands/prepare.c b/src/backend/commands/prepare.c index dc6d43ec6d..5b3f777f2c 100644 --- a/src/backend/commands/prepare.c +++ b/src/backend/commands/prepare.c @@ -91,7 +91,7 @@ PrepareQuery(PrepareStmt *stmt, const char *queryString, * to see the unmodified raw parse tree. */ plansource = CreateCachedPlan(rawstmt, queryString, - CreateCommandTag(stmt->query)); + CreateCommandTag(stmt->query), NULL); /* Transform list of TypeNames to array of type OIDs */ nargs = list_length(stmt->argtypes); @@ -243,7 +243,7 @@ ExecuteQuery(ExecuteStmt *stmt, IntoClause *intoClause, entry->plansource->query_string); /* Replan if needed, and increment plan refcount for portal */ - cplan = GetCachedPlan(entry->plansource, paramLI, false); + cplan = GetCachedPlan(entry->plansource, paramLI, false, NULL); plan_list = cplan->stmt_list; /* @@ -551,7 +551,7 @@ FetchPreparedStatementTargetList(PreparedStatement *stmt) List *tlist; /* Get the plan's primary targetlist */ - tlist = CachedPlanGetTargetList(stmt->plansource); + tlist = CachedPlanGetTargetList(stmt->plansource, NULL); /* Copy into caller's context in case plan gets invalidated */ return copyObject(tlist); @@ -629,7 +629,8 @@ DropAllPreparedStatements(void) */ void ExplainExecuteQuery(ExecuteStmt *execstmt, IntoClause *into, ExplainState *es, - const char *queryString, ParamListInfo params) + const char *queryString, ParamListInfo params, + QueryEnvironment *queryEnv) { PreparedStatement *entry; const char *query_string; @@ -668,7 +669,7 @@ ExplainExecuteQuery(ExecuteStmt *execstmt, IntoClause *into, ExplainState *es, } /* Replan if needed, and acquire a transient refcount */ - cplan = GetCachedPlan(entry->plansource, paramLI, true); + cplan = GetCachedPlan(entry->plansource, paramLI, true, queryEnv); INSTR_TIME_SET_CURRENT(planduration); INSTR_TIME_SUBTRACT(planduration, planstart); @@ -681,9 +682,11 @@ ExplainExecuteQuery(ExecuteStmt *execstmt, IntoClause *into, ExplainState *es, PlannedStmt *pstmt = castNode(PlannedStmt, lfirst(p)); if (pstmt->commandType != CMD_UTILITY) - ExplainOnePlan(pstmt, into, es, query_string, paramLI, &planduration); + ExplainOnePlan(pstmt, into, es, query_string, paramLI, queryEnv, + &planduration); else - ExplainOneUtility(pstmt->utilityStmt, into, es, query_string, paramLI); + ExplainOneUtility(pstmt->utilityStmt, into, es, query_string, + paramLI, queryEnv); /* No need for CommandCounterIncrement, as ExplainOnePlan did it */ diff --git a/src/backend/commands/schemacmds.c b/src/backend/commands/schemacmds.c index 722b965d65..93425babbe 100644 --- a/src/backend/commands/schemacmds.c +++ b/src/backend/commands/schemacmds.c @@ -194,6 +194,7 @@ CreateSchemaCommand(CreateSchemaStmt *stmt, const char *queryString, queryString, PROCESS_UTILITY_SUBCOMMAND, NULL, + NULL, None_Receiver, NULL); diff --git a/src/backend/commands/trigger.c b/src/backend/commands/trigger.c index f3b1a52682..ebf23a0d94 100644 --- a/src/backend/commands/trigger.c +++ b/src/backend/commands/trigger.c @@ -354,6 +354,13 @@ CreateTrigger(CreateTrigStmt *stmt, const char *queryString, * adjustments will be needed below. */ + if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("\"%s\" is a partitioned table", + RelationGetRelationName(rel)), + errdetail("Triggers on partitioned tables cannot have transition tables."))); + if (stmt->timing != TRIGGER_TYPE_AFTER) ereport(ERROR, (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), @@ -1173,7 +1180,7 @@ ConvertTriggerToFK(CreateTrigStmt *stmt, Oid funcoid) /* ... and execute it */ ProcessUtility(wrapper, "(generated ALTER TABLE ADD FOREIGN KEY command)", - PROCESS_UTILITY_SUBCOMMAND, NULL, + PROCESS_UTILITY_SUBCOMMAND, NULL, NULL, None_Receiver, NULL); /* Remove the matched item from the list */ diff --git a/src/backend/commands/view.c b/src/backend/commands/view.c index 35e25db7dc..6909a67e77 100644 --- a/src/backend/commands/view.c +++ b/src/backend/commands/view.c @@ -436,7 +436,7 @@ DefineView(ViewStmt *stmt, const char *queryString, rawstmt->stmt_location = stmt_location; rawstmt->stmt_len = stmt_len; - viewParse = parse_analyze(rawstmt, queryString, NULL, 0); + viewParse = parse_analyze(rawstmt, queryString, NULL, 0, NULL); /* * The grammar should ensure that the result is a single SELECT Query. diff --git a/src/backend/executor/Makefile b/src/backend/executor/Makefile index d1c1324399..083b20f3fe 100644 --- a/src/backend/executor/Makefile +++ b/src/backend/executor/Makefile @@ -25,7 +25,8 @@ OBJS = execAmi.o execCurrent.o execExpr.o execExprInterp.o \ nodeMaterial.o nodeMergeAppend.o nodeMergejoin.o nodeModifyTable.o \ nodeNestloop.o nodeProjectSet.o nodeRecursiveunion.o nodeResult.o \ nodeSamplescan.o nodeSeqscan.o nodeSetOp.o nodeSort.o nodeUnique.o \ - nodeValuesscan.o nodeCtescan.o nodeWorktablescan.o \ + nodeValuesscan.o \ + nodeCtescan.o nodeNamedtuplestorescan.o nodeWorktablescan.o \ nodeGroup.o nodeSubplan.o nodeSubqueryscan.o nodeTidscan.o \ nodeForeignscan.o nodeWindowAgg.o tstoreReceiver.o tqueue.o spi.o \ nodeTableFuncscan.o diff --git a/src/backend/executor/execAmi.c b/src/backend/executor/execAmi.c index 5d59f95a91..7e85c66da3 100644 --- a/src/backend/executor/execAmi.c +++ b/src/backend/executor/execAmi.c @@ -38,6 +38,7 @@ #include "executor/nodeMergeAppend.h" #include "executor/nodeMergejoin.h" #include "executor/nodeModifyTable.h" +#include "executor/nodeNamedtuplestorescan.h" #include "executor/nodeNestloop.h" #include "executor/nodeProjectSet.h" #include "executor/nodeRecursiveunion.h" @@ -211,6 +212,10 @@ ExecReScan(PlanState *node) ExecReScanCteScan((CteScanState *) node); break; + case T_NamedTuplestoreScanState: + ExecReScanNamedTuplestoreScan((NamedTuplestoreScanState *) node); + break; + case T_WorkTableScanState: ExecReScanWorkTableScan((WorkTableScanState *) node); break; @@ -571,6 +576,7 @@ ExecMaterializesOutput(NodeTag plantype) case T_FunctionScan: case T_TableFuncScan: case T_CteScan: + case T_NamedTuplestoreScan: case T_WorkTableScan: case T_Sort: return true; diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c index f2995f2e7b..920b12072f 100644 --- a/src/backend/executor/execMain.c +++ b/src/backend/executor/execMain.c @@ -198,6 +198,11 @@ standard_ExecutorStart(QueryDesc *queryDesc, int eflags) estate->es_sourceText = queryDesc->sourceText; + /* + * Fill in the query environment, if any, from queryDesc. + */ + estate->es_queryEnv = queryDesc->queryEnv; + /* * If non-read-only query, set the command ID to mark output tuples with */ diff --git a/src/backend/executor/execParallel.c b/src/backend/executor/execParallel.c index b91b663c46..469a32c7b0 100644 --- a/src/backend/executor/execParallel.c +++ b/src/backend/executor/execParallel.c @@ -710,7 +710,7 @@ ExecParallelGetQueryDesc(shm_toc *toc, DestReceiver *receiver, return CreateQueryDesc(pstmt, queryString, GetActiveSnapshot(), InvalidSnapshot, - receiver, paramLI, instrument_options); + receiver, paramLI, NULL, instrument_options); } /* diff --git a/src/backend/executor/execProcnode.c b/src/backend/executor/execProcnode.c index 80c77addb8..486ddf1762 100644 --- a/src/backend/executor/execProcnode.c +++ b/src/backend/executor/execProcnode.c @@ -101,6 +101,7 @@ #include "executor/nodeMergeAppend.h" #include "executor/nodeMergejoin.h" #include "executor/nodeModifyTable.h" +#include "executor/nodeNamedtuplestorescan.h" #include "executor/nodeNestloop.h" #include "executor/nodeProjectSet.h" #include "executor/nodeRecursiveunion.h" @@ -256,6 +257,11 @@ ExecInitNode(Plan *node, EState *estate, int eflags) estate, eflags); break; + case T_NamedTuplestoreScan: + result = (PlanState *) ExecInitNamedTuplestoreScan((NamedTuplestoreScan *) node, + estate, eflags); + break; + case T_WorkTableScan: result = (PlanState *) ExecInitWorkTableScan((WorkTableScan *) node, estate, eflags); @@ -483,6 +489,10 @@ ExecProcNode(PlanState *node) result = ExecCteScan((CteScanState *) node); break; + case T_NamedTuplestoreScanState: + result = ExecNamedTuplestoreScan((NamedTuplestoreScanState *) node); + break; + case T_WorkTableScanState: result = ExecWorkTableScan((WorkTableScanState *) node); break; @@ -751,6 +761,10 @@ ExecEndNode(PlanState *node) ExecEndCteScan((CteScanState *) node); break; + case T_NamedTuplestoreScanState: + ExecEndNamedTuplestoreScan((NamedTuplestoreScanState *) node); + break; + case T_WorkTableScanState: ExecEndWorkTableScan((WorkTableScanState *) node); break; diff --git a/src/backend/executor/execUtils.c b/src/backend/executor/execUtils.c index 2613ffbb71..ce7b064217 100644 --- a/src/backend/executor/execUtils.c +++ b/src/backend/executor/execUtils.c @@ -120,6 +120,8 @@ CreateExecutorState(void) estate->es_param_list_info = NULL; estate->es_param_exec_vals = NULL; + estate->es_queryEnv = NULL; + estate->es_query_cxt = qcontext; estate->es_tupleTable = NIL; diff --git a/src/backend/executor/functions.c b/src/backend/executor/functions.c index 3e4b0191c7..3cadf95304 100644 --- a/src/backend/executor/functions.c +++ b/src/backend/executor/functions.c @@ -713,7 +713,8 @@ init_sql_fcache(FmgrInfo *finfo, Oid collation, bool lazyEvalOK) queryTree_sublist = pg_analyze_and_rewrite_params(parsetree, fcache->src, (ParserSetupHook) sql_fn_parser_setup, - fcache->pinfo); + fcache->pinfo, + NULL); queryTree_list = lappend(queryTree_list, queryTree_sublist); flat_query_list = list_concat(flat_query_list, list_copy(queryTree_sublist)); @@ -809,7 +810,9 @@ postquel_start(execution_state *es, SQLFunctionCachePtr fcache) GetActiveSnapshot(), InvalidSnapshot, dest, - fcache->paramLI, 0); + fcache->paramLI, + es->qd ? es->qd->queryEnv : NULL, + 0); /* Utility commands don't need Executor. */ if (es->qd->operation != CMD_UTILITY) @@ -846,6 +849,7 @@ postquel_getnext(execution_state *es, SQLFunctionCachePtr fcache) fcache->src, PROCESS_UTILITY_QUERY, es->qd->params, + es->qd->queryEnv, es->qd->dest, NULL); result = true; /* never stops early */ diff --git a/src/backend/executor/nodeNamedtuplestorescan.c b/src/backend/executor/nodeNamedtuplestorescan.c new file mode 100644 index 0000000000..917b05197a --- /dev/null +++ b/src/backend/executor/nodeNamedtuplestorescan.c @@ -0,0 +1,198 @@ +/*------------------------------------------------------------------------- + * + * nodeNamedtuplestorescan.c + * routines to handle NamedTuplestoreScan nodes. + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/executor/nodeNamedtuplestorescan.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "executor/execdebug.h" +#include "executor/nodeNamedtuplestorescan.h" +#include "miscadmin.h" +#include "utils/queryenvironment.h" + +static TupleTableSlot *NamedTuplestoreScanNext(NamedTuplestoreScanState *node); + +/* ---------------------------------------------------------------- + * NamedTuplestoreScanNext + * + * This is a workhorse for ExecNamedTuplestoreScan + * ---------------------------------------------------------------- + */ +static TupleTableSlot * +NamedTuplestoreScanNext(NamedTuplestoreScanState *node) +{ + TupleTableSlot *slot; + + /* We intentionally do not support backward scan. */ + Assert(ScanDirectionIsForward(node->ss.ps.state->es_direction)); + + /* + * Get the next tuple from tuplestore. Return NULL if no more tuples. + */ + slot = node->ss.ss_ScanTupleSlot; + (void) tuplestore_gettupleslot(node->relation, true, false, slot); + return slot; +} + +/* + * NamedTuplestoreScanRecheck -- access method routine to recheck a tuple in + * EvalPlanQual + */ +static bool +NamedTuplestoreScanRecheck(NamedTuplestoreScanState *node, TupleTableSlot *slot) +{ + /* nothing to check */ + return true; +} + +/* ---------------------------------------------------------------- + * ExecNamedTuplestoreScan(node) + * + * Scans the CTE sequentially and returns the next qualifying tuple. + * We call the ExecScan() routine and pass it the appropriate + * access method functions. + * ---------------------------------------------------------------- + */ +TupleTableSlot * +ExecNamedTuplestoreScan(NamedTuplestoreScanState *node) +{ + return ExecScan(&node->ss, + (ExecScanAccessMtd) NamedTuplestoreScanNext, + (ExecScanRecheckMtd) NamedTuplestoreScanRecheck); +} + + +/* ---------------------------------------------------------------- + * ExecInitNamedTuplestoreScan + * ---------------------------------------------------------------- + */ +NamedTuplestoreScanState * +ExecInitNamedTuplestoreScan(NamedTuplestoreScan *node, EState *estate, int eflags) +{ + NamedTuplestoreScanState *scanstate; + EphemeralNamedRelation enr; + + /* check for unsupported flags */ + Assert(!(eflags & (EXEC_FLAG_BACKWARD | EXEC_FLAG_MARK))); + + /* + * NamedTuplestoreScan should not have any children. + */ + Assert(outerPlan(node) == NULL); + Assert(innerPlan(node) == NULL); + + /* + * create new NamedTuplestoreScanState for node + */ + scanstate = makeNode(NamedTuplestoreScanState); + scanstate->ss.ps.plan = (Plan *) node; + scanstate->ss.ps.state = estate; + + enr = get_ENR(estate->es_queryEnv, node->enrname); + if (!enr) + elog(ERROR, "executor could not find named tuplestore \"%s\"", + node->enrname); + + Assert(enr->reldata); + scanstate->relation = (Tuplestorestate *) enr->reldata; + scanstate->tupdesc = ENRMetadataGetTupDesc(&(enr->md)); + scanstate->readptr = + tuplestore_alloc_read_pointer(scanstate->relation, 0); + + /* + * The new read pointer copies its position from read pointer 0, which + * could be anywhere, so explicitly rewind it. + */ + tuplestore_rescan(scanstate->relation); + + /* + * XXX: Should we add a function to free that read pointer when done? + * This was attempted, but it did not improve performance or memory usage + * in any tested cases. + */ + + /* + * Miscellaneous initialization + * + * create expression context for node + */ + ExecAssignExprContext(estate, &scanstate->ss.ps); + + /* + * initialize child expressions + */ + scanstate->ss.ps.qual = + ExecInitQual(node->scan.plan.qual, (PlanState *) scanstate); + + /* + * tuple table initialization + */ + ExecInitResultTupleSlot(estate, &scanstate->ss.ps); + ExecInitScanTupleSlot(estate, &scanstate->ss); + + /* + * The scan tuple type is specified for the tuplestore. + */ + ExecAssignScanType(&scanstate->ss, scanstate->tupdesc); + + /* + * Initialize result tuple type and projection info. + */ + ExecAssignResultTypeFromTL(&scanstate->ss.ps); + ExecAssignScanProjectionInfo(&scanstate->ss); + + return scanstate; +} + +/* ---------------------------------------------------------------- + * ExecEndNamedTuplestoreScan + * + * frees any storage allocated through C routines. + * ---------------------------------------------------------------- + */ +void +ExecEndNamedTuplestoreScan(NamedTuplestoreScanState *node) +{ + /* + * Free exprcontext + */ + ExecFreeExprContext(&node->ss.ps); + + /* + * clean out the tuple table + */ + ExecClearTuple(node->ss.ps.ps_ResultTupleSlot); + ExecClearTuple(node->ss.ss_ScanTupleSlot); +} + +/* ---------------------------------------------------------------- + * ExecReScanNamedTuplestoreScan + * + * Rescans the relation. + * ---------------------------------------------------------------- + */ +void +ExecReScanNamedTuplestoreScan(NamedTuplestoreScanState *node) +{ + Tuplestorestate *tuplestorestate = node->relation; + + ExecClearTuple(node->ss.ps.ps_ResultTupleSlot); + + ExecScanReScan(&node->ss); + + /* + * Rewind my own pointer. + */ + tuplestore_select_read_pointer(tuplestorestate, node->readptr); + tuplestore_rescan(tuplestorestate); +} diff --git a/src/backend/executor/spi.c b/src/backend/executor/spi.c index eeaa4805e4..54c022d013 100644 --- a/src/backend/executor/spi.c +++ b/src/backend/executor/spi.c @@ -122,6 +122,7 @@ SPI_connect(void) _SPI_current->procCxt = NULL; /* in case we fail to create 'em */ _SPI_current->execCxt = NULL; _SPI_current->connectSubid = GetCurrentSubTransactionId(); + _SPI_current->queryEnv = NULL; /* * Create memory contexts for this procedure @@ -1193,7 +1194,7 @@ SPI_cursor_open_internal(const char *name, SPIPlanPtr plan, */ /* Replan if needed, and increment plan refcount for portal */ - cplan = GetCachedPlan(plansource, paramLI, false); + cplan = GetCachedPlan(plansource, paramLI, false, _SPI_current->queryEnv); stmt_list = cplan->stmt_list; /* Pop the error context stack */ @@ -1532,6 +1533,10 @@ SPI_result_code_string(int code) return "SPI_ERROR_NOOUTFUNC"; case SPI_ERROR_TYPUNKNOWN: return "SPI_ERROR_TYPUNKNOWN"; + case SPI_ERROR_REL_DUPLICATE: + return "SPI_ERROR_REL_DUPLICATE"; + case SPI_ERROR_REL_NOT_FOUND: + return "SPI_ERROR_REL_NOT_FOUND"; case SPI_OK_CONNECT: return "SPI_OK_CONNECT"; case SPI_OK_FINISH: @@ -1560,6 +1565,10 @@ SPI_result_code_string(int code) return "SPI_OK_UPDATE_RETURNING"; case SPI_OK_REWRITTEN: return "SPI_OK_REWRITTEN"; + case SPI_OK_REL_REGISTER: + return "SPI_OK_REL_REGISTER"; + case SPI_OK_REL_UNREGISTER: + return "SPI_OK_REL_UNREGISTER"; } /* Unrecognized code ... return something useful ... */ sprintf(buf, "Unrecognized SPI code %d", code); @@ -1615,7 +1624,8 @@ SPI_plan_get_cached_plan(SPIPlanPtr plan) error_context_stack = &spierrcontext; /* Get the generic plan for the query */ - cplan = GetCachedPlan(plansource, NULL, plan->saved); + cplan = GetCachedPlan(plansource, NULL, plan->saved, + _SPI_current->queryEnv); Assert(cplan == plansource->gplan); /* Pop the error context stack */ @@ -1767,7 +1777,8 @@ _SPI_prepare_plan(const char *src, SPIPlanPtr plan) */ plansource = CreateCachedPlan(parsetree, src, - CreateCommandTag(parsetree->stmt)); + CreateCommandTag(parsetree->stmt), + _SPI_current->queryEnv); /* * Parameter datatypes are driven by parserSetup hook if provided, @@ -1779,14 +1790,16 @@ _SPI_prepare_plan(const char *src, SPIPlanPtr plan) stmt_list = pg_analyze_and_rewrite_params(parsetree, src, plan->parserSetup, - plan->parserSetupArg); + plan->parserSetupArg, + _SPI_current->queryEnv); } else { stmt_list = pg_analyze_and_rewrite(parsetree, src, plan->argtypes, - plan->nargs); + plan->nargs, + _SPI_current->queryEnv); } /* Finish filling in the CachedPlanSource */ @@ -1975,14 +1988,16 @@ _SPI_execute_plan(SPIPlanPtr plan, ParamListInfo paramLI, stmt_list = pg_analyze_and_rewrite_params(parsetree, src, plan->parserSetup, - plan->parserSetupArg); + plan->parserSetupArg, + _SPI_current->queryEnv); } else { stmt_list = pg_analyze_and_rewrite(parsetree, src, plan->argtypes, - plan->nargs); + plan->nargs, + _SPI_current->queryEnv); } /* Finish filling in the CachedPlanSource */ @@ -2001,7 +2016,7 @@ _SPI_execute_plan(SPIPlanPtr plan, ParamListInfo paramLI, * Replan if needed, and increment plan refcount. If it's a saved * plan, the refcount must be backed by the CurrentResourceOwner. */ - cplan = GetCachedPlan(plansource, paramLI, plan->saved); + cplan = GetCachedPlan(plansource, paramLI, plan->saved, _SPI_current->queryEnv); stmt_list = cplan->stmt_list; /* @@ -2081,7 +2096,8 @@ _SPI_execute_plan(SPIPlanPtr plan, ParamListInfo paramLI, plansource->query_string, snap, crosscheck_snapshot, dest, - paramLI, 0); + paramLI, _SPI_current->queryEnv, + 0); res = _SPI_pquery(qdesc, fire_triggers, canSetTag ? tcount : 0); FreeQueryDesc(qdesc); @@ -2094,6 +2110,7 @@ _SPI_execute_plan(SPIPlanPtr plan, ParamListInfo paramLI, plansource->query_string, PROCESS_UTILITY_QUERY, paramLI, + _SPI_current->queryEnv, dest, completionTag); @@ -2619,3 +2636,84 @@ _SPI_save_plan(SPIPlanPtr plan) return newplan; } + +/* + * Internal lookup of ephemeral named relation by name. + */ +static EphemeralNamedRelation +_SPI_find_ENR_by_name(const char *name) +{ + /* internal static function; any error is bug in SPI itself */ + Assert(name != NULL); + + /* fast exit if no tuplestores have been added */ + if (_SPI_current->queryEnv == NULL) + return NULL; + + return get_ENR(_SPI_current->queryEnv, name); +} + +/* + * Register an ephemeral named relation for use by the planner and executor on + * subsequent calls using this SPI connection. + */ +int +SPI_register_relation(EphemeralNamedRelation enr) +{ + EphemeralNamedRelation match; + int res; + + if (enr == NULL || enr->md.name == NULL) + return SPI_ERROR_ARGUMENT; + + res = _SPI_begin_call(false); /* keep current memory context */ + if (res < 0) + return res; + + match = _SPI_find_ENR_by_name(enr->md.name); + if (match) + res = SPI_ERROR_REL_DUPLICATE; + else + { + if (_SPI_current->queryEnv == NULL) + _SPI_current->queryEnv = create_queryEnv(); + + register_ENR(_SPI_current->queryEnv, enr); + res = SPI_OK_REL_REGISTER; + } + + _SPI_end_call(false); + + return res; +} + +/* + * Unregister an ephemeral named relation by name. This will probably be a + * rarely used function, since SPI_finish will clear it automatically. + */ +int +SPI_unregister_relation(const char *name) +{ + EphemeralNamedRelation match; + int res; + + if (name == NULL) + return SPI_ERROR_ARGUMENT; + + res = _SPI_begin_call(false); /* keep current memory context */ + if (res < 0) + return res; + + match = _SPI_find_ENR_by_name(name); + if (match) + { + unregister_ENR(_SPI_current->queryEnv, match->md.name); + res = SPI_OK_REL_UNREGISTER; + } + else + res = SPI_ERROR_REL_NOT_FOUND; + + _SPI_end_call(false); + + return res; +} diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index 1c88d601bd..61bc5025e2 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -682,6 +682,27 @@ _copyCteScan(const CteScan *from) return newnode; } +/* + * _copyNamedTuplestoreScan + */ +static NamedTuplestoreScan * +_copyNamedTuplestoreScan(const NamedTuplestoreScan *from) +{ + NamedTuplestoreScan *newnode = makeNode(NamedTuplestoreScan); + + /* + * copy node superclass fields + */ + CopyScanFields((const Scan *) from, (Scan *) newnode); + + /* + * copy remainder of node + */ + COPY_STRING_FIELD(enrname); + + return newnode; +} + /* * _copyWorkTableScan */ @@ -2265,6 +2286,7 @@ _copyRangeTblEntry(const RangeTblEntry *from) COPY_STRING_FIELD(ctename); COPY_SCALAR_FIELD(ctelevelsup); COPY_SCALAR_FIELD(self_reference); + COPY_STRING_FIELD(enrname); COPY_NODE_FIELD(coltypes); COPY_NODE_FIELD(coltypmods); COPY_NODE_FIELD(colcollations); @@ -4706,6 +4728,9 @@ copyObjectImpl(const void *from) case T_CteScan: retval = _copyCteScan(from); break; + case T_NamedTuplestoreScan: + retval = _copyNamedTuplestoreScan(from); + break; case T_WorkTableScan: retval = _copyWorkTableScan(from); break; diff --git a/src/backend/nodes/nodeFuncs.c b/src/backend/nodes/nodeFuncs.c index 6e52eb7231..d5293a1a78 100644 --- a/src/backend/nodes/nodeFuncs.c +++ b/src/backend/nodes/nodeFuncs.c @@ -2321,6 +2321,7 @@ range_table_walker(List *rtable, return true; break; case RTE_CTE: + case RTE_NAMEDTUPLESTORE: /* nothing to do */ break; case RTE_SUBQUERY: @@ -3135,6 +3136,7 @@ range_table_mutator(List *rtable, /* we don't bother to copy eref, aliases, etc; OK? */ break; case RTE_CTE: + case RTE_NAMEDTUPLESTORE: /* nothing to do */ break; case RTE_SUBQUERY: diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 0b45c25a49..766ca49216 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -631,6 +631,16 @@ _outCteScan(StringInfo str, const CteScan *node) WRITE_INT_FIELD(cteParam); } +static void +_outNamedTuplestoreScan(StringInfo str, const NamedTuplestoreScan *node) +{ + WRITE_NODE_TYPE("NAMEDTUPLESTORESCAN"); + + _outScanInfo(str, (const Scan *) node); + + WRITE_STRING_FIELD(enrname); +} + static void _outWorkTableScan(StringInfo str, const WorkTableScan *node) { @@ -3024,6 +3034,13 @@ _outRangeTblEntry(StringInfo str, const RangeTblEntry *node) WRITE_NODE_FIELD(coltypmods); WRITE_NODE_FIELD(colcollations); break; + case RTE_NAMEDTUPLESTORE: + WRITE_STRING_FIELD(enrname); + WRITE_OID_FIELD(relid); + WRITE_NODE_FIELD(coltypes); + WRITE_NODE_FIELD(coltypmods); + WRITE_NODE_FIELD(colcollations); + break; default: elog(ERROR, "unrecognized RTE kind: %d", (int) node->rtekind); break; @@ -3621,6 +3638,9 @@ outNode(StringInfo str, const void *obj) case T_CteScan: _outCteScan(str, obj); break; + case T_NamedTuplestoreScan: + _outNamedTuplestoreScan(str, obj); + break; case T_WorkTableScan: _outWorkTableScan(str, obj); break; diff --git a/src/backend/nodes/print.c b/src/backend/nodes/print.c index dfb8bfa803..380e8b71f2 100644 --- a/src/backend/nodes/print.c +++ b/src/backend/nodes/print.c @@ -291,6 +291,10 @@ print_rt(const List *rtable) printf("%d\t%s\t[cte]", i, rte->eref->aliasname); break; + case RTE_NAMEDTUPLESTORE: + printf("%d\t%s\t[tuplestore]", + i, rte->eref->aliasname); + break; default: printf("%d\t%s\t[unknown rtekind]", i, rte->eref->aliasname); diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index 474f221a75..766f2d8db1 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -1355,6 +1355,13 @@ _readRangeTblEntry(void) READ_NODE_FIELD(coltypmods); READ_NODE_FIELD(colcollations); break; + case RTE_NAMEDTUPLESTORE: + READ_STRING_FIELD(enrname); + READ_OID_FIELD(relid); + READ_NODE_FIELD(coltypes); + READ_NODE_FIELD(coltypmods); + READ_NODE_FIELD(colcollations); + break; default: elog(ERROR, "unrecognized RTE kind: %d", (int) local_node->rtekind); diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index a1e1a87c29..343b35aa32 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -111,6 +111,8 @@ static void set_tablefunc_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte); static void set_cte_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte); +static void set_namedtuplestore_pathlist(PlannerInfo *root, RelOptInfo *rel, + RangeTblEntry *rte); static void set_worktable_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte); static RelOptInfo *make_rel_from_joinlist(PlannerInfo *root, List *joinlist); @@ -396,6 +398,9 @@ set_rel_size(PlannerInfo *root, RelOptInfo *rel, else set_cte_pathlist(root, rel, rte); break; + case RTE_NAMEDTUPLESTORE: + set_namedtuplestore_pathlist(root, rel, rte); + break; default: elog(ERROR, "unexpected rtekind: %d", (int) rel->rtekind); break; @@ -464,6 +469,9 @@ set_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, case RTE_CTE: /* CTE reference --- fully handled during set_rel_size */ break; + case RTE_NAMEDTUPLESTORE: + /* tuplestore reference --- fully handled during set_rel_size */ + break; default: elog(ERROR, "unexpected rtekind: %d", (int) rel->rtekind); break; @@ -639,6 +647,13 @@ set_rel_consider_parallel(PlannerInfo *root, RelOptInfo *rel, * executed only once. */ return; + + case RTE_NAMEDTUPLESTORE: + /* + * tuplestore cannot be shared, at least without more + * infrastructure to support that. + */ + return; } /* @@ -2089,6 +2104,36 @@ set_cte_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) add_path(rel, create_ctescan_path(root, rel, required_outer)); } +/* + * set_namedtuplestore_pathlist + * Build the (single) access path for a named tuplestore RTE + * + * There's no need for a separate set_namedtuplestore_size phase, since we + * don't support join-qual-parameterized paths for tuplestores. + */ +static void +set_namedtuplestore_pathlist(PlannerInfo *root, RelOptInfo *rel, + RangeTblEntry *rte) +{ + Relids required_outer; + + /* Mark rel with estimated output rows, width, etc */ + set_namedtuplestore_size_estimates(root, rel); + + /* + * We don't support pushing join clauses into the quals of a tuplestore + * scan, but it could still have required parameterization due to LATERAL + * refs in its tlist. + */ + required_outer = rel->lateral_relids; + + /* Generate appropriate path */ + add_path(rel, create_namedtuplestorescan_path(root, rel, required_outer)); + + /* Select cheapest path (pretty easy in this case...) */ + set_cheapest(rel); +} + /* * set_worktable_pathlist * Build the (single) access path for a self-reference CTE RTE diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index 92de2b7d48..ed07e2f655 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -1516,6 +1516,43 @@ cost_ctescan(Path *path, PlannerInfo *root, path->total_cost = startup_cost + run_cost; } +/* + * cost_namedtuplestorescan + * Determines and returns the cost of scanning a named tuplestore. + */ +void +cost_namedtuplestorescan(Path *path, PlannerInfo *root, + RelOptInfo *baserel, ParamPathInfo *param_info) +{ + Cost startup_cost = 0; + Cost run_cost = 0; + QualCost qpqual_cost; + Cost cpu_per_tuple; + + /* Should only be applied to base relations that are Tuplestores */ + Assert(baserel->relid > 0); + Assert(baserel->rtekind == RTE_NAMEDTUPLESTORE); + + /* Mark the path with the correct row estimate */ + if (param_info) + path->rows = param_info->ppi_rows; + else + path->rows = baserel->rows; + + /* Charge one CPU tuple cost per row for tuplestore manipulation */ + cpu_per_tuple = cpu_tuple_cost; + + /* Add scanning CPU costs */ + get_restriction_qual_cost(root, baserel, param_info, &qpqual_cost); + + startup_cost += qpqual_cost.startup; + cpu_per_tuple += cpu_tuple_cost + qpqual_cost.per_tuple; + run_cost += cpu_per_tuple * baserel->tuples; + + path->startup_cost = startup_cost; + path->total_cost = startup_cost + run_cost; +} + /* * cost_recursive_union * Determines and returns the cost of performing a recursive union, @@ -4684,6 +4721,39 @@ set_cte_size_estimates(PlannerInfo *root, RelOptInfo *rel, double cte_rows) set_baserel_size_estimates(root, rel); } +/* + * set_namedtuplestore_size_estimates + * Set the size estimates for a base relation that is a tuplestore reference. + * + * The rel's targetlist and restrictinfo list must have been constructed + * already. + * + * We set the same fields as set_baserel_size_estimates. + */ +void +set_namedtuplestore_size_estimates(PlannerInfo *root, RelOptInfo *rel) +{ + RangeTblEntry *rte; + + /* Should only be applied to base relations that are tuplestore references */ + Assert(rel->relid > 0); + rte = planner_rt_fetch(rel->relid, root); + Assert(rte->rtekind == RTE_NAMEDTUPLESTORE); + + /* + * Use the estimate provided by the code which is generating the named + * tuplestore. In some cases, the actual number might be available; in + * others the same plan will be re-used, so a "typical" value might be + * estimated and used. + */ + rel->tuples = rte->enrtuples; + if (rel->tuples < 0) + rel->tuples = 1000; + + /* Now estimate number of output rows, etc */ + set_baserel_size_estimates(root, rel); +} + /* * set_foreign_size_estimates * Set the size estimates for a base relation that is a foreign table. diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index ed06a8de78..2a78595e1f 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -139,6 +139,8 @@ static TableFuncScan *create_tablefuncscan_plan(PlannerInfo *root, Path *best_pa List *tlist, List *scan_clauses); static CteScan *create_ctescan_plan(PlannerInfo *root, Path *best_path, List *tlist, List *scan_clauses); +static NamedTuplestoreScan *create_namedtuplestorescan_plan(PlannerInfo *root, + Path *best_path, List *tlist, List *scan_clauses); static WorkTableScan *create_worktablescan_plan(PlannerInfo *root, Path *best_path, List *tlist, List *scan_clauses); static ForeignScan *create_foreignscan_plan(PlannerInfo *root, ForeignPath *best_path, @@ -197,6 +199,8 @@ static TableFuncScan *make_tablefuncscan(List *qptlist, List *qpqual, Index scanrelid, TableFunc *tablefunc); static CteScan *make_ctescan(List *qptlist, List *qpqual, Index scanrelid, int ctePlanId, int cteParam); +static NamedTuplestoreScan *make_namedtuplestorescan(List *qptlist, List *qpqual, + Index scanrelid, char *enrname); static WorkTableScan *make_worktablescan(List *qptlist, List *qpqual, Index scanrelid, int wtParam); static Append *make_append(List *appendplans, List *tlist, List *partitioned_rels); @@ -366,6 +370,7 @@ create_plan_recurse(PlannerInfo *root, Path *best_path, int flags) case T_ValuesScan: case T_CteScan: case T_WorkTableScan: + case T_NamedTuplestoreScan: case T_ForeignScan: case T_CustomScan: plan = create_scan_plan(root, best_path, flags); @@ -668,6 +673,13 @@ create_scan_plan(PlannerInfo *root, Path *best_path, int flags) scan_clauses); break; + case T_NamedTuplestoreScan: + plan = (Plan *) create_namedtuplestorescan_plan(root, + best_path, + tlist, + scan_clauses); + break; + case T_WorkTableScan: plan = (Plan *) create_worktablescan_plan(root, best_path, @@ -3285,6 +3297,45 @@ create_ctescan_plan(PlannerInfo *root, Path *best_path, return scan_plan; } +/* + * create_namedtuplestorescan_plan + * Returns a tuplestorescan plan for the base relation scanned by + * 'best_path' with restriction clauses 'scan_clauses' and targetlist + * 'tlist'. + */ +static NamedTuplestoreScan * +create_namedtuplestorescan_plan(PlannerInfo *root, Path *best_path, + List *tlist, List *scan_clauses) +{ + NamedTuplestoreScan *scan_plan; + Index scan_relid = best_path->parent->relid; + RangeTblEntry *rte; + + Assert(scan_relid > 0); + rte = planner_rt_fetch(scan_relid, root); + Assert(rte->rtekind == RTE_NAMEDTUPLESTORE); + + /* Sort clauses into best execution order */ + scan_clauses = order_qual_clauses(root, scan_clauses); + + /* Reduce RestrictInfo list to bare expressions; ignore pseudoconstants */ + scan_clauses = extract_actual_clauses(scan_clauses, false); + + /* Replace any outer-relation variables with nestloop params */ + if (best_path->param_info) + { + scan_clauses = (List *) + replace_nestloop_params(root, (Node *) scan_clauses); + } + + scan_plan = make_namedtuplestorescan(tlist, scan_clauses, scan_relid, + rte->enrname); + + copy_generic_path_info(&scan_plan->scan.plan, best_path); + + return scan_plan; +} + /* * create_worktablescan_plan * Returns a worktablescan plan for the base relation scanned by 'best_path' @@ -5120,6 +5171,26 @@ make_ctescan(List *qptlist, return node; } +static NamedTuplestoreScan * +make_namedtuplestorescan(List *qptlist, + List *qpqual, + Index scanrelid, + char *enrname) +{ + NamedTuplestoreScan *node = makeNode(NamedTuplestoreScan); + Plan *plan = &node->scan.plan; + + /* cost should be inserted by caller */ + plan->targetlist = qptlist; + plan->qual = qpqual; + plan->lefttree = NULL; + plan->righttree = NULL; + node->scan.scanrelid = scanrelid; + node->enrname = enrname; + + return node; +} + static WorkTableScan * make_worktablescan(List *qptlist, List *qpqual, diff --git a/src/backend/optimizer/plan/setrefs.c b/src/backend/optimizer/plan/setrefs.c index 4e3f6ee960..cdb8e95deb 100644 --- a/src/backend/optimizer/plan/setrefs.c +++ b/src/backend/optimizer/plan/setrefs.c @@ -591,6 +591,17 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) fix_scan_list(root, splan->scan.plan.qual, rtoffset); } break; + case T_NamedTuplestoreScan: + { + NamedTuplestoreScan *splan = (NamedTuplestoreScan *) plan; + + splan->scan.scanrelid += rtoffset; + splan->scan.plan.targetlist = + fix_scan_list(root, splan->scan.plan.targetlist, rtoffset); + splan->scan.plan.qual = + fix_scan_list(root, splan->scan.plan.qual, rtoffset); + } + break; case T_WorkTableScan: { WorkTableScan *splan = (WorkTableScan *) plan; @@ -2571,6 +2582,11 @@ extract_query_dependencies_walker(Node *node, PlannerInfo *context) if (rte->rtekind == RTE_RELATION) context->glob->relationOids = lappend_oid(context->glob->relationOids, rte->relid); + else if (rte->rtekind == RTE_NAMEDTUPLESTORE && + OidIsValid(rte->relid)) + context->glob->relationOids = + lappend_oid(context->glob->relationOids, + rte->relid); } /* And recurse into the query's subexpressions */ diff --git a/src/backend/optimizer/plan/subselect.c b/src/backend/optimizer/plan/subselect.c index db0e5b31e2..87cc44d678 100644 --- a/src/backend/optimizer/plan/subselect.c +++ b/src/backend/optimizer/plan/subselect.c @@ -2476,6 +2476,10 @@ finalize_plan(PlannerInfo *root, Plan *plan, Bitmapset *valid_params, context.paramids = bms_add_members(context.paramids, scan_params); break; + case T_NamedTuplestoreScan: + context.paramids = bms_add_members(context.paramids, scan_params); + break; + case T_ForeignScan: { ForeignScan *fscan = (ForeignScan *) plan; diff --git a/src/backend/optimizer/prep/prepjointree.c b/src/backend/optimizer/prep/prepjointree.c index 348c6b791f..749ea805f8 100644 --- a/src/backend/optimizer/prep/prepjointree.c +++ b/src/backend/optimizer/prep/prepjointree.c @@ -1123,6 +1123,7 @@ pull_up_simple_subquery(PlannerInfo *root, Node *jtnode, RangeTblEntry *rte, break; case RTE_JOIN: case RTE_CTE: + case RTE_NAMEDTUPLESTORE: /* these can't contain any lateral references */ break; } @@ -1977,6 +1978,7 @@ replace_vars_in_jointree(Node *jtnode, break; case RTE_JOIN: case RTE_CTE: + case RTE_NAMEDTUPLESTORE: /* these shouldn't be marked LATERAL */ Assert(false); break; diff --git a/src/backend/optimizer/util/clauses.c b/src/backend/optimizer/util/clauses.c index a578867cce..59d71c1b32 100644 --- a/src/backend/optimizer/util/clauses.c +++ b/src/backend/optimizer/util/clauses.c @@ -4910,7 +4910,7 @@ inline_set_returning_function(PlannerInfo *root, RangeTblEntry *rte) querytree_list = pg_analyze_and_rewrite_params(linitial(raw_parsetree_list), src, (ParserSetupHook) sql_fn_parser_setup, - pinfo); + pinfo, NULL); if (list_length(querytree_list) != 1) goto fail; querytree = linitial(querytree_list); diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index c6298072c9..8536212177 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -1892,6 +1892,32 @@ create_ctescan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer) return pathnode; } +/* + * create_namedtuplestorescan_path + * Creates a path corresponding to a scan of a named tuplestore, returning + * the pathnode. + */ +Path * +create_namedtuplestorescan_path(PlannerInfo *root, RelOptInfo *rel, + Relids required_outer) +{ + Path *pathnode = makeNode(Path); + + pathnode->pathtype = T_NamedTuplestoreScan; + pathnode->parent = rel; + pathnode->pathtarget = rel->reltarget; + pathnode->param_info = get_baserel_parampathinfo(root, rel, + required_outer); + pathnode->parallel_aware = false; + pathnode->parallel_safe = rel->consider_parallel; + pathnode->parallel_workers = 0; + pathnode->pathkeys = NIL; /* result is always unordered */ + + cost_namedtuplestorescan(pathnode, root, rel, pathnode->param_info); + + return pathnode; +} + /* * create_worktablescan_path * Creates a path corresponding to a scan of a self-reference CTE, diff --git a/src/backend/optimizer/util/plancat.c b/src/backend/optimizer/util/plancat.c index cc88dcc28e..1cd21c0fdc 100644 --- a/src/backend/optimizer/util/plancat.c +++ b/src/backend/optimizer/util/plancat.c @@ -1446,9 +1446,9 @@ relation_excluded_by_constraints(PlannerInfo *root, * dropped cols. * * We also support building a "physical" tlist for subqueries, functions, - * values lists, table expressions and CTEs, since the same optimization can - * occur in SubqueryScan, FunctionScan, ValuesScan, CteScan, TableFunc - * and WorkTableScan nodes. + * values lists, table expressions, and CTEs, since the same optimization can + * occur in SubqueryScan, FunctionScan, ValuesScan, CteScan, TableFunc, + * NamedTuplestoreScan, and WorkTableScan nodes. */ List * build_physical_tlist(PlannerInfo *root, RelOptInfo *rel) @@ -1523,6 +1523,7 @@ build_physical_tlist(PlannerInfo *root, RelOptInfo *rel) case RTE_TABLEFUNC: case RTE_VALUES: case RTE_CTE: + case RTE_NAMEDTUPLESTORE: /* Not all of these can have dropped cols, but share code anyway */ expandRTE(rte, varno, 0, -1, true /* include dropped */ , NULL, &colvars); diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index 6ab78545c3..7912df0baa 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -156,6 +156,7 @@ build_simple_rel(PlannerInfo *root, int relid, RelOptKind reloptkind) case RTE_TABLEFUNC: case RTE_VALUES: case RTE_CTE: + case RTE_NAMEDTUPLESTORE: /* * Subquery, function, tablefunc, or values list --- set up attr diff --git a/src/backend/parser/Makefile b/src/backend/parser/Makefile index df9a9fbb35..4b97f83803 100644 --- a/src/backend/parser/Makefile +++ b/src/backend/parser/Makefile @@ -14,8 +14,9 @@ override CPPFLAGS := -I. -I$(srcdir) $(CPPFLAGS) OBJS= analyze.o gram.o scan.o parser.o \ parse_agg.o parse_clause.o parse_coerce.o parse_collate.o parse_cte.o \ - parse_expr.o parse_func.o parse_node.o parse_oper.o parse_param.o \ - parse_relation.o parse_target.o parse_type.o parse_utilcmd.o scansup.o + parse_enr.o parse_expr.o parse_func.o parse_node.o parse_oper.o \ + parse_param.o parse_relation.o parse_target.o parse_type.o \ + parse_utilcmd.o scansup.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/parser/analyze.c b/src/backend/parser/analyze.c index 8f11c46621..811fccaec9 100644 --- a/src/backend/parser/analyze.c +++ b/src/backend/parser/analyze.c @@ -94,7 +94,8 @@ static bool test_raw_expression_coverage(Node *node, void *context); */ Query * parse_analyze(RawStmt *parseTree, const char *sourceText, - Oid *paramTypes, int numParams) + Oid *paramTypes, int numParams, + QueryEnvironment *queryEnv) { ParseState *pstate = make_parsestate(NULL); Query *query; @@ -106,6 +107,8 @@ parse_analyze(RawStmt *parseTree, const char *sourceText, if (numParams > 0) parse_fixed_parameters(pstate, paramTypes, numParams); + pstate->p_queryEnv = queryEnv; + query = transformTopLevelStmt(pstate, parseTree); if (post_parse_analyze_hook) @@ -2799,6 +2802,15 @@ transformLockingClause(ParseState *pstate, Query *qry, LockingClause *lc, LCS_asString(lc->strength)), parser_errposition(pstate, thisrel->location))); break; + case RTE_NAMEDTUPLESTORE: + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + /*------ + translator: %s is a SQL row locking clause such as FOR UPDATE */ + errmsg("%s cannot be applied to a named tuplestore", + LCS_asString(lc->strength)), + parser_errposition(pstate, thisrel->location))); + break; default: elog(ERROR, "unrecognized RTE type: %d", (int) rte->rtekind); diff --git a/src/backend/parser/parse_clause.c b/src/backend/parser/parse_clause.c index 4f391d2d41..e268a127d1 100644 --- a/src/backend/parser/parse_clause.c +++ b/src/backend/parser/parse_clause.c @@ -59,9 +59,12 @@ static Node *transformJoinUsingClause(ParseState *pstate, List *leftVars, List *rightVars); static Node *transformJoinOnClause(ParseState *pstate, JoinExpr *j, List *namespace); +static RangeTblEntry *getRTEForSpecialRelationTypes(ParseState *pstate, + RangeVar *rv); static RangeTblEntry *transformTableEntry(ParseState *pstate, RangeVar *r); static RangeTblEntry *transformCTEReference(ParseState *pstate, RangeVar *r, CommonTableExpr *cte, Index levelsup); +static RangeTblEntry *transformENRReference(ParseState *pstate, RangeVar *r); static RangeTblEntry *transformRangeSubselect(ParseState *pstate, RangeSubselect *r); static RangeTblEntry *transformRangeFunction(ParseState *pstate, @@ -181,6 +184,14 @@ setTargetTable(ParseState *pstate, RangeVar *relation, RangeTblEntry *rte; int rtindex; + /* So far special relations are immutable; so they cannot be targets. */ + rte = getRTEForSpecialRelationTypes(pstate, relation); + if (rte != NULL) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("relation \"%s\" cannot be the target of a modifying statement", + relation->relname))); + /* Close old target; this could only happen for multi-action rules */ if (pstate->p_target_relation != NULL) heap_close(pstate->p_target_relation, NoLock); @@ -434,6 +445,20 @@ transformCTEReference(ParseState *pstate, RangeVar *r, return rte; } +/* + * transformENRReference --- transform a RangeVar that references an ephemeral + * named relation + */ +static RangeTblEntry * +transformENRReference(ParseState *pstate, RangeVar *r) +{ + RangeTblEntry *rte; + + rte = addRangeTableEntryForENR(pstate, r, true); + + return rte; +} + /* * transformRangeSubselect --- transform a sub-SELECT appearing in FROM */ @@ -1021,6 +1046,24 @@ transformRangeTableSample(ParseState *pstate, RangeTableSample *rts) return tablesample; } + +static RangeTblEntry * +getRTEForSpecialRelationTypes(ParseState *pstate, RangeVar *rv) +{ + + CommonTableExpr *cte; + Index levelsup; + RangeTblEntry *rte = NULL; + + cte = scanNameSpaceForCTE(pstate, rv->relname, &levelsup); + if (cte) + rte = transformCTEReference(pstate, rv, cte, levelsup); + if (!rte && scanNameSpaceForENR(pstate, rv->relname)) + rte = transformENRReference(pstate, rv); + + return rte; +} + /* * transformFromClauseItem - * Transform a FROM-clause item, adding any required entries to the @@ -1055,18 +1098,14 @@ transformFromClauseItem(ParseState *pstate, Node *n, RangeTblEntry *rte = NULL; int rtindex; - /* if it is an unqualified name, it might be a CTE reference */ + /* + * if it is an unqualified name, it might be a CTE or tuplestore + * reference + */ if (!rv->schemaname) - { - CommonTableExpr *cte; - Index levelsup; - - cte = scanNameSpaceForCTE(pstate, rv->relname, &levelsup); - if (cte) - rte = transformCTEReference(pstate, rv, cte, levelsup); - } + rte = getRTEForSpecialRelationTypes(pstate, rv); - /* if not found as a CTE, must be a table reference */ + /* if not found above, must be a table reference */ if (!rte) rte = transformTableEntry(pstate, rv); diff --git a/src/backend/parser/parse_enr.c b/src/backend/parser/parse_enr.c new file mode 100644 index 0000000000..1cfcf65a51 --- /dev/null +++ b/src/backend/parser/parse_enr.c @@ -0,0 +1,29 @@ +/*------------------------------------------------------------------------- + * + * parse_enr.c + * parser support routines dealing with ephemeral named relations + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/parser/parse_enr.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "parser/parse_enr.h" + +bool +name_matches_visible_ENR(ParseState *pstate, const char *refname) +{ + return (get_visible_ENR_metadata(pstate->p_queryEnv, refname) != NULL); +} + +EphemeralNamedRelationMetadata +get_visible_ENR(ParseState *pstate, const char *refname) +{ + return get_visible_ENR_metadata(pstate->p_queryEnv, refname); +} diff --git a/src/backend/parser/parse_node.c b/src/backend/parser/parse_node.c index 30cc7dadca..34006c70cd 100644 --- a/src/backend/parser/parse_node.c +++ b/src/backend/parser/parse_node.c @@ -62,6 +62,8 @@ make_parsestate(ParseState *parentParseState) pstate->p_paramref_hook = parentParseState->p_paramref_hook; pstate->p_coerce_param_hook = parentParseState->p_coerce_param_hook; pstate->p_ref_hook_state = parentParseState->p_ref_hook_state; + /* query environment stays in context for the whole parse analysis */ + pstate->p_queryEnv = parentParseState->p_queryEnv; } return pstate; diff --git a/src/backend/parser/parse_relation.c b/src/backend/parser/parse_relation.c index 2c19e0cbf5..7db13f37f7 100644 --- a/src/backend/parser/parse_relation.c +++ b/src/backend/parser/parse_relation.c @@ -25,6 +25,7 @@ #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "parser/parsetree.h" +#include "parser/parse_enr.h" #include "parser/parse_relation.h" #include "parser/parse_type.h" #include "utils/builtins.h" @@ -281,6 +282,16 @@ isFutureCTE(ParseState *pstate, const char *refname) return false; } +/* + * Search the query's ephemeral named relation namespace for a relation + * matching the given unqualified refname. + */ +bool +scanNameSpaceForENR(ParseState *pstate, const char *refname) +{ + return name_matches_visible_ENR(pstate, refname); +} + /* * searchRangeTableForRel * See if any RangeTblEntry could possibly match the RangeVar. @@ -302,6 +313,7 @@ searchRangeTableForRel(ParseState *pstate, RangeVar *relation) const char *refname = relation->relname; Oid relId = InvalidOid; CommonTableExpr *cte = NULL; + bool isenr = false; Index ctelevelsup = 0; Index levelsup; @@ -318,11 +330,16 @@ searchRangeTableForRel(ParseState *pstate, RangeVar *relation) * unlocked. */ if (!relation->schemaname) + { cte = scanNameSpaceForCTE(pstate, refname, &ctelevelsup); - if (!cte) + if (!cte) + isenr = scanNameSpaceForENR(pstate, refname); + } + + if (!cte && !isenr) relId = RangeVarGetRelid(relation, NoLock, true); - /* Now look for RTEs matching either the relation/CTE or the alias */ + /* Now look for RTEs matching either the relation/CTE/ENR or the alias */ for (levelsup = 0; pstate != NULL; pstate = pstate->parentParseState, levelsup++) @@ -342,6 +359,10 @@ searchRangeTableForRel(ParseState *pstate, RangeVar *relation) rte->ctelevelsup + levelsup == ctelevelsup && strcmp(rte->ctename, refname) == 0) return rte; + if (rte->rtekind == RTE_NAMEDTUPLESTORE && + isenr && + strcmp(rte->enrname, refname) == 0) + return rte; if (strcmp(rte->eref->aliasname, refname) == 0) return rte; } @@ -1138,13 +1159,18 @@ parserOpenTable(ParseState *pstate, const RangeVar *relation, int lockmode) relation->schemaname, relation->relname))); else { + /* + * An unqualified name might be a named ephemeral relation. + */ + if (get_visible_ENR_metadata(pstate->p_queryEnv, relation->relname)) + rel = NULL; /* * An unqualified name might have been meant as a reference to * some not-yet-in-scope CTE. The bare "does not exist" message * has proven remarkably unhelpful for figuring out such problems, * so we take pains to offer a specific hint. */ - if (isFutureCTE(pstate, relation->relname)) + else if (isFutureCTE(pstate, relation->relname)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_TABLE), errmsg("relation \"%s\" does not exist", @@ -1940,6 +1966,102 @@ addRangeTableEntryForCTE(ParseState *pstate, return rte; } +/* + * Add an entry for an ephemeral named relation reference to the pstate's + * range table (p_rtable). + * + * It is expected that the RangeVar, which up until now is only known to be an + * ephemeral named relation, will (in conjunction with the QueryEnvironment in + * the ParseState), create a RangeTblEntry for a specific *kind* of ephemeral + * named relation, based on enrtype. + * + * This is much like addRangeTableEntry() except that it makes an RTE for an + * ephemeral named relation. + */ +RangeTblEntry * +addRangeTableEntryForENR(ParseState *pstate, + RangeVar *rv, + bool inFromCl) +{ + RangeTblEntry *rte = makeNode(RangeTblEntry); + Alias *alias = rv->alias; + char *refname = alias ? alias->aliasname : rv->relname; + EphemeralNamedRelationMetadata enrmd = + get_visible_ENR(pstate, rv->relname); + TupleDesc tupdesc; + int attno; + + Assert(enrmd != NULL); + + switch (enrmd->enrtype) + { + case ENR_NAMED_TUPLESTORE: + rte->rtekind = RTE_NAMEDTUPLESTORE; + break; + + default: + elog(ERROR, "unexpected enrtype of %i", enrmd->enrtype); + return NULL; /* for fussy compilers */ + } + + /* + * Record dependency on a relation. This allows plans to be invalidated + * if they access transition tables linked to a table that is altered. + */ + rte->relid = enrmd->reliddesc; + + /* + * Build the list of effective column names using user-supplied aliases + * and/or actual column names. Also build the cannibalized fields. + */ + tupdesc = ENRMetadataGetTupDesc(enrmd); + rte->eref = makeAlias(refname, NIL); + buildRelationAliases(tupdesc, alias, rte->eref); + rte->enrname = enrmd->name; + rte->enrtuples = enrmd->enrtuples; + rte->coltypes = NIL; + rte->coltypmods = NIL; + rte->colcollations = NIL; + for (attno = 1; attno <= tupdesc->natts; ++attno) + { + if (tupdesc->attrs[attno - 1]->atttypid == InvalidOid && + !(tupdesc->attrs[attno - 1]->attisdropped)) + elog(ERROR, "atttypid was invalid for column which has not been dropped from \"%s\"", + rv->relname); + rte->coltypes = + lappend_oid(rte->coltypes, + tupdesc->attrs[attno - 1]->atttypid); + rte->coltypmods = + lappend_int(rte->coltypmods, + tupdesc->attrs[attno - 1]->atttypmod); + rte->colcollations = + lappend_oid(rte->colcollations, + tupdesc->attrs[attno - 1]->attcollation); + } + + /* + * Set flags and access permissions. + * + * ENRs are never checked for access rights. + */ + rte->lateral = false; + rte->inh = false; /* never true for ENRs */ + rte->inFromCl = inFromCl; + + rte->requiredPerms = 0; + rte->checkAsUser = InvalidOid; + rte->selectedCols = NULL; + + /* + * Add completed RTE to pstate's range table list, but not to join list + * nor namespace --- caller must do that if appropriate. + */ + if (pstate != NULL) + pstate->p_rtable = lappend(pstate->p_rtable, rte); + + return rte; +} + /* * Has the specified refname been selected FOR UPDATE/FOR SHARE? @@ -2292,6 +2414,7 @@ expandRTE(RangeTblEntry *rte, int rtindex, int sublevels_up, case RTE_TABLEFUNC: case RTE_VALUES: case RTE_CTE: + case RTE_NAMEDTUPLESTORE: { /* Tablefunc, Values or CTE RTE */ ListCell *aliasp_item = list_head(rte->eref->colnames); @@ -2705,6 +2828,7 @@ get_rte_attribute_type(RangeTblEntry *rte, AttrNumber attnum, case RTE_TABLEFUNC: case RTE_VALUES: case RTE_CTE: + case RTE_NAMEDTUPLESTORE: { /* * tablefunc, VALUES or CTE RTE --- get type info from lists @@ -2762,6 +2886,19 @@ get_rte_attribute_is_dropped(RangeTblEntry *rte, AttrNumber attnum) */ result = false; break; + case RTE_NAMEDTUPLESTORE: + { + Assert(rte->enrname); + + /* + * We checked when we loaded ctecoltypes for the tuplestore + * that InvalidOid was only used for dropped columns, so it is + * safe to count on that here. + */ + result = + (list_nth(rte->coltypes, attnum - 1) != InvalidOid); + } + break; case RTE_JOIN: { /* diff --git a/src/backend/parser/parse_target.c b/src/backend/parser/parse_target.c index 3b84140a9b..c46c3b38a4 100644 --- a/src/backend/parser/parse_target.c +++ b/src/backend/parser/parse_target.c @@ -397,6 +397,7 @@ markTargetListOrigin(ParseState *pstate, TargetEntry *tle, case RTE_FUNCTION: case RTE_VALUES: case RTE_TABLEFUNC: + case RTE_NAMEDTUPLESTORE: /* not a simple relation, leave it unmarked */ break; case RTE_CTE: @@ -1505,6 +1506,7 @@ expandRecordVariable(ParseState *pstate, Var *var, int levelsup) { case RTE_RELATION: case RTE_VALUES: + case RTE_NAMEDTUPLESTORE: /* * This case should not occur: a column of a table or values list diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index 3055b483b1..139c4c0f68 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -642,7 +642,8 @@ pg_parse_query(const char *query_string) */ List * pg_analyze_and_rewrite(RawStmt *parsetree, const char *query_string, - Oid *paramTypes, int numParams) + Oid *paramTypes, int numParams, + QueryEnvironment *queryEnv) { Query *query; List *querytree_list; @@ -655,7 +656,8 @@ pg_analyze_and_rewrite(RawStmt *parsetree, const char *query_string, if (log_parser_stats) ResetUsage(); - query = parse_analyze(parsetree, query_string, paramTypes, numParams); + query = parse_analyze(parsetree, query_string, paramTypes, numParams, + queryEnv); if (log_parser_stats) ShowUsage("PARSE ANALYSIS STATISTICS"); @@ -679,7 +681,8 @@ List * pg_analyze_and_rewrite_params(RawStmt *parsetree, const char *query_string, ParserSetupHook parserSetup, - void *parserSetupArg) + void *parserSetupArg, + QueryEnvironment *queryEnv) { ParseState *pstate; Query *query; @@ -697,6 +700,7 @@ pg_analyze_and_rewrite_params(RawStmt *parsetree, pstate = make_parsestate(NULL); pstate->p_sourcetext = query_string; + pstate->p_queryEnv = queryEnv; (*parserSetup) (pstate, parserSetupArg); query = transformTopLevelStmt(pstate, parsetree); @@ -1024,7 +1028,7 @@ exec_simple_query(const char *query_string) oldcontext = MemoryContextSwitchTo(MessageContext); querytree_list = pg_analyze_and_rewrite(parsetree, query_string, - NULL, 0); + NULL, 0, NULL); plantree_list = pg_plan_queries(querytree_list, CURSOR_OPT_PARALLEL_OK, NULL); @@ -1314,7 +1318,8 @@ exec_parse_message(const char *query_string, /* string to execute */ * Create the CachedPlanSource before we do parse analysis, since it * needs to see the unmodified raw parse tree. */ - psrc = CreateCachedPlan(raw_parse_tree, query_string, commandTag); + psrc = CreateCachedPlan(raw_parse_tree, query_string, commandTag, + NULL); /* * Set up a snapshot if parse analysis will need one. @@ -1366,7 +1371,8 @@ exec_parse_message(const char *query_string, /* string to execute */ /* Empty input string. This is legal. */ raw_parse_tree = NULL; commandTag = NULL; - psrc = CreateCachedPlan(raw_parse_tree, query_string, commandTag); + psrc = CreateCachedPlan(raw_parse_tree, query_string, commandTag, + NULL); querytree_list = NIL; } @@ -1769,7 +1775,7 @@ exec_bind_message(StringInfo input_message) * will be generated in MessageContext. The plan refcount will be * assigned to the Portal, so it will be released at portal destruction. */ - cplan = GetCachedPlan(psrc, params, false); + cplan = GetCachedPlan(psrc, params, false, NULL); /* * Now we can define the portal. @@ -2367,7 +2373,7 @@ exec_describe_statement_message(const char *stmt_name) List *tlist; /* Get the plan's primary targetlist */ - tlist = CachedPlanGetTargetList(psrc); + tlist = CachedPlanGetTargetList(psrc, NULL); SendRowDescriptionMessage(psrc->resultDesc, tlist, NULL); } diff --git a/src/backend/tcop/pquery.c b/src/backend/tcop/pquery.c index f538b7787c..988c9ff43c 100644 --- a/src/backend/tcop/pquery.c +++ b/src/backend/tcop/pquery.c @@ -38,6 +38,7 @@ Portal ActivePortal = NULL; static void ProcessQuery(PlannedStmt *plan, const char *sourceText, ParamListInfo params, + QueryEnvironment *queryEnv, DestReceiver *dest, char *completionTag); static void FillPortalStore(Portal portal, bool isTopLevel); @@ -69,6 +70,7 @@ CreateQueryDesc(PlannedStmt *plannedstmt, Snapshot crosscheck_snapshot, DestReceiver *dest, ParamListInfo params, + QueryEnvironment *queryEnv, int instrument_options) { QueryDesc *qd = (QueryDesc *) palloc(sizeof(QueryDesc)); @@ -81,6 +83,7 @@ CreateQueryDesc(PlannedStmt *plannedstmt, qd->crosscheck_snapshot = RegisterSnapshot(crosscheck_snapshot); qd->dest = dest; /* output dest */ qd->params = params; /* parameter values passed into query */ + qd->queryEnv = queryEnv; qd->instrument_options = instrument_options; /* instrumentation * wanted? */ @@ -135,6 +138,7 @@ static void ProcessQuery(PlannedStmt *plan, const char *sourceText, ParamListInfo params, + QueryEnvironment *queryEnv, DestReceiver *dest, char *completionTag) { @@ -145,7 +149,7 @@ ProcessQuery(PlannedStmt *plan, */ queryDesc = CreateQueryDesc(plan, sourceText, GetActiveSnapshot(), InvalidSnapshot, - dest, params, 0); + dest, params, queryEnv, 0); /* * Call ExecutorStart to prepare the plan for execution @@ -498,6 +502,7 @@ PortalStart(Portal portal, ParamListInfo params, InvalidSnapshot, None_Receiver, params, + portal->queryEnv, 0); /* @@ -1175,6 +1180,7 @@ PortalRunUtility(Portal portal, PlannedStmt *pstmt, portal->sourceText, isTopLevel ? PROCESS_UTILITY_TOPLEVEL : PROCESS_UTILITY_QUERY, portal->portalParams, + portal->queryEnv, dest, completionTag); @@ -1281,6 +1287,7 @@ PortalRunMulti(Portal portal, ProcessQuery(pstmt, portal->sourceText, portal->portalParams, + portal->queryEnv, dest, completionTag); } else @@ -1289,6 +1296,7 @@ PortalRunMulti(Portal portal, ProcessQuery(pstmt, portal->sourceText, portal->portalParams, + portal->queryEnv, altdest, NULL); } diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index 584f4f13cc..b610c8e7ce 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -78,6 +78,7 @@ static void ProcessUtilitySlow(ParseState *pstate, const char *queryString, ProcessUtilityContext context, ParamListInfo params, + QueryEnvironment *queryEnv, DestReceiver *dest, char *completionTag); static void ExecDropStmt(DropStmt *stmt, bool isTopLevel); @@ -333,6 +334,7 @@ ProcessUtility(PlannedStmt *pstmt, const char *queryString, ProcessUtilityContext context, ParamListInfo params, + QueryEnvironment *queryEnv, DestReceiver *dest, char *completionTag) { @@ -347,11 +349,11 @@ ProcessUtility(PlannedStmt *pstmt, */ if (ProcessUtility_hook) (*ProcessUtility_hook) (pstmt, queryString, - context, params, + context, params, queryEnv, dest, completionTag); else standard_ProcessUtility(pstmt, queryString, - context, params, + context, params, queryEnv, dest, completionTag); } @@ -371,6 +373,7 @@ standard_ProcessUtility(PlannedStmt *pstmt, const char *queryString, ProcessUtilityContext context, ParamListInfo params, + QueryEnvironment *queryEnv, DestReceiver *dest, char *completionTag) { @@ -672,7 +675,8 @@ standard_ProcessUtility(PlannedStmt *pstmt, break; case T_ExplainStmt: - ExplainQuery(pstate, (ExplainStmt *) parsetree, queryString, params, dest); + ExplainQuery(pstate, (ExplainStmt *) parsetree, queryString, params, + queryEnv, dest); break; case T_AlterSystemStmt: @@ -819,7 +823,7 @@ standard_ProcessUtility(PlannedStmt *pstmt, if (EventTriggerSupportsGrantObjectType(stmt->objtype)) ProcessUtilitySlow(pstate, pstmt, queryString, - context, params, + context, params, queryEnv, dest, completionTag); else ExecuteGrantStmt(stmt); @@ -832,7 +836,7 @@ standard_ProcessUtility(PlannedStmt *pstmt, if (EventTriggerSupportsObjectType(stmt->removeType)) ProcessUtilitySlow(pstate, pstmt, queryString, - context, params, + context, params, queryEnv, dest, completionTag); else ExecDropStmt(stmt, isTopLevel); @@ -845,7 +849,7 @@ standard_ProcessUtility(PlannedStmt *pstmt, if (EventTriggerSupportsObjectType(stmt->renameType)) ProcessUtilitySlow(pstate, pstmt, queryString, - context, params, + context, params, queryEnv, dest, completionTag); else ExecRenameStmt(stmt); @@ -858,7 +862,7 @@ standard_ProcessUtility(PlannedStmt *pstmt, if (EventTriggerSupportsObjectType(stmt->objectType)) ProcessUtilitySlow(pstate, pstmt, queryString, - context, params, + context, params, queryEnv, dest, completionTag); else ExecAlterObjectDependsStmt(stmt, NULL); @@ -871,7 +875,7 @@ standard_ProcessUtility(PlannedStmt *pstmt, if (EventTriggerSupportsObjectType(stmt->objectType)) ProcessUtilitySlow(pstate, pstmt, queryString, - context, params, + context, params, queryEnv, dest, completionTag); else ExecAlterObjectSchemaStmt(stmt, NULL); @@ -884,7 +888,7 @@ standard_ProcessUtility(PlannedStmt *pstmt, if (EventTriggerSupportsObjectType(stmt->objectType)) ProcessUtilitySlow(pstate, pstmt, queryString, - context, params, + context, params, queryEnv, dest, completionTag); else ExecAlterOwnerStmt(stmt); @@ -897,7 +901,7 @@ standard_ProcessUtility(PlannedStmt *pstmt, if (EventTriggerSupportsObjectType(stmt->objtype)) ProcessUtilitySlow(pstate, pstmt, queryString, - context, params, + context, params, queryEnv, dest, completionTag); else CommentObject(stmt); @@ -910,7 +914,7 @@ standard_ProcessUtility(PlannedStmt *pstmt, if (EventTriggerSupportsObjectType(stmt->objtype)) ProcessUtilitySlow(pstate, pstmt, queryString, - context, params, + context, params, queryEnv, dest, completionTag); else ExecSecLabelStmt(stmt); @@ -920,7 +924,7 @@ standard_ProcessUtility(PlannedStmt *pstmt, default: /* All other statement types have event trigger support */ ProcessUtilitySlow(pstate, pstmt, queryString, - context, params, + context, params, queryEnv, dest, completionTag); break; } @@ -939,6 +943,7 @@ ProcessUtilitySlow(ParseState *pstate, const char *queryString, ProcessUtilityContext context, ParamListInfo params, + QueryEnvironment *queryEnv, DestReceiver *dest, char *completionTag) { @@ -1062,6 +1067,7 @@ ProcessUtilitySlow(ParseState *pstate, queryString, PROCESS_UTILITY_SUBCOMMAND, params, + NULL, None_Receiver, NULL); } @@ -1140,6 +1146,7 @@ ProcessUtilitySlow(ParseState *pstate, queryString, PROCESS_UTILITY_SUBCOMMAND, params, + NULL, None_Receiver, NULL); EventTriggerAlterTableStart(parsetree); @@ -1438,7 +1445,8 @@ ProcessUtilitySlow(ParseState *pstate, case T_CreateTableAsStmt: address = ExecCreateTableAs((CreateTableAsStmt *) parsetree, - queryString, params, completionTag); + queryString, params, queryEnv, + completionTag); break; case T_RefreshMatViewStmt: diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c index c2681ced2a..0c1a201ecb 100644 --- a/src/backend/utils/adt/ruleutils.c +++ b/src/backend/utils/adt/ruleutils.c @@ -6710,6 +6710,7 @@ get_name_for_var_field(Var *var, int fieldno, { case RTE_RELATION: case RTE_VALUES: + case RTE_NAMEDTUPLESTORE: /* * This case should not occur: a column of a table or values list diff --git a/src/backend/utils/cache/plancache.c b/src/backend/utils/cache/plancache.c index 043085d3a7..a116d5ed63 100644 --- a/src/backend/utils/cache/plancache.c +++ b/src/backend/utils/cache/plancache.c @@ -88,10 +88,11 @@ static CachedPlanSource *first_saved_plan = NULL; static void ReleaseGenericPlan(CachedPlanSource *plansource); -static List *RevalidateCachedQuery(CachedPlanSource *plansource); +static List *RevalidateCachedQuery(CachedPlanSource *plansource, + QueryEnvironment *queryEnv); static bool CheckCachedPlan(CachedPlanSource *plansource); static CachedPlan *BuildCachedPlan(CachedPlanSource *plansource, List *qlist, - ParamListInfo boundParams); + ParamListInfo boundParams, QueryEnvironment *queryEnv); static bool choose_custom_plan(CachedPlanSource *plansource, ParamListInfo boundParams); static double cached_plan_cost(CachedPlan *plan, bool include_planner); @@ -150,7 +151,8 @@ InitPlanCache(void) CachedPlanSource * CreateCachedPlan(RawStmt *raw_parse_tree, const char *query_string, - const char *commandTag) + const char *commandTag, + QueryEnvironment *queryEnv) { CachedPlanSource *plansource; MemoryContext source_context; @@ -553,7 +555,8 @@ ReleaseGenericPlan(CachedPlanSource *plansource) * a tree copying step in a subsequent BuildCachedPlan call.) */ static List * -RevalidateCachedQuery(CachedPlanSource *plansource) +RevalidateCachedQuery(CachedPlanSource *plansource, + QueryEnvironment *queryEnv) { bool snapshot_set; RawStmt *rawtree; @@ -685,12 +688,14 @@ RevalidateCachedQuery(CachedPlanSource *plansource) tlist = pg_analyze_and_rewrite_params(rawtree, plansource->query_string, plansource->parserSetup, - plansource->parserSetupArg); + plansource->parserSetupArg, + queryEnv); else tlist = pg_analyze_and_rewrite(rawtree, plansource->query_string, plansource->param_types, - plansource->num_params); + plansource->num_params, + queryEnv); /* Release snapshot if we got one */ if (snapshot_set) @@ -875,7 +880,7 @@ CheckCachedPlan(CachedPlanSource *plansource) */ static CachedPlan * BuildCachedPlan(CachedPlanSource *plansource, List *qlist, - ParamListInfo boundParams) + ParamListInfo boundParams, QueryEnvironment *queryEnv) { CachedPlan *plan; List *plist; @@ -899,7 +904,7 @@ BuildCachedPlan(CachedPlanSource *plansource, List *qlist, * safety, let's treat it as real and redo the RevalidateCachedQuery call. */ if (!plansource->is_valid) - qlist = RevalidateCachedQuery(plansource); + qlist = RevalidateCachedQuery(plansource, queryEnv); /* * If we don't already have a copy of the querytree list that can be @@ -1129,7 +1134,7 @@ cached_plan_cost(CachedPlan *plan, bool include_planner) */ CachedPlan * GetCachedPlan(CachedPlanSource *plansource, ParamListInfo boundParams, - bool useResOwner) + bool useResOwner, QueryEnvironment *queryEnv) { CachedPlan *plan = NULL; List *qlist; @@ -1143,7 +1148,7 @@ GetCachedPlan(CachedPlanSource *plansource, ParamListInfo boundParams, elog(ERROR, "cannot apply ResourceOwner to non-saved cached plan"); /* Make sure the querytree list is valid and we have parse-time locks */ - qlist = RevalidateCachedQuery(plansource); + qlist = RevalidateCachedQuery(plansource, queryEnv); /* Decide whether to use a custom plan */ customplan = choose_custom_plan(plansource, boundParams); @@ -1159,7 +1164,7 @@ GetCachedPlan(CachedPlanSource *plansource, ParamListInfo boundParams, else { /* Build a new generic plan */ - plan = BuildCachedPlan(plansource, qlist, NULL); + plan = BuildCachedPlan(plansource, qlist, NULL, queryEnv); /* Just make real sure plansource->gplan is clear */ ReleaseGenericPlan(plansource); /* Link the new generic plan into the plansource */ @@ -1204,7 +1209,7 @@ GetCachedPlan(CachedPlanSource *plansource, ParamListInfo boundParams, if (customplan) { /* Build a custom plan */ - plan = BuildCachedPlan(plansource, qlist, boundParams); + plan = BuildCachedPlan(plansource, qlist, boundParams, queryEnv); /* Accumulate total costs of custom plans, but 'ware overflow */ if (plansource->num_custom_plans < INT_MAX) { @@ -1418,7 +1423,8 @@ CachedPlanIsValid(CachedPlanSource *plansource) * within the cached plan, and may disappear next time the plan is updated. */ List * -CachedPlanGetTargetList(CachedPlanSource *plansource) +CachedPlanGetTargetList(CachedPlanSource *plansource, + QueryEnvironment *queryEnv) { Query *pstmt; @@ -1434,7 +1440,7 @@ CachedPlanGetTargetList(CachedPlanSource *plansource) return NIL; /* Make sure the querytree list is valid and we have parse-time locks */ - RevalidateCachedQuery(plansource); + RevalidateCachedQuery(plansource, queryEnv); /* Get the primary statement and find out what it returns */ pstmt = QueryListGetPrimaryStmt(plansource->query_list); diff --git a/src/backend/utils/misc/Makefile b/src/backend/utils/misc/Makefile index 45cdf76ec2..a53fcdf188 100644 --- a/src/backend/utils/misc/Makefile +++ b/src/backend/utils/misc/Makefile @@ -15,8 +15,8 @@ include $(top_builddir)/src/Makefile.global override CPPFLAGS := -I. -I$(srcdir) $(CPPFLAGS) OBJS = backend_random.o guc.o help_config.o pg_config.o pg_controldata.o \ - pg_rusage.o ps_status.o rls.o sampling.o superuser.o timeout.o \ - tzparser.o + pg_rusage.o ps_status.o queryenvironment.o rls.o sampling.o \ + superuser.o timeout.o tzparser.o # This location might depend on the installation directories. Therefore # we can't substitute it into pg_config.h. diff --git a/src/backend/utils/misc/queryenvironment.c b/src/backend/utils/misc/queryenvironment.c new file mode 100644 index 0000000000..a0b10d402b --- /dev/null +++ b/src/backend/utils/misc/queryenvironment.c @@ -0,0 +1,144 @@ +/*------------------------------------------------------------------------- + * + * queryenvironment.c + * Query environment, to store context-specific values like ephemeral named + * relations. Initial use is for named tuplestores for delta information + * from "normal" relations. + * + * The initial implementation uses a list because the number of such relations + * in any one context is expected to be very small. If that becomes a + * performance problem, the implementation can be changed with no other impact + * on callers, since this is an opaque structure. This is the reason to + * require a create function. + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/backend/utils/misc/queryenvironment.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/heapam.h" +#include "utils/queryenvironment.h" +#include "utils/rel.h" + +/* + * Private state of a query environment. + */ +struct QueryEnvironment +{ + List *namedRelList; +}; + + +QueryEnvironment * +create_queryEnv() +{ + return (QueryEnvironment *) palloc0(sizeof(QueryEnvironment)); +} + +EphemeralNamedRelationMetadata +get_visible_ENR_metadata(QueryEnvironment *queryEnv, const char *refname) +{ + EphemeralNamedRelation enr; + + Assert(refname != NULL); + + if (queryEnv == NULL) + return NULL; + + enr = get_ENR(queryEnv, refname); + + if (enr) + return &(enr->md); + + return NULL; +} + +/* + * Register a named relation for use in the given environment. + * + * If this is intended exclusively for planning purposes, the tstate field can + * be left NULL; + */ +void +register_ENR(QueryEnvironment *queryEnv, EphemeralNamedRelation enr) +{ + Assert(enr != NULL); + Assert(get_ENR(queryEnv, enr->md.name) == NULL); + + queryEnv->namedRelList = lappend(queryEnv->namedRelList, enr); +} + +/* + * Unregister an ephemeral relation by name. This will probably be a rarely + * used function, but seems like it should be provided "just in case". + */ +void +unregister_ENR(QueryEnvironment *queryEnv, const char *name) +{ + EphemeralNamedRelation match; + + match = get_ENR(queryEnv, name); + if (match) + queryEnv->namedRelList = list_delete(queryEnv->namedRelList, match); +} + +/* + * This returns an ENR if there is a name match in the given collection. It + * must quietly return NULL if no match is found. + */ +EphemeralNamedRelation +get_ENR(QueryEnvironment *queryEnv, const char *name) +{ + ListCell *lc; + + Assert(name != NULL); + + if (queryEnv == NULL) + return NULL; + + foreach(lc, queryEnv->namedRelList) + { + EphemeralNamedRelation enr = (EphemeralNamedRelation) lfirst(lc); + + if (strcmp(enr->md.name, name) == 0) + return enr; + } + + return NULL; +} + +/* + * Gets the TupleDesc for a Ephemeral Named Relation, based on which field was + * filled. + * + * When the TupleDesc is based on a relation from the catalogs, we count on + * that relation being used at the same time, so that appropriate locks will + * already be held. Locking here would be too late anyway. + */ +TupleDesc +ENRMetadataGetTupDesc(EphemeralNamedRelationMetadata enrmd) +{ + TupleDesc tupdesc; + + /* One, and only one, of these fields must be filled. */ + Assert((enrmd->reliddesc == InvalidOid) != (enrmd->tupdesc == NULL)); + + if (enrmd->tupdesc != NULL) + tupdesc = enrmd->tupdesc; + else + { + Relation relation; + + relation = heap_open(enrmd->reliddesc, NoLock); + tupdesc = relation->rd_att; + heap_close(relation, NoLock); + } + + return tupdesc; +} diff --git a/src/backend/utils/sort/tuplestore.c b/src/backend/utils/sort/tuplestore.c index 84abf5f67e..b3f6be7457 100644 --- a/src/backend/utils/sort/tuplestore.c +++ b/src/backend/utils/sort/tuplestore.c @@ -109,6 +109,7 @@ struct Tuplestorestate bool truncated; /* tuplestore_trim has removed tuples? */ int64 availMem; /* remaining memory available, in bytes */ int64 allowedMem; /* total memory allowed, in bytes */ + int64 tuples; /* number of tuples added */ BufFile *myfile; /* underlying file, or NULL if none */ MemoryContext context; /* memory context for holding tuples */ ResourceOwner resowner; /* resowner for holding temp files */ @@ -267,6 +268,7 @@ tuplestore_begin_common(int eflags, bool interXact, int maxKBytes) state->memtupdeleted = 0; state->memtupcount = 0; + state->tuples = 0; /* * Initial size of array must be more than ALLOCSET_SEPARATE_THRESHOLD; @@ -433,6 +435,7 @@ tuplestore_clear(Tuplestorestate *state) state->truncated = false; state->memtupdeleted = 0; state->memtupcount = 0; + state->tuples = 0; readptr = state->readptrs; for (i = 0; i < state->readptrcount; readptr++, i++) { @@ -533,6 +536,18 @@ tuplestore_select_read_pointer(Tuplestorestate *state, int ptr) state->activeptr = ptr; } +/* + * tuplestore_tuple_count + * + * Returns the number of tuples added since creation or the last + * tuplestore_clear(). + */ +int64 +tuplestore_tuple_count(Tuplestorestate *state) +{ + return state->tuples; +} + /* * tuplestore_ateof * @@ -753,6 +768,8 @@ tuplestore_puttuple_common(Tuplestorestate *state, void *tuple) int i; ResourceOwner oldowner; + state->tuples++; + switch (state->status) { case TSS_INMEM: diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index d067b757b0..48c5a570a0 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201703311 +#define CATALOG_VERSION_NO 201703312 #endif diff --git a/src/include/commands/createas.h b/src/include/commands/createas.h index 7b78cc4a13..0fb9990e04 100644 --- a/src/include/commands/createas.h +++ b/src/include/commands/createas.h @@ -18,10 +18,11 @@ #include "nodes/params.h" #include "nodes/parsenodes.h" #include "tcop/dest.h" +#include "utils/queryenvironment.h" extern ObjectAddress ExecCreateTableAs(CreateTableAsStmt *stmt, const char *queryString, - ParamListInfo params, char *completionTag); + ParamListInfo params, QueryEnvironment *queryEnv, char *completionTag); extern int GetIntoRelEFlags(IntoClause *intoClause); diff --git a/src/include/commands/explain.h b/src/include/commands/explain.h index 9191e186c1..b77f81db97 100644 --- a/src/include/commands/explain.h +++ b/src/include/commands/explain.h @@ -62,19 +62,20 @@ extern PGDLLIMPORT explain_get_index_name_hook_type explain_get_index_name_hook; extern void ExplainQuery(ParseState *pstate, ExplainStmt *stmt, const char *queryString, - ParamListInfo params, DestReceiver *dest); + ParamListInfo params, QueryEnvironment *queryEnv, DestReceiver *dest); extern ExplainState *NewExplainState(void); extern TupleDesc ExplainResultDesc(ExplainStmt *stmt); extern void ExplainOneUtility(Node *utilityStmt, IntoClause *into, - ExplainState *es, - const char *queryString, ParamListInfo params); + ExplainState *es, const char *queryString, + ParamListInfo params, QueryEnvironment *queryEnv); extern void ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es, const char *queryString, - ParamListInfo params, const instr_time *planduration); + ParamListInfo params, QueryEnvironment *queryEnv, + const instr_time *planduration); extern void ExplainPrintPlan(ExplainState *es, QueryDesc *queryDesc); extern void ExplainPrintTriggers(ExplainState *es, QueryDesc *queryDesc); diff --git a/src/include/commands/prepare.h b/src/include/commands/prepare.h index d8d22edbbc..c60e6f30b8 100644 --- a/src/include/commands/prepare.h +++ b/src/include/commands/prepare.h @@ -42,8 +42,8 @@ extern void ExecuteQuery(ExecuteStmt *stmt, IntoClause *intoClause, DestReceiver *dest, char *completionTag); extern void DeallocateQuery(DeallocateStmt *stmt); extern void ExplainExecuteQuery(ExecuteStmt *execstmt, IntoClause *into, - ExplainState *es, - const char *queryString, ParamListInfo params); + ExplainState *es, const char *queryString, + ParamListInfo params, QueryEnvironment *queryEnv); /* Low-level access to stored prepared statements */ extern void StorePreparedStatement(const char *stmt_name, diff --git a/src/include/executor/execdesc.h b/src/include/executor/execdesc.h index 87e7ca8508..37de6f2011 100644 --- a/src/include/executor/execdesc.h +++ b/src/include/executor/execdesc.h @@ -40,6 +40,7 @@ typedef struct QueryDesc Snapshot crosscheck_snapshot; /* crosscheck for RI update/delete */ DestReceiver *dest; /* the destination for tuple output */ ParamListInfo params; /* param values being passed in */ + QueryEnvironment *queryEnv; /* query environment passed in */ int instrument_options; /* OR of InstrumentOption flags */ /* These fields are set by ExecutorStart */ @@ -61,6 +62,7 @@ extern QueryDesc *CreateQueryDesc(PlannedStmt *plannedstmt, Snapshot crosscheck_snapshot, DestReceiver *dest, ParamListInfo params, + QueryEnvironment *queryEnv, int instrument_options); extern void FreeQueryDesc(QueryDesc *qdesc); diff --git a/src/include/executor/nodeNamedtuplestorescan.h b/src/include/executor/nodeNamedtuplestorescan.h new file mode 100644 index 0000000000..9ef477e7ff --- /dev/null +++ b/src/include/executor/nodeNamedtuplestorescan.h @@ -0,0 +1,24 @@ +/*------------------------------------------------------------------------- + * + * nodeNamedtuplestorescan.h + * + * + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/executor/nodeNamedtuplestorescan.h + * + *------------------------------------------------------------------------- + */ +#ifndef NODENAMEDTUPLESTORESCAN_H +#define NODENAMEDTUPLESTORESCAN_H + +#include "nodes/execnodes.h" + +extern NamedTuplestoreScanState *ExecInitNamedTuplestoreScan(NamedTuplestoreScan *node, EState *estate, int eflags); +extern TupleTableSlot *ExecNamedTuplestoreScan(NamedTuplestoreScanState *node); +extern void ExecEndNamedTuplestoreScan(NamedTuplestoreScanState *node); +extern void ExecReScanNamedTuplestoreScan(NamedTuplestoreScanState *node); + +#endif /* NODENAMEDTUPLESTORESCAN_H */ diff --git a/src/include/executor/spi.h b/src/include/executor/spi.h index a18ae63245..e2e8bb9553 100644 --- a/src/include/executor/spi.h +++ b/src/include/executor/spi.h @@ -43,6 +43,8 @@ typedef struct _SPI_plan *SPIPlanPtr; #define SPI_ERROR_NOATTRIBUTE (-9) #define SPI_ERROR_NOOUTFUNC (-10) #define SPI_ERROR_TYPUNKNOWN (-11) +#define SPI_ERROR_REL_DUPLICATE (-12) +#define SPI_ERROR_REL_NOT_FOUND (-13) #define SPI_OK_CONNECT 1 #define SPI_OK_FINISH 2 @@ -58,6 +60,8 @@ typedef struct _SPI_plan *SPIPlanPtr; #define SPI_OK_DELETE_RETURNING 12 #define SPI_OK_UPDATE_RETURNING 13 #define SPI_OK_REWRITTEN 14 +#define SPI_OK_REL_REGISTER 15 +#define SPI_OK_REL_UNREGISTER 16 /* These used to be functions, now just no-ops for backwards compatibility */ #define SPI_push() ((void) 0) @@ -146,6 +150,9 @@ extern void SPI_scroll_cursor_fetch(Portal, FetchDirection direction, long count extern void SPI_scroll_cursor_move(Portal, FetchDirection direction, long count); extern void SPI_cursor_close(Portal portal); +extern int SPI_register_relation(EphemeralNamedRelation enr); +extern int SPI_unregister_relation(const char *name); + extern void AtEOXact_SPI(bool isCommit); extern void AtEOSubXact_SPI(bool isCommit, SubTransactionId mySubid); diff --git a/src/include/executor/spi_priv.h b/src/include/executor/spi_priv.h index db8b59c387..49aa7c94e7 100644 --- a/src/include/executor/spi_priv.h +++ b/src/include/executor/spi_priv.h @@ -14,6 +14,7 @@ #define SPI_PRIV_H #include "executor/spi.h" +#include "utils/queryenvironment.h" #define _SPI_PLAN_MAGIC 569278163 @@ -31,6 +32,7 @@ typedef struct MemoryContext execCxt; /* executor context */ MemoryContext savedcxt; /* context of SPI_connect's caller */ SubTransactionId connectSubid; /* ID of connecting subtransaction */ + QueryEnvironment *queryEnv; /* query environment setup for SPI level */ } _SPI_connection; /* diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 11a68500ee..fa992449f4 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -22,6 +22,7 @@ #include "nodes/params.h" #include "nodes/plannodes.h" #include "utils/hsearch.h" +#include "utils/queryenvironment.h" #include "utils/reltrigger.h" #include "utils/sortsupport.h" #include "utils/tuplestore.h" @@ -431,6 +432,8 @@ typedef struct EState ParamListInfo es_param_list_info; /* values of external params */ ParamExecData *es_param_exec_vals; /* values of internal params */ + QueryEnvironment *es_queryEnv; /* query environment */ + /* Other working state: */ MemoryContext es_query_cxt; /* per-query context in which EState lives */ @@ -1445,6 +1448,24 @@ typedef struct CteScanState bool eof_cte; /* reached end of CTE query? */ } CteScanState; +/* ---------------- + * NamedTuplestoreScanState information + * + * NamedTuplestoreScan nodes are used to scan a Tuplestore created and + * named prior to execution of the query. An example is a transition + * table for an AFTER trigger. + * + * Multiple NamedTuplestoreScan nodes can read out from the same Tuplestore. + * ---------------- + */ +typedef struct NamedTuplestoreScanState +{ + ScanState ss; /* its first field is NodeTag */ + int readptr; /* index of my tuplestore read pointer */ + TupleDesc tupdesc; /* format of the tuples in the tuplestore */ + Tuplestorestate *relation; /* the rows */ +} NamedTuplestoreScanState; + /* ---------------- * WorkTableScanState information * diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index 963ce45ae3..177853b3bf 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -63,6 +63,7 @@ typedef enum NodeTag T_ValuesScan, T_TableFuncScan, T_CteScan, + T_NamedTuplestoreScan, T_WorkTableScan, T_ForeignScan, T_CustomScan, @@ -114,6 +115,7 @@ typedef enum NodeTag T_TableFuncScanState, T_ValuesScanState, T_CteScanState, + T_NamedTuplestoreScanState, T_WorkTableScanState, T_ForeignScanState, T_CustomScanState, diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 3a71dd5b37..b2afd50818 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -906,7 +906,8 @@ typedef enum RTEKind RTE_FUNCTION, /* function in FROM */ RTE_TABLEFUNC, /* TableFunc(.., column list) */ RTE_VALUES, /* VALUES (), (), ... */ - RTE_CTE /* common table expr (WITH list element) */ + RTE_CTE, /* common table expr (WITH list element) */ + RTE_NAMEDTUPLESTORE /* tuplestore, e.g. for AFTER triggers */ } RTEKind; typedef struct RangeTblEntry @@ -993,6 +994,9 @@ typedef struct RangeTblEntry List *coltypmods; /* integer list of column typmods */ List *colcollations; /* OID list of column collation OIDs */ + char *enrname; /* name of ephemeral named relation */ + double enrtuples; /* estimated or actual from caller */ + /* * Fields valid in all RTEs: */ diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h index 6e531b6238..a2dd26f8a9 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h @@ -527,6 +527,16 @@ typedef struct CteScan int cteParam; /* ID of Param representing CTE output */ } CteScan; +/* ---------------- + * NamedTuplestoreScan node + * ---------------- + */ +typedef struct NamedTuplestoreScan +{ + Scan scan; + char *enrname; /* Name given to Ephemeral Named Relation */ +} NamedTuplestoreScan; + /* ---------------- * WorkTableScan node * ---------------- diff --git a/src/include/optimizer/cost.h b/src/include/optimizer/cost.h index d9a9b12a06..6909359bcf 100644 --- a/src/include/optimizer/cost.h +++ b/src/include/optimizer/cost.h @@ -98,6 +98,8 @@ extern void cost_tablefuncscan(Path *path, PlannerInfo *root, RelOptInfo *baserel, ParamPathInfo *param_info); extern void cost_ctescan(Path *path, PlannerInfo *root, RelOptInfo *baserel, ParamPathInfo *param_info); +extern void cost_namedtuplestorescan(Path *path, PlannerInfo *root, + RelOptInfo *baserel, ParamPathInfo *param_info); extern void cost_recursive_union(Path *runion, Path *nrterm, Path *rterm); extern void cost_sort(Path *path, PlannerInfo *root, List *pathkeys, Cost input_cost, double tuples, int width, @@ -187,6 +189,7 @@ extern void set_values_size_estimates(PlannerInfo *root, RelOptInfo *rel); extern void set_cte_size_estimates(PlannerInfo *root, RelOptInfo *rel, double cte_rows); extern void set_tablefunc_size_estimates(PlannerInfo *root, RelOptInfo *rel); +extern void set_namedtuplestore_size_estimates(PlannerInfo *root, RelOptInfo *rel); extern void set_foreign_size_estimates(PlannerInfo *root, RelOptInfo *rel); extern PathTarget *set_pathtarget_cost_width(PlannerInfo *root, PathTarget *target); extern double compute_bitmap_pages(PlannerInfo *root, RelOptInfo *baserel, diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index c72c7e02cb..82d4e8701c 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -100,6 +100,8 @@ extern Path *create_tablefuncscan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer); extern Path *create_ctescan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer); +extern Path *create_namedtuplestorescan_path(PlannerInfo *root, RelOptInfo *rel, + Relids required_outer); extern Path *create_worktablescan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer); extern ForeignPath *create_foreignscan_path(PlannerInfo *root, RelOptInfo *rel, diff --git a/src/include/parser/analyze.h b/src/include/parser/analyze.h index 17259409a7..9b33ba5dfd 100644 --- a/src/include/parser/analyze.h +++ b/src/include/parser/analyze.h @@ -23,7 +23,7 @@ extern PGDLLIMPORT post_parse_analyze_hook_type post_parse_analyze_hook; extern Query *parse_analyze(RawStmt *parseTree, const char *sourceText, - Oid *paramTypes, int numParams); + Oid *paramTypes, int numParams, QueryEnvironment *queryEnv); extern Query *parse_analyze_varparams(RawStmt *parseTree, const char *sourceText, Oid **paramTypes, int *numParams); diff --git a/src/include/parser/parse_enr.h b/src/include/parser/parse_enr.h new file mode 100644 index 0000000000..48a7576f2c --- /dev/null +++ b/src/include/parser/parse_enr.h @@ -0,0 +1,22 @@ +/*------------------------------------------------------------------------- + * + * parse_enr.h + * Internal definitions for parser + * + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/parser/parse_enr.h + * + *------------------------------------------------------------------------- + */ +#ifndef PARSE_ENR_H +#define PARSE_ENR_H + +#include "parser/parse_node.h" + +extern bool name_matches_visible_ENR(ParseState *pstate, const char *refname); +extern EphemeralNamedRelationMetadata get_visible_ENR(ParseState *pstate, const char *refname); + +#endif /* PARSE_ENR_H */ diff --git a/src/include/parser/parse_node.h b/src/include/parser/parse_node.h index 3a25d9598d..1035bad322 100644 --- a/src/include/parser/parse_node.h +++ b/src/include/parser/parse_node.h @@ -15,6 +15,7 @@ #define PARSE_NODE_H #include "nodes/parsenodes.h" +#include "utils/queryenvironment.h" #include "utils/relcache.h" @@ -188,6 +189,8 @@ struct ParseState bool p_resolve_unknowns; /* resolve unknown-type SELECT outputs * as type text */ + QueryEnvironment *p_queryEnv; /* curr env, incl refs to enclosing env */ + /* Flags telling about things found in the query: */ bool p_hasAggs; bool p_hasWindowFuncs; diff --git a/src/include/parser/parse_relation.h b/src/include/parser/parse_relation.h index 515c06cfef..2f42cc8ef0 100644 --- a/src/include/parser/parse_relation.h +++ b/src/include/parser/parse_relation.h @@ -42,6 +42,7 @@ extern RangeTblEntry *refnameRangeTblEntry(ParseState *pstate, extern CommonTableExpr *scanNameSpaceForCTE(ParseState *pstate, const char *refname, Index *ctelevelsup); +extern bool scanNameSpaceForENR(ParseState *pstate, const char *refname); extern void checkNameSpaceConflicts(ParseState *pstate, List *namespace1, List *namespace2); extern int RTERangeTablePosn(ParseState *pstate, @@ -107,6 +108,9 @@ extern RangeTblEntry *addRangeTableEntryForCTE(ParseState *pstate, Index levelsup, RangeVar *rv, bool inFromCl); +extern RangeTblEntry *addRangeTableEntryForENR(ParseState *pstate, + RangeVar *rv, + bool inFromCl); extern bool isLockedRefname(ParseState *pstate, const char *refname); extern void addRTEtoQuery(ParseState *pstate, RangeTblEntry *rte, bool addToJoinList, diff --git a/src/include/tcop/tcopprot.h b/src/include/tcop/tcopprot.h index 1958be85b7..f1a34a1c72 100644 --- a/src/include/tcop/tcopprot.h +++ b/src/include/tcop/tcopprot.h @@ -24,6 +24,7 @@ #include "nodes/plannodes.h" #include "storage/procsignal.h" #include "utils/guc.h" +#include "utils/queryenvironment.h" /* Required daylight between max_stack_depth and the kernel limit, in bytes */ @@ -49,11 +50,13 @@ extern int log_statement; extern List *pg_parse_query(const char *query_string); extern List *pg_analyze_and_rewrite(RawStmt *parsetree, const char *query_string, - Oid *paramTypes, int numParams); + Oid *paramTypes, int numParams, + QueryEnvironment *queryEnv); extern List *pg_analyze_and_rewrite_params(RawStmt *parsetree, const char *query_string, ParserSetupHook parserSetup, - void *parserSetupArg); + void *parserSetupArg, + QueryEnvironment *queryEnv); extern PlannedStmt *pg_plan_query(Query *querytree, int cursorOptions, ParamListInfo boundParams); extern List *pg_plan_queries(List *querytrees, int cursorOptions, diff --git a/src/include/tcop/utility.h b/src/include/tcop/utility.h index 4f8d353900..90f1215aec 100644 --- a/src/include/tcop/utility.h +++ b/src/include/tcop/utility.h @@ -26,15 +26,18 @@ typedef enum /* Hook for plugins to get control in ProcessUtility() */ typedef void (*ProcessUtility_hook_type) (PlannedStmt *pstmt, const char *queryString, ProcessUtilityContext context, - ParamListInfo params, + ParamListInfo params, + QueryEnvironment *queryEnv, DestReceiver *dest, char *completionTag); extern PGDLLIMPORT ProcessUtility_hook_type ProcessUtility_hook; extern void ProcessUtility(PlannedStmt *pstmt, const char *queryString, ProcessUtilityContext context, ParamListInfo params, + QueryEnvironment *queryEnv, DestReceiver *dest, char *completionTag); extern void standard_ProcessUtility(PlannedStmt *pstmt, const char *queryString, ProcessUtilityContext context, ParamListInfo params, + QueryEnvironment *queryEnv, DestReceiver *dest, char *completionTag); extern bool UtilityReturnsTuples(Node *parsetree); diff --git a/src/include/utils/plancache.h b/src/include/utils/plancache.h index 84952d56e7..48d4ac94b2 100644 --- a/src/include/utils/plancache.h +++ b/src/include/utils/plancache.h @@ -17,6 +17,7 @@ #include "access/tupdesc.h" #include "nodes/params.h" +#include "utils/queryenvironment.h" /* Forward declaration, to avoid including parsenodes.h here */ struct RawStmt; @@ -148,7 +149,8 @@ extern void ResetPlanCache(void); extern CachedPlanSource *CreateCachedPlan(struct RawStmt *raw_parse_tree, const char *query_string, - const char *commandTag); + const char *commandTag, + QueryEnvironment *queryEnv); extern CachedPlanSource *CreateOneShotCachedPlan(struct RawStmt *raw_parse_tree, const char *query_string, const char *commandTag); @@ -172,11 +174,13 @@ extern CachedPlanSource *CopyCachedPlan(CachedPlanSource *plansource); extern bool CachedPlanIsValid(CachedPlanSource *plansource); -extern List *CachedPlanGetTargetList(CachedPlanSource *plansource); +extern List *CachedPlanGetTargetList(CachedPlanSource *plansource, + QueryEnvironment *queryEnv); extern CachedPlan *GetCachedPlan(CachedPlanSource *plansource, ParamListInfo boundParams, - bool useResOwner); + bool useResOwner, + QueryEnvironment *queryEnv); extern void ReleaseCachedPlan(CachedPlan *plan, bool useResOwner); #endif /* PLANCACHE_H */ diff --git a/src/include/utils/portal.h b/src/include/utils/portal.h index e7c5a8bd09..ef3898c98c 100644 --- a/src/include/utils/portal.h +++ b/src/include/utils/portal.h @@ -137,6 +137,7 @@ typedef struct PortalData CachedPlan *cplan; /* CachedPlan, if stmts are from one */ ParamListInfo portalParams; /* params to pass to query */ + QueryEnvironment *queryEnv; /* environment for query */ /* Features/options */ PortalStrategy strategy; /* see above */ diff --git a/src/include/utils/queryenvironment.h b/src/include/utils/queryenvironment.h new file mode 100644 index 0000000000..b4f65a1976 --- /dev/null +++ b/src/include/utils/queryenvironment.h @@ -0,0 +1,74 @@ +/*------------------------------------------------------------------------- + * + * queryenvironment.h + * Access to functions to mutate the query environment and retrieve the + * actual data related to entries (if any). + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/utils/queryenvironment.h + * + *------------------------------------------------------------------------- + */ +#ifndef QUERYENVIRONMENT_H +#define QUERYENVIRONMENT_H + +#include "access/tupdesc.h" + + +typedef enum EphemeralNameRelationType +{ + ENR_NAMED_TUPLESTORE /* named tuplestore relation; e.g., deltas */ +} EphemeralNameRelationType; + +/* + * Some ephemeral named relations must match some relation (e.g., trigger + * transition tables), so to properly handle cached plans and DDL, we should + * carry the OID of that relation. In other cases an ENR might be independent + * of any relation which is stored in the system catalogs, so we need to be + * able to directly store the TupleDesc. We never need both. + */ +typedef struct EphemeralNamedRelationMetadataData +{ + char *name; /* name used to identify the relation */ + + /* only one of the next two fields should be used */ + Oid reliddesc; /* oid of relation to get tupdesc */ + TupleDesc tupdesc; /* description of result rows */ + + EphemeralNameRelationType enrtype; /* to identify type of relation */ + double enrtuples; /* estimated number of tuples */ +} EphemeralNamedRelationMetadataData; + +typedef EphemeralNamedRelationMetadataData *EphemeralNamedRelationMetadata; + +/* + * Ephemeral Named Relation data; used for parsing named relations not in the + * catalog, like transition tables in AFTER triggers. + */ +typedef struct EphemeralNamedRelationData +{ + EphemeralNamedRelationMetadataData md; + void *reldata; /* structure for execution-time access to data */ +} EphemeralNamedRelationData; + +typedef EphemeralNamedRelationData *EphemeralNamedRelation; + +/* + * This is an opaque structure outside of queryenvironment.c itself. The + * intention is to be able to change the implementation or add new context + * features without needing to change existing code for use of existing + * features. + */ +typedef struct QueryEnvironment QueryEnvironment; + + +extern QueryEnvironment *create_queryEnv(void); +extern EphemeralNamedRelationMetadata get_visible_ENR_metadata(QueryEnvironment *queryEnv, const char *refname); +extern void register_ENR(QueryEnvironment *queryEnv, EphemeralNamedRelation enr); +extern void unregister_ENR(QueryEnvironment *queryEnv, const char *name); +extern EphemeralNamedRelation get_ENR(QueryEnvironment *queryEnv, const char *name); +extern TupleDesc ENRMetadataGetTupDesc(EphemeralNamedRelationMetadata enrmd); + +#endif /* QUERYENVIRONMENT_H */ diff --git a/src/include/utils/tuplestore.h b/src/include/utils/tuplestore.h index a52a547037..b31ede882b 100644 --- a/src/include/utils/tuplestore.h +++ b/src/include/utils/tuplestore.h @@ -78,6 +78,8 @@ extern bool tuplestore_advance(Tuplestorestate *state, bool forward); extern bool tuplestore_skiptuples(Tuplestorestate *state, int64 ntuples, bool forward); +extern int64 tuplestore_tuple_count(Tuplestorestate *state); + extern bool tuplestore_ateof(Tuplestorestate *state); extern void tuplestore_rescan(Tuplestorestate *state); diff --git a/src/test/regress/expected/with.out b/src/test/regress/expected/with.out index 3b7f689a98..fdcc4970a1 100644 --- a/src/test/regress/expected/with.out +++ b/src/test/regress/expected/with.out @@ -2272,3 +2272,6 @@ with ordinality as (select 1 as x) select * from ordinality; 1 (1 row) +-- check sane response to attempt to modify CTE relation +WITH d AS (SELECT 42) INSERT INTO d VALUES (1); +ERROR: relation "d" cannot be the target of a modifying statement diff --git a/src/test/regress/sql/with.sql b/src/test/regress/sql/with.sql index 08ddc8bae0..8ae5184d0f 100644 --- a/src/test/regress/sql/with.sql +++ b/src/test/regress/sql/with.sql @@ -1028,3 +1028,6 @@ DROP RULE y_rule ON y; create table foo (with baz); -- fail, WITH is a reserved word create table foo (with ordinality); -- fail, WITH is a reserved word with ordinality as (select 1 as x) select * from ordinality; + +-- check sane response to attempt to modify CTE relation +WITH d AS (SELECT 42) INSERT INTO d VALUES (1); -- cgit v1.2.3 From 59702716324ab9c07b02fb005dcf14c7f48c4632 Mon Sep 17 00:00:00 2001 From: Kevin Grittner Date: Fri, 31 Mar 2017 23:30:08 -0500 Subject: Add transition table support to plpgsql. Kevin Grittner and Thomas Munro Reviewed by Heikki Linnakangas, David Fetter, and Thomas Munro with valuable comments and suggestions from many others --- doc/src/sgml/ref/create_trigger.sgml | 5 + src/pl/plpgsql/src/pl_comp.c | 13 +- src/pl/plpgsql/src/pl_exec.c | 47 ++++++ src/pl/plpgsql/src/plpgsql.h | 14 +- src/test/regress/expected/plpgsql.out | 287 +++++++++++++++++++++++++++++++++ src/test/regress/expected/triggers.out | 24 +++ src/test/regress/sql/plpgsql.sql | 283 ++++++++++++++++++++++++++++++++ src/test/regress/sql/triggers.sql | 23 +++ 8 files changed, 685 insertions(+), 11 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/create_trigger.sgml b/doc/src/sgml/ref/create_trigger.sgml index c9a9fa6ba2..e22e42e7dc 100644 --- a/doc/src/sgml/ref/create_trigger.sgml +++ b/doc/src/sgml/ref/create_trigger.sgml @@ -322,6 +322,11 @@ UPDATE OF column_name1 [, column_name2 The (unqualified) name to be used within the trigger for this relation. + + + So far only triggers written in C or PL/pgSQL support this. + + diff --git a/src/pl/plpgsql/src/pl_comp.c b/src/pl/plpgsql/src/pl_comp.c index bed343ea0c..a6375511f6 100644 --- a/src/pl/plpgsql/src/pl_comp.c +++ b/src/pl/plpgsql/src/pl_comp.c @@ -589,11 +589,11 @@ do_compile(FunctionCallInfo fcinfo, errmsg("trigger functions cannot have declared arguments"), errhint("The arguments of the trigger can be accessed through TG_NARGS and TG_ARGV instead."))); - /* Add the record for referencing NEW */ + /* Add the record for referencing NEW ROW */ rec = plpgsql_build_record("new", 0, true); function->new_varno = rec->dno; - /* Add the record for referencing OLD */ + /* Add the record for referencing OLD ROW */ rec = plpgsql_build_record("old", 0, true); function->old_varno = rec->dno; @@ -2453,15 +2453,16 @@ compute_function_hashkey(FunctionCallInfo fcinfo, hashkey->isTrigger = CALLED_AS_TRIGGER(fcinfo); /* - * if trigger, get relation OID. In validation mode we do not know what - * relation is intended to be used, so we leave trigrelOid zero; the hash - * entry built in this case will never really be used. + * if trigger, get its OID. In validation mode we do not know what + * relation or transition table names are intended to be used, so we leave + * trigOid zero; the hash entry built in this case will never really be + * used. */ if (hashkey->isTrigger && !forValidator) { TriggerData *trigdata = (TriggerData *) fcinfo->context; - hashkey->trigrelOid = RelationGetRelid(trigdata->tg_relation); + hashkey->trigOid = trigdata->tg_trigger->tgoid; } /* get input collation, if known */ diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c index c27935b51b..43da986fc0 100644 --- a/src/pl/plpgsql/src/pl_exec.c +++ b/src/pl/plpgsql/src/pl_exec.c @@ -689,6 +689,47 @@ plpgsql_exec_trigger(PLpgSQL_function *func, else elog(ERROR, "unrecognized trigger action: not INSERT, DELETE, or UPDATE"); + /* + * Capture the NEW and OLD transition TABLE tuplestores (if specified for + * this trigger). + */ + if (trigdata->tg_newtable || trigdata->tg_oldtable) + { + estate.queryEnv = create_queryEnv(); + if (trigdata->tg_newtable) + { + EphemeralNamedRelation enr = + palloc(sizeof(EphemeralNamedRelationData)); + int rc PG_USED_FOR_ASSERTS_ONLY; + + enr->md.name = trigdata->tg_trigger->tgnewtable; + enr->md.reliddesc = RelationGetRelid(trigdata->tg_relation); + enr->md.tupdesc = NULL; + enr->md.enrtype = ENR_NAMED_TUPLESTORE; + enr->md.enrtuples = tuplestore_tuple_count(trigdata->tg_newtable); + enr->reldata = trigdata->tg_newtable; + register_ENR(estate.queryEnv, enr); + rc = SPI_register_relation(enr); + Assert(rc >= 0); + } + if (trigdata->tg_oldtable) + { + EphemeralNamedRelation enr = + palloc(sizeof(EphemeralNamedRelationData)); + int rc PG_USED_FOR_ASSERTS_ONLY; + + enr->md.name = trigdata->tg_trigger->tgoldtable; + enr->md.reliddesc = RelationGetRelid(trigdata->tg_relation); + enr->md.tupdesc = NULL; + enr->md.enrtype = ENR_NAMED_TUPLESTORE; + enr->md.enrtuples = tuplestore_tuple_count(trigdata->tg_oldtable); + enr->reldata = trigdata->tg_oldtable; + register_ENR(estate.queryEnv, enr); + rc = SPI_register_relation(enr); + Assert(rc >= 0); + } + } + /* * Assign the special tg_ variables */ @@ -3442,6 +3483,9 @@ plpgsql_estate_setup(PLpgSQL_execstate *estate, estate->paramLI->paramMask = NULL; estate->params_dirty = false; + /* default tuplestore cache to "none" */ + estate->queryEnv = NULL; + /* set up for use of appropriate simple-expression EState and cast hash */ if (simple_eval_estate) { @@ -7329,6 +7373,9 @@ exec_dynquery_with_params(PLpgSQL_execstate *estate, /* Release transient data */ MemoryContextReset(stmt_mcontext); + /* Make sure the portal knows about any named tuplestores. */ + portal->queryEnv = estate->queryEnv; + return portal; } diff --git a/src/pl/plpgsql/src/plpgsql.h b/src/pl/plpgsql/src/plpgsql.h index b7e103b514..43a62ef34e 100644 --- a/src/pl/plpgsql/src/plpgsql.h +++ b/src/pl/plpgsql/src/plpgsql.h @@ -20,6 +20,7 @@ #include "commands/event_trigger.h" #include "commands/trigger.h" #include "executor/spi.h" +#include "utils/queryenvironment.h" /********************************************************************** * Definitions @@ -780,12 +781,12 @@ typedef struct PLpgSQL_func_hashkey /* be careful that pad bytes in this struct get zeroed! */ /* - * For a trigger function, the OID of the relation triggered on is part of - * the hash key --- we want to compile the trigger separately for each - * relation it is used with, in case the rowtype is different. Zero if - * not called as a trigger. + * For a trigger function, the OID of the trigger is part of the hash key + * --- we want to compile the trigger function separately for each trigger + * it is used with, in case the rowtype or transition table names are + * different. Zero if not called as a trigger. */ - Oid trigrelOid; + Oid trigOid; /* * We must include the input collation as part of the hash key too, @@ -910,6 +911,9 @@ typedef struct PLpgSQL_execstate ParamListInfo paramLI; bool params_dirty; /* T if any resettable datum has been passed */ + /* custom environment for parsing/execution of query for this context */ + QueryEnvironment *queryEnv; + /* EState to use for "simple" expression evaluation */ EState *simple_eval_estate; diff --git a/src/test/regress/expected/plpgsql.out b/src/test/regress/expected/plpgsql.out index 04848c10a2..93b71c7c78 100644 --- a/src/test/regress/expected/plpgsql.out +++ b/src/test/regress/expected/plpgsql.out @@ -5684,3 +5684,290 @@ end; $$; ERROR: value for domain plpgsql_arr_domain violates check constraint "plpgsql_arr_domain_check" CONTEXT: PL/pgSQL function inline_code_block line 4 at assignment +-- +-- test usage of transition tables in AFTER triggers +-- +CREATE TABLE transition_table_base (id int PRIMARY KEY, val text); +CREATE FUNCTION transition_table_base_ins_func() + RETURNS trigger + LANGUAGE plpgsql +AS $$ +DECLARE + t text; + l text; +BEGIN + t = ''; + FOR l IN EXECUTE + $q$ + EXPLAIN (TIMING off, COSTS off, VERBOSE on) + SELECT * FROM newtable + $q$ LOOP + t = t || l || E'\n'; + END LOOP; + + RAISE INFO '%', t; + RETURN new; +END; +$$; +CREATE TRIGGER transition_table_base_ins_trig + AFTER INSERT ON transition_table_base + REFERENCING OLD TABLE AS oldtable NEW TABLE AS newtable + FOR EACH STATEMENT + EXECUTE PROCEDURE transition_table_base_ins_func(); +ERROR: OLD TABLE can only be specified for a DELETE or UPDATE trigger +CREATE TRIGGER transition_table_base_ins_trig + AFTER INSERT ON transition_table_base + REFERENCING NEW TABLE AS newtable + FOR EACH STATEMENT + EXECUTE PROCEDURE transition_table_base_ins_func(); +INSERT INTO transition_table_base VALUES (1, 'One'), (2, 'Two'); +INFO: Named Tuplestore Scan + Output: id, val + +INSERT INTO transition_table_base VALUES (3, 'Three'), (4, 'Four'); +INFO: Named Tuplestore Scan + Output: id, val + +CREATE OR REPLACE FUNCTION transition_table_base_upd_func() + RETURNS trigger + LANGUAGE plpgsql +AS $$ +DECLARE + t text; + l text; +BEGIN + t = ''; + FOR l IN EXECUTE + $q$ + EXPLAIN (TIMING off, COSTS off, VERBOSE on) + SELECT * FROM oldtable ot FULL JOIN newtable nt USING (id) + $q$ LOOP + t = t || l || E'\n'; + END LOOP; + + RAISE INFO '%', t; + RETURN new; +END; +$$; +CREATE TRIGGER transition_table_base_upd_trig + AFTER UPDATE ON transition_table_base + REFERENCING OLD TABLE AS oldtable NEW TABLE AS newtable + FOR EACH STATEMENT + EXECUTE PROCEDURE transition_table_base_upd_func(); +UPDATE transition_table_base + SET val = '*' || val || '*' + WHERE id BETWEEN 2 AND 3; +INFO: Hash Full Join + Output: COALESCE(ot.id, nt.id), ot.val, nt.val + Hash Cond: (ot.id = nt.id) + -> Named Tuplestore Scan + Output: ot.id, ot.val + -> Hash + Output: nt.id, nt.val + -> Named Tuplestore Scan + Output: nt.id, nt.val + +CREATE TABLE transition_table_level1 +( + level1_no serial NOT NULL , + level1_node_name varchar(255), + PRIMARY KEY (level1_no) +) WITHOUT OIDS; +CREATE TABLE transition_table_level2 +( + level2_no serial NOT NULL , + parent_no int NOT NULL, + level1_node_name varchar(255), + PRIMARY KEY (level2_no) +) WITHOUT OIDS; +CREATE TABLE transition_table_status +( + level int NOT NULL, + node_no int NOT NULL, + status int, + PRIMARY KEY (level, node_no) +) WITHOUT OIDS; +CREATE FUNCTION transition_table_level1_ri_parent_del_func() + RETURNS TRIGGER + LANGUAGE plpgsql +AS $$ + DECLARE n bigint; + BEGIN + PERFORM FROM p JOIN transition_table_level2 c ON c.parent_no = p.level1_no; + IF FOUND THEN + RAISE EXCEPTION 'RI error'; + END IF; + RETURN NULL; + END; +$$; +CREATE TRIGGER transition_table_level1_ri_parent_del_trigger + AFTER DELETE ON transition_table_level1 + REFERENCING OLD TABLE AS p + FOR EACH STATEMENT EXECUTE PROCEDURE + transition_table_level1_ri_parent_del_func(); +CREATE FUNCTION transition_table_level1_ri_parent_upd_func() + RETURNS TRIGGER + LANGUAGE plpgsql +AS $$ + DECLARE + x int; + BEGIN + WITH p AS (SELECT level1_no, sum(delta) cnt + FROM (SELECT level1_no, 1 AS delta FROM i + UNION ALL + SELECT level1_no, -1 AS delta FROM d) w + GROUP BY level1_no + HAVING sum(delta) < 0) + SELECT level1_no + FROM p JOIN transition_table_level2 c ON c.parent_no = p.level1_no + INTO x; + IF FOUND THEN + RAISE EXCEPTION 'RI error'; + END IF; + RETURN NULL; + END; +$$; +CREATE TRIGGER transition_table_level1_ri_parent_upd_trigger + AFTER UPDATE ON transition_table_level1 + REFERENCING OLD TABLE AS d NEW TABLE AS i + FOR EACH STATEMENT EXECUTE PROCEDURE + transition_table_level1_ri_parent_upd_func(); +CREATE FUNCTION transition_table_level2_ri_child_insupd_func() + RETURNS TRIGGER + LANGUAGE plpgsql +AS $$ + BEGIN + PERFORM FROM i + LEFT JOIN transition_table_level1 p + ON p.level1_no IS NOT NULL AND p.level1_no = i.parent_no + WHERE p.level1_no IS NULL; + IF FOUND THEN + RAISE EXCEPTION 'RI error'; + END IF; + RETURN NULL; + END; +$$; +CREATE TRIGGER transition_table_level2_ri_child_insupd_trigger + AFTER INSERT OR UPDATE ON transition_table_level2 + REFERENCING NEW TABLE AS i + FOR EACH STATEMENT EXECUTE PROCEDURE + transition_table_level2_ri_child_insupd_func(); +-- create initial test data +INSERT INTO transition_table_level1 (level1_no) + SELECT generate_series(1,200); +ANALYZE transition_table_level1; +INSERT INTO transition_table_level2 (level2_no, parent_no) + SELECT level2_no, level2_no / 50 + 1 AS parent_no + FROM generate_series(1,9999) level2_no; +ANALYZE transition_table_level2; +INSERT INTO transition_table_status (level, node_no, status) + SELECT 1, level1_no, 0 FROM transition_table_level1; +INSERT INTO transition_table_status (level, node_no, status) + SELECT 2, level2_no, 0 FROM transition_table_level2; +ANALYZE transition_table_status; +INSERT INTO transition_table_level1(level1_no) + SELECT generate_series(201,1000); +ANALYZE transition_table_level1; +-- behave reasonably if someone tries to modify a transition table +CREATE FUNCTION transition_table_level2_bad_usage_func() + RETURNS TRIGGER + LANGUAGE plpgsql +AS $$ + BEGIN + INSERT INTO d VALUES (1000000, 1000000, 'x'); + RETURN NULL; + END; +$$; +CREATE TRIGGER transition_table_level2_bad_usage_trigger + AFTER DELETE ON transition_table_level2 + REFERENCING OLD TABLE AS d + FOR EACH STATEMENT EXECUTE PROCEDURE + transition_table_level2_bad_usage_func(); +DELETE FROM transition_table_level2 + WHERE level2_no BETWEEN 301 AND 305; +ERROR: relation "d" cannot be the target of a modifying statement +CONTEXT: SQL statement "INSERT INTO d VALUES (1000000, 1000000, 'x')" +PL/pgSQL function transition_table_level2_bad_usage_func() line 3 at SQL statement +DROP TRIGGER transition_table_level2_bad_usage_trigger + ON transition_table_level2; +-- attempt modifications which would break RI (should all fail) +DELETE FROM transition_table_level1 + WHERE level1_no = 25; +ERROR: RI error +CONTEXT: PL/pgSQL function transition_table_level1_ri_parent_del_func() line 6 at RAISE +UPDATE transition_table_level1 SET level1_no = -1 + WHERE level1_no = 30; +ERROR: RI error +CONTEXT: PL/pgSQL function transition_table_level1_ri_parent_upd_func() line 15 at RAISE +INSERT INTO transition_table_level2 (level2_no, parent_no) + VALUES (10000, 10000); +ERROR: RI error +CONTEXT: PL/pgSQL function transition_table_level2_ri_child_insupd_func() line 8 at RAISE +UPDATE transition_table_level2 SET parent_no = 2000 + WHERE level2_no = 40; +ERROR: RI error +CONTEXT: PL/pgSQL function transition_table_level2_ri_child_insupd_func() line 8 at RAISE +-- attempt modifications which would not break RI (should all succeed) +DELETE FROM transition_table_level1 + WHERE level1_no BETWEEN 201 AND 1000; +DELETE FROM transition_table_level1 + WHERE level1_no BETWEEN 100000000 AND 100000010; +SELECT count(*) FROM transition_table_level1; + count +------- + 200 +(1 row) + +DELETE FROM transition_table_level2 + WHERE level2_no BETWEEN 211 AND 220; +SELECT count(*) FROM transition_table_level2; + count +------- + 9989 +(1 row) + +CREATE TABLE alter_table_under_transition_tables +( + id int PRIMARY KEY, + name text +); +CREATE FUNCTION alter_table_under_transition_tables_upd_func() + RETURNS TRIGGER + LANGUAGE plpgsql +AS $$ +BEGIN + RAISE WARNING 'old table = %, new table = %', + (SELECT string_agg(id || '=' || name, ',') FROM d), + (SELECT string_agg(id || '=' || name, ',') FROM i); + RAISE NOTICE 'one = %', (SELECT 1 FROM alter_table_under_transition_tables LIMIT 1); + RETURN NULL; +END; +$$; +CREATE TRIGGER alter_table_under_transition_tables_upd_trigger + AFTER UPDATE ON alter_table_under_transition_tables + REFERENCING OLD TABLE AS d NEW TABLE AS i + FOR EACH STATEMENT EXECUTE PROCEDURE + alter_table_under_transition_tables_upd_func(); +INSERT INTO alter_table_under_transition_tables + VALUES (1, '1'), (2, '2'), (3, '3'); +UPDATE alter_table_under_transition_tables + SET name = name || name; +WARNING: old table = 1=1,2=2,3=3, new table = 1=11,2=22,3=33 +NOTICE: one = 1 +-- now change 'name' to an integer to see what happens... +ALTER TABLE alter_table_under_transition_tables + ALTER COLUMN name TYPE int USING name::integer; +UPDATE alter_table_under_transition_tables + SET name = (name::text || name::text)::integer; +WARNING: old table = 1=11,2=22,3=33, new table = 1=1111,2=2222,3=3333 +NOTICE: one = 1 +-- now drop column 'name' +ALTER TABLE alter_table_under_transition_tables + DROP column name; +UPDATE alter_table_under_transition_tables + SET id = id; +ERROR: column "name" does not exist +LINE 1: SELECT (SELECT string_agg(id || '=' || name, ',') FROM d) + ^ +QUERY: SELECT (SELECT string_agg(id || '=' || name, ',') FROM d) +CONTEXT: PL/pgSQL function alter_table_under_transition_tables_upd_func() line 3 at RAISE diff --git a/src/test/regress/expected/triggers.out b/src/test/regress/expected/triggers.out index f408475f33..4b0b3b7c42 100644 --- a/src/test/regress/expected/triggers.out +++ b/src/test/regress/expected/triggers.out @@ -1763,3 +1763,27 @@ select * from upsert; drop table upsert; drop function upsert_before_func(); drop function upsert_after_func(); +-- +-- Verify that triggers are prevented on partitioned tables if they would +-- access row data (ROW and STATEMENT-with-transition-table) +-- +create table my_table (i int) partition by list (i); +create table my_table_42 partition of my_table for values in (42); +create function my_trigger_function() returns trigger as $$ begin end; $$ language plpgsql; +create trigger my_trigger before update on my_table for each row execute procedure my_trigger_function(); +ERROR: "my_table" is a partitioned table +DETAIL: Partitioned tables cannot have ROW triggers. +create trigger my_trigger after update on my_table referencing old table as old_table + for each statement execute procedure my_trigger_function(); +ERROR: "my_table" is a partitioned table +DETAIL: Triggers on partitioned tables cannot have transition tables. +-- +-- Verify that triggers are allowed on partitions +-- +create trigger my_trigger before update on my_table_42 for each row execute procedure my_trigger_function(); +drop trigger my_trigger on my_table_42; +create trigger my_trigger after update on my_table_42 referencing old table as old_table + for each statement execute procedure my_trigger_function(); +drop trigger my_trigger on my_table_42; +drop table my_table_42; +drop table my_table; diff --git a/src/test/regress/sql/plpgsql.sql b/src/test/regress/sql/plpgsql.sql index 31dcbdffdd..628a9d126e 100644 --- a/src/test/regress/sql/plpgsql.sql +++ b/src/test/regress/sql/plpgsql.sql @@ -4475,3 +4475,286 @@ begin v_test := 0 || v_test; -- fail end; $$; + +-- +-- test usage of transition tables in AFTER triggers +-- + +CREATE TABLE transition_table_base (id int PRIMARY KEY, val text); + +CREATE FUNCTION transition_table_base_ins_func() + RETURNS trigger + LANGUAGE plpgsql +AS $$ +DECLARE + t text; + l text; +BEGIN + t = ''; + FOR l IN EXECUTE + $q$ + EXPLAIN (TIMING off, COSTS off, VERBOSE on) + SELECT * FROM newtable + $q$ LOOP + t = t || l || E'\n'; + END LOOP; + + RAISE INFO '%', t; + RETURN new; +END; +$$; + +CREATE TRIGGER transition_table_base_ins_trig + AFTER INSERT ON transition_table_base + REFERENCING OLD TABLE AS oldtable NEW TABLE AS newtable + FOR EACH STATEMENT + EXECUTE PROCEDURE transition_table_base_ins_func(); + +CREATE TRIGGER transition_table_base_ins_trig + AFTER INSERT ON transition_table_base + REFERENCING NEW TABLE AS newtable + FOR EACH STATEMENT + EXECUTE PROCEDURE transition_table_base_ins_func(); + +INSERT INTO transition_table_base VALUES (1, 'One'), (2, 'Two'); +INSERT INTO transition_table_base VALUES (3, 'Three'), (4, 'Four'); + +CREATE OR REPLACE FUNCTION transition_table_base_upd_func() + RETURNS trigger + LANGUAGE plpgsql +AS $$ +DECLARE + t text; + l text; +BEGIN + t = ''; + FOR l IN EXECUTE + $q$ + EXPLAIN (TIMING off, COSTS off, VERBOSE on) + SELECT * FROM oldtable ot FULL JOIN newtable nt USING (id) + $q$ LOOP + t = t || l || E'\n'; + END LOOP; + + RAISE INFO '%', t; + RETURN new; +END; +$$; + +CREATE TRIGGER transition_table_base_upd_trig + AFTER UPDATE ON transition_table_base + REFERENCING OLD TABLE AS oldtable NEW TABLE AS newtable + FOR EACH STATEMENT + EXECUTE PROCEDURE transition_table_base_upd_func(); + +UPDATE transition_table_base + SET val = '*' || val || '*' + WHERE id BETWEEN 2 AND 3; + +CREATE TABLE transition_table_level1 +( + level1_no serial NOT NULL , + level1_node_name varchar(255), + PRIMARY KEY (level1_no) +) WITHOUT OIDS; + +CREATE TABLE transition_table_level2 +( + level2_no serial NOT NULL , + parent_no int NOT NULL, + level1_node_name varchar(255), + PRIMARY KEY (level2_no) +) WITHOUT OIDS; + +CREATE TABLE transition_table_status +( + level int NOT NULL, + node_no int NOT NULL, + status int, + PRIMARY KEY (level, node_no) +) WITHOUT OIDS; + +CREATE FUNCTION transition_table_level1_ri_parent_del_func() + RETURNS TRIGGER + LANGUAGE plpgsql +AS $$ + DECLARE n bigint; + BEGIN + PERFORM FROM p JOIN transition_table_level2 c ON c.parent_no = p.level1_no; + IF FOUND THEN + RAISE EXCEPTION 'RI error'; + END IF; + RETURN NULL; + END; +$$; + +CREATE TRIGGER transition_table_level1_ri_parent_del_trigger + AFTER DELETE ON transition_table_level1 + REFERENCING OLD TABLE AS p + FOR EACH STATEMENT EXECUTE PROCEDURE + transition_table_level1_ri_parent_del_func(); + +CREATE FUNCTION transition_table_level1_ri_parent_upd_func() + RETURNS TRIGGER + LANGUAGE plpgsql +AS $$ + DECLARE + x int; + BEGIN + WITH p AS (SELECT level1_no, sum(delta) cnt + FROM (SELECT level1_no, 1 AS delta FROM i + UNION ALL + SELECT level1_no, -1 AS delta FROM d) w + GROUP BY level1_no + HAVING sum(delta) < 0) + SELECT level1_no + FROM p JOIN transition_table_level2 c ON c.parent_no = p.level1_no + INTO x; + IF FOUND THEN + RAISE EXCEPTION 'RI error'; + END IF; + RETURN NULL; + END; +$$; + +CREATE TRIGGER transition_table_level1_ri_parent_upd_trigger + AFTER UPDATE ON transition_table_level1 + REFERENCING OLD TABLE AS d NEW TABLE AS i + FOR EACH STATEMENT EXECUTE PROCEDURE + transition_table_level1_ri_parent_upd_func(); + +CREATE FUNCTION transition_table_level2_ri_child_insupd_func() + RETURNS TRIGGER + LANGUAGE plpgsql +AS $$ + BEGIN + PERFORM FROM i + LEFT JOIN transition_table_level1 p + ON p.level1_no IS NOT NULL AND p.level1_no = i.parent_no + WHERE p.level1_no IS NULL; + IF FOUND THEN + RAISE EXCEPTION 'RI error'; + END IF; + RETURN NULL; + END; +$$; + +CREATE TRIGGER transition_table_level2_ri_child_insupd_trigger + AFTER INSERT OR UPDATE ON transition_table_level2 + REFERENCING NEW TABLE AS i + FOR EACH STATEMENT EXECUTE PROCEDURE + transition_table_level2_ri_child_insupd_func(); + +-- create initial test data +INSERT INTO transition_table_level1 (level1_no) + SELECT generate_series(1,200); +ANALYZE transition_table_level1; + +INSERT INTO transition_table_level2 (level2_no, parent_no) + SELECT level2_no, level2_no / 50 + 1 AS parent_no + FROM generate_series(1,9999) level2_no; +ANALYZE transition_table_level2; + +INSERT INTO transition_table_status (level, node_no, status) + SELECT 1, level1_no, 0 FROM transition_table_level1; + +INSERT INTO transition_table_status (level, node_no, status) + SELECT 2, level2_no, 0 FROM transition_table_level2; +ANALYZE transition_table_status; + +INSERT INTO transition_table_level1(level1_no) + SELECT generate_series(201,1000); +ANALYZE transition_table_level1; + +-- behave reasonably if someone tries to modify a transition table +CREATE FUNCTION transition_table_level2_bad_usage_func() + RETURNS TRIGGER + LANGUAGE plpgsql +AS $$ + BEGIN + INSERT INTO d VALUES (1000000, 1000000, 'x'); + RETURN NULL; + END; +$$; + +CREATE TRIGGER transition_table_level2_bad_usage_trigger + AFTER DELETE ON transition_table_level2 + REFERENCING OLD TABLE AS d + FOR EACH STATEMENT EXECUTE PROCEDURE + transition_table_level2_bad_usage_func(); + +DELETE FROM transition_table_level2 + WHERE level2_no BETWEEN 301 AND 305; + +DROP TRIGGER transition_table_level2_bad_usage_trigger + ON transition_table_level2; + +-- attempt modifications which would break RI (should all fail) +DELETE FROM transition_table_level1 + WHERE level1_no = 25; + +UPDATE transition_table_level1 SET level1_no = -1 + WHERE level1_no = 30; + +INSERT INTO transition_table_level2 (level2_no, parent_no) + VALUES (10000, 10000); + +UPDATE transition_table_level2 SET parent_no = 2000 + WHERE level2_no = 40; + + +-- attempt modifications which would not break RI (should all succeed) +DELETE FROM transition_table_level1 + WHERE level1_no BETWEEN 201 AND 1000; + +DELETE FROM transition_table_level1 + WHERE level1_no BETWEEN 100000000 AND 100000010; + +SELECT count(*) FROM transition_table_level1; + +DELETE FROM transition_table_level2 + WHERE level2_no BETWEEN 211 AND 220; + +SELECT count(*) FROM transition_table_level2; + +CREATE TABLE alter_table_under_transition_tables +( + id int PRIMARY KEY, + name text +); + +CREATE FUNCTION alter_table_under_transition_tables_upd_func() + RETURNS TRIGGER + LANGUAGE plpgsql +AS $$ +BEGIN + RAISE WARNING 'old table = %, new table = %', + (SELECT string_agg(id || '=' || name, ',') FROM d), + (SELECT string_agg(id || '=' || name, ',') FROM i); + RAISE NOTICE 'one = %', (SELECT 1 FROM alter_table_under_transition_tables LIMIT 1); + RETURN NULL; +END; +$$; + +CREATE TRIGGER alter_table_under_transition_tables_upd_trigger + AFTER UPDATE ON alter_table_under_transition_tables + REFERENCING OLD TABLE AS d NEW TABLE AS i + FOR EACH STATEMENT EXECUTE PROCEDURE + alter_table_under_transition_tables_upd_func(); + +INSERT INTO alter_table_under_transition_tables + VALUES (1, '1'), (2, '2'), (3, '3'); +UPDATE alter_table_under_transition_tables + SET name = name || name; + +-- now change 'name' to an integer to see what happens... +ALTER TABLE alter_table_under_transition_tables + ALTER COLUMN name TYPE int USING name::integer; +UPDATE alter_table_under_transition_tables + SET name = (name::text || name::text)::integer; + +-- now drop column 'name' +ALTER TABLE alter_table_under_transition_tables + DROP column name; +UPDATE alter_table_under_transition_tables + SET id = id; diff --git a/src/test/regress/sql/triggers.sql b/src/test/regress/sql/triggers.sql index b6de1b3256..4473ce0518 100644 --- a/src/test/regress/sql/triggers.sql +++ b/src/test/regress/sql/triggers.sql @@ -1240,3 +1240,26 @@ select * from upsert; drop table upsert; drop function upsert_before_func(); drop function upsert_after_func(); + +-- +-- Verify that triggers are prevented on partitioned tables if they would +-- access row data (ROW and STATEMENT-with-transition-table) +-- + +create table my_table (i int) partition by list (i); +create table my_table_42 partition of my_table for values in (42); +create function my_trigger_function() returns trigger as $$ begin end; $$ language plpgsql; +create trigger my_trigger before update on my_table for each row execute procedure my_trigger_function(); +create trigger my_trigger after update on my_table referencing old table as old_table + for each statement execute procedure my_trigger_function(); + +-- +-- Verify that triggers are allowed on partitions +-- +create trigger my_trigger before update on my_table_42 for each row execute procedure my_trigger_function(); +drop trigger my_trigger on my_table_42; +create trigger my_trigger after update on my_table_42 referencing old table as old_table + for each statement execute procedure my_trigger_function(); +drop trigger my_trigger on my_table_42; +drop table my_table_42; +drop table my_table; -- cgit v1.2.3 From 7526e10224f0792201e99631567bbe44492bbde4 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Sat, 1 Apr 2017 14:00:53 -0300 Subject: BRIN auto-summarization Previously, only VACUUM would cause a page range to get initially summarized by BRIN indexes, which for some use cases takes too much time since the inserts occur. To avoid the delay, have brininsert request a summarization run for the previous range as soon as the first tuple is inserted into the first page of the next range. Autovacuum is in charge of processing these requests, after doing all the regular vacuuming/ analyzing work on tables. This doesn't impose any new tasks on autovacuum, because autovacuum was already in charge of doing summarizations. The only actual effect is to change the timing, i.e. that it occurs earlier. For this reason, we don't go any great lengths to record these requests very robustly; if they are lost because of a server crash or restart, they will happen at a later time anyway. Most of the new code here is in autovacuum, which can now be told about "work items" to process. This can be used for other things such as GIN pending list cleaning, perhaps visibility map bit setting, both of which are currently invoked during vacuum, but do not really depend on vacuum taking place. The requests are at the page range level, a granularity for which we did not have SQL-level access; we only had index-level summarization requests via brin_summarize_new_values(). It seems reasonable to add SQL-level access to range-level summarization too, so add a function brin_summarize_range() to do that. Authors: Álvaro Herrera, based on sketch from Simon Riggs. Reviewed-by: Thomas Munro. Discussion: https://postgr.es/m/20170301045823.vneqdqkmsd4as4ds@alvherre.pgsql --- doc/src/sgml/brin.sgml | 9 +- doc/src/sgml/func.sgml | 10 +- doc/src/sgml/ref/create_index.sgml | 12 +- src/backend/access/brin/brin.c | 124 ++++++++-- src/backend/access/brin/brin_revmap.c | 6 +- src/backend/access/common/reloptions.c | 9 + src/backend/postmaster/autovacuum.c | 430 ++++++++++++++++++++++++++++++++- src/include/access/brin.h | 5 + src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_proc.h | 2 + src/include/postmaster/autovacuum.h | 12 + src/test/regress/expected/brin.out | 48 ++++ src/test/regress/sql/brin.sql | 28 +++ 13 files changed, 672 insertions(+), 25 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/brin.sgml b/doc/src/sgml/brin.sgml index 5bf11dc2d1..5140a38baa 100644 --- a/doc/src/sgml/brin.sgml +++ b/doc/src/sgml/brin.sgml @@ -74,9 +74,14 @@ tuple; those tuples remain unsummarized until a summarization run is invoked later, creating initial summaries. This process can be invoked manually using the - brin_summarize_new_values(regclass) function, - or automatically when VACUUM processes the table. + brin_summarize_range(regclass, bigint) or + brin_summarize_new_values(regclass) functions; + automatically when VACUUM processes the table; + or by automatic summarization executed by autovacuum, as insertions + occur. (This last trigger is disabled by default and can be enabled + with the autosummarize parameter.) + diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 6887eabd0e..25c18d107c 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -19683,6 +19683,13 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); integer summarize page ranges not already summarized + + + brin_summarize_range(index regclass, blockNumber bigint) + + integer + summarize the page range covering the given block, if not already summarized + gin_clean_pending_list(index regclass) @@ -19700,7 +19707,8 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); that are not currently summarized by the index; for any such range it creates a new summary index tuple by scanning the table pages. It returns the number of new page range summaries that were inserted - into the index. + into the index. brin_summarize_range does the same, except + it only summarizes the range that covers the given block number. diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 7163b032b1..83ee7d3f25 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -382,7 +382,7 @@ CREATE [ UNIQUE ] INDEX [ CONCURRENTLY ] [ [ IF NOT EXISTS ] - BRIN indexes accept a different parameter: + BRIN indexes accept different parameters: @@ -396,6 +396,16 @@ CREATE [ UNIQUE ] INDEX [ CONCURRENTLY ] [ [ IF NOT EXISTS ] + + + autosummarize + + + Defines whether a summarization run is invoked for the previous page + range whenever an insertion is detected on the next one. + + + diff --git a/src/backend/access/brin/brin.c b/src/backend/access/brin/brin.c index b22563bf7c..86e73b6242 100644 --- a/src/backend/access/brin/brin.c +++ b/src/backend/access/brin/brin.c @@ -26,6 +26,7 @@ #include "catalog/pg_am.h" #include "miscadmin.h" #include "pgstat.h" +#include "postmaster/autovacuum.h" #include "storage/bufmgr.h" #include "storage/freespace.h" #include "utils/builtins.h" @@ -60,10 +61,12 @@ typedef struct BrinOpaque BrinDesc *bo_bdesc; } BrinOpaque; +#define BRIN_ALL_BLOCKRANGES InvalidBlockNumber + static BrinBuildState *initialize_brin_buildstate(Relation idxRel, BrinRevmap *revmap, BlockNumber pagesPerRange); static void terminate_brin_buildstate(BrinBuildState *state); -static void brinsummarize(Relation index, Relation heapRel, +static void brinsummarize(Relation index, Relation heapRel, BlockNumber pageRange, double *numSummarized, double *numExisting); static void form_and_insert_tuple(BrinBuildState *state); static void union_tuples(BrinDesc *bdesc, BrinMemTuple *a, @@ -126,8 +129,11 @@ brinhandler(PG_FUNCTION_ARGS) * with those of the new tuple. If the tuple values are not consistent with * the summary tuple, we need to update the index tuple. * + * If autosummarization is enabled, check if we need to summarize the previous + * page range. + * * If the range is not currently summarized (i.e. the revmap returns NULL for - * it), there's nothing to do. + * it), there's nothing to do for this tuple. */ bool brininsert(Relation idxRel, Datum *values, bool *nulls, @@ -136,30 +142,59 @@ brininsert(Relation idxRel, Datum *values, bool *nulls, IndexInfo *indexInfo) { BlockNumber pagesPerRange; + BlockNumber origHeapBlk; + BlockNumber heapBlk; BrinDesc *bdesc = (BrinDesc *) indexInfo->ii_AmCache; BrinRevmap *revmap; Buffer buf = InvalidBuffer; MemoryContext tupcxt = NULL; MemoryContext oldcxt = CurrentMemoryContext; + bool autosummarize = BrinGetAutoSummarize(idxRel); revmap = brinRevmapInitialize(idxRel, &pagesPerRange, NULL); + /* + * origHeapBlk is the block number where the insertion occurred. heapBlk + * is the first block in the corresponding page range. + */ + origHeapBlk = ItemPointerGetBlockNumber(heaptid); + heapBlk = (origHeapBlk / pagesPerRange) * pagesPerRange; + for (;;) { bool need_insert = false; OffsetNumber off; BrinTuple *brtup; BrinMemTuple *dtup; - BlockNumber heapBlk; int keyno; CHECK_FOR_INTERRUPTS(); - heapBlk = ItemPointerGetBlockNumber(heaptid); - /* normalize the block number to be the first block in the range */ - heapBlk = (heapBlk / pagesPerRange) * pagesPerRange; - brtup = brinGetTupleForHeapBlock(revmap, heapBlk, &buf, &off, NULL, - BUFFER_LOCK_SHARE, NULL); + /* + * If auto-summarization is enabled and we just inserted the first + * tuple into the first block of a new non-first page range, request a + * summarization run of the previous range. + */ + if (autosummarize && + heapBlk > 0 && + heapBlk == origHeapBlk && + ItemPointerGetOffsetNumber(heaptid) == FirstOffsetNumber) + { + BlockNumber lastPageRange = heapBlk - 1; + BrinTuple *lastPageTuple; + + lastPageTuple = + brinGetTupleForHeapBlock(revmap, lastPageRange, &buf, &off, + NULL, BUFFER_LOCK_SHARE, NULL); + if (!lastPageTuple) + AutoVacuumRequestWork(AVW_BRINSummarizeRange, + RelationGetRelid(idxRel), + lastPageRange); + brin_free_tuple(lastPageTuple); + } + + brtup = brinGetTupleForHeapBlock(revmap, heapBlk, &buf, &off, + NULL, BUFFER_LOCK_SHARE, NULL); /* if range is unsummarized, there's nothing to do */ if (!brtup) @@ -747,7 +782,7 @@ brinvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats) brin_vacuum_scan(info->index, info->strategy); - brinsummarize(info->index, heapRel, + brinsummarize(info->index, heapRel, BRIN_ALL_BLOCKRANGES, &stats->num_index_tuples, &stats->num_index_tuples); heap_close(heapRel, AccessShareLock); @@ -765,7 +800,8 @@ brinoptions(Datum reloptions, bool validate) BrinOptions *rdopts; int numoptions; static const relopt_parse_elt tab[] = { - {"pages_per_range", RELOPT_TYPE_INT, offsetof(BrinOptions, pagesPerRange)} + {"pages_per_range", RELOPT_TYPE_INT, offsetof(BrinOptions, pagesPerRange)}, + {"autosummarize", RELOPT_TYPE_BOOL, offsetof(BrinOptions, autosummarize)} }; options = parseRelOptions(reloptions, validate, RELOPT_KIND_BRIN, @@ -791,13 +827,40 @@ brinoptions(Datum reloptions, bool validate) */ Datum brin_summarize_new_values(PG_FUNCTION_ARGS) +{ + Datum relation = PG_GETARG_DATUM(0); + + return DirectFunctionCall2(brin_summarize_range, + relation, + Int64GetDatum((int64) BRIN_ALL_BLOCKRANGES)); +} + +/* + * SQL-callable function to summarize the indicated page range, if not already + * summarized. If the second argument is BRIN_ALL_BLOCKRANGES, all + * unsummarized ranges are summarized. + */ +Datum +brin_summarize_range(PG_FUNCTION_ARGS) { Oid indexoid = PG_GETARG_OID(0); + int64 heapBlk64 = PG_GETARG_INT64(1); + BlockNumber heapBlk; Oid heapoid; Relation indexRel; Relation heapRel; double numSummarized = 0; + if (heapBlk64 > BRIN_ALL_BLOCKRANGES || heapBlk64 < 0) + { + char *blk = psprintf(INT64_FORMAT, heapBlk64); + + ereport(ERROR, + (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), + errmsg("block number out of range: %s", blk))); + } + heapBlk = (BlockNumber) heapBlk64; + /* * We must lock table before index to avoid deadlocks. However, if the * passed indexoid isn't an index then IndexGetRelation() will fail. @@ -837,7 +900,7 @@ brin_summarize_new_values(PG_FUNCTION_ARGS) RelationGetRelationName(indexRel)))); /* OK, do it */ - brinsummarize(indexRel, heapRel, &numSummarized, NULL); + brinsummarize(indexRel, heapRel, heapBlk, &numSummarized, NULL); relation_close(indexRel, ShareUpdateExclusiveLock); relation_close(heapRel, ShareUpdateExclusiveLock); @@ -1063,17 +1126,17 @@ summarize_range(IndexInfo *indexInfo, BrinBuildState *state, Relation heapRel, } /* - * Scan a complete BRIN index, and summarize each page range that's not already - * summarized. The index and heap must have been locked by caller in at - * least ShareUpdateExclusiveLock mode. + * Summarize page ranges that are not already summarized. If pageRange is + * BRIN_ALL_BLOCKRANGES then the whole table is scanned; otherwise, only the + * page range containing the given heap page number is scanned. * * For each new index tuple inserted, *numSummarized (if not NULL) is * incremented; for each existing tuple, *numExisting (if not NULL) is * incremented. */ static void -brinsummarize(Relation index, Relation heapRel, double *numSummarized, - double *numExisting) +brinsummarize(Relation index, Relation heapRel, BlockNumber pageRange, + double *numSummarized, double *numExisting) { BrinRevmap *revmap; BrinBuildState *state = NULL; @@ -1082,15 +1145,40 @@ brinsummarize(Relation index, Relation heapRel, double *numSummarized, BlockNumber heapBlk; BlockNumber pagesPerRange; Buffer buf; + BlockNumber startBlk; + BlockNumber endBlk; + + /* determine range of pages to process; nothing to do for an empty table */ + heapNumBlocks = RelationGetNumberOfBlocks(heapRel); + if (heapNumBlocks == 0) + return; revmap = brinRevmapInitialize(index, &pagesPerRange, NULL); + if (pageRange == BRIN_ALL_BLOCKRANGES) + { + startBlk = 0; + endBlk = heapNumBlocks; + } + else + { + startBlk = (pageRange / pagesPerRange) * pagesPerRange; + /* Nothing to do if start point is beyond end of table */ + if (startBlk > heapNumBlocks) + { + brinRevmapTerminate(revmap); + return; + } + endBlk = startBlk + pagesPerRange; + if (endBlk > heapNumBlocks) + endBlk = heapNumBlocks; + } + /* * Scan the revmap to find unsummarized items. */ buf = InvalidBuffer; - heapNumBlocks = RelationGetNumberOfBlocks(heapRel); - for (heapBlk = 0; heapBlk < heapNumBlocks; heapBlk += pagesPerRange) + for (heapBlk = startBlk; heapBlk < endBlk; heapBlk += pagesPerRange) { BrinTuple *tup; OffsetNumber off; diff --git a/src/backend/access/brin/brin_revmap.c b/src/backend/access/brin/brin_revmap.c index 0de6999c2b..5d45b48fd9 100644 --- a/src/backend/access/brin/brin_revmap.c +++ b/src/backend/access/brin/brin_revmap.c @@ -205,7 +205,11 @@ brinGetTupleForHeapBlock(BrinRevmap *revmap, BlockNumber heapBlk, /* normalize the heap block number to be the first page in the range */ heapBlk = (heapBlk / revmap->rm_pagesPerRange) * revmap->rm_pagesPerRange; - /* Compute the revmap page number we need */ + /* + * Compute the revmap page number we need. If Invalid is returned (i.e., + * the revmap page hasn't been created yet), the requested page range is + * not summarized. + */ mapBlk = revmap_get_blkno(revmap, heapBlk); if (mapBlk == InvalidBlockNumber) { diff --git a/src/backend/access/common/reloptions.c b/src/backend/access/common/reloptions.c index de7507aa68..6d1f22f049 100644 --- a/src/backend/access/common/reloptions.c +++ b/src/backend/access/common/reloptions.c @@ -92,6 +92,15 @@ static relopt_bool boolRelOpts[] = { + { + { + "autosummarize", + "Enables automatic summarization on this BRIN index", + RELOPT_KIND_BRIN, + AccessExclusiveLock + }, + false + }, { { "autovacuum_enabled", diff --git a/src/backend/postmaster/autovacuum.c b/src/backend/postmaster/autovacuum.c index 33ca749b52..89dd3b321b 100644 --- a/src/backend/postmaster/autovacuum.c +++ b/src/backend/postmaster/autovacuum.c @@ -92,7 +92,9 @@ #include "storage/procsignal.h" #include "storage/sinvaladt.h" #include "tcop/tcopprot.h" +#include "utils/dsa.h" #include "utils/fmgroids.h" +#include "utils/fmgrprotos.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/ps_status.h" @@ -252,9 +254,10 @@ typedef enum * av_runningWorkers the WorkerInfo non-free queue * av_startingWorker pointer to WorkerInfo currently being started (cleared by * the worker itself as soon as it's up and running) + * av_dsa_handle handle for allocatable shared memory * * This struct is protected by AutovacuumLock, except for av_signal and parts - * of the worker list (see above). + * of the worker list (see above). av_dsa_handle is readable unlocked. *------------- */ typedef struct @@ -264,6 +267,8 @@ typedef struct dlist_head av_freeWorkers; dlist_head av_runningWorkers; WorkerInfo av_startingWorker; + dsa_handle av_dsa_handle; + dsa_pointer av_workitems; } AutoVacuumShmemStruct; static AutoVacuumShmemStruct *AutoVacuumShmem; @@ -278,6 +283,32 @@ static MemoryContext DatabaseListCxt = NULL; /* Pointer to my own WorkerInfo, valid on each worker */ static WorkerInfo MyWorkerInfo = NULL; +/* + * Autovacuum workitem array, stored in AutoVacuumShmem->av_workitems. This + * list is mostly protected by AutovacuumLock, except that if it's marked + * 'active' other processes must not modify the work-identifying members, + * though changing the list pointers is okay. + */ +typedef struct AutoVacuumWorkItem +{ + AutoVacuumWorkItemType avw_type; + Oid avw_database; + Oid avw_relation; + BlockNumber avw_blockNumber; + bool avw_active; + dsa_pointer avw_next; /* doubly linked list pointers */ + dsa_pointer avw_prev; +} AutoVacuumWorkItem; + +#define NUM_WORKITEMS 256 +typedef struct +{ + dsa_pointer avs_usedItems; + dsa_pointer avs_freeItems; +} AutovacWorkItems; + +static dsa_area *AutoVacuumDSA = NULL; + /* PID of launcher, valid only in worker while shutting down */ int AutovacuumLauncherPid = 0; @@ -316,11 +347,16 @@ static AutoVacOpts *extract_autovac_opts(HeapTuple tup, static PgStat_StatTabEntry *get_pgstat_tabentry_relid(Oid relid, bool isshared, PgStat_StatDBEntry *shared, PgStat_StatDBEntry *dbentry); +static void perform_work_item(AutoVacuumWorkItem *workitem); static void autovac_report_activity(autovac_table *tab); +static void autovac_report_workitem(AutoVacuumWorkItem *workitem, + const char *nspname, const char *relname); static void av_sighup_handler(SIGNAL_ARGS); static void avl_sigusr2_handler(SIGNAL_ARGS); static void avl_sigterm_handler(SIGNAL_ARGS); static void autovac_refresh_stats(void); +static void remove_wi_from_list(dsa_pointer *list, dsa_pointer wi_ptr); +static void add_wi_to_list(dsa_pointer *list, dsa_pointer wi_ptr); @@ -574,6 +610,28 @@ AutoVacLauncherMain(int argc, char *argv[]) */ rebuild_database_list(InvalidOid); + /* + * Set up our DSA so that backends can install work-item requests. It may + * already exist as created by a previous launcher. + */ + if (!AutoVacuumShmem->av_dsa_handle) + { + LWLockAcquire(AutovacuumLock, LW_EXCLUSIVE); + AutoVacuumDSA = dsa_create(AutovacuumLock->tranche); + /* make sure it doesn't go away even if we do */ + dsa_pin(AutoVacuumDSA); + dsa_pin_mapping(AutoVacuumDSA); + AutoVacuumShmem->av_dsa_handle = dsa_get_handle(AutoVacuumDSA); + /* delay array allocation until first request */ + AutoVacuumShmem->av_workitems = InvalidDsaPointer; + LWLockRelease(AutovacuumLock); + } + else + { + AutoVacuumDSA = dsa_attach(AutoVacuumShmem->av_dsa_handle); + dsa_pin_mapping(AutoVacuumDSA); + } + /* loop until shutdown request */ while (!got_SIGTERM) { @@ -1617,6 +1675,14 @@ AutoVacWorkerMain(int argc, char *argv[]) { char dbname[NAMEDATALEN]; + if (AutoVacuumShmem->av_dsa_handle) + { + /* First use of DSA in this worker, so attach to it */ + Assert(!AutoVacuumDSA); + AutoVacuumDSA = dsa_attach(AutoVacuumShmem->av_dsa_handle); + dsa_pin_mapping(AutoVacuumDSA); + } + /* * Report autovac startup to the stats collector. We deliberately do * this before InitPostgres, so that the last_autovac_time will get @@ -2466,6 +2532,69 @@ deleted: VacuumCostLimit = stdVacuumCostLimit; } + /* + * Perform additional work items, as requested by backends. + */ + if (AutoVacuumShmem->av_workitems) + { + dsa_pointer wi_ptr; + AutovacWorkItems *workitems; + + LWLockAcquire(AutovacuumLock, LW_EXCLUSIVE); + + /* + * Scan the list of pending items, and process the inactive ones in + * our database. + */ + workitems = (AutovacWorkItems *) + dsa_get_address(AutoVacuumDSA, AutoVacuumShmem->av_workitems); + wi_ptr = workitems->avs_usedItems; + + while (wi_ptr != InvalidDsaPointer) + { + AutoVacuumWorkItem *workitem; + + workitem = (AutoVacuumWorkItem *) + dsa_get_address(AutoVacuumDSA, wi_ptr); + + if (workitem->avw_database == MyDatabaseId && !workitem->avw_active) + { + dsa_pointer next_ptr; + + /* claim this one */ + workitem->avw_active = true; + + LWLockRelease(AutovacuumLock); + + perform_work_item(workitem); + + /* + * Check for config changes before acquiring lock for further + * jobs. + */ + CHECK_FOR_INTERRUPTS(); + if (got_SIGHUP) + { + got_SIGHUP = false; + ProcessConfigFile(PGC_SIGHUP); + } + + LWLockAcquire(AutovacuumLock, LW_EXCLUSIVE); + + /* Put the array item back for the next user */ + next_ptr = workitem->avw_next; + remove_wi_from_list(&workitems->avs_usedItems, wi_ptr); + add_wi_to_list(&workitems->avs_freeItems, wi_ptr); + wi_ptr = next_ptr; + } + else + wi_ptr = workitem->avw_next; + } + + /* all done */ + LWLockRelease(AutovacuumLock); + } + /* * We leak table_toast_map here (among other things), but since we're * going away soon, it's not a problem. @@ -2498,6 +2627,103 @@ deleted: CommitTransactionCommand(); } +/* + * Execute a previously registered work item. + */ +static void +perform_work_item(AutoVacuumWorkItem *workitem) +{ + char *cur_datname = NULL; + char *cur_nspname = NULL; + char *cur_relname = NULL; + + /* + * Note we do not store table info in MyWorkerInfo, since this is not + * vacuuming proper. + */ + + /* + * Save the relation name for a possible error message, to avoid a catalog + * lookup in case of an error. If any of these return NULL, then the + * relation has been dropped since last we checked; skip it. Note: they + * must live in a long-lived memory context because we call vacuum and + * analyze in different transactions. + */ + + cur_relname = get_rel_name(workitem->avw_relation); + cur_nspname = get_namespace_name(get_rel_namespace(workitem->avw_relation)); + cur_datname = get_database_name(MyDatabaseId); + if (!cur_relname || !cur_nspname || !cur_datname) + goto deleted2; + + autovac_report_workitem(workitem, cur_nspname, cur_datname); + + /* + * We will abort the current work item if something errors out, and + * continue with the next one; in particular, this happens if we are + * interrupted with SIGINT. Note that this means that the work item list + * can be lossy. + */ + PG_TRY(); + { + /* have at it */ + MemoryContextSwitchTo(TopTransactionContext); + + switch (workitem->avw_type) + { + case AVW_BRINSummarizeRange: + DirectFunctionCall2(brin_summarize_range, + ObjectIdGetDatum(workitem->avw_relation), + Int64GetDatum((int64) workitem->avw_blockNumber)); + break; + default: + elog(WARNING, "unrecognized work item found: type %d", + workitem->avw_type); + break; + } + + /* + * Clear a possible query-cancel signal, to avoid a late reaction to + * an automatically-sent signal because of vacuuming the current table + * (we're done with it, so it would make no sense to cancel at this + * point.) + */ + QueryCancelPending = false; + } + PG_CATCH(); + { + /* + * Abort the transaction, start a new one, and proceed with the next + * table in our list. + */ + HOLD_INTERRUPTS(); + errcontext("processing work entry for relation \"%s.%s.%s\"", + cur_datname, cur_nspname, cur_relname); + EmitErrorReport(); + + /* this resets the PGXACT flags too */ + AbortOutOfAnyTransaction(); + FlushErrorState(); + MemoryContextResetAndDeleteChildren(PortalContext); + + /* restart our transaction for the following operations */ + StartTransactionCommand(); + RESUME_INTERRUPTS(); + } + PG_END_TRY(); + + /* We intentionally do not set did_vacuum here */ + + /* be tidy */ +deleted2: + if (cur_datname) + pfree(cur_datname); + if (cur_nspname) + pfree(cur_nspname); + if (cur_relname) + pfree(cur_relname); +} + /* * extract_autovac_opts * @@ -2945,6 +3171,45 @@ autovac_report_activity(autovac_table *tab) pgstat_report_activity(STATE_RUNNING, activity); } +/* + * autovac_report_workitem + * Report to pgstat that autovacuum is processing a work item + */ +static void +autovac_report_workitem(AutoVacuumWorkItem *workitem, + const char *nspname, const char *relname) +{ + char activity[MAX_AUTOVAC_ACTIV_LEN + 12 + 2]; + char blk[12 + 2]; + int len; + + switch (workitem->avw_type) + { + case AVW_BRINSummarizeRange: + snprintf(activity, MAX_AUTOVAC_ACTIV_LEN, + "autovacuum: BRIN summarize"); + break; + } + + /* + * Report the qualified name of the relation, and the block number if any + */ + len = strlen(activity); + + if (BlockNumberIsValid(workitem->avw_blockNumber)) + snprintf(blk, sizeof(blk), " %u", workitem->avw_blockNumber); + else + blk[0] = '\0'; + + snprintf(activity + len, MAX_AUTOVAC_ACTIV_LEN - len, + " %s.%s%s", nspname, relname, blk); + + /* Set statement_timestamp() to current time for pg_stat_activity */ + SetCurrentStatementStartTimestamp(); + + pgstat_report_activity(STATE_RUNNING, activity); +} + /* * AutoVacuumingActive * Check GUC vars and report whether the autovacuum process should be @@ -2958,6 +3223,113 @@ AutoVacuumingActive(void) return true; } +/* + * Request one work item to the next autovacuum run processing our database. + */ +void +AutoVacuumRequestWork(AutoVacuumWorkItemType type, Oid relationId, + BlockNumber blkno) +{ + AutovacWorkItems *workitems; + dsa_pointer wi_ptr; + AutoVacuumWorkItem *workitem; + + LWLockAcquire(AutovacuumLock, LW_EXCLUSIVE); + + /* + * It may be useful to de-duplicate the list upon insertion. For the only + * currently existing caller, this is not necessary. + */ + + /* First use in this process? Set up DSA */ + if (!AutoVacuumDSA) + { + if (!AutoVacuumShmem->av_dsa_handle) + { + /* autovacuum launcher not started; nothing can be done */ + LWLockRelease(AutovacuumLock); + return; + } + AutoVacuumDSA = dsa_attach(AutoVacuumShmem->av_dsa_handle); + dsa_pin_mapping(AutoVacuumDSA); + } + + /* First use overall? Allocate work items array */ + if (AutoVacuumShmem->av_workitems == InvalidDsaPointer) + { + int i; + AutovacWorkItems *workitems; + + AutoVacuumShmem->av_workitems = + dsa_allocate_extended(AutoVacuumDSA, + sizeof(AutovacWorkItems) + + NUM_WORKITEMS * sizeof(AutoVacuumWorkItem), + DSA_ALLOC_NO_OOM); + /* if out of memory, silently disregard the request */ + if (AutoVacuumShmem->av_workitems == InvalidDsaPointer) + { + LWLockRelease(AutovacuumLock); + dsa_detach(AutoVacuumDSA); + AutoVacuumDSA = NULL; + return; + } + + /* Initialize each array entry as a member of the free list */ + workitems = dsa_get_address(AutoVacuumDSA, AutoVacuumShmem->av_workitems); + + workitems->avs_usedItems = InvalidDsaPointer; + workitems->avs_freeItems = InvalidDsaPointer; + for (i = 0; i < NUM_WORKITEMS; i++) + { + /* XXX surely there is a simpler way to do this */ + wi_ptr = AutoVacuumShmem->av_workitems + sizeof(AutovacWorkItems) + + sizeof(AutoVacuumWorkItem) * i; + workitem = (AutoVacuumWorkItem *) dsa_get_address(AutoVacuumDSA, wi_ptr); + + workitem->avw_type = 0; + workitem->avw_database = InvalidOid; + workitem->avw_relation = InvalidOid; + workitem->avw_active = false; + + /* put this item in the free list */ + workitem->avw_next = workitems->avs_freeItems; + workitems->avs_freeItems = wi_ptr; + } + } + + workitems = (AutovacWorkItems *) + dsa_get_address(AutoVacuumDSA, AutoVacuumShmem->av_workitems); + + /* If array is full, disregard the request */ + if (workitems->avs_freeItems == InvalidDsaPointer) + { + LWLockRelease(AutovacuumLock); + dsa_detach(AutoVacuumDSA); + AutoVacuumDSA = NULL; + return; + } + + /* remove workitem struct from free list ... */ + wi_ptr = workitems->avs_freeItems; + remove_wi_from_list(&workitems->avs_freeItems, wi_ptr); + + /* ... initialize it ... */ + workitem = dsa_get_address(AutoVacuumDSA, wi_ptr); + workitem->avw_type = type; + workitem->avw_database = MyDatabaseId; + workitem->avw_relation = relationId; + workitem->avw_blockNumber = blkno; + workitem->avw_active = false; + + /* ... and put it on autovacuum's to-do list */ + add_wi_to_list(&workitems->avs_usedItems, wi_ptr); + + LWLockRelease(AutovacuumLock); + + dsa_detach(AutoVacuumDSA); + AutoVacuumDSA = NULL; +} + /* * autovac_init * This is called at postmaster initialization. @@ -3079,3 +3451,59 @@ autovac_refresh_stats(void) pgstat_clear_snapshot(); } + +/* + * Simplistic open-coded list implementation for objects stored in DSA. + * Each item is doubly linked, but we have no tail pointer, and the "prev" + * element of the first item is null, not the list. + */ + +/* + * Remove a work item from the given list. + */ +static void +remove_wi_from_list(dsa_pointer *list, dsa_pointer wi_ptr) +{ + AutoVacuumWorkItem *workitem = dsa_get_address(AutoVacuumDSA, wi_ptr); + dsa_pointer next = workitem->avw_next; + dsa_pointer prev = workitem->avw_prev; + + workitem->avw_next = workitem->avw_prev = InvalidDsaPointer; + + if (next != InvalidDsaPointer) + { + workitem = dsa_get_address(AutoVacuumDSA, next); + workitem->avw_prev = prev; + } + + if (prev != InvalidDsaPointer) + { + workitem = dsa_get_address(AutoVacuumDSA, prev); + workitem->avw_next = next; + } + else + *list = next; +} + +/* + * Add a workitem to the given list + */ +static void +add_wi_to_list(dsa_pointer *list, dsa_pointer wi_ptr) +{ + if (*list == InvalidDsaPointer) + { + /* list is empty; item is now singleton */ + *list = wi_ptr; + } + else + { + AutoVacuumWorkItem *workitem = dsa_get_address(AutoVacuumDSA, wi_ptr); + AutoVacuumWorkItem *old = dsa_get_address(AutoVacuumDSA, *list); + + /* Put item at head of list */ + workitem->avw_next = *list; + old->avw_prev = wi_ptr; + *list = wi_ptr; + } +} diff --git a/src/include/access/brin.h b/src/include/access/brin.h index 896824a0cf..3f4c29bdcb 100644 --- a/src/include/access/brin.h +++ b/src/include/access/brin.h @@ -22,6 +22,7 @@ typedef struct BrinOptions { int32 vl_len_; /* varlena header (do not touch directly!) */ BlockNumber pagesPerRange; + bool autosummarize; } BrinOptions; #define BRIN_DEFAULT_PAGES_PER_RANGE 128 @@ -29,5 +30,9 @@ typedef struct BrinOptions ((relation)->rd_options ? \ ((BrinOptions *) (relation)->rd_options)->pagesPerRange : \ BRIN_DEFAULT_PAGES_PER_RANGE) +#define BrinGetAutoSummarize(relation) \ + ((relation)->rd_options ? \ + ((BrinOptions *) (relation)->rd_options)->autosummarize : \ + false) #endif /* BRIN_H */ diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 48c5a570a0..fa3dcacd32 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201703312 +#define CATALOG_VERSION_NO 201704011 #endif diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index 1132a6052e..1b7ab2a997 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -564,6 +564,8 @@ DATA(insert OID = 335 ( brinhandler PGNSP PGUID 12 1 0 0 0 f f f f t f v s 1 0 DESCR("brin index access method handler"); DATA(insert OID = 3952 ( brin_summarize_new_values PGNSP PGUID 12 1 0 0 0 f f f f t f v s 1 0 23 "2205" _null_ _null_ _null_ _null_ _null_ brin_summarize_new_values _null_ _null_ _null_ )); DESCR("brin: standalone scan new table pages"); +DATA(insert OID = 3999 ( brin_summarize_range PGNSP PGUID 12 1 0 0 0 f f f f t f v s 2 0 23 "2205 20" _null_ _null_ _null_ _null_ _null_ brin_summarize_range _null_ _null_ _null_ )); +DESCR("brin: standalone scan new table pages"); DATA(insert OID = 338 ( amvalidate PGNSP PGUID 12 1 0 0 0 f f f f t f v s 1 0 16 "26" _null_ _null_ _null_ _null_ _null_ amvalidate _null_ _null_ _null_ )); DESCR("validate an operator class"); diff --git a/src/include/postmaster/autovacuum.h b/src/include/postmaster/autovacuum.h index 99d7f09ef9..174e91a64a 100644 --- a/src/include/postmaster/autovacuum.h +++ b/src/include/postmaster/autovacuum.h @@ -14,6 +14,15 @@ #ifndef AUTOVACUUM_H #define AUTOVACUUM_H +/* + * Other processes can request specific work from autovacuum, identified by + * AutoVacuumWorkItem elements. + */ +typedef enum +{ + AVW_BRINSummarizeRange +} AutoVacuumWorkItemType; + /* GUC variables */ extern bool autovacuum_start_daemon; @@ -60,6 +69,9 @@ extern void AutovacuumWorkerIAm(void); extern void AutovacuumLauncherIAm(void); #endif +extern void AutoVacuumRequestWork(AutoVacuumWorkItemType type, + Oid relationId, BlockNumber blkno); + /* shared memory stuff */ extern Size AutoVacuumShmemSize(void); extern void AutoVacuumShmemInit(void); diff --git a/src/test/regress/expected/brin.out b/src/test/regress/expected/brin.out index f0008dd31b..3e0a0381ed 100644 --- a/src/test/regress/expected/brin.out +++ b/src/test/regress/expected/brin.out @@ -406,3 +406,51 @@ SELECT brin_summarize_new_values('brinidx'); -- ok, no change expected 0 (1 row) +-- Test brin_summarize_range +CREATE TABLE brin_summarize ( + value int +) WITH (fillfactor=10, autovacuum_enabled=false); +CREATE INDEX brin_summarize_idx ON brin_summarize USING brin (value) WITH (pages_per_range=2); +-- Fill a few pages +DO $$ +DECLARE curtid tid; +BEGIN + LOOP + INSERT INTO brin_summarize VALUES (1) RETURNING ctid INTO curtid; + EXIT WHEN curtid > tid '(2, 0)'; + END LOOP; +END; +$$; +-- summarize one range +SELECT brin_summarize_range('brin_summarize_idx', 0); + brin_summarize_range +---------------------- + 1 +(1 row) + +-- nothing: already summarized +SELECT brin_summarize_range('brin_summarize_idx', 1); + brin_summarize_range +---------------------- + 0 +(1 row) + +-- summarize one range +SELECT brin_summarize_range('brin_summarize_idx', 2); + brin_summarize_range +---------------------- + 1 +(1 row) + +-- nothing: page doesn't exist in table +SELECT brin_summarize_range('brin_summarize_idx', 4294967295); + brin_summarize_range +---------------------- + 0 +(1 row) + +-- invalid block number values +SELECT brin_summarize_range('brin_summarize_idx', -1); +ERROR: block number out of range: -1 +SELECT brin_summarize_range('brin_summarize_idx', 4294967296); +ERROR: block number out of range: 4294967296 diff --git a/src/test/regress/sql/brin.sql b/src/test/regress/sql/brin.sql index 5bf53873f7..da73df3659 100644 --- a/src/test/regress/sql/brin.sql +++ b/src/test/regress/sql/brin.sql @@ -409,3 +409,31 @@ UPDATE brintest SET textcol = '' WHERE textcol IS NOT NULL; SELECT brin_summarize_new_values('brintest'); -- error, not an index SELECT brin_summarize_new_values('tenk1_unique1'); -- error, not a BRIN index SELECT brin_summarize_new_values('brinidx'); -- ok, no change expected + +-- Test brin_summarize_range +CREATE TABLE brin_summarize ( + value int +) WITH (fillfactor=10, autovacuum_enabled=false); +CREATE INDEX brin_summarize_idx ON brin_summarize USING brin (value) WITH (pages_per_range=2); +-- Fill a few pages +DO $$ +DECLARE curtid tid; +BEGIN + LOOP + INSERT INTO brin_summarize VALUES (1) RETURNING ctid INTO curtid; + EXIT WHEN curtid > tid '(2, 0)'; + END LOOP; +END; +$$; + +-- summarize one range +SELECT brin_summarize_range('brin_summarize_idx', 0); +-- nothing: already summarized +SELECT brin_summarize_range('brin_summarize_idx', 1); +-- summarize one range +SELECT brin_summarize_range('brin_summarize_idx', 2); +-- nothing: page doesn't exist in table +SELECT brin_summarize_range('brin_summarize_idx', 4294967295); +-- invalid block number values +SELECT brin_summarize_range('brin_summarize_idx', -1); +SELECT brin_summarize_range('brin_summarize_idx', 4294967296); -- cgit v1.2.3 From c655899ba9ae2a0d24e99c797167c33e0cfa0820 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Sat, 1 Apr 2017 16:10:04 -0300 Subject: BRIN de-summarization When the BRIN summary tuple for a page range becomes too "wide" for the values actually stored in the table (because the tuples that were present originally are no longer present due to updates or deletes), it can be useful to remove the outdated summary tuple, so that a future summarization can install a tighter summary. This commit introduces a SQL-callable interface to do so. Author: Álvaro Herrera Reviewed-by: Eiji Seki Discussion: https://postgr.es/m/20170228045643.n2ri74ara4fhhfxf@alvherre.pgsql --- doc/src/sgml/brin.sgml | 4 + doc/src/sgml/func.sgml | 15 ++++ src/backend/access/brin/brin.c | 74 +++++++++++++++++ src/backend/access/brin/brin_revmap.c | 140 ++++++++++++++++++++++++++++++++- src/backend/access/brin/brin_xlog.c | 43 ++++++++++ src/backend/access/rmgrdesc/brindesc.c | 10 +++ src/include/access/brin_revmap.h | 1 + src/include/access/brin_xlog.h | 20 ++++- src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_proc.h | 2 + src/test/regress/expected/brin.out | 27 +++++++ src/test/regress/sql/brin.sql | 7 ++ 12 files changed, 340 insertions(+), 5 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/brin.sgml b/doc/src/sgml/brin.sgml index 5140a38baa..ad11109775 100644 --- a/doc/src/sgml/brin.sgml +++ b/doc/src/sgml/brin.sgml @@ -80,6 +80,10 @@ or by automatic summarization executed by autovacuum, as insertions occur. (This last trigger is disabled by default and can be enabled with the autosummarize parameter.) + Conversely, a range can be de-summarized using the + brin_desummarize_range(regclass, bigint) range, + which is useful when the index tuple is no longer a very good + representation because the existing values have changed. diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 25c18d107c..19329dd103 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -19660,6 +19660,14 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); gin_clean_pending_list + + brin_summarize_range + + + + brin_desummarize_range + + shows the functions available for index maintenance tasks. @@ -19690,6 +19698,13 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); integer summarize the page range covering the given block, if not already summarized + + + brin_desummarize_range(index regclass, blockNumber bigint) + + integer + de-summarize the page range covering the given block, if summarized + gin_clean_pending_list(index regclass) diff --git a/src/backend/access/brin/brin.c b/src/backend/access/brin/brin.c index 86e73b6242..649f3488c2 100644 --- a/src/backend/access/brin/brin.c +++ b/src/backend/access/brin/brin.c @@ -908,6 +908,80 @@ brin_summarize_range(PG_FUNCTION_ARGS) PG_RETURN_INT32((int32) numSummarized); } +/* + * SQL-callable interface to mark a range as no longer summarized + */ +Datum +brin_desummarize_range(PG_FUNCTION_ARGS) +{ + Oid indexoid = PG_GETARG_OID(0); + int64 heapBlk64 = PG_GETARG_INT64(1); + BlockNumber heapBlk; + Oid heapoid; + Relation heapRel; + Relation indexRel; + bool done; + + if (heapBlk64 > MaxBlockNumber || heapBlk64 < 0) + { + char *blk = psprintf(INT64_FORMAT, heapBlk64); + + ereport(ERROR, + (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), + errmsg("block number out of range: %s", blk))); + } + heapBlk = (BlockNumber) heapBlk64; + + /* + * We must lock table before index to avoid deadlocks. However, if the + * passed indexoid isn't an index then IndexGetRelation() will fail. + * Rather than emitting a not-very-helpful error message, postpone + * complaining, expecting that the is-it-an-index test below will fail. + */ + heapoid = IndexGetRelation(indexoid, true); + if (OidIsValid(heapoid)) + heapRel = heap_open(heapoid, ShareUpdateExclusiveLock); + else + heapRel = NULL; + + indexRel = index_open(indexoid, ShareUpdateExclusiveLock); + + /* Must be a BRIN index */ + if (indexRel->rd_rel->relkind != RELKIND_INDEX || + indexRel->rd_rel->relam != BRIN_AM_OID) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("\"%s\" is not a BRIN index", + RelationGetRelationName(indexRel)))); + + /* User must own the index (comparable to privileges needed for VACUUM) */ + if (!pg_class_ownercheck(indexoid, GetUserId())) + aclcheck_error(ACLCHECK_NOT_OWNER, ACL_KIND_CLASS, + RelationGetRelationName(indexRel)); + + /* + * Since we did the IndexGetRelation call above without any lock, it's + * barely possible that a race against an index drop/recreation could have + * netted us the wrong table. Recheck. + */ + if (heapRel == NULL || heapoid != IndexGetRelation(indexoid, false)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_TABLE), + errmsg("could not open parent table of index %s", + RelationGetRelationName(indexRel)))); + + /* the revmap does the hard work */ + do { + done = brinRevmapDesummarizeRange(indexRel, heapBlk); + } + while (!done); + + relation_close(indexRel, ShareUpdateExclusiveLock); + relation_close(heapRel, ShareUpdateExclusiveLock); + + PG_RETURN_VOID(); +} + /* * Build a BrinDesc used to create or scan a BRIN index */ diff --git a/src/backend/access/brin/brin_revmap.c b/src/backend/access/brin/brin_revmap.c index 5d45b48fd9..35e53a2bac 100644 --- a/src/backend/access/brin/brin_revmap.c +++ b/src/backend/access/brin/brin_revmap.c @@ -168,9 +168,12 @@ brinSetHeapBlockItemptr(Buffer buf, BlockNumber pagesPerRange, iptr = (ItemPointerData *) contents->rm_tids; iptr += HEAPBLK_TO_REVMAP_INDEX(pagesPerRange, heapBlk); - ItemPointerSet(iptr, - ItemPointerGetBlockNumber(&tid), - ItemPointerGetOffsetNumber(&tid)); + if (ItemPointerIsValid(&tid)) + ItemPointerSet(iptr, + ItemPointerGetBlockNumber(&tid), + ItemPointerGetOffsetNumber(&tid)); + else + ItemPointerSetInvalid(iptr); } /* @@ -304,6 +307,137 @@ brinGetTupleForHeapBlock(BrinRevmap *revmap, BlockNumber heapBlk, return NULL; } +/* + * Delete an index tuple, marking a page range as unsummarized. + * + * Index must be locked in ShareUpdateExclusiveLock mode. + * + * Return FALSE if caller should retry. + */ +bool +brinRevmapDesummarizeRange(Relation idxrel, BlockNumber heapBlk) +{ + BrinRevmap *revmap; + BlockNumber pagesPerRange; + RevmapContents *contents; + ItemPointerData *iptr; + ItemPointerData invalidIptr; + BlockNumber revmapBlk; + Buffer revmapBuf; + Buffer regBuf; + Page revmapPg; + Page regPg; + OffsetNumber revmapOffset; + OffsetNumber regOffset; + ItemId lp; + BrinTuple *tup; + + revmap = brinRevmapInitialize(idxrel, &pagesPerRange, NULL); + + revmapBlk = revmap_get_blkno(revmap, heapBlk); + if (!BlockNumberIsValid(revmapBlk)) + { + /* revmap page doesn't exist: range not summarized, we're done */ + brinRevmapTerminate(revmap); + return true; + } + + /* Lock the revmap page, obtain the index tuple pointer from it */ + revmapBuf = brinLockRevmapPageForUpdate(revmap, heapBlk); + revmapPg = BufferGetPage(revmapBuf); + revmapOffset = HEAPBLK_TO_REVMAP_INDEX(revmap->rm_pagesPerRange, heapBlk); + + contents = (RevmapContents *) PageGetContents(revmapPg); + iptr = contents->rm_tids; + iptr += revmapOffset; + + if (!ItemPointerIsValid(iptr)) + { + /* no index tuple: range not summarized, we're done */ + LockBuffer(revmapBuf, BUFFER_LOCK_UNLOCK); + brinRevmapTerminate(revmap); + return true; + } + + regBuf = ReadBuffer(idxrel, ItemPointerGetBlockNumber(iptr)); + LockBuffer(regBuf, BUFFER_LOCK_EXCLUSIVE); + regPg = BufferGetPage(regBuf); + + /* if this is no longer a regular page, tell caller to start over */ + if (!BRIN_IS_REGULAR_PAGE(regPg)) + { + LockBuffer(revmapBuf, BUFFER_LOCK_UNLOCK); + LockBuffer(regBuf, BUFFER_LOCK_UNLOCK); + brinRevmapTerminate(revmap); + return false; + } + + regOffset = ItemPointerGetOffsetNumber(iptr); + if (regOffset > PageGetMaxOffsetNumber(regPg)) + ereport(ERROR, + (errcode(ERRCODE_INDEX_CORRUPTED), + errmsg("corrupted BRIN index: inconsistent range map"))); + + lp = PageGetItemId(regPg, regOffset); + if (!ItemIdIsUsed(lp)) + ereport(ERROR, + (errcode(ERRCODE_INDEX_CORRUPTED), + errmsg("corrupted BRIN index: inconsistent range map"))); + tup = (BrinTuple *) PageGetItem(regPg, lp); + /* XXX apply sanity checks? Might as well delete a bogus tuple ... */ + + /* + * We're only removing data, not reading it, so there's no need to + * TestForOldSnapshot here. + */ + + /* + * Because of SUE lock, this function shouldn't run concurrently with + * summarization. Placeholder tuples can only exist as leftovers from + * crashed summarization, so if we detect any, we complain but proceed. + */ + if (BrinTupleIsPlaceholder(tup)) + ereport(WARNING, + (errmsg("leftover placeholder tuple detected in BRIN index \"%s\", deleting", + RelationGetRelationName(idxrel)))); + + START_CRIT_SECTION(); + + ItemPointerSetInvalid(&invalidIptr); + brinSetHeapBlockItemptr(revmapBuf, revmap->rm_pagesPerRange, heapBlk, + invalidIptr); + PageIndexTupleDeleteNoCompact(regPg, regOffset); + /* XXX record free space in FSM? */ + + MarkBufferDirty(regBuf); + MarkBufferDirty(revmapBuf); + + if (RelationNeedsWAL(idxrel)) + { + xl_brin_desummarize xlrec; + XLogRecPtr recptr; + + xlrec.heapBlk = heapBlk; + xlrec.regOffset = regOffset; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, SizeOfBrinDesummarize); + XLogRegisterBuffer(0, revmapBuf, 0); + XLogRegisterBuffer(1, regBuf, REGBUF_STANDARD); + recptr = XLogInsert(RM_BRIN_ID, XLOG_BRIN_DESUMMARIZE); + PageSetLSN(revmapPg, recptr); + PageSetLSN(regPg, recptr); + } + + END_CRIT_SECTION(); + + UnlockReleaseBuffer(regBuf); + LockBuffer(revmapBuf, BUFFER_LOCK_UNLOCK); + brinRevmapTerminate(revmap); + + return true; +} + /* * Given a heap block number, find the corresponding physical revmap block * number and return it. If the revmap page hasn't been allocated yet, return diff --git a/src/backend/access/brin/brin_xlog.c b/src/backend/access/brin/brin_xlog.c index f416bacc3f..8f5b5ceb3f 100644 --- a/src/backend/access/brin/brin_xlog.c +++ b/src/backend/access/brin/brin_xlog.c @@ -254,6 +254,46 @@ brin_xlog_revmap_extend(XLogReaderState *record) UnlockReleaseBuffer(metabuf); } +static void +brin_xlog_desummarize_page(XLogReaderState *record) +{ + XLogRecPtr lsn = record->EndRecPtr; + xl_brin_desummarize *xlrec; + Buffer buffer; + XLogRedoAction action; + + xlrec = (xl_brin_desummarize *) XLogRecGetData(record); + + /* Update the revmap */ + action = XLogReadBufferForRedo(record, 0, &buffer); + if (action == BLK_NEEDS_REDO) + { + ItemPointerData iptr; + + ItemPointerSetInvalid(&iptr); + brinSetHeapBlockItemptr(buffer, xlrec->pagesPerRange, xlrec->heapBlk, iptr); + + PageSetLSN(BufferGetPage(buffer), lsn); + MarkBufferDirty(buffer); + } + if (BufferIsValid(buffer)) + UnlockReleaseBuffer(buffer); + + /* remove the leftover entry from the regular page */ + action = XLogReadBufferForRedo(record, 1, &buffer); + if (action == BLK_NEEDS_REDO) + { + Page regPg = BufferGetPage(buffer); + + PageIndexTupleDeleteNoCompact(regPg, xlrec->regOffset); + + PageSetLSN(regPg, lsn); + MarkBufferDirty(buffer); + } + if (BufferIsValid(buffer)) + UnlockReleaseBuffer(buffer); +} + void brin_redo(XLogReaderState *record) { @@ -276,6 +316,9 @@ brin_redo(XLogReaderState *record) case XLOG_BRIN_REVMAP_EXTEND: brin_xlog_revmap_extend(record); break; + case XLOG_BRIN_DESUMMARIZE: + brin_xlog_desummarize_page(record); + break; default: elog(PANIC, "brin_redo: unknown op code %u", info); } diff --git a/src/backend/access/rmgrdesc/brindesc.c b/src/backend/access/rmgrdesc/brindesc.c index b58cb5bde9..8eb5275a8b 100644 --- a/src/backend/access/rmgrdesc/brindesc.c +++ b/src/backend/access/rmgrdesc/brindesc.c @@ -61,6 +61,13 @@ brin_desc(StringInfo buf, XLogReaderState *record) appendStringInfo(buf, "targetBlk %u", xlrec->targetBlk); } + else if (info == XLOG_BRIN_DESUMMARIZE) + { + xl_brin_desummarize *xlrec = (xl_brin_desummarize *) rec; + + appendStringInfo(buf, "pagesPerRange %u, heapBlk %u, page offset %u", + xlrec->pagesPerRange, xlrec->heapBlk, xlrec->regOffset); + } } const char * @@ -91,6 +98,9 @@ brin_identify(uint8 info) case XLOG_BRIN_REVMAP_EXTEND: id = "REVMAP_EXTEND"; break; + case XLOG_BRIN_DESUMMARIZE: + id = "DESUMMARIZE"; + break; } return id; diff --git a/src/include/access/brin_revmap.h b/src/include/access/brin_revmap.h index 2ec4169f6d..7fdcf877f4 100644 --- a/src/include/access/brin_revmap.h +++ b/src/include/access/brin_revmap.h @@ -36,5 +36,6 @@ extern void brinSetHeapBlockItemptr(Buffer rmbuf, BlockNumber pagesPerRange, extern BrinTuple *brinGetTupleForHeapBlock(BrinRevmap *revmap, BlockNumber heapBlk, Buffer *buf, OffsetNumber *off, Size *size, int mode, Snapshot snapshot); +extern bool brinRevmapDesummarizeRange(Relation idxrel, BlockNumber heapBlk); #endif /* BRIN_REVMAP_H */ diff --git a/src/include/access/brin_xlog.h b/src/include/access/brin_xlog.h index 33ceb34ea5..89ed334a01 100644 --- a/src/include/access/brin_xlog.h +++ b/src/include/access/brin_xlog.h @@ -33,7 +33,7 @@ #define XLOG_BRIN_UPDATE 0x20 #define XLOG_BRIN_SAMEPAGE_UPDATE 0x30 #define XLOG_BRIN_REVMAP_EXTEND 0x40 -#define XLOG_BRIN_REVMAP_VACUUM 0x50 +#define XLOG_BRIN_DESUMMARIZE 0x50 #define XLOG_BRIN_OPMASK 0x70 /* @@ -124,6 +124,24 @@ typedef struct xl_brin_revmap_extend #define SizeOfBrinRevmapExtend (offsetof(xl_brin_revmap_extend, targetBlk) + \ sizeof(BlockNumber)) +/* + * This is what we need to know about a range de-summarization + * + * Backup block 0: revmap page + * Backup block 1: regular page + */ +typedef struct xl_brin_desummarize +{ + BlockNumber pagesPerRange; + /* page number location to set to invalid */ + OffsetNumber heapBlk; + /* offset of item to delete in regular index page */ + OffsetNumber regOffset; +} xl_brin_desummarize; + +#define SizeOfBrinDesummarize (offsetof(xl_brin_desummarize, regOffset) + \ + sizeof(OffsetNumber)) + extern void brin_redo(XLogReaderState *record); extern void brin_desc(StringInfo buf, XLogReaderState *record); diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index fa3dcacd32..1db7a4d715 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201704011 +#define CATALOG_VERSION_NO 201704012 #endif diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index 1b7ab2a997..711211d2e6 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -566,6 +566,8 @@ DATA(insert OID = 3952 ( brin_summarize_new_values PGNSP PGUID 12 1 0 0 0 f f f DESCR("brin: standalone scan new table pages"); DATA(insert OID = 3999 ( brin_summarize_range PGNSP PGUID 12 1 0 0 0 f f f f t f v s 2 0 23 "2205 20" _null_ _null_ _null_ _null_ _null_ brin_summarize_range _null_ _null_ _null_ )); DESCR("brin: standalone scan new table pages"); +DATA(insert OID = 4014 ( brin_desummarize_range PGNSP PGUID 12 1 0 0 0 f f f f t f v s 2 0 2278 "2205 20" _null_ _null_ _null_ _null_ _null_ brin_desummarize_range _null_ _null_ _null_ )); +DESCR("brin: desummarize page range"); DATA(insert OID = 338 ( amvalidate PGNSP PGUID 12 1 0 0 0 f f f f t f v s 1 0 16 "26" _null_ _null_ _null_ _null_ _null_ amvalidate _null_ _null_ _null_ )); DESCR("validate an operator class"); diff --git a/src/test/regress/expected/brin.out b/src/test/regress/expected/brin.out index 3b9c0db833..a40f87aea0 100644 --- a/src/test/regress/expected/brin.out +++ b/src/test/regress/expected/brin.out @@ -392,6 +392,12 @@ INSERT INTO brintest SELECT format('%s/%s%s', odd, even, tenthous)::pg_lsn, box(point(odd, even), point(thousand, twothousand)) FROM tenk1 ORDER BY unique2 LIMIT 5 OFFSET 5; +SELECT brin_desummarize_range('brinidx', 0); + brin_desummarize_range +------------------------ + +(1 row) + VACUUM brintest; -- force a summarization cycle in brinidx UPDATE brintest SET int8col = int8col * int4col; UPDATE brintest SET textcol = '' WHERE textcol IS NOT NULL; @@ -406,6 +412,27 @@ SELECT brin_summarize_new_values('brinidx'); -- ok, no change expected 0 (1 row) +-- Tests for brin_desummarize_range +SELECT brin_desummarize_range('brinidx', -1); -- error, invalid range +ERROR: block number out of range: -1 +SELECT brin_desummarize_range('brinidx', 0); + brin_desummarize_range +------------------------ + +(1 row) + +SELECT brin_desummarize_range('brinidx', 0); + brin_desummarize_range +------------------------ + +(1 row) + +SELECT brin_desummarize_range('brinidx', 100000000); + brin_desummarize_range +------------------------ + +(1 row) + -- Test brin_summarize_range CREATE TABLE brin_summarize ( value int diff --git a/src/test/regress/sql/brin.sql b/src/test/regress/sql/brin.sql index da73df3659..521b22fe56 100644 --- a/src/test/regress/sql/brin.sql +++ b/src/test/regress/sql/brin.sql @@ -400,6 +400,7 @@ INSERT INTO brintest SELECT box(point(odd, even), point(thousand, twothousand)) FROM tenk1 ORDER BY unique2 LIMIT 5 OFFSET 5; +SELECT brin_desummarize_range('brinidx', 0); VACUUM brintest; -- force a summarization cycle in brinidx UPDATE brintest SET int8col = int8col * int4col; @@ -410,6 +411,12 @@ SELECT brin_summarize_new_values('brintest'); -- error, not an index SELECT brin_summarize_new_values('tenk1_unique1'); -- error, not a BRIN index SELECT brin_summarize_new_values('brinidx'); -- ok, no change expected +-- Tests for brin_desummarize_range +SELECT brin_desummarize_range('brinidx', -1); -- error, invalid range +SELECT brin_desummarize_range('brinidx', 0); +SELECT brin_desummarize_range('brinidx', 0); +SELECT brin_desummarize_range('brinidx', 100000000); + -- Test brin_summarize_range CREATE TABLE brin_summarize ( value int -- cgit v1.2.3 From f833c847b8fa4782efab45c8371d3cee64292d9b Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 1 Apr 2017 21:44:54 -0400 Subject: Allow psql variable substitution to occur in backtick command strings. Previously, text between backquotes in a psql metacommand's arguments was always passed to the shell literally. That considerably hobbles the usefulness of the feature for scripting, so we'd foreseen for a long time that we'd someday want to allow substitution of psql variables into the shell command. IMO the addition of \if metacommands has brought us to that point, since \if can greatly benefit from some sort of client-side expression evaluation capability, and psql itself is not going to grow any such thing in time for v10. Hence, this patch. It allows :VARIABLE to be replaced by the exact contents of the named variable, while :'VARIABLE' is replaced by the variable's contents suitably quoted to become a single shell-command argument. (The quoting rules for that are different from those for SQL literals, so this is a bit of an abuse of the :'VARIABLE' notation, but I doubt anyone will be confused.) As with other situations in psql, no substitution occurs if the word following a colon is not a known variable name. That limits the risk of compatibility problems for existing psql scripts; but the risk isn't zero, so this needs to be called out in the v10 release notes. Discussion: https://postgr.es/m/9561.1490895211@sss.pgh.pa.us --- doc/src/sgml/ref/psql-ref.sgml | 29 ++++++++--- src/bin/psql/common.c | 98 +++++++++++++++++++++++++------------ src/bin/psql/common.h | 3 +- src/bin/psql/psqlscanslash.l | 52 +++++++++++++++++--- src/fe_utils/psqlscan.l | 13 ++--- src/fe_utils/string_utils.c | 33 +++++++++---- src/include/fe_utils/psqlscan.h | 15 ++++-- src/include/fe_utils/psqlscan_int.h | 2 +- src/include/fe_utils/string_utils.h | 1 + 9 files changed, 180 insertions(+), 66 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml index b51b11baa3..ad463e71c1 100644 --- a/doc/src/sgml/ref/psql-ref.sgml +++ b/doc/src/sgml/ref/psql-ref.sgml @@ -769,18 +769,33 @@ testdb=> quotes that single character, whatever it is. - - Within an argument, text that is enclosed in backquotes - (`) is taken as a command line that is passed to the - shell. The output of the command (with any trailing newline removed) - replaces the backquoted text. - - If an unquoted colon (:) followed by a psql variable name appears within an argument, it is replaced by the variable's value, as described in . + The forms :'variable_name' and + :"variable_name" described there + work as well. + + + + Within an argument, text that is enclosed in backquotes + (`) is taken as a command line that is passed to the + shell. The output of the command (with any trailing newline removed) + replaces the backquoted text. Within the text enclosed in backquotes, + no special quoting or other processing occurs, except that appearances + of :variable_name where + variable_name is a psql variable name + are replaced by the variable's value. Also, appearances of + :'variable_name' are replaced by the + variable's value suitably quoted to become a single shell command + argument. (The latter form is almost always preferable, unless you are + very sure of what is in the variable.) Because carriage return and line + feed characters cannot be safely quoted on all platforms, the + :'variable_name' form prints an + error message and does not substitute the variable value when such + characters appear in the value. diff --git a/src/bin/psql/common.c b/src/bin/psql/common.c index b06ae9779d..a2f1259c1e 100644 --- a/src/bin/psql/common.c +++ b/src/bin/psql/common.c @@ -116,19 +116,19 @@ setQFout(const char *fname) * If the specified variable exists, return its value as a string (malloc'd * and expected to be freed by the caller); else return NULL. * - * If "escape" is true, return the value suitably quoted and escaped, - * as an identifier or string literal depending on "as_ident". - * (Failure in escaping should lead to returning NULL.) + * If "quote" isn't PQUOTE_PLAIN, then return the value suitably quoted and + * escaped for the specified quoting requirement. (Failure in escaping + * should lead to printing an error and returning NULL.) * * "passthrough" is the pointer previously given to psql_scan_set_passthrough. * In psql, passthrough points to a ConditionalStack, which we check to * determine whether variable expansion is allowed. */ char * -psql_get_variable(const char *varname, bool escape, bool as_ident, +psql_get_variable(const char *varname, PsqlScanQuoteType quote, void *passthrough) { - char *result; + char *result = NULL; const char *value; /* In an inactive \if branch, suppress all variable substitutions */ @@ -139,40 +139,74 @@ psql_get_variable(const char *varname, bool escape, bool as_ident, if (!value) return NULL; - if (escape) + switch (quote) { - char *escaped_value; + case PQUOTE_PLAIN: + result = pg_strdup(value); + break; + case PQUOTE_SQL_LITERAL: + case PQUOTE_SQL_IDENT: + { + /* + * For these cases, we use libpq's quoting functions, which + * assume the string is in the connection's client encoding. + */ + char *escaped_value; - if (!pset.db) - { - psql_error("cannot escape without active connection\n"); - return NULL; - } + if (!pset.db) + { + psql_error("cannot escape without active connection\n"); + return NULL; + } - if (as_ident) - escaped_value = - PQescapeIdentifier(pset.db, value, strlen(value)); - else - escaped_value = - PQescapeLiteral(pset.db, value, strlen(value)); + if (quote == PQUOTE_SQL_LITERAL) + escaped_value = + PQescapeLiteral(pset.db, value, strlen(value)); + else + escaped_value = + PQescapeIdentifier(pset.db, value, strlen(value)); - if (escaped_value == NULL) - { - const char *error = PQerrorMessage(pset.db); + if (escaped_value == NULL) + { + const char *error = PQerrorMessage(pset.db); - psql_error("%s", error); - return NULL; - } + psql_error("%s", error); + return NULL; + } - /* - * Rather than complicate the lexer's API with a notion of which - * free() routine to use, just pay the price of an extra strdup(). - */ - result = pg_strdup(escaped_value); - PQfreemem(escaped_value); + /* + * Rather than complicate the lexer's API with a notion of + * which free() routine to use, just pay the price of an extra + * strdup(). + */ + result = pg_strdup(escaped_value); + PQfreemem(escaped_value); + break; + } + case PQUOTE_SHELL_ARG: + { + /* + * For this we use appendShellStringNoError, which is + * encoding-agnostic, which is fine since the shell probably + * is too. In any case, the only special character is "'", + * which is not known to appear in valid multibyte characters. + */ + PQExpBufferData buf; + + initPQExpBuffer(&buf); + if (!appendShellStringNoError(&buf, value)) + { + psql_error("shell command argument contains a newline or carriage return: \"%s\"\n", + value); + free(buf.data); + return NULL; + } + result = buf.data; + break; + } + + /* No default: we want a compiler warning for missing cases */ } - else - result = pg_strdup(value); return result; } diff --git a/src/bin/psql/common.h b/src/bin/psql/common.h index 3d8b8da7fe..1ceb8ae386 100644 --- a/src/bin/psql/common.h +++ b/src/bin/psql/common.h @@ -12,11 +12,12 @@ #include "libpq-fe.h" #include "fe_utils/print.h" +#include "fe_utils/psqlscan.h" extern bool openQueryOutputFile(const char *fname, FILE **fout, bool *is_pipe); extern bool setQFout(const char *fname); -extern char *psql_get_variable(const char *varname, bool escape, bool as_ident, +extern char *psql_get_variable(const char *varname, PsqlScanQuoteType quote, void *passthrough); extern void psql_error(const char *fmt,...) pg_attribute_printf(1, 2); diff --git a/src/bin/psql/psqlscanslash.l b/src/bin/psql/psqlscanslash.l index 319afdc744..db7a1b9eea 100644 --- a/src/bin/psql/psqlscanslash.l +++ b/src/bin/psql/psqlscanslash.l @@ -242,8 +242,7 @@ other . yytext + 1, yyleng - 1); value = cur_state->callbacks->get_variable(varname, - false, - false, + PQUOTE_PLAIN, cur_state->cb_passthrough); free(varname); @@ -268,14 +267,16 @@ other . } :'{variable_char}+' { - psqlscan_escape_variable(cur_state, yytext, yyleng, false); + psqlscan_escape_variable(cur_state, yytext, yyleng, + PQUOTE_SQL_LITERAL); *option_quote = ':'; unquoted_option_chars = 0; } :\"{variable_char}+\" { - psqlscan_escape_variable(cur_state, yytext, yyleng, true); + psqlscan_escape_variable(cur_state, yytext, yyleng, + PQUOTE_SQL_IDENT); *option_quote = ':'; unquoted_option_chars = 0; } @@ -337,9 +338,8 @@ other . { /* - * backticked text: copy everything until next backquote, then evaluate. - * - * XXX Possible future behavioral change: substitute for :VARIABLE? + * backticked text: copy everything until next backquote (expanding + * variable references, but doing nought else), then evaluate. */ "`" { @@ -350,6 +350,44 @@ other . BEGIN(xslasharg); } +:{variable_char}+ { + /* Possible psql variable substitution */ + if (cur_state->callbacks->get_variable == NULL) + ECHO; + else + { + char *varname; + char *value; + + varname = psqlscan_extract_substring(cur_state, + yytext + 1, + yyleng - 1); + value = cur_state->callbacks->get_variable(varname, + PQUOTE_PLAIN, + cur_state->cb_passthrough); + free(varname); + + if (value) + { + appendPQExpBufferStr(output_buf, value); + free(value); + } + else + ECHO; + } + } + +:'{variable_char}+' { + psqlscan_escape_variable(cur_state, yytext, yyleng, + PQUOTE_SHELL_ARG); + } + +:'{variable_char}* { + /* Throw back everything but the colon */ + yyless(1); + ECHO; + } + {other}|\n { ECHO; } } diff --git a/src/fe_utils/psqlscan.l b/src/fe_utils/psqlscan.l index 19b3e57aa4..27689d72da 100644 --- a/src/fe_utils/psqlscan.l +++ b/src/fe_utils/psqlscan.l @@ -699,8 +699,7 @@ other . yyleng - 1); if (cur_state->callbacks->get_variable) value = cur_state->callbacks->get_variable(varname, - false, - false, + PQUOTE_PLAIN, cur_state->cb_passthrough); else value = NULL; @@ -737,11 +736,13 @@ other . } :'{variable_char}+' { - psqlscan_escape_variable(cur_state, yytext, yyleng, false); + psqlscan_escape_variable(cur_state, yytext, yyleng, + PQUOTE_SQL_LITERAL); } :\"{variable_char}+\" { - psqlscan_escape_variable(cur_state, yytext, yyleng, true); + psqlscan_escape_variable(cur_state, yytext, yyleng, + PQUOTE_SQL_IDENT); } /* @@ -1415,7 +1416,7 @@ psqlscan_extract_substring(PsqlScanState state, const char *txt, int len) */ void psqlscan_escape_variable(PsqlScanState state, const char *txt, int len, - bool as_ident) + PsqlScanQuoteType quote) { char *varname; char *value; @@ -1423,7 +1424,7 @@ psqlscan_escape_variable(PsqlScanState state, const char *txt, int len, /* Variable lookup. */ varname = psqlscan_extract_substring(state, txt + 2, len - 3); if (state->callbacks->get_variable) - value = state->callbacks->get_variable(varname, true, as_ident, + value = state->callbacks->get_variable(varname, quote, state->cb_passthrough); else value = NULL; diff --git a/src/fe_utils/string_utils.c b/src/fe_utils/string_utils.c index d1a9ddc4c6..dc84d32a09 100644 --- a/src/fe_utils/string_utils.c +++ b/src/fe_utils/string_utils.c @@ -425,13 +425,30 @@ appendByteaLiteral(PQExpBuffer buf, const unsigned char *str, size_t length, * arguments containing LF or CR characters. A future major release should * reject those characters in CREATE ROLE and CREATE DATABASE, because use * there eventually leads to errors here. + * + * appendShellString() simply prints an error and dies if LF or CR appears. + * appendShellStringNoError() omits those characters from the result, and + * returns false if there were any. */ void appendShellString(PQExpBuffer buf, const char *str) +{ + if (!appendShellStringNoError(buf, str)) + { + fprintf(stderr, + _("shell command argument contains a newline or carriage return: \"%s\"\n"), + str); + exit(EXIT_FAILURE); + } +} + +bool +appendShellStringNoError(PQExpBuffer buf, const char *str) { #ifdef WIN32 int backslash_run_length = 0; #endif + bool ok = true; const char *p; /* @@ -442,7 +459,7 @@ appendShellString(PQExpBuffer buf, const char *str) strspn(str, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_./:") == strlen(str)) { appendPQExpBufferStr(buf, str); - return; + return ok; } #ifndef WIN32 @@ -451,10 +468,8 @@ appendShellString(PQExpBuffer buf, const char *str) { if (*p == '\n' || *p == '\r') { - fprintf(stderr, - _("shell command argument contains a newline or carriage return: \"%s\"\n"), - str); - exit(EXIT_FAILURE); + ok = false; + continue; } if (*p == '\'') @@ -481,10 +496,8 @@ appendShellString(PQExpBuffer buf, const char *str) { if (*p == '\n' || *p == '\r') { - fprintf(stderr, - _("shell command argument contains a newline or carriage return: \"%s\"\n"), - str); - exit(EXIT_FAILURE); + ok = false; + continue; } /* Change N backslashes before a double quote to 2N+1 backslashes. */ @@ -524,6 +537,8 @@ appendShellString(PQExpBuffer buf, const char *str) } appendPQExpBufferStr(buf, "^\""); #endif /* WIN32 */ + + return ok; } diff --git a/src/include/fe_utils/psqlscan.h b/src/include/fe_utils/psqlscan.h index 0cc632b821..e9c8143925 100644 --- a/src/include/fe_utils/psqlscan.h +++ b/src/include/fe_utils/psqlscan.h @@ -48,13 +48,22 @@ typedef enum _promptStatus PROMPT_COPY } promptStatus_t; +/* Quoting request types for get_variable() callback */ +typedef enum +{ + PQUOTE_PLAIN, /* just return the actual value */ + PQUOTE_SQL_LITERAL, /* add quotes to make a valid SQL literal */ + PQUOTE_SQL_IDENT, /* quote if needed to make a SQL identifier */ + PQUOTE_SHELL_ARG /* quote if needed to be safe in a shell cmd */ +} PsqlScanQuoteType; + /* Callback functions to be used by the lexer */ typedef struct PsqlScanCallbacks { - /* Fetch value of a variable, as a pfree'able string; NULL if unknown */ + /* Fetch value of a variable, as a free'able string; NULL if unknown */ /* This pointer can be NULL if no variable substitution is wanted */ - char *(*get_variable) (const char *varname, bool escape, - bool as_ident, void *passthrough); + char *(*get_variable) (const char *varname, PsqlScanQuoteType quote, + void *passthrough); /* Print an error message someplace appropriate */ /* (very old gcc versions don't support attributes on function pointers) */ #if defined(__GNUC__) && __GNUC__ < 4 diff --git a/src/include/fe_utils/psqlscan_int.h b/src/include/fe_utils/psqlscan_int.h index b4044e806a..af62f5ebdf 100644 --- a/src/include/fe_utils/psqlscan_int.h +++ b/src/include/fe_utils/psqlscan_int.h @@ -141,6 +141,6 @@ extern char *psqlscan_extract_substring(PsqlScanState state, const char *txt, int len); extern void psqlscan_escape_variable(PsqlScanState state, const char *txt, int len, - bool as_ident); + PsqlScanQuoteType quote); #endif /* PSQLSCAN_INT_H */ diff --git a/src/include/fe_utils/string_utils.h b/src/include/fe_utils/string_utils.h index 6fb7f5e30e..c68234335e 100644 --- a/src/include/fe_utils/string_utils.h +++ b/src/include/fe_utils/string_utils.h @@ -42,6 +42,7 @@ extern void appendByteaLiteral(PQExpBuffer buf, bool std_strings); extern void appendShellString(PQExpBuffer buf, const char *str); +extern bool appendShellStringNoError(PQExpBuffer buf, const char *str); extern void appendConnStrVal(PQExpBuffer buf, const char *str); extern void appendPsqlMetaConnect(PQExpBuffer buf, const char *dbname); -- cgit v1.2.3 From 68dba97a4dea5c5c915e31978a475107c17c458d Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 2 Apr 2017 18:26:37 -0400 Subject: Document psql's behavior of recalling the previously executed query. Various psql slash commands that normally act on the current query buffer will automatically recall and re-use the most recently executed SQL command instead, if the current query buffer is empty. Although this behavior is ancient (dating apparently to commit 77a472993), it was documented nowhere in the psql reference page. For that matter, we'd never bothered to define the concept of "current query buffer" explicitly. Fix that. Do some wordsmithing on relevant command descriptions to improve clarity and consistency. Discussion: https://postgr.es/m/9b4ea968-753f-4b5f-b46c-d7d3bf7c8f90@manitou-mail.org --- doc/src/sgml/ref/psql-ref.sgml | 73 ++++++++++++++++++++++++++++++------------ 1 file changed, 52 insertions(+), 21 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml index ad463e71c1..8f43a1c8b3 100644 --- a/doc/src/sgml/ref/psql-ref.sgml +++ b/doc/src/sgml/ref/psql-ref.sgml @@ -823,6 +823,14 @@ testdb=> continue beyond the end of the line. + + Many of the meta-commands act on the current query buffer. + This is simply a buffer holding whatever SQL command text has been typed + but not yet sent to the server for execution. This will include previous + input lines as well as any text appearing before the meta-command on the + same line. + + The following meta-commands are defined: @@ -1713,22 +1721,28 @@ testdb=> If filename is - specified, the file is edited; after the editor exits, its - content is copied back to the query buffer. If no filename is given, the current query buffer is copied to a temporary file which is then edited in the same + fashion. Or, if the current query buffer is empty, the most recently + executed query is copied to a temporary file and edited in the same fashion. - The new query buffer is then re-parsed according to the normal - rules of psql, where the whole buffer - is treated as a single line. (Thus you cannot make scripts this - way. Use \i for that.) This means that - if the query ends with (or contains) a semicolon, it is - immediately executed. Otherwise it will merely wait in the - query buffer; type semicolon or \g to send it, or - \r to cancel. + The new contents of the query buffer are then re-parsed according to + the normal rules of psql, treating the + whole buffer as a single line. Any complete queries are immediately + executed; that is, if the query buffer contains or ends with a + semicolon, everything up to that point is executed. Whatever remains + will wait in the query buffer; type semicolon or \g to + send it, or \r to cancel it by clearing the query buffer. + Treating the buffer as a single line primarily affects meta-commands: + whatever is in the buffer after a meta-command will be taken as + argument(s) to the meta-command, even if it spans multiple lines. + (Thus you cannot make meta-command-using scripts this way. + Use \i for that.) @@ -1888,16 +1902,17 @@ Tue Oct 26 21:40:57 CEST 1999 \g [ |command ] - Sends the current query input buffer to the server, and - optionally stores the query's output in filename or pipes the output - to the shell command command. The file or command is - written to only if the query successfully returns zero or more tuples, - not if the query fails or is a non-data-returning SQL command. + Sends the current query buffer to the server for execution. + If an argument is given, the query's output is written to the named + file or piped to the given shell command, instead of displaying it as + usual. The file or command is written to only if the query + successfully returns zero or more tuples, not if the query fails or + is a non-data-returning SQL command. - A bare \g is essentially equivalent to a semicolon. + If the current query buffer is empty, the most recently sent query is + re-executed instead. Except for that behavior, \g + without an argument is essentially equivalent to a semicolon. A \g with argument is a one-shot alternative to the \o command. @@ -1922,7 +1937,7 @@ Tue Oct 26 21:40:57 CEST 1999 - Sends the current query input buffer to the server, then treats + Sends the current query buffer to the server, then treats each column of each row of the query's output (if any) as a SQL statement to be executed. For example, to create an index on each column of my_table: @@ -1955,6 +1970,10 @@ CREATE INDEX timing, and other query execution features apply to each generated query as well. + + If the current query buffer is empty, the most recently sent query + is re-executed instead. + @@ -1964,7 +1983,7 @@ CREATE INDEX - Sends the current query input buffer to the server and stores the + Sends the current query buffer to the server and stores the query's output into psql variables (see ). The query to be executed must return exactly one row. Each column of @@ -1996,6 +2015,10 @@ hello 10 If the query fails or does not return one row, no variables are changed. + + If the current query buffer is empty, the most recently sent query + is re-executed instead. + @@ -2302,6 +2325,8 @@ lo_import 152801 Print the current query buffer to the standard output. + If the current query buffer is empty, the most recently executed query + is printed instead. @@ -2970,9 +2995,11 @@ testdb=> \setenv LESS -imx4F \w or \write |command - Outputs the current query buffer to the file filename or pipes it to the shell command command. + If the current query buffer is empty, the most recently executed query + is written instead. @@ -2988,6 +3015,10 @@ testdb=> \setenv LESS -imx4F displayed with a header that includes the \pset title string (if any), the time as of query start, and the delay interval. + + If the current query buffer is empty, the most recently sent query + is re-executed instead. + -- cgit v1.2.3 From ffac5998b4c18920f86d80f1bddbde9ebcf0a314 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 2 Apr 2017 19:01:53 -0400 Subject: Doc: clarify behavior of OT_WHOLE_LINE and OT_FILEPIPE psql slash commands. This is another bit of ancient behavior that was documented poorly (in a couple of cases) or not at all (in several others). Discussion: https://postgr.es/m/9b4ea968-753f-4b5f-b46c-d7d3bf7c8f90@manitou-mail.org --- doc/src/sgml/ref/psql-ref.sgml | 88 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 79 insertions(+), 9 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml index 8f43a1c8b3..3b86612862 100644 --- a/doc/src/sgml/ref/psql-ref.sgml +++ b/doc/src/sgml/ref/psql-ref.sgml @@ -989,8 +989,10 @@ testdb=> command. All options other than the data source/destination are as specified for . Because of this, special parsing rules apply to the \copy - command. In particular, psql's variable substitution - rules and backslash escapes do not apply. + meta-command. Unlike most other meta-commands, the entire remainder + of the line is always taken to be the arguments of \copy, + and neither variable interpolation nor backquote expansion are + performed in the arguments. @@ -1820,6 +1822,13 @@ Tue Oct 26 21:40:57 CEST 1999 line of the file.) + + Unlike most other meta-commands, the entire remainder of the line is + always taken to be the argument(s) of \ef, and neither + variable interpolation nor backquote expansion are performed in the + arguments. + + See under psql will position the cursor on the specified line of the view definition. + + + Unlike most other meta-commands, the entire remainder of the line is + always taken to be the argument(s) of \ev, and neither + variable interpolation nor backquote expansion are performed in the + arguments. + @@ -1916,6 +1932,14 @@ Tue Oct 26 21:40:57 CEST 1999 A \g with argument is a one-shot alternative to the \o command. + + If the argument begins with |, then the entire remainder + of the line is taken to be + the command to execute, + and neither variable interpolation nor backquote expansion are + performed in it. The rest of the line is simply passed literally to + the shell. + @@ -2035,6 +2059,13 @@ hello 10 SQL commands is shown. + + Unlike most other meta-commands, the entire remainder of the line is + always taken to be the argument(s) of \help, and neither + variable interpolation nor backquote expansion are performed in the + arguments. + + To simplify typing, commands that consists of several words do @@ -2303,10 +2334,20 @@ lo_import 152801 specified, the query output is reset to the standard output. - Query results includes all tables, command + + If the argument begins with |, then the entire remainder + of the line is taken to be + the command to execute, + and neither variable interpolation nor backquote expansion are + performed in it. The rest of the line is simply passed literally to + the shell. + + + + Query results includes all tables, command responses, and notices obtained from the database server, as well as output of various backslash commands that query the - database (such as \d), but not error + database (such as \d); but not error messages. @@ -2907,6 +2948,13 @@ testdb=> \setenv LESS -imx4F output lines are numbered, with the first line of the function body being line 1. + + + Unlike most other meta-commands, the entire remainder of the line is + always taken to be the argument(s) of \sf, and neither + variable interpolation nor backquote expansion are performed in the + arguments. + @@ -2926,6 +2974,13 @@ testdb=> \setenv LESS -imx4F If + is appended to the command name, then the output lines are numbered from 1. + + + Unlike most other meta-commands, the entire remainder of the line is + always taken to be the argument(s) of \sv, and neither + variable interpolation nor backquote expansion are performed in the + arguments. + @@ -3001,6 +3056,15 @@ testdb=> \setenv LESS -imx4F If the current query buffer is empty, the most recently executed query is written instead. + + + If the argument begins with |, then the entire remainder + of the line is taken to be + the command to execute, + and neither variable interpolation nor backquote expansion are + performed in it. The rest of the line is simply passed literally to + the shell. + @@ -3057,11 +3121,17 @@ testdb=> \setenv LESS -imx4F \! [ command ] - Escapes to a separate shell or executes the shell command - command. The - arguments are not further interpreted; the shell will see them - as-is. In particular, the variable substitution rules and - backslash escapes do not apply. + With no argument, escapes to a sub-shell; psql + resumes when the sub-shell exits. With an argument, executes the + shell command command. + + + + Unlike most other meta-commands, the entire remainder of the line is + always taken to be the argument(s) of \!, and neither + variable interpolation nor backquote expansion are performed in the + arguments. The rest of the line is simply passed literally to the + shell. -- cgit v1.2.3 From 70da87d334ebe14fbbefe98827df5a8e1400444e Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Mon, 3 Apr 2017 10:50:32 -0400 Subject: doc: Change xref style to number only Change the style of links generated by xrefs to section number only, as it was with DSSSL, instead of number and title, as is the default of the XSLT stylesheets. Our documentation is mostly written expecting the old style, so keep that for the time being, per discussion. --- doc/src/sgml/stylesheet-common.xsl | 1 + 1 file changed, 1 insertion(+) (limited to 'doc/src') diff --git a/doc/src/sgml/stylesheet-common.xsl b/doc/src/sgml/stylesheet-common.xsl index 3d0651a823..658a5ac5e1 100644 --- a/doc/src/sgml/stylesheet-common.xsl +++ b/doc/src/sgml/stylesheet-common.xsl @@ -39,6 +39,7 @@ 1 + -- cgit v1.2.3 From 334bf9c77d0630c65d83f59b42b007cf9523cacf Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Mon, 3 Apr 2017 23:10:16 -0400 Subject: Further corrections and improvements to partitioning documentation. Amit Langote Discussion: http://postgr.es/m/80f6b049-e882-f6c3-f82c-f44baa94d369@lab.ntt.co.jp --- doc/src/sgml/ddl.sgml | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ddl.sgml b/doc/src/sgml/ddl.sgml index 5109778196..340c961b3f 100644 --- a/doc/src/sgml/ddl.sgml +++ b/doc/src/sgml/ddl.sgml @@ -2932,7 +2932,7 @@ VALUES ('Albany', NULL, NULL, 'NY'); tables and partitions. For example, a partition cannot have any parents other than the partitioned table it is a partition of, nor can a regular table inherit from a partitioned table making the latter its parent. - That means partitioned table and partitions do not participate in + That means partitioned tables and partitions do not participate in inheritance with regular tables. Since a partition hierarchy consisting of the partitioned table and its partitions is still an inheritance hierarchy, all the normal rules of inheritance apply as described in @@ -3036,11 +3036,12 @@ CREATE TABLE measurement ( You may decide to use multiple columns in the partition key for range partitioning, if desired. Of course, this will often result in a larger - number of partitions, each of which is individually smaller. - criteria. Using fewer columns may lead to coarser-grained - A query accessing the partitioned table will have - to scan fewer partitions if the conditions involve some or all of these - columns. For example, consider a table range partitioned using columns + number of partitions, each of which is individually smaller. On the + other hand, using fewer columns may lead to a coarser-grained + partitioning criteria with smaller number of partitions. A query + accessing the partitioned table will have to scan fewer partitions if + the conditions involve some or all of these columns. + For example, consider a table range partitioned using columns lastname and firstname (in that order) as the partition key. @@ -3167,8 +3168,8 @@ CREATE INDEX ON measurement_y2008m01 (logdate); - The simplest option for removing old data is simply to drop the partition - that is no longer necessary: + The simplest option for removing old data is to drop the partition that + is no longer necessary: DROP TABLE measurement_y2006m02; @@ -3595,8 +3596,8 @@ DO INSTEAD Partition Maintenance - To remove old data quickly, simply to drop the partition that is no - longer necessary: + To remove old data quickly, simply drop the partition that is no longer + necessary: DROP TABLE measurement_y2006m02; @@ -3692,7 +3693,7 @@ ANALYZE measurement; Triggers or rules will be needed to route rows to the desired partition, unless the application is explicitly aware of the partitioning scheme. Triggers may be complicated to write, and will - be much slower than the tuple routing performed interally by + be much slower than the tuple routing performed internally by declarative partitioning. -- cgit v1.2.3 From ea69a0dead5128c421140dc53fac165ba4af8520 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Mon, 3 Apr 2017 23:46:33 -0400 Subject: Expand hash indexes more gradually. Since hash indexes typically have very few overflow pages, adding a new splitpoint essentially doubles the on-disk size of the index, which can lead to large and abrupt increases in disk usage (and perhaps long delays on occasion). To mitigate this problem to some degree, divide larger splitpoints into four equal phases. This means that, for example, instead of growing from 4GB to 8GB all at once, a hash index will now grow from 4GB to 5GB to 6GB to 7GB to 8GB, which is perhaps still not as smooth as we'd like but certainly an improvement. This changes the on-disk format of the metapage, so bump HASH_VERSION from 2 to 3. This will force a REINDEX of all existing hash indexes, but that's probably a good idea anyway. First, hash indexes from pre-10 versions of PostgreSQL could easily be corrupted, and we don't want to confuse corruption carried over from an older release with any corruption caused despite the new write-ahead logging in v10. Second, it will let us remove some backward-compatibility code added by commit 293e24e507838733aba4748b514536af2d39d7f2. Mithun Cy, reviewed by Amit Kapila, Jesper Pedersen and me. Regression test outputs updated by me. Discussion: http://postgr.es/m/CAD__OuhG6F1gQLCgMQNnMNgoCvOLQZz9zKYJQNYvYmmJoM42gA@mail.gmail.com Discussion: http://postgr.es/m/CA+TgmoYty0jCf-pa+m+vYUJ716+AxM7nv_syvyanyf5O-L_i2A@mail.gmail.com --- contrib/pageinspect/expected/hash.out | 4 +- contrib/pgstattuple/expected/pgstattuple.out | 4 +- doc/src/sgml/pageinspect.sgml | 6 +-- src/backend/access/hash/README | 62 ++++++++++++++++---------- src/backend/access/hash/hashovfl.c | 9 ++-- src/backend/access/hash/hashpage.c | 62 +++++++++++++------------- src/backend/access/hash/hashsort.c | 27 +++++++++--- src/backend/access/hash/hashutil.c | 65 ++++++++++++++++++++++++++++ src/backend/utils/sort/tuplesort.c | 37 +++++++++++----- src/include/access/hash.h | 24 ++++++++-- src/include/utils/tuplesort.h | 4 +- 11 files changed, 218 insertions(+), 86 deletions(-) (limited to 'doc/src') diff --git a/contrib/pageinspect/expected/hash.out b/contrib/pageinspect/expected/hash.out index 3ba01f6ca3..39374158b1 100644 --- a/contrib/pageinspect/expected/hash.out +++ b/contrib/pageinspect/expected/hash.out @@ -45,7 +45,7 @@ lowmask, ovflpoint, firstfree, nmaps, procid, spares, mapp FROM hash_metapage_info(get_raw_page('test_hash_a_idx', 0)); -[ RECORD 1 ]---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- magic | 105121344 -version | 2 +version | 3 ntuples | 1 bsize | 8152 bmsize | 4096 @@ -57,7 +57,7 @@ ovflpoint | 2 firstfree | 0 nmaps | 1 procid | 450 -spares | {0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0} +spares | {0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0} mapp | {5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0} SELECT magic, version, ntuples, bsize, bmsize, bmshift, maxbucket, highmask, diff --git a/contrib/pgstattuple/expected/pgstattuple.out b/contrib/pgstattuple/expected/pgstattuple.out index 2c3515b4ef..86a7c78fbb 100644 --- a/contrib/pgstattuple/expected/pgstattuple.out +++ b/contrib/pgstattuple/expected/pgstattuple.out @@ -134,7 +134,7 @@ create index test_hashidx on test using hash (b); select * from pgstathashindex('test_hashidx'); version | bucket_pages | overflow_pages | bitmap_pages | zero_pages | live_items | dead_items | free_percent ---------+--------------+----------------+--------------+------------+------------+------------+-------------- - 2 | 4 | 0 | 1 | 0 | 0 | 0 | 100 + 3 | 4 | 0 | 1 | 0 | 0 | 0 | 100 (1 row) -- these should error with the wrong type @@ -235,7 +235,7 @@ select pgstatindex('test_partition_idx'); select pgstathashindex('test_partition_hash_idx'); pgstathashindex --------------------- - (2,8,0,1,0,0,0,100) + (3,8,0,1,0,0,0,100) (1 row) drop table test_partitioned; diff --git a/doc/src/sgml/pageinspect.sgml b/doc/src/sgml/pageinspect.sgml index 9f41bb21eb..c87b160bf4 100644 --- a/doc/src/sgml/pageinspect.sgml +++ b/doc/src/sgml/pageinspect.sgml @@ -658,7 +658,7 @@ test=# SELECT * FROM hash_bitmap_info('con_hash_index', 2052); test=# SELECT * FROM hash_metapage_info(get_raw_page('con_hash_index', 0)); -[ RECORD 1 ]----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- magic | 105121344 -version | 2 +version | 3 ntuples | 500500 ffactor | 40 bsize | 8152 @@ -667,11 +667,11 @@ bmshift | 15 maxbucket | 12512 highmask | 16383 lowmask | 8191 -ovflpoint | 14 +ovflpoint | 28 firstfree | 1204 nmaps | 1 procid | 450 -spares | {0,0,0,0,0,0,1,1,1,1,1,4,59,704,1204,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0} +spares | {0,0,0,0,0,0,1,1,1,1,1,1,1,1,3,4,4,4,45,55,58,59,508,567,628,704,1193,1202,1204,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0} mapp | {65,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0} diff --git a/src/backend/access/hash/README b/src/backend/access/hash/README index 1541438354..c8a0ec78a9 100644 --- a/src/backend/access/hash/README +++ b/src/backend/access/hash/README @@ -58,35 +58,51 @@ rules to support a variable number of overflow pages while not having to move primary bucket pages around after they are created. Primary bucket pages (henceforth just "bucket pages") are allocated in -power-of-2 groups, called "split points" in the code. Buckets 0 and 1 -are created when the index is initialized. At the first split, buckets 2 -and 3 are allocated; when bucket 4 is needed, buckets 4-7 are allocated; -when bucket 8 is needed, buckets 8-15 are allocated; etc. All the bucket -pages of a power-of-2 group appear consecutively in the index. This -addressing scheme allows the physical location of a bucket page to be -computed from the bucket number relatively easily, using only a small -amount of control information. We take the log2() of the bucket number -to determine which split point S the bucket belongs to, and then simply -add "hashm_spares[S] + 1" (where hashm_spares[] is an array stored in the -metapage) to compute the physical address. hashm_spares[S] can be -interpreted as the total number of overflow pages that have been allocated -before the bucket pages of splitpoint S. hashm_spares[0] is always 0, -so that buckets 0 and 1 (which belong to splitpoint 0) always appear at -block numbers 1 and 2, just after the meta page. We always have -hashm_spares[N] <= hashm_spares[N+1], since the latter count includes the -former. The difference between the two represents the number of overflow -pages appearing between the bucket page groups of splitpoints N and N+1. - +power-of-2 groups, called "split points" in the code. That means at every new +splitpoint we double the existing number of buckets. Allocating huge chunks +of bucket pages all at once isn't optimal and we will take ages to consume +those. To avoid this exponential growth of index size, we did use a trick to +break up allocation of buckets at the splitpoint into 4 equal phases. If +(2 ^ x) are the total buckets need to be allocated at a splitpoint (from now on +we shall call this as a splitpoint group), then we allocate 1/4th (2 ^ (x - 2)) +of total buckets at each phase of splitpoint group. Next quarter of allocation +will only happen if buckets of the previous phase have been already consumed. +For the initial splitpoint groups < 10 we will allocate all of their buckets in +single phase only, as number of buckets allocated at initial groups are small +in numbers. And for the groups >= 10 the allocation process is distributed +among four equal phases. At group 10 we allocate (2 ^ 9) buckets in 4 +different phases {2 ^ 7, 2 ^ 7, 2 ^ 7, 2 ^ 7}, the numbers in curly braces +indicate the number of buckets allocated within each phase of splitpoint group +10. And, for splitpoint group 11 and 12 allocation phases will be +{2 ^ 8, 2 ^ 8, 2 ^ 8, 2 ^ 8} and {2 ^ 9, 2 ^ 9, 2 ^ 9, 2 ^ 9} respectively. We +can see that at each splitpoint group we double the total number of buckets +from the previous group but in an incremental phase. The bucket pages +allocated within one phase of a splitpoint group will appear consecutively in +the index. This addressing scheme allows the physical location of a bucket +page to be computed from the bucket number relatively easily, using only a +small amount of control information. If we look at the function +_hash_spareindex for a given bucket number we first compute the +splitpoint group it belongs to and then the phase to which the bucket belongs +to. Adding them we get the global splitpoint phase number S to which the +bucket belongs and then simply add "hashm_spares[S] + 1" (where hashm_spares[] +is an array stored in the metapage) with given bucket number to compute its +physical address. The hashm_spares[S] can be interpreted as the total number +of overflow pages that have been allocated before the bucket pages of +splitpoint phase S. The hashm_spares[0] is always 0, so that buckets 0 and 1 +always appear at block numbers 1 and 2, just after the meta page. We always +have hashm_spares[N] <= hashm_spares[N+1], since the latter count includes the +former. The difference between the two represents the number of overflow pages +appearing between the bucket page groups of splitpoints phase N and N+1. (Note: the above describes what happens when filling an initially minimally -sized hash index. In practice, we try to estimate the required index size -and allocate a suitable number of splitpoints immediately, to avoid +sized hash index. In practice, we try to estimate the required index size and +allocate a suitable number of splitpoints phases immediately, to avoid expensive re-splitting during initial index build.) When S splitpoints exist altogether, the array entries hashm_spares[0] through hashm_spares[S] are valid; hashm_spares[S] records the current total number of overflow pages. New overflow pages are created as needed at the end of the index, and recorded by incrementing hashm_spares[S]. -When it is time to create a new splitpoint's worth of bucket pages, we +When it is time to create a new splitpoint phase's worth of bucket pages, we copy hashm_spares[S] into hashm_spares[S+1] and increment S (which is stored in the hashm_ovflpoint field of the meta page). This has the effect of reserving the correct number of bucket pages at the end of the @@ -101,7 +117,7 @@ We have to allow the case "greater than" because it's possible that during an index extension we crash after allocating filesystem space and before updating the metapage. Note that on filesystems that allow "holes" in files, it's entirely likely that pages before the logical EOF are not yet -allocated: when we allocate a new splitpoint's worth of bucket pages, we +allocated: when we allocate a new splitpoint phase's worth of bucket pages, we physically zero the last such page to force the EOF up, and the first such page will be used immediately, but the intervening pages are not written until needed. diff --git a/src/backend/access/hash/hashovfl.c b/src/backend/access/hash/hashovfl.c index a3cae21c60..41ef654862 100644 --- a/src/backend/access/hash/hashovfl.c +++ b/src/backend/access/hash/hashovfl.c @@ -49,7 +49,7 @@ bitno_to_blkno(HashMetaPage metap, uint32 ovflbitnum) * Convert to absolute page number by adding the number of bucket pages * that exist before this split point. */ - return (BlockNumber) ((1 << i) + ovflbitnum); + return (BlockNumber) (_hash_get_totalbuckets(i) + ovflbitnum); } /* @@ -67,14 +67,15 @@ _hash_ovflblkno_to_bitno(HashMetaPage metap, BlockNumber ovflblkno) /* Determine the split number containing this page */ for (i = 1; i <= splitnum; i++) { - if (ovflblkno <= (BlockNumber) (1 << i)) + if (ovflblkno <= (BlockNumber) _hash_get_totalbuckets(i)) break; /* oops */ - bitnum = ovflblkno - (1 << i); + bitnum = ovflblkno - _hash_get_totalbuckets(i); /* * bitnum has to be greater than number of overflow page added in * previous split point. The overflow page at this splitnum (i) if any - * should start from ((2 ^ i) + metap->hashm_spares[i - 1] + 1). + * should start from (_hash_get_totalbuckets(i) + + * metap->hashm_spares[i - 1] + 1). */ if (bitnum > metap->hashm_spares[i - 1] && bitnum <= metap->hashm_spares[i]) diff --git a/src/backend/access/hash/hashpage.c b/src/backend/access/hash/hashpage.c index 61ca2ecf55..b5a1c7ed28 100644 --- a/src/backend/access/hash/hashpage.c +++ b/src/backend/access/hash/hashpage.c @@ -502,14 +502,15 @@ _hash_init_metabuffer(Buffer buf, double num_tuples, RegProcedure procid, Page page; double dnumbuckets; uint32 num_buckets; - uint32 log2_num_buckets; + uint32 spare_index; uint32 i; /* * Choose the number of initial bucket pages to match the fill factor * given the estimated number of tuples. We round up the result to the - * next power of 2, however, and always force at least 2 bucket pages. The - * upper limit is determined by considerations explained in + * total number of buckets which has to be allocated before using its + * _hashm_spare element. However always force at least 2 bucket pages. + * The upper limit is determined by considerations explained in * _hash_expandtable(). */ dnumbuckets = num_tuples / ffactor; @@ -518,11 +519,10 @@ _hash_init_metabuffer(Buffer buf, double num_tuples, RegProcedure procid, else if (dnumbuckets >= (double) 0x40000000) num_buckets = 0x40000000; else - num_buckets = ((uint32) 1) << _hash_log2((uint32) dnumbuckets); + num_buckets = _hash_get_totalbuckets(_hash_spareindex(dnumbuckets)); - log2_num_buckets = _hash_log2(num_buckets); - Assert(num_buckets == (((uint32) 1) << log2_num_buckets)); - Assert(log2_num_buckets < HASH_MAX_SPLITPOINTS); + spare_index = _hash_spareindex(num_buckets); + Assert(spare_index < HASH_MAX_SPLITPOINTS); page = BufferGetPage(buf); if (initpage) @@ -563,18 +563,23 @@ _hash_init_metabuffer(Buffer buf, double num_tuples, RegProcedure procid, /* * We initialize the index with N buckets, 0 .. N-1, occupying physical - * blocks 1 to N. The first freespace bitmap page is in block N+1. Since - * N is a power of 2, we can set the masks this way: + * blocks 1 to N. The first freespace bitmap page is in block N+1. */ - metap->hashm_maxbucket = metap->hashm_lowmask = num_buckets - 1; - metap->hashm_highmask = (num_buckets << 1) - 1; + metap->hashm_maxbucket = num_buckets - 1; + + /* + * Set highmask as next immediate ((2 ^ x) - 1), which should be sufficient + * to cover num_buckets. + */ + metap->hashm_highmask = (1 << (_hash_log2(num_buckets + 1))) - 1; + metap->hashm_lowmask = (metap->hashm_highmask >> 1); MemSet(metap->hashm_spares, 0, sizeof(metap->hashm_spares)); MemSet(metap->hashm_mapp, 0, sizeof(metap->hashm_mapp)); /* Set up mapping for one spare page after the initial splitpoints */ - metap->hashm_spares[log2_num_buckets] = 1; - metap->hashm_ovflpoint = log2_num_buckets; + metap->hashm_spares[spare_index] = 1; + metap->hashm_ovflpoint = spare_index; metap->hashm_firstfree = 0; /* @@ -773,25 +778,25 @@ restart_expand: start_nblkno = BUCKET_TO_BLKNO(metap, new_bucket); /* - * If the split point is increasing (hashm_maxbucket's log base 2 - * increases), we need to allocate a new batch of bucket pages. + * If the split point is increasing we need to allocate a new batch of + * bucket pages. */ - spare_ndx = _hash_log2(new_bucket + 1); + spare_ndx = _hash_spareindex(new_bucket + 1); if (spare_ndx > metap->hashm_ovflpoint) { + uint32 buckets_to_add; + Assert(spare_ndx == metap->hashm_ovflpoint + 1); /* - * The number of buckets in the new splitpoint is equal to the total - * number already in existence, i.e. new_bucket. Currently this maps - * one-to-one to blocks required, but someday we may need a more - * complicated calculation here. We treat allocation of buckets as a - * separate WAL-logged action. Even if we fail after this operation, - * won't leak bucket pages; rather, the next split will consume this - * space. In any case, even without failure we don't use all the space - * in one split operation. + * We treat allocation of buckets as a separate WAL-logged action. + * Even if we fail after this operation, won't leak bucket pages; + * rather, the next split will consume this space. In any case, even + * without failure we don't use all the space in one split + * operation. */ - if (!_hash_alloc_buckets(rel, start_nblkno, new_bucket)) + buckets_to_add = _hash_get_totalbuckets(spare_ndx) - new_bucket; + if (!_hash_alloc_buckets(rel, start_nblkno, buckets_to_add)) { /* can't split due to BlockNumber overflow */ _hash_relbuf(rel, buf_oblkno); @@ -836,10 +841,9 @@ restart_expand: } /* - * If the split point is increasing (hashm_maxbucket's log base 2 - * increases), we need to adjust the hashm_spares[] array and - * hashm_ovflpoint so that future overflow pages will be created beyond - * this new batch of bucket pages. + * If the split point is increasing we need to adjust the hashm_spares[] + * array and hashm_ovflpoint so that future overflow pages will be created + * beyond this new batch of bucket pages. */ if (spare_ndx > metap->hashm_ovflpoint) { diff --git a/src/backend/access/hash/hashsort.c b/src/backend/access/hash/hashsort.c index 0e0f393711..41d615df8b 100644 --- a/src/backend/access/hash/hashsort.c +++ b/src/backend/access/hash/hashsort.c @@ -37,7 +37,15 @@ struct HSpool { Tuplesortstate *sortstate; /* state data for tuplesort.c */ Relation index; - uint32 hash_mask; /* bitmask for hash codes */ + + /* + * We sort the hash keys based on the buckets they belong to. Below masks + * are used in _hash_hashkey2bucket to determine the bucket of given hash + * key. + */ + uint32 high_mask; + uint32 low_mask; + uint32 max_buckets; }; @@ -56,11 +64,12 @@ _h_spoolinit(Relation heap, Relation index, uint32 num_buckets) * num_buckets buckets in the index, the appropriate mask can be computed * as follows. * - * Note: at present, the passed-in num_buckets is always a power of 2, so - * we could just compute num_buckets - 1. We prefer not to assume that - * here, though. + * NOTE : This hash mask calculation should be in sync with similar + * calculation in _hash_init_metabuffer. */ - hspool->hash_mask = (((uint32) 1) << _hash_log2(num_buckets)) - 1; + hspool->high_mask = (((uint32) 1) << _hash_log2(num_buckets + 1)) - 1; + hspool->low_mask = (hspool->high_mask >> 1); + hspool->max_buckets = num_buckets - 1; /* * We size the sort area as maintenance_work_mem rather than work_mem to @@ -69,7 +78,9 @@ _h_spoolinit(Relation heap, Relation index, uint32 num_buckets) */ hspool->sortstate = tuplesort_begin_index_hash(heap, index, - hspool->hash_mask, + hspool->high_mask, + hspool->low_mask, + hspool->max_buckets, maintenance_work_mem, false); @@ -122,7 +133,9 @@ _h_indexbuild(HSpool *hspool, Relation heapRel) #ifdef USE_ASSERT_CHECKING uint32 lasthashkey = hashkey; - hashkey = _hash_get_indextuple_hashkey(itup) & hspool->hash_mask; + hashkey = _hash_hashkey2bucket(_hash_get_indextuple_hashkey(itup), + hspool->max_buckets, hspool->high_mask, + hspool->low_mask); Assert(hashkey >= lasthashkey); #endif diff --git a/src/backend/access/hash/hashutil.c b/src/backend/access/hash/hashutil.c index 2e9971920b..d679cf0aed 100644 --- a/src/backend/access/hash/hashutil.c +++ b/src/backend/access/hash/hashutil.c @@ -149,6 +149,71 @@ _hash_log2(uint32 num) return i; } +/* + * _hash_spareindex -- returns spare index / global splitpoint phase of the + * bucket + */ +uint32 +_hash_spareindex(uint32 num_bucket) +{ + uint32 splitpoint_group; + uint32 splitpoint_phases; + + splitpoint_group = _hash_log2(num_bucket); + + if (splitpoint_group < HASH_SPLITPOINT_GROUPS_WITH_ONE_PHASE) + return splitpoint_group; + + /* account for single-phase groups */ + splitpoint_phases = HASH_SPLITPOINT_GROUPS_WITH_ONE_PHASE; + + /* account for multi-phase groups before splitpoint_group */ + splitpoint_phases += + ((splitpoint_group - HASH_SPLITPOINT_GROUPS_WITH_ONE_PHASE) << + HASH_SPLITPOINT_PHASE_BITS); + + /* account for phases within current group */ + splitpoint_phases += + (((num_bucket - 1) >> (HASH_SPLITPOINT_PHASE_BITS + 1)) & + HASH_SPLITPOINT_PHASE_MASK); /* to 0-based value. */ + + return splitpoint_phases; +} + +/* + * _hash_get_totalbuckets -- returns total number of buckets allocated till + * the given splitpoint phase. + */ +uint32 +_hash_get_totalbuckets(uint32 splitpoint_phase) +{ + uint32 splitpoint_group; + uint32 total_buckets; + uint32 phases_within_splitpoint_group; + + if (splitpoint_phase < HASH_SPLITPOINT_GROUPS_WITH_ONE_PHASE) + return (1 << splitpoint_phase); + + /* get splitpoint's group */ + splitpoint_group = HASH_SPLITPOINT_GROUPS_WITH_ONE_PHASE; + splitpoint_group += + ((splitpoint_phase - HASH_SPLITPOINT_GROUPS_WITH_ONE_PHASE) >> + HASH_SPLITPOINT_PHASE_BITS); + + /* account for buckets before splitpoint_group */ + total_buckets = (1 << (splitpoint_group - 1)); + + /* account for buckets within splitpoint_group */ + phases_within_splitpoint_group = + (((splitpoint_phase - HASH_SPLITPOINT_GROUPS_WITH_ONE_PHASE) & + HASH_SPLITPOINT_PHASE_MASK) + 1); /* from 0-based to 1-based */ + total_buckets += + (((1 << (splitpoint_group - 1)) >> HASH_SPLITPOINT_PHASE_BITS) * + phases_within_splitpoint_group); + + return total_buckets; +} + /* * _hash_checkpage -- sanity checks on the format of all hash pages * diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c index e1e692d5f0..65cda52714 100644 --- a/src/backend/utils/sort/tuplesort.c +++ b/src/backend/utils/sort/tuplesort.c @@ -127,6 +127,7 @@ #include "access/htup_details.h" #include "access/nbtree.h" +#include "access/hash.h" #include "catalog/index.h" #include "catalog/pg_am.h" #include "commands/tablespace.h" @@ -473,7 +474,9 @@ struct Tuplesortstate bool enforceUnique; /* complain if we find duplicate tuples */ /* These are specific to the index_hash subcase: */ - uint32 hash_mask; /* mask for sortable part of hash code */ + uint32 high_mask; /* masks for sortable part of hash code */ + uint32 low_mask; + uint32 max_buckets; /* * These variables are specific to the Datum case; they are set by @@ -991,7 +994,9 @@ tuplesort_begin_index_btree(Relation heapRel, Tuplesortstate * tuplesort_begin_index_hash(Relation heapRel, Relation indexRel, - uint32 hash_mask, + uint32 high_mask, + uint32 low_mask, + uint32 max_buckets, int workMem, bool randomAccess) { Tuplesortstate *state = tuplesort_begin_common(workMem, randomAccess); @@ -1002,8 +1007,11 @@ tuplesort_begin_index_hash(Relation heapRel, #ifdef TRACE_SORT if (trace_sort) elog(LOG, - "begin index sort: hash_mask = 0x%x, workMem = %d, randomAccess = %c", - hash_mask, + "begin index sort: high_mask = 0x%x, low_mask = 0x%x, " + "max_buckets = 0x%x, workMem = %d, randomAccess = %c", + high_mask, + low_mask, + max_buckets, workMem, randomAccess ? 't' : 'f'); #endif @@ -1017,7 +1025,9 @@ tuplesort_begin_index_hash(Relation heapRel, state->heapRel = heapRel; state->indexRel = indexRel; - state->hash_mask = hash_mask; + state->high_mask = high_mask; + state->low_mask = low_mask; + state->max_buckets = max_buckets; MemoryContextSwitchTo(oldcontext); @@ -4157,8 +4167,8 @@ static int comparetup_index_hash(const SortTuple *a, const SortTuple *b, Tuplesortstate *state) { - uint32 hash1; - uint32 hash2; + Bucket bucket1; + Bucket bucket2; IndexTuple tuple1; IndexTuple tuple2; @@ -4167,13 +4177,16 @@ comparetup_index_hash(const SortTuple *a, const SortTuple *b, * that the first column of the index tuple is the hash key. */ Assert(!a->isnull1); - hash1 = DatumGetUInt32(a->datum1) & state->hash_mask; + bucket1 = _hash_hashkey2bucket(DatumGetUInt32(a->datum1), + state->max_buckets, state->high_mask, + state->low_mask); Assert(!b->isnull1); - hash2 = DatumGetUInt32(b->datum1) & state->hash_mask; - - if (hash1 > hash2) + bucket2 = _hash_hashkey2bucket(DatumGetUInt32(b->datum1), + state->max_buckets, state->high_mask, + state->low_mask); + if (bucket1 > bucket2) return 1; - else if (hash1 < hash2) + else if (bucket1 < bucket2) return -1; /* diff --git a/src/include/access/hash.h b/src/include/access/hash.h index eb1df57291..fcc3957036 100644 --- a/src/include/access/hash.h +++ b/src/include/access/hash.h @@ -36,7 +36,7 @@ typedef uint32 Bucket; #define InvalidBucket ((Bucket) 0xFFFFFFFF) #define BUCKET_TO_BLKNO(metap,B) \ - ((BlockNumber) ((B) + ((B) ? (metap)->hashm_spares[_hash_log2((B)+1)-1] : 0)) + 1) + ((BlockNumber) ((B) + ((B) ? (metap)->hashm_spares[_hash_spareindex((B)+1)-1] : 0)) + 1) /* * Special space for hash index pages. @@ -158,7 +158,8 @@ typedef HashScanOpaqueData *HashScanOpaque; #define HASH_METAPAGE 0 /* metapage is always block 0 */ #define HASH_MAGIC 0x6440640 -#define HASH_VERSION 2 /* 2 signifies only hash key value is stored */ +#define HASH_VERSION 3 /* 3 signifies multi-phased bucket allocation + * to reduce doubling */ /* * spares[] holds the number of overflow pages currently allocated at or @@ -176,13 +177,28 @@ typedef HashScanOpaqueData *HashScanOpaque; * * The limitation on the size of spares[] comes from the fact that there's * no point in having more than 2^32 buckets with only uint32 hashcodes. + * (Note: The value of HASH_MAX_SPLITPOINTS which is the size of spares[] is + * adjusted in such a way to accommodate multi phased allocation of buckets + * after HASH_SPLITPOINT_GROUPS_WITH_ONE_PHASE). + * * There is no particular upper limit on the size of mapp[], other than * needing to fit into the metapage. (With 8K block size, 128 bitmaps * limit us to 64 GB of overflow space...) */ -#define HASH_MAX_SPLITPOINTS 32 #define HASH_MAX_BITMAPS 128 +#define HASH_SPLITPOINT_PHASE_BITS 2 +#define HASH_SPLITPOINT_PHASES_PER_GRP (1 << HASH_SPLITPOINT_PHASE_BITS) +#define HASH_SPLITPOINT_PHASE_MASK (HASH_SPLITPOINT_PHASES_PER_GRP - 1) +#define HASH_SPLITPOINT_GROUPS_WITH_ONE_PHASE 10 + +/* defines max number of splitpoit phases a hash index can have */ +#define HASH_MAX_SPLITPOINT_GROUP 32 +#define HASH_MAX_SPLITPOINTS \ + (((HASH_MAX_SPLITPOINT_GROUP - HASH_SPLITPOINT_GROUPS_WITH_ONE_PHASE) * \ + HASH_SPLITPOINT_PHASES_PER_GRP) + \ + HASH_SPLITPOINT_GROUPS_WITH_ONE_PHASE) + typedef struct HashMetaPageData { uint32 hashm_magic; /* magic no. for hash tables */ @@ -382,6 +398,8 @@ extern uint32 _hash_datum2hashkey_type(Relation rel, Datum key, Oid keytype); extern Bucket _hash_hashkey2bucket(uint32 hashkey, uint32 maxbucket, uint32 highmask, uint32 lowmask); extern uint32 _hash_log2(uint32 num); +extern uint32 _hash_spareindex(uint32 num_bucket); +extern uint32 _hash_get_totalbuckets(uint32 splitpoint_phase); extern void _hash_checkpage(Relation rel, Buffer buf, int flags); extern uint32 _hash_get_indextuple_hashkey(IndexTuple itup); extern bool _hash_convert_tuple(Relation index, diff --git a/src/include/utils/tuplesort.h b/src/include/utils/tuplesort.h index 5b3f4752f4..9719db42b9 100644 --- a/src/include/utils/tuplesort.h +++ b/src/include/utils/tuplesort.h @@ -72,7 +72,9 @@ extern Tuplesortstate *tuplesort_begin_index_btree(Relation heapRel, int workMem, bool randomAccess); extern Tuplesortstate *tuplesort_begin_index_hash(Relation heapRel, Relation indexRel, - uint32 hash_mask, + uint32 high_mask, + uint32 low_mask, + uint32 max_buckets, int workMem, bool randomAccess); extern Tuplesortstate *tuplesort_begin_datum(Oid datumType, Oid sortOperator, Oid sortCollation, -- cgit v1.2.3 From 60a0b2ec8943451186dfa22907f88334d97cb2e0 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 4 Apr 2017 12:36:15 -0400 Subject: Adjust min/max values when changing sequence type When changing the type of a sequence, adjust the min/max values of the sequence if it looks like the previous values were the default values. Previously, it would leave the old values in place, requiring manual adjustments even in the usual/default cases. Reviewed-by: Michael Paquier Reviewed-by: Vitaly Burovoy --- doc/src/sgml/ref/alter_sequence.sgml | 13 ++++++---- src/backend/commands/sequence.c | 43 ++++++++++++++++++++++++++-------- src/test/regress/expected/sequence.out | 40 +++++++++++++++++++++++++++---- src/test/regress/sql/sequence.sql | 22 ++++++++++++++--- 4 files changed, 96 insertions(+), 22 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/alter_sequence.sgml b/doc/src/sgml/ref/alter_sequence.sgml index 252a668189..5c912ab892 100644 --- a/doc/src/sgml/ref/alter_sequence.sgml +++ b/doc/src/sgml/ref/alter_sequence.sgml @@ -94,10 +94,15 @@ ALTER SEQUENCE [ IF EXISTS ] name S - Note that changing the data type does not automatically change the - minimum and maximum values. You can use the clauses NO - MINVALUE and NO MAXVALUE to adjust the - minimum and maximum values to the range of the new data type. + Changing the data type automatically changes the minimum and maximum + values of the sequence if and only if the previous minimum and maximum + values were the minimum or maximum value of the old data type (in + other words, if the sequence had been created using NO + MINVALUE or NO MAXVALUE, implicitly or + explicitly). Otherwise, the minimum and maximum values are preserved, + unless new values are given as part of the same command. If the + minimum and maximum values do not fit into the new data type, an error + will be generated. diff --git a/src/backend/commands/sequence.c b/src/backend/commands/sequence.c index d547db714e..89b810bbb7 100644 --- a/src/backend/commands/sequence.c +++ b/src/backend/commands/sequence.c @@ -1232,6 +1232,8 @@ init_params(ParseState *pstate, List *options, bool isInit, DefElem *cache_value = NULL; DefElem *is_cycled = NULL; ListCell *option; + bool reset_max_value = false; + bool reset_min_value = false; *owned_by = NIL; @@ -1335,13 +1337,34 @@ init_params(ParseState *pstate, List *options, bool isInit, /* AS type */ if (as_type != NULL) { - seqform->seqtypid = typenameTypeId(pstate, defGetTypeName(as_type)); - if (seqform->seqtypid != INT2OID && - seqform->seqtypid != INT4OID && - seqform->seqtypid != INT8OID) + Oid newtypid = typenameTypeId(pstate, defGetTypeName(as_type)); + + if (newtypid != INT2OID && + newtypid != INT4OID && + newtypid != INT8OID) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("sequence type must be smallint, integer, or bigint"))); + + if (!isInit) + { + /* + * When changing type and the old sequence min/max values were the + * min/max of the old type, adjust sequence min/max values to + * min/max of new type. (Otherwise, the user chose explicit + * min/max values, which we'll leave alone.) + */ + if ((seqform->seqtypid == INT2OID && seqform->seqmax == PG_INT16_MAX) || + (seqform->seqtypid == INT4OID && seqform->seqmax == PG_INT32_MAX) || + (seqform->seqtypid == INT8OID && seqform->seqmax == PG_INT64_MAX)) + reset_max_value = true; + if ((seqform->seqtypid == INT2OID && seqform->seqmin == PG_INT16_MIN) || + (seqform->seqtypid == INT4OID && seqform->seqmin == PG_INT32_MIN) || + (seqform->seqtypid == INT8OID && seqform->seqmin == PG_INT64_MIN)) + reset_min_value = true; + } + + seqform->seqtypid = newtypid; } else if (isInit) seqform->seqtypid = INT8OID; @@ -1375,9 +1398,9 @@ init_params(ParseState *pstate, List *options, bool isInit, seqform->seqmax = defGetInt64(max_value); seqdataform->log_cnt = 0; } - else if (isInit || max_value != NULL) + else if (isInit || max_value != NULL || reset_max_value) { - if (seqform->seqincrement > 0) + if (seqform->seqincrement > 0 || reset_max_value) { /* ascending seq */ if (seqform->seqtypid == INT2OID) @@ -1412,11 +1435,9 @@ init_params(ParseState *pstate, List *options, bool isInit, seqform->seqmin = defGetInt64(min_value); seqdataform->log_cnt = 0; } - else if (isInit || min_value != NULL) + else if (isInit || min_value != NULL || reset_min_value) { - if (seqform->seqincrement > 0) - seqform->seqmin = 1; /* ascending seq */ - else + if (seqform->seqincrement < 0 || reset_min_value) { /* descending seq */ if (seqform->seqtypid == INT2OID) @@ -1426,6 +1447,8 @@ init_params(ParseState *pstate, List *options, bool isInit, else seqform->seqmin = PG_INT64_MIN; } + else + seqform->seqmin = 1; /* ascending seq */ seqdataform->log_cnt = 0; } diff --git a/src/test/regress/expected/sequence.out b/src/test/regress/expected/sequence.out index f339489151..1d8d02b800 100644 --- a/src/test/regress/expected/sequence.out +++ b/src/test/regress/expected/sequence.out @@ -32,19 +32,35 @@ DROP TABLE sequence_test_table; CREATE SEQUENCE sequence_test5 AS integer; CREATE SEQUENCE sequence_test6 AS smallint; CREATE SEQUENCE sequence_test7 AS bigint; +CREATE SEQUENCE sequence_test8 AS integer MAXVALUE 100000; +CREATE SEQUENCE sequence_test9 AS integer INCREMENT BY -1; +CREATE SEQUENCE sequence_test10 AS integer MINVALUE -100000 START 1; +CREATE SEQUENCE sequence_test11 AS smallint; +CREATE SEQUENCE sequence_test12 AS smallint INCREMENT -1; +CREATE SEQUENCE sequence_test13 AS smallint MINVALUE -32768; +CREATE SEQUENCE sequence_test14 AS smallint MAXVALUE 32767 INCREMENT -1; CREATE SEQUENCE sequence_testx AS text; ERROR: sequence type must be smallint, integer, or bigint CREATE SEQUENCE sequence_testx AS nosuchtype; ERROR: type "nosuchtype" does not exist LINE 1: CREATE SEQUENCE sequence_testx AS nosuchtype; ^ -ALTER SEQUENCE sequence_test5 AS smallint; -- fails -ERROR: MAXVALUE (2147483647) is out of range for sequence data type smallint -ALTER SEQUENCE sequence_test5 AS smallint NO MINVALUE NO MAXVALUE; CREATE SEQUENCE sequence_testx AS smallint MAXVALUE 100000; ERROR: MAXVALUE (100000) is out of range for sequence data type smallint CREATE SEQUENCE sequence_testx AS smallint MINVALUE -100000; ERROR: MINVALUE (-100000) is out of range for sequence data type smallint +ALTER SEQUENCE sequence_test5 AS smallint; -- success, max will be adjusted +ALTER SEQUENCE sequence_test8 AS smallint; -- fail, max has to be adjusted +ERROR: MAXVALUE (100000) is out of range for sequence data type smallint +ALTER SEQUENCE sequence_test8 AS smallint MAXVALUE 20000; -- ok now +ALTER SEQUENCE sequence_test9 AS smallint; -- success, min will be adjusted +ALTER SEQUENCE sequence_test10 AS smallint; -- fail, min has to be adjusted +ERROR: MINVALUE (-100000) is out of range for sequence data type smallint +ALTER SEQUENCE sequence_test10 AS smallint MINVALUE -20000; -- ok now +ALTER SEQUENCE sequence_test11 AS int; -- max will be adjusted +ALTER SEQUENCE sequence_test12 AS int; -- min will be adjusted +ALTER SEQUENCE sequence_test13 AS int; -- min and max will be adjusted +ALTER SEQUENCE sequence_test14 AS int; -- min and max will be adjusted --- --- test creation of SERIAL column --- @@ -459,19 +475,26 @@ SELECT * FROM information_schema.sequences ORDER BY sequence_name ASC; sequence_catalog | sequence_schema | sequence_name | data_type | numeric_precision | numeric_precision_radix | numeric_scale | start_value | minimum_value | maximum_value | increment | cycle_option ------------------+-----------------+--------------------+-----------+-------------------+-------------------------+---------------+-------------+----------------------+---------------------+-----------+-------------- + regression | public | sequence_test10 | smallint | 16 | 2 | 0 | 1 | -20000 | 32767 | 1 | NO + regression | public | sequence_test11 | integer | 32 | 2 | 0 | 1 | 1 | 2147483647 | 1 | NO + regression | public | sequence_test12 | integer | 32 | 2 | 0 | -1 | -2147483648 | -1 | -1 | NO + regression | public | sequence_test13 | integer | 32 | 2 | 0 | -32768 | -2147483648 | 2147483647 | 1 | NO + regression | public | sequence_test14 | integer | 32 | 2 | 0 | 32767 | -2147483648 | 2147483647 | -1 | NO regression | public | sequence_test2 | bigint | 64 | 2 | 0 | 32 | 5 | 36 | 4 | YES regression | public | sequence_test3 | bigint | 64 | 2 | 0 | 1 | 1 | 9223372036854775807 | 1 | NO regression | public | sequence_test4 | bigint | 64 | 2 | 0 | -1 | -9223372036854775808 | -1 | -1 | NO regression | public | sequence_test5 | smallint | 16 | 2 | 0 | 1 | 1 | 32767 | 1 | NO regression | public | sequence_test6 | smallint | 16 | 2 | 0 | 1 | 1 | 32767 | 1 | NO regression | public | sequence_test7 | bigint | 64 | 2 | 0 | 1 | 1 | 9223372036854775807 | 1 | NO + regression | public | sequence_test8 | smallint | 16 | 2 | 0 | 1 | 1 | 20000 | 1 | NO + regression | public | sequence_test9 | smallint | 16 | 2 | 0 | -1 | -32768 | -1 | -1 | NO regression | public | serialtest1_f2_foo | integer | 32 | 2 | 0 | 1 | 1 | 2147483647 | 1 | NO regression | public | serialtest2_f2_seq | integer | 32 | 2 | 0 | 1 | 1 | 2147483647 | 1 | NO regression | public | serialtest2_f3_seq | smallint | 16 | 2 | 0 | 1 | 1 | 32767 | 1 | NO regression | public | serialtest2_f4_seq | smallint | 16 | 2 | 0 | 1 | 1 | 32767 | 1 | NO regression | public | serialtest2_f5_seq | bigint | 64 | 2 | 0 | 1 | 1 | 9223372036854775807 | 1 | NO regression | public | serialtest2_f6_seq | bigint | 64 | 2 | 0 | 1 | 1 | 9223372036854775807 | 1 | NO -(12 rows) +(19 rows) SELECT schemaname, sequencename, start_value, min_value, max_value, increment_by, cycle, cache_size, last_value FROM pg_sequences @@ -479,19 +502,26 @@ WHERE sequencename ~ ANY(ARRAY['sequence_test', 'serialtest']) ORDER BY sequencename ASC; schemaname | sequencename | start_value | min_value | max_value | increment_by | cycle | cache_size | last_value ------------+--------------------+-------------+----------------------+---------------------+--------------+-------+------------+------------ + public | sequence_test10 | 1 | -20000 | 32767 | 1 | f | 1 | + public | sequence_test11 | 1 | 1 | 2147483647 | 1 | f | 1 | + public | sequence_test12 | -1 | -2147483648 | -1 | -1 | f | 1 | + public | sequence_test13 | -32768 | -2147483648 | 2147483647 | 1 | f | 1 | + public | sequence_test14 | 32767 | -2147483648 | 2147483647 | -1 | f | 1 | public | sequence_test2 | 32 | 5 | 36 | 4 | t | 1 | 5 public | sequence_test3 | 1 | 1 | 9223372036854775807 | 1 | f | 1 | public | sequence_test4 | -1 | -9223372036854775808 | -1 | -1 | f | 1 | -1 public | sequence_test5 | 1 | 1 | 32767 | 1 | f | 1 | public | sequence_test6 | 1 | 1 | 32767 | 1 | f | 1 | public | sequence_test7 | 1 | 1 | 9223372036854775807 | 1 | f | 1 | + public | sequence_test8 | 1 | 1 | 20000 | 1 | f | 1 | + public | sequence_test9 | -1 | -32768 | -1 | -1 | f | 1 | public | serialtest1_f2_foo | 1 | 1 | 2147483647 | 1 | f | 1 | 3 public | serialtest2_f2_seq | 1 | 1 | 2147483647 | 1 | f | 1 | 2 public | serialtest2_f3_seq | 1 | 1 | 32767 | 1 | f | 1 | 2 public | serialtest2_f4_seq | 1 | 1 | 32767 | 1 | f | 1 | 2 public | serialtest2_f5_seq | 1 | 1 | 9223372036854775807 | 1 | f | 1 | 2 public | serialtest2_f6_seq | 1 | 1 | 9223372036854775807 | 1 | f | 1 | 2 -(12 rows) +(19 rows) SELECT * FROM pg_sequence_parameters('sequence_test4'::regclass); start_value | minimum_value | maximum_value | increment | cycle_option | cache_size | data_type diff --git a/src/test/regress/sql/sequence.sql b/src/test/regress/sql/sequence.sql index 0fbd255967..74663d7351 100644 --- a/src/test/regress/sql/sequence.sql +++ b/src/test/regress/sql/sequence.sql @@ -23,15 +23,31 @@ DROP TABLE sequence_test_table; CREATE SEQUENCE sequence_test5 AS integer; CREATE SEQUENCE sequence_test6 AS smallint; CREATE SEQUENCE sequence_test7 AS bigint; +CREATE SEQUENCE sequence_test8 AS integer MAXVALUE 100000; +CREATE SEQUENCE sequence_test9 AS integer INCREMENT BY -1; +CREATE SEQUENCE sequence_test10 AS integer MINVALUE -100000 START 1; +CREATE SEQUENCE sequence_test11 AS smallint; +CREATE SEQUENCE sequence_test12 AS smallint INCREMENT -1; +CREATE SEQUENCE sequence_test13 AS smallint MINVALUE -32768; +CREATE SEQUENCE sequence_test14 AS smallint MAXVALUE 32767 INCREMENT -1; CREATE SEQUENCE sequence_testx AS text; CREATE SEQUENCE sequence_testx AS nosuchtype; -ALTER SEQUENCE sequence_test5 AS smallint; -- fails -ALTER SEQUENCE sequence_test5 AS smallint NO MINVALUE NO MAXVALUE; - CREATE SEQUENCE sequence_testx AS smallint MAXVALUE 100000; CREATE SEQUENCE sequence_testx AS smallint MINVALUE -100000; +ALTER SEQUENCE sequence_test5 AS smallint; -- success, max will be adjusted +ALTER SEQUENCE sequence_test8 AS smallint; -- fail, max has to be adjusted +ALTER SEQUENCE sequence_test8 AS smallint MAXVALUE 20000; -- ok now +ALTER SEQUENCE sequence_test9 AS smallint; -- success, min will be adjusted +ALTER SEQUENCE sequence_test10 AS smallint; -- fail, min has to be adjusted +ALTER SEQUENCE sequence_test10 AS smallint MINVALUE -20000; -- ok now + +ALTER SEQUENCE sequence_test11 AS int; -- max will be adjusted +ALTER SEQUENCE sequence_test12 AS int; -- min will be adjusted +ALTER SEQUENCE sequence_test13 AS int; -- min and max will be adjusted +ALTER SEQUENCE sequence_test14 AS int; -- min and max will be adjusted + --- --- test creation of SERIAL column --- -- cgit v1.2.3 From e75a78656bcaa3faff6b85891ca69d45dd21023f Mon Sep 17 00:00:00 2001 From: Andrew Dunstan Date: Tue, 4 Apr 2017 16:50:13 -0400 Subject: Clarify documentation of to_tsvector(json(b)) Per gripe from Sven R. Kunze --- doc/src/sgml/func.sgml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 19329dd103..e3d852cdc7 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -9569,7 +9569,10 @@ CREATE TYPE rainbow AS ENUM ('red', 'orange', 'yellow', 'green', 'blue', 'purple to_tsvector( config regconfig , document json(b)) tsvector - reduce document text to tsvector + + reduce each string value in the document to a tsvector, and then + concatentate those in document order to produce a single tsvector + to_tsvector('english', '{"a": "The Fat Rats"}'::json) 'fat':2 'rat':3 -- cgit v1.2.3 From 5ebeb579b9b281dba5f8415b2fbda86fdae7b366 Mon Sep 17 00:00:00 2001 From: Kevin Grittner Date: Tue, 4 Apr 2017 18:36:39 -0500 Subject: Follow-on cleanup for the transition table patch. Commit 59702716 added transition table support to PL/pgsql so that SQL queries in trigger functions could access those transient tables. In order to provide the same level of support for PL/perl, PL/python and PL/tcl, refactor the relevant code into a new function SPI_register_trigger_data. Call the new function in the trigger handler of all four PLs, and document it as a public SPI function so that authors of out-of-tree PLs can do the same. Also get rid of a second QueryEnvironment object that was maintained by PL/pgsql. That was previously used to deal with cursors, but the same approach wasn't appropriate for PLs that are less tangled up with core code. Instead, have SPI_cursor_open install the connection's current QueryEnvironment, as already happens for SPI_execute_plan. While in the docs, remove the note that transition tables were only supported in C and PL/pgSQL triggers, and correct some ommissions. Thomas Munro with some work by Kevin Grittner (mostly docs) --- doc/src/sgml/ref/create_trigger.sgml | 10 +-- doc/src/sgml/spi.sgml | 125 ++++++++++++++++++++++++++ doc/src/sgml/trigger.sgml | 54 +++++++++-- src/backend/executor/spi.c | 52 +++++++++++ src/include/executor/spi.h | 3 + src/pl/plperl/expected/plperl_trigger.out | 29 ++++++ src/pl/plperl/plperl.c | 7 ++ src/pl/plperl/sql/plperl_trigger.sql | 32 +++++++ src/pl/plpgsql/src/pl_exec.c | 49 +--------- src/pl/plpgsql/src/plpgsql.h | 4 - src/pl/plpython/expected/plpython_trigger.out | 21 +++++ src/pl/plpython/plpy_exec.c | 5 ++ src/pl/plpython/sql/plpython_trigger.sql | 24 +++++ src/pl/tcl/expected/pltcl_queries.out | 21 +++++ src/pl/tcl/pltcl.c | 5 ++ src/pl/tcl/sql/pltcl_queries.sql | 20 +++++ 16 files changed, 398 insertions(+), 63 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/create_trigger.sgml b/doc/src/sgml/ref/create_trigger.sgml index e22e42e7dc..24195b3849 100644 --- a/doc/src/sgml/ref/create_trigger.sgml +++ b/doc/src/sgml/ref/create_trigger.sgml @@ -8,6 +8,11 @@ PostgreSQL documentation CREATE TRIGGER + + transition tables + ephemeral named relation + + CREATE TRIGGER 7 @@ -322,11 +327,6 @@ UPDATE OF column_name1 [, column_name2 The (unqualified) name to be used within the trigger for this relation. - - - So far only triggers written in C or PL/pgSQL support this. - - diff --git a/doc/src/sgml/spi.sgml b/doc/src/sgml/spi.sgml index af7eada2e1..86be87c0fd 100644 --- a/doc/src/sgml/spi.sgml +++ b/doc/src/sgml/spi.sgml @@ -2644,6 +2644,11 @@ SPIPlanPtr SPI_saveplan(SPIPlanPtr plan) SPI_register_relation + + ephemeral named relation + registering with SPI + + SPI_register_relation 3 @@ -2746,6 +2751,11 @@ int SPI_register_relation(EphemeralNamedRelation enr) SPI_unregister_relation + + ephemeral named relation + unregistering from SPI + + SPI_unregister_relation 3 @@ -2843,6 +2853,121 @@ int SPI_unregister_relation(const char * name) + + SPI_register_trigger_data + + + ephemeral named relation + registering with SPI + + + + transition tables + implementation in PLs + + + + SPI_register_trigger_data + 3 + + + + SPI_register_trigger_data + make ephemeral trigger data available in SPI queries + + + + +int SPI_register_trigger_data(TriggerData *tdata) + + + + + Description + + + SPI_register_trigger_data makes any ephemeral + relations captured by a trigger available to queries planned and executed + through the current SPI connection. Currently, this means the transition + tables captured by an AFTER trigger defined with a + REFERENCING OLD/NEW TABLE AS ... clause. This function + should be called by a PL trigger handler function after connecting. + + + + + Arguments + + + + TriggerData *tdata + + + the TriggerData object passed to a trigger + handler function as fcinfo->context + + + + + + + + Return Value + + + If the execution of the command was successful then the following + (nonnegative) value will be returned: + + + + SPI_OK_TD_REGISTER + + + if the captured trigger data (if any) has been successfully registered + + + + + + + + On error, one of the following negative values is returned: + + + + SPI_ERROR_ARGUMENT + + + if tdata is NULL + + + + + + SPI_ERROR_UNCONNECTED + + + if called from an unconnected procedure + + + + + + SPI_ERROR_REL_DUPLICATE + + + if the name of any trigger data transient relation is already + registered for this connection + + + + + + + + + + diff --git a/doc/src/sgml/trigger.sgml b/doc/src/sgml/trigger.sgml index 8f724c8664..2a718d7f47 100644 --- a/doc/src/sgml/trigger.sgml +++ b/doc/src/sgml/trigger.sgml @@ -395,6 +395,11 @@ in C + + transition tables + referencing from C trigger + + This section describes the low-level details of the interface to a trigger function. This information is only needed when writing @@ -438,14 +443,16 @@ CALLED_AS_TRIGGER(fcinfo) typedef struct TriggerData { - NodeTag type; - TriggerEvent tg_event; - Relation tg_relation; - HeapTuple tg_trigtuple; - HeapTuple tg_newtuple; - Trigger *tg_trigger; - Buffer tg_trigtuplebuf; - Buffer tg_newtuplebuf; + NodeTag type; + TriggerEvent tg_event; + Relation tg_relation; + HeapTuple tg_trigtuple; + HeapTuple tg_newtuple; + Trigger *tg_trigger; + Buffer tg_trigtuplebuf; + Buffer tg_newtuplebuf; + Tuplestorestate *tg_oldtable; + Tuplestorestate *tg_newtable; } TriggerData; @@ -629,6 +636,8 @@ typedef struct Trigger int16 *tgattr; char **tgargs; char *tgqual; + char *tgoldtable; + char *tgnewtable; } Trigger; @@ -662,9 +671,38 @@ typedef struct Trigger + + tg_oldtable + + + A pointer to a structure of type Tuplestorestate + containing zero or more rows in the format specified by + tg_relation, or a NULL pointer + if there is no OLD TABLE transition relation. + + + + + + tg_newtable + + + A pointer to a structure of type Tuplestorestate + containing zero or more rows in the format specified by + tg_relation, or a NULL pointer + if there is no NEW TABLE transition relation. + + + + + + To allow queries issued through SPI to reference transition tables, see + . + + A trigger function must return either a HeapTuple pointer or a NULL pointer diff --git a/src/backend/executor/spi.c b/src/backend/executor/spi.c index 2f07a444b4..3a89ccd913 100644 --- a/src/backend/executor/spi.c +++ b/src/backend/executor/spi.c @@ -1257,6 +1257,9 @@ SPI_cursor_open_internal(const char *name, SPIPlanPtr plan, errdetail("Scrollable cursors must be READ ONLY."))); } + /* Make current query environment available to portal at execution time. */ + portal->queryEnv = _SPI_current->queryEnv; + /* * If told to be read-only, or in parallel mode, verify that this query is * in fact read-only. This can't be done earlier because we need to look @@ -2716,3 +2719,52 @@ SPI_unregister_relation(const char *name) return res; } + +/* + * Register the transient relations from 'tdata' using this SPI connection. + * This should be called by PL implementations' trigger handlers after + * connecting, in order to make transition tables visible to any queries run + * in this connection. + */ +int +SPI_register_trigger_data(TriggerData *tdata) +{ + if (tdata == NULL) + return SPI_ERROR_ARGUMENT; + + if (tdata->tg_newtable) + { + EphemeralNamedRelation enr = + palloc(sizeof(EphemeralNamedRelationData)); + int rc; + + enr->md.name = tdata->tg_trigger->tgnewtable; + enr->md.reliddesc = tdata->tg_relation->rd_id; + enr->md.tupdesc = NULL; + enr->md.enrtype = ENR_NAMED_TUPLESTORE; + enr->md.enrtuples = tuplestore_tuple_count(tdata->tg_newtable); + enr->reldata = tdata->tg_newtable; + rc = SPI_register_relation(enr); + if (rc != SPI_OK_REL_REGISTER) + return rc; + } + + if (tdata->tg_oldtable) + { + EphemeralNamedRelation enr = + palloc(sizeof(EphemeralNamedRelationData)); + int rc; + + enr->md.name = tdata->tg_trigger->tgoldtable; + enr->md.reliddesc = tdata->tg_relation->rd_id; + enr->md.tupdesc = NULL; + enr->md.enrtype = ENR_NAMED_TUPLESTORE; + enr->md.enrtuples = tuplestore_tuple_count(tdata->tg_oldtable); + enr->reldata = tdata->tg_oldtable; + rc = SPI_register_relation(enr); + if (rc != SPI_OK_REL_REGISTER) + return rc; + } + + return SPI_OK_TD_REGISTER; +} diff --git a/src/include/executor/spi.h b/src/include/executor/spi.h index e2e8bb9553..94a805d477 100644 --- a/src/include/executor/spi.h +++ b/src/include/executor/spi.h @@ -13,6 +13,7 @@ #ifndef SPI_H #define SPI_H +#include "commands/trigger.h" #include "lib/ilist.h" #include "nodes/parsenodes.h" #include "utils/portal.h" @@ -62,6 +63,7 @@ typedef struct _SPI_plan *SPIPlanPtr; #define SPI_OK_REWRITTEN 14 #define SPI_OK_REL_REGISTER 15 #define SPI_OK_REL_UNREGISTER 16 +#define SPI_OK_TD_REGISTER 17 /* These used to be functions, now just no-ops for backwards compatibility */ #define SPI_push() ((void) 0) @@ -152,6 +154,7 @@ extern void SPI_cursor_close(Portal portal); extern int SPI_register_relation(EphemeralNamedRelation enr); extern int SPI_unregister_relation(const char *name); +extern int SPI_register_trigger_data(TriggerData *tdata); extern void AtEOXact_SPI(bool isCommit); extern void AtEOSubXact_SPI(bool isCommit, SubTransactionId mySubid); diff --git a/src/pl/plperl/expected/plperl_trigger.out b/src/pl/plperl/expected/plperl_trigger.out index 5e3860ef97..28011cd9f6 100644 --- a/src/pl/plperl/expected/plperl_trigger.out +++ b/src/pl/plperl/expected/plperl_trigger.out @@ -241,6 +241,35 @@ $$ LANGUAGE plperl; SELECT direct_trigger(); ERROR: trigger functions can only be called as triggers CONTEXT: compilation of PL/Perl function "direct_trigger" +-- check that SQL run in trigger code can see transition tables +CREATE TABLE transition_table_test (id int, name text); +INSERT INTO transition_table_test VALUES (1, 'a'); +CREATE FUNCTION transition_table_test_f() RETURNS trigger LANGUAGE plperl AS +$$ + my $cursor = spi_query("SELECT * FROM old_table"); + my $row = spi_fetchrow($cursor); + defined($row) || die "expected a row"; + elog(INFO, "old: " . $row->{id} . " -> " . $row->{name}); + my $row = spi_fetchrow($cursor); + !defined($row) || die "expected no more rows"; + + my $cursor = spi_query("SELECT * FROM new_table"); + my $row = spi_fetchrow($cursor); + defined($row) || die "expected a row"; + elog(INFO, "new: " . $row->{id} . " -> " . $row->{name}); + my $row = spi_fetchrow($cursor); + !defined($row) || die "expected no more rows"; + + return undef; +$$; +CREATE TRIGGER a_t AFTER UPDATE ON transition_table_test + REFERENCING OLD TABLE AS old_table NEW TABLE AS new_table + FOR EACH STATEMENT EXECUTE PROCEDURE transition_table_test_f(); +UPDATE transition_table_test SET name = 'b'; +INFO: old: 1 -> a +INFO: new: 1 -> b +DROP TABLE transition_table_test; +DROP FUNCTION transition_table_test_f(); -- test plperl command triggers create or replace function perlsnitch() returns event_trigger language plperl as $$ elog(NOTICE, "perlsnitch: " . $_TD->{event} . " " . $_TD->{tag} . " "); diff --git a/src/pl/plperl/plperl.c b/src/pl/plperl/plperl.c index 00979cba74..1de770b250 100644 --- a/src/pl/plperl/plperl.c +++ b/src/pl/plperl/plperl.c @@ -2442,11 +2442,18 @@ plperl_trigger_handler(PG_FUNCTION_ARGS) SV *svTD; HV *hvTD; ErrorContextCallback pl_error_context; + TriggerData *tdata; + int rc PG_USED_FOR_ASSERTS_ONLY; /* Connect to SPI manager */ if (SPI_connect() != SPI_OK_CONNECT) elog(ERROR, "could not connect to SPI manager"); + /* Make transition tables visible to this SPI connection */ + tdata = (TriggerData *) fcinfo->context; + rc = SPI_register_trigger_data(tdata); + Assert(rc >= 0); + /* Find or compile the function */ prodesc = compile_plperl_function(fcinfo->flinfo->fn_oid, true, false); current_call_data->prodesc = prodesc; diff --git a/src/pl/plperl/sql/plperl_trigger.sql b/src/pl/plperl/sql/plperl_trigger.sql index a375b401ea..624193b9d0 100644 --- a/src/pl/plperl/sql/plperl_trigger.sql +++ b/src/pl/plperl/sql/plperl_trigger.sql @@ -170,6 +170,38 @@ $$ LANGUAGE plperl; SELECT direct_trigger(); +-- check that SQL run in trigger code can see transition tables + +CREATE TABLE transition_table_test (id int, name text); +INSERT INTO transition_table_test VALUES (1, 'a'); + +CREATE FUNCTION transition_table_test_f() RETURNS trigger LANGUAGE plperl AS +$$ + my $cursor = spi_query("SELECT * FROM old_table"); + my $row = spi_fetchrow($cursor); + defined($row) || die "expected a row"; + elog(INFO, "old: " . $row->{id} . " -> " . $row->{name}); + my $row = spi_fetchrow($cursor); + !defined($row) || die "expected no more rows"; + + my $cursor = spi_query("SELECT * FROM new_table"); + my $row = spi_fetchrow($cursor); + defined($row) || die "expected a row"; + elog(INFO, "new: " . $row->{id} . " -> " . $row->{name}); + my $row = spi_fetchrow($cursor); + !defined($row) || die "expected no more rows"; + + return undef; +$$; + +CREATE TRIGGER a_t AFTER UPDATE ON transition_table_test + REFERENCING OLD TABLE AS old_table NEW TABLE AS new_table + FOR EACH STATEMENT EXECUTE PROCEDURE transition_table_test_f(); +UPDATE transition_table_test SET name = 'b'; + +DROP TABLE transition_table_test; +DROP FUNCTION transition_table_test_f(); + -- test plperl command triggers create or replace function perlsnitch() returns event_trigger language plperl as $$ elog(NOTICE, "perlsnitch: " . $_TD->{event} . " " . $_TD->{tag} . " "); diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c index 43da986fc0..8d7c7caa1c 100644 --- a/src/pl/plpgsql/src/pl_exec.c +++ b/src/pl/plpgsql/src/pl_exec.c @@ -689,46 +689,9 @@ plpgsql_exec_trigger(PLpgSQL_function *func, else elog(ERROR, "unrecognized trigger action: not INSERT, DELETE, or UPDATE"); - /* - * Capture the NEW and OLD transition TABLE tuplestores (if specified for - * this trigger). - */ - if (trigdata->tg_newtable || trigdata->tg_oldtable) - { - estate.queryEnv = create_queryEnv(); - if (trigdata->tg_newtable) - { - EphemeralNamedRelation enr = - palloc(sizeof(EphemeralNamedRelationData)); - int rc PG_USED_FOR_ASSERTS_ONLY; - - enr->md.name = trigdata->tg_trigger->tgnewtable; - enr->md.reliddesc = RelationGetRelid(trigdata->tg_relation); - enr->md.tupdesc = NULL; - enr->md.enrtype = ENR_NAMED_TUPLESTORE; - enr->md.enrtuples = tuplestore_tuple_count(trigdata->tg_newtable); - enr->reldata = trigdata->tg_newtable; - register_ENR(estate.queryEnv, enr); - rc = SPI_register_relation(enr); - Assert(rc >= 0); - } - if (trigdata->tg_oldtable) - { - EphemeralNamedRelation enr = - palloc(sizeof(EphemeralNamedRelationData)); - int rc PG_USED_FOR_ASSERTS_ONLY; - - enr->md.name = trigdata->tg_trigger->tgoldtable; - enr->md.reliddesc = RelationGetRelid(trigdata->tg_relation); - enr->md.tupdesc = NULL; - enr->md.enrtype = ENR_NAMED_TUPLESTORE; - enr->md.enrtuples = tuplestore_tuple_count(trigdata->tg_oldtable); - enr->reldata = trigdata->tg_oldtable; - register_ENR(estate.queryEnv, enr); - rc = SPI_register_relation(enr); - Assert(rc >= 0); - } - } + /* Make transition tables visible to this SPI connection */ + rc = SPI_register_trigger_data(trigdata); + Assert(rc >= 0); /* * Assign the special tg_ variables @@ -3483,9 +3446,6 @@ plpgsql_estate_setup(PLpgSQL_execstate *estate, estate->paramLI->paramMask = NULL; estate->params_dirty = false; - /* default tuplestore cache to "none" */ - estate->queryEnv = NULL; - /* set up for use of appropriate simple-expression EState and cast hash */ if (simple_eval_estate) { @@ -7373,9 +7333,6 @@ exec_dynquery_with_params(PLpgSQL_execstate *estate, /* Release transient data */ MemoryContextReset(stmt_mcontext); - /* Make sure the portal knows about any named tuplestores. */ - portal->queryEnv = estate->queryEnv; - return portal; } diff --git a/src/pl/plpgsql/src/plpgsql.h b/src/pl/plpgsql/src/plpgsql.h index 43a62ef34e..abb19035b6 100644 --- a/src/pl/plpgsql/src/plpgsql.h +++ b/src/pl/plpgsql/src/plpgsql.h @@ -20,7 +20,6 @@ #include "commands/event_trigger.h" #include "commands/trigger.h" #include "executor/spi.h" -#include "utils/queryenvironment.h" /********************************************************************** * Definitions @@ -911,9 +910,6 @@ typedef struct PLpgSQL_execstate ParamListInfo paramLI; bool params_dirty; /* T if any resettable datum has been passed */ - /* custom environment for parsing/execution of query for this context */ - QueryEnvironment *queryEnv; - /* EState to use for "simple" expression evaluation */ EState *simple_eval_estate; diff --git a/src/pl/plpython/expected/plpython_trigger.out b/src/pl/plpython/expected/plpython_trigger.out index 4148963f3a..d7ab8ac6b8 100644 --- a/src/pl/plpython/expected/plpython_trigger.out +++ b/src/pl/plpython/expected/plpython_trigger.out @@ -503,3 +503,24 @@ SELECT * FROM b; 1234 (1 row) +-- check that SQL run in trigger code can see transition tables +CREATE TABLE transition_table_test (id int, name text); +INSERT INTO transition_table_test VALUES (1, 'a'); +CREATE FUNCTION transition_table_test_f() RETURNS trigger LANGUAGE plpythonu AS +$$ + rv = plpy.execute("SELECT * FROM old_table") + assert(rv.nrows() == 1) + plpy.info("old: " + str(rv[0]["id"]) + " -> " + rv[0]["name"]) + rv = plpy.execute("SELECT * FROM new_table") + assert(rv.nrows() == 1) + plpy.info("new: " + str(rv[0]["id"]) + " -> " + rv[0]["name"]) + return None +$$; +CREATE TRIGGER a_t AFTER UPDATE ON transition_table_test + REFERENCING OLD TABLE AS old_table NEW TABLE AS new_table + FOR EACH STATEMENT EXECUTE PROCEDURE transition_table_test_f(); +UPDATE transition_table_test SET name = 'b'; +INFO: old: 1 -> a +INFO: new: 1 -> b +DROP TABLE transition_table_test; +DROP FUNCTION transition_table_test_f(); diff --git a/src/pl/plpython/plpy_exec.c b/src/pl/plpython/plpy_exec.c index 697a0e1cc0..7ccd2c8235 100644 --- a/src/pl/plpython/plpy_exec.c +++ b/src/pl/plpython/plpy_exec.c @@ -345,6 +345,11 @@ PLy_exec_trigger(FunctionCallInfo fcinfo, PLyProcedure *proc) PG_TRY(); { + int rc PG_USED_FOR_ASSERTS_ONLY; + + rc = SPI_register_trigger_data(tdata); + Assert(rc >= 0); + plargs = PLy_trigger_build_args(fcinfo, proc, &rv); plrv = PLy_procedure_call(proc, "TD", plargs); diff --git a/src/pl/plpython/sql/plpython_trigger.sql b/src/pl/plpython/sql/plpython_trigger.sql index a054fe729b..79c24b714b 100644 --- a/src/pl/plpython/sql/plpython_trigger.sql +++ b/src/pl/plpython/sql/plpython_trigger.sql @@ -406,3 +406,27 @@ SELECT * FROM a; DROP TABLE a; INSERT INTO b DEFAULT VALUES; SELECT * FROM b; + +-- check that SQL run in trigger code can see transition tables + +CREATE TABLE transition_table_test (id int, name text); +INSERT INTO transition_table_test VALUES (1, 'a'); + +CREATE FUNCTION transition_table_test_f() RETURNS trigger LANGUAGE plpythonu AS +$$ + rv = plpy.execute("SELECT * FROM old_table") + assert(rv.nrows() == 1) + plpy.info("old: " + str(rv[0]["id"]) + " -> " + rv[0]["name"]) + rv = plpy.execute("SELECT * FROM new_table") + assert(rv.nrows() == 1) + plpy.info("new: " + str(rv[0]["id"]) + " -> " + rv[0]["name"]) + return None +$$; + +CREATE TRIGGER a_t AFTER UPDATE ON transition_table_test + REFERENCING OLD TABLE AS old_table NEW TABLE AS new_table + FOR EACH STATEMENT EXECUTE PROCEDURE transition_table_test_f(); +UPDATE transition_table_test SET name = 'b'; + +DROP TABLE transition_table_test; +DROP FUNCTION transition_table_test_f(); diff --git a/src/pl/tcl/expected/pltcl_queries.out b/src/pl/tcl/expected/pltcl_queries.out index 7300b315d6..5f50f46887 100644 --- a/src/pl/tcl/expected/pltcl_queries.out +++ b/src/pl/tcl/expected/pltcl_queries.out @@ -660,3 +660,24 @@ select tcl_eval($$ (1 row) +-- test transition table visibility +create table transition_table_test (id int, name text); +insert into transition_table_test values (1, 'a'); +create function transition_table_test_f() returns trigger language pltcl as +$$ + spi_exec -array C "SELECT id, name FROM old_table" { + elog INFO "old: $C(id) -> $C(name)" + } + spi_exec -array C "SELECT id, name FROM new_table" { + elog INFO "new: $C(id) -> $C(name)" + } + return OK +$$; +CREATE TRIGGER a_t AFTER UPDATE ON transition_table_test + REFERENCING OLD TABLE AS old_table NEW TABLE AS new_table + FOR EACH STATEMENT EXECUTE PROCEDURE transition_table_test_f(); +update transition_table_test set name = 'b'; +INFO: old: 1 -> a +INFO: new: 1 -> b +drop table transition_table_test; +drop function transition_table_test_f(); diff --git a/src/pl/tcl/pltcl.c b/src/pl/tcl/pltcl.c index b8fcf0673d..5a48e5d175 100644 --- a/src/pl/tcl/pltcl.c +++ b/src/pl/tcl/pltcl.c @@ -1039,6 +1039,7 @@ pltcl_trigger_handler(PG_FUNCTION_ARGS, pltcl_call_state *call_state, const char *result; int result_Objc; Tcl_Obj **result_Objv; + int rc PG_USED_FOR_ASSERTS_ONLY; call_state->trigdata = trigdata; @@ -1046,6 +1047,10 @@ pltcl_trigger_handler(PG_FUNCTION_ARGS, pltcl_call_state *call_state, if (SPI_connect() != SPI_OK_CONNECT) elog(ERROR, "could not connect to SPI manager"); + /* Make transition tables visible to this SPI connection */ + rc = SPI_register_trigger_data(trigdata); + Assert(rc >= 0); + /* Find or compile the function */ prodesc = compile_pltcl_function(fcinfo->flinfo->fn_oid, RelationGetRelid(trigdata->tg_relation), diff --git a/src/pl/tcl/sql/pltcl_queries.sql b/src/pl/tcl/sql/pltcl_queries.sql index 29ed616cd0..dabd8cd35f 100644 --- a/src/pl/tcl/sql/pltcl_queries.sql +++ b/src/pl/tcl/sql/pltcl_queries.sql @@ -216,3 +216,23 @@ select tcl_eval($$ after 100 {set ::tcl_vwait 1} vwait ::tcl_vwait unset -nocomplain ::tcl_vwait$$); + +-- test transition table visibility +create table transition_table_test (id int, name text); +insert into transition_table_test values (1, 'a'); +create function transition_table_test_f() returns trigger language pltcl as +$$ + spi_exec -array C "SELECT id, name FROM old_table" { + elog INFO "old: $C(id) -> $C(name)" + } + spi_exec -array C "SELECT id, name FROM new_table" { + elog INFO "new: $C(id) -> $C(name)" + } + return OK +$$; +CREATE TRIGGER a_t AFTER UPDATE ON transition_table_test + REFERENCING OLD TABLE AS old_table NEW TABLE AS new_table + FOR EACH STATEMENT EXECUTE PROCEDURE transition_table_test_f(); +update transition_table_test set name = 'b'; +drop table transition_table_test; +drop function transition_table_test_f(); -- cgit v1.2.3 From 193f5f9e913f3ead6609cc99be82578e0dedd409 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 4 Apr 2017 23:48:49 -0400 Subject: pageinspect: Add bt_page_items function with bytea argument Author: Tomas Vondra Reviewed-by: Ashutosh Sharma --- contrib/pageinspect/btreefuncs.c | 198 +++++++++++++++++++------- contrib/pageinspect/expected/btree.out | 13 ++ contrib/pageinspect/pageinspect--1.5--1.6.sql | 14 ++ contrib/pageinspect/sql/btree.sql | 4 + doc/src/sgml/pageinspect.sgml | 32 +++++ src/include/access/nbtree.h | 1 + 6 files changed, 211 insertions(+), 51 deletions(-) (limited to 'doc/src') diff --git a/contrib/pageinspect/btreefuncs.c b/contrib/pageinspect/btreefuncs.c index 3d69aa9d37..02440eca47 100644 --- a/contrib/pageinspect/btreefuncs.c +++ b/contrib/pageinspect/btreefuncs.c @@ -41,6 +41,7 @@ PG_FUNCTION_INFO_V1(bt_metap); PG_FUNCTION_INFO_V1(bt_page_items); +PG_FUNCTION_INFO_V1(bt_page_items_bytea); PG_FUNCTION_INFO_V1(bt_page_stats); #define IS_INDEX(r) ((r)->rd_rel->relkind == RELKIND_INDEX) @@ -235,14 +236,6 @@ bt_page_stats(PG_FUNCTION_ARGS) PG_RETURN_DATUM(result); } -/*------------------------------------------------------- - * bt_page_items() - * - * Get IndexTupleData set in a btree page - * - * Usage: SELECT * FROM bt_page_items('t1_pkey', 1); - *------------------------------------------------------- - */ /* * cross-call data structure for SRF @@ -253,14 +246,72 @@ struct user_args OffsetNumber offset; }; +/*------------------------------------------------------- + * bt_page_print_tuples() + * + * Form a tuple describing index tuple at a given offset + * ------------------------------------------------------ + */ +static Datum +bt_page_print_tuples(FuncCallContext *fctx, Page page, OffsetNumber offset) +{ + char *values[6]; + HeapTuple tuple; + ItemId id; + IndexTuple itup; + int j; + int off; + int dlen; + char *dump; + char *ptr; + + id = PageGetItemId(page, offset); + + if (!ItemIdIsValid(id)) + elog(ERROR, "invalid ItemId"); + + itup = (IndexTuple) PageGetItem(page, id); + + j = 0; + values[j++] = psprintf("%d", offset); + values[j++] = psprintf("(%u,%u)", + ItemPointerGetBlockNumberNoCheck(&itup->t_tid), + ItemPointerGetOffsetNumberNoCheck(&itup->t_tid)); + values[j++] = psprintf("%d", (int) IndexTupleSize(itup)); + values[j++] = psprintf("%c", IndexTupleHasNulls(itup) ? 't' : 'f'); + values[j++] = psprintf("%c", IndexTupleHasVarwidths(itup) ? 't' : 'f'); + + ptr = (char *) itup + IndexInfoFindDataOffset(itup->t_info); + dlen = IndexTupleSize(itup) - IndexInfoFindDataOffset(itup->t_info); + dump = palloc0(dlen * 3 + 1); + values[j] = dump; + for (off = 0; off < dlen; off++) + { + if (off > 0) + *dump++ = ' '; + sprintf(dump, "%02x", *(ptr + off) & 0xff); + dump += 2; + } + + tuple = BuildTupleFromCStrings(fctx->attinmeta, values); + + return HeapTupleGetDatum(tuple); +} + +/*------------------------------------------------------- + * bt_page_items() + * + * Get IndexTupleData set in a btree page + * + * Usage: SELECT * FROM bt_page_items('t1_pkey', 1); + *------------------------------------------------------- + */ Datum bt_page_items(PG_FUNCTION_ARGS) { text *relname = PG_GETARG_TEXT_PP(0); uint32 blkno = PG_GETARG_UINT32(1); Datum result; - char *values[6]; - HeapTuple tuple; FuncCallContext *fctx; MemoryContext mctx; struct user_args *uargs; @@ -345,47 +396,8 @@ bt_page_items(PG_FUNCTION_ARGS) if (fctx->call_cntr < fctx->max_calls) { - ItemId id; - IndexTuple itup; - int j; - int off; - int dlen; - char *dump; - char *ptr; - - id = PageGetItemId(uargs->page, uargs->offset); - - if (!ItemIdIsValid(id)) - elog(ERROR, "invalid ItemId"); - - itup = (IndexTuple) PageGetItem(uargs->page, id); - - j = 0; - values[j++] = psprintf("%d", uargs->offset); - values[j++] = psprintf("(%u,%u)", - ItemPointerGetBlockNumberNoCheck(&itup->t_tid), - ItemPointerGetOffsetNumberNoCheck(&itup->t_tid)); - values[j++] = psprintf("%d", (int) IndexTupleSize(itup)); - values[j++] = psprintf("%c", IndexTupleHasNulls(itup) ? 't' : 'f'); - values[j++] = psprintf("%c", IndexTupleHasVarwidths(itup) ? 't' : 'f'); - - ptr = (char *) itup + IndexInfoFindDataOffset(itup->t_info); - dlen = IndexTupleSize(itup) - IndexInfoFindDataOffset(itup->t_info); - dump = palloc0(dlen * 3 + 1); - values[j] = dump; - for (off = 0; off < dlen; off++) - { - if (off > 0) - *dump++ = ' '; - sprintf(dump, "%02x", *(ptr + off) & 0xff); - dump += 2; - } - - tuple = BuildTupleFromCStrings(fctx->attinmeta, values); - result = HeapTupleGetDatum(tuple); - - uargs->offset = uargs->offset + 1; - + result = bt_page_print_tuples(fctx, uargs->page, uargs->offset); + uargs->offset++; SRF_RETURN_NEXT(fctx, result); } else @@ -396,6 +408,90 @@ bt_page_items(PG_FUNCTION_ARGS) } } +/*------------------------------------------------------- + * bt_page_items_bytea() + * + * Get IndexTupleData set in a btree page + * + * Usage: SELECT * FROM bt_page_items(get_raw_page('t1_pkey', 1)); + *------------------------------------------------------- + */ + +Datum +bt_page_items_bytea(PG_FUNCTION_ARGS) +{ + bytea *raw_page = PG_GETARG_BYTEA_P(0); + Datum result; + FuncCallContext *fctx; + struct user_args *uargs; + int raw_page_size; + + if (!superuser()) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + (errmsg("must be superuser to use pageinspect functions")))); + + if (SRF_IS_FIRSTCALL()) + { + BTPageOpaque opaque; + MemoryContext mctx; + TupleDesc tupleDesc; + + raw_page_size = VARSIZE(raw_page) - VARHDRSZ; + + if (raw_page_size < SizeOfPageHeaderData) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("input page too small (%d bytes)", raw_page_size))); + + fctx = SRF_FIRSTCALL_INIT(); + mctx = MemoryContextSwitchTo(fctx->multi_call_memory_ctx); + + uargs = palloc(sizeof(struct user_args)); + + uargs->page = VARDATA(raw_page); + + uargs->offset = FirstOffsetNumber; + + opaque = (BTPageOpaque) PageGetSpecialPointer(uargs->page); + + if (P_ISMETA(opaque)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("block is a meta page"))); + + if (P_ISDELETED(opaque)) + elog(NOTICE, "page is deleted"); + + fctx->max_calls = PageGetMaxOffsetNumber(uargs->page); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupleDesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + + fctx->attinmeta = TupleDescGetAttInMetadata(tupleDesc); + + fctx->user_fctx = uargs; + + MemoryContextSwitchTo(mctx); + } + + fctx = SRF_PERCALL_SETUP(); + uargs = fctx->user_fctx; + + if (fctx->call_cntr < fctx->max_calls) + { + result = bt_page_print_tuples(fctx, uargs->page, uargs->offset); + uargs->offset++; + SRF_RETURN_NEXT(fctx, result); + } + else + { + pfree(uargs); + SRF_RETURN_DONE(fctx); + } +} + /* ------------------------------------------------ * bt_metap() diff --git a/contrib/pageinspect/expected/btree.out b/contrib/pageinspect/expected/btree.out index 82a49e3d6c..67b103add3 100644 --- a/contrib/pageinspect/expected/btree.out +++ b/contrib/pageinspect/expected/btree.out @@ -42,4 +42,17 @@ data | 01 00 00 00 00 00 00 01 SELECT * FROM bt_page_items('test1_a_idx', 2); ERROR: block number out of range +SELECT * FROM bt_page_items(get_raw_page('test1_a_idx', 0)); +ERROR: block is a meta page +SELECT * FROM bt_page_items(get_raw_page('test1_a_idx', 1)); +-[ RECORD 1 ]----------------------- +itemoffset | 1 +ctid | (0,1) +itemlen | 16 +nulls | f +vars | f +data | 01 00 00 00 00 00 00 01 + +SELECT * FROM bt_page_items(get_raw_page('test1_a_idx', 2)); +ERROR: block number 2 is out of range for relation "test1_a_idx" DROP TABLE test1; diff --git a/contrib/pageinspect/pageinspect--1.5--1.6.sql b/contrib/pageinspect/pageinspect--1.5--1.6.sql index 6ac2a8011d..c4356288db 100644 --- a/contrib/pageinspect/pageinspect--1.5--1.6.sql +++ b/contrib/pageinspect/pageinspect--1.5--1.6.sql @@ -83,3 +83,17 @@ CREATE FUNCTION page_checksum(IN page bytea, IN blkno int4) RETURNS smallint AS 'MODULE_PATHNAME', 'page_checksum' LANGUAGE C STRICT PARALLEL SAFE; + +-- +-- bt_page_items_bytea() +-- +CREATE FUNCTION bt_page_items(IN page bytea, + OUT itemoffset smallint, + OUT ctid tid, + OUT itemlen smallint, + OUT nulls bool, + OUT vars bool, + OUT data text) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'bt_page_items_bytea' +LANGUAGE C STRICT PARALLEL SAFE; diff --git a/contrib/pageinspect/sql/btree.sql b/contrib/pageinspect/sql/btree.sql index 1eafc3249c..8eac64c7b3 100644 --- a/contrib/pageinspect/sql/btree.sql +++ b/contrib/pageinspect/sql/btree.sql @@ -14,4 +14,8 @@ SELECT * FROM bt_page_items('test1_a_idx', 0); SELECT * FROM bt_page_items('test1_a_idx', 1); SELECT * FROM bt_page_items('test1_a_idx', 2); +SELECT * FROM bt_page_items(get_raw_page('test1_a_idx', 0)); +SELECT * FROM bt_page_items(get_raw_page('test1_a_idx', 1)); +SELECT * FROM bt_page_items(get_raw_page('test1_a_idx', 2)); + DROP TABLE test1; diff --git a/doc/src/sgml/pageinspect.sgml b/doc/src/sgml/pageinspect.sgml index c87b160bf4..ccdaf3e0ac 100644 --- a/doc/src/sgml/pageinspect.sgml +++ b/doc/src/sgml/pageinspect.sgml @@ -333,6 +333,38 @@ test=# SELECT * FROM bt_page_items('pg_cast_oid_index', 1); + + + + bt_page_items(page bytea) returns setof record + + bt_page_items + + + + + + It is also possible to pass a page to bt_page_items + as a bytea value. A page image obtained + with get_raw_page should be passed as argument. So + the last example could also be rewritten like this: + +test=# SELECT * FROM bt_page_items(get_raw_page('pg_cast_oid_index', 1)); + itemoffset | ctid | itemlen | nulls | vars | data +------------+---------+---------+-------+------+------------- + 1 | (0,1) | 12 | f | f | 23 27 00 00 + 2 | (0,2) | 12 | f | f | 24 27 00 00 + 3 | (0,3) | 12 | f | f | 25 27 00 00 + 4 | (0,4) | 12 | f | f | 26 27 00 00 + 5 | (0,5) | 12 | f | f | 27 27 00 00 + 6 | (0,6) | 12 | f | f | 28 27 00 00 + 7 | (0,7) | 12 | f | f | 29 27 00 00 + 8 | (0,8) | 12 | f | f | 2a 27 00 00 + + All the other details are the same as explained in the previous item. + + + diff --git a/src/include/access/nbtree.h b/src/include/access/nbtree.h index f9304dbe1d..15771ce9e0 100644 --- a/src/include/access/nbtree.h +++ b/src/include/access/nbtree.h @@ -176,6 +176,7 @@ typedef struct BTMetaPageData #define P_ISLEAF(opaque) ((opaque)->btpo_flags & BTP_LEAF) #define P_ISROOT(opaque) ((opaque)->btpo_flags & BTP_ROOT) #define P_ISDELETED(opaque) ((opaque)->btpo_flags & BTP_DELETED) +#define P_ISMETA(opaque) ((opaque)->btpo_flags & BTP_META) #define P_ISHALFDEAD(opaque) ((opaque)->btpo_flags & BTP_HALF_DEAD) #define P_IGNORE(opaque) ((opaque)->btpo_flags & (BTP_DELETED|BTP_HALF_DEAD)) #define P_HAS_GARBAGE(opaque) ((opaque)->btpo_flags & BTP_HAS_GARBAGE) -- cgit v1.2.3 From afd79873a0b151bfbe5312acebfe361da09c11fd Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 5 Apr 2017 00:38:25 -0400 Subject: Capitalize names of PLs consistently Author: Daniel Gustafsson --- contrib/postgres_fdw/expected/postgres_fdw.out | 2 +- contrib/postgres_fdw/sql/postgres_fdw.sql | 2 +- doc/src/sgml/install-windows.sgml | 2 +- doc/src/sgml/parallel.sgml | 4 ++-- doc/src/sgml/plpgsql.sgml | 2 +- doc/src/sgml/release-7.4.sgml | 4 ++-- doc/src/sgml/release-8.0.sgml | 8 ++++---- doc/src/sgml/release-8.1.sgml | 8 ++++---- doc/src/sgml/release-8.2.sgml | 4 ++-- doc/src/sgml/release-8.3.sgml | 4 ++-- doc/src/sgml/release-8.4.sgml | 6 +++--- doc/src/sgml/release-9.4.sgml | 2 +- doc/src/sgml/release-old.sgml | 6 +++--- doc/src/sgml/xfunc.sgml | 2 +- src/backend/access/transam/README.parallel | 2 +- src/backend/executor/spi.c | 4 ++-- src/backend/parser/gram.y | 6 +++--- src/bin/initdb/initdb.c | 2 +- src/include/parser/scanner.h | 2 +- src/pl/plpgsql/src/Makefile | 2 +- src/pl/plpgsql/src/pl_gram.y | 4 ++-- src/pl/tcl/Makefile | 2 +- src/pl/tcl/pltcl.c | 2 +- 23 files changed, 41 insertions(+), 41 deletions(-) (limited to 'doc/src') diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 1a9e6c87f6..7657f0bf86 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -3879,7 +3879,7 @@ SELECT oid, * FROM ft_pg_type WHERE typname = 'int4'; (1 row) -- =================================================================== --- used in pl/pgsql function +-- used in PL/pgSQL function -- =================================================================== CREATE OR REPLACE FUNCTION f_test(p_c1 int) RETURNS int AS $$ DECLARE diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index cf70ca2c01..423eb024bb 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -952,7 +952,7 @@ SELECT oid, * FROM ft_pg_type WHERE typname = 'int4'; SELECT oid, * FROM ft_pg_type WHERE typname = 'int4'; -- =================================================================== --- used in pl/pgsql function +-- used in PL/pgSQL function -- =================================================================== CREATE OR REPLACE FUNCTION f_test(p_c1 int) RETURNS int AS $$ DECLARE diff --git a/doc/src/sgml/install-windows.sgml b/doc/src/sgml/install-windows.sgml index ecec0a60c7..7a9cf7f893 100644 --- a/doc/src/sgml/install-windows.sgml +++ b/doc/src/sgml/install-windows.sgml @@ -205,7 +205,7 @@ $ENV{MSBFLAGS}="/m"; ActiveState TCL - Required for building PL/TCL (Note: version + Required for building PL/Tcl (Note: version 8.4 is required, the free Standard Distribution is sufficient). diff --git a/doc/src/sgml/parallel.sgml b/doc/src/sgml/parallel.sgml index 2ea5c34ba2..a65129078c 100644 --- a/doc/src/sgml/parallel.sgml +++ b/doc/src/sgml/parallel.sgml @@ -149,7 +149,7 @@ EXPLAIN SELECT * FROM pgbench_accounts WHERE filler LIKE '%x%'; which the system thinks that partial or incremental execution might occur, no parallel plan is generated. For example, a cursor created using DECLARE CURSOR will never use - a parallel plan. Similarly, a PL/pgsql loop of the form + a parallel plan. Similarly, a PL/pgSQL loop of the form FOR x IN query LOOP .. END LOOP will never use a parallel plan, because the parallel query system is unable to verify that the code in the loop is safe to execute while parallel query is @@ -478,7 +478,7 @@ EXPLAIN SELECT * FROM pgbench_accounts WHERE filler LIKE '%x%'; Functions and aggregates must be marked PARALLEL UNSAFE if they write to the database, access sequences, change the transaction state - even temporarily (e.g. a PL/pgsql function which establishes an + even temporarily (e.g. a PL/pgSQL function which establishes an EXCEPTION block to catch errors), or make persistent changes to settings. Similarly, functions must be marked PARALLEL RESTRICTED if they access temporary tables, client connection state, diff --git a/doc/src/sgml/plpgsql.sgml b/doc/src/sgml/plpgsql.sgml index d356deb9f5..a6088e9c07 100644 --- a/doc/src/sgml/plpgsql.sgml +++ b/doc/src/sgml/plpgsql.sgml @@ -4851,7 +4851,7 @@ a_output := a_output || $$ if v_$$ || referrer_keys.kind || $$ like '$$ To aid the user in finding instances of simple but common problems before - they cause harm, PL/PgSQL provides additional + they cause harm, PL/pgSQL provides additional checks. When enabled, depending on the configuration, they can be used to emit either a WARNING or an ERROR during the compilation of a function. A function which has received diff --git a/doc/src/sgml/release-7.4.sgml b/doc/src/sgml/release-7.4.sgml index 649c82b17e..b4a217af2c 100644 --- a/doc/src/sgml/release-7.4.sgml +++ b/doc/src/sgml/release-7.4.sgml @@ -242,14 +242,14 @@ - Update pl/perl's ppport.h for modern Perl versions + Update PL/Perl's ppport.h for modern Perl versions (Andrew) - Fix assorted memory leaks in pl/python (Andreas Freund, Tom) + Fix assorted memory leaks in PL/Python (Andreas Freund, Tom) diff --git a/doc/src/sgml/release-8.0.sgml b/doc/src/sgml/release-8.0.sgml index becd5090cc..6224187ab4 100644 --- a/doc/src/sgml/release-8.0.sgml +++ b/doc/src/sgml/release-8.0.sgml @@ -312,14 +312,14 @@ - Update pl/perl's ppport.h for modern Perl versions + Update PL/Perl's ppport.h for modern Perl versions (Andrew) - Fix assorted memory leaks in pl/python (Andreas Freund, Tom) + Fix assorted memory leaks in PL/Python (Andreas Freund, Tom) @@ -2439,9 +2439,9 @@ when the data directory is not specified (Magnus) (Neil) Recover properly if error occurs during argument passing -in PL/python (Neil) +in PL/Python (Neil) -Fix PL/perl's handling of locales on +Fix PL/Perl's handling of locales on Win32 to match the backend (Andrew) Fix crash when log_min_messages is set to diff --git a/doc/src/sgml/release-8.1.sgml b/doc/src/sgml/release-8.1.sgml index 05b07ade99..49a4fe2ad8 100644 --- a/doc/src/sgml/release-8.1.sgml +++ b/doc/src/sgml/release-8.1.sgml @@ -539,14 +539,14 @@ - Update pl/perl's ppport.h for modern Perl versions + Update PL/Perl's ppport.h for modern Perl versions (Andrew) - Fix assorted memory leaks in pl/python (Andreas Freund, Tom) + Fix assorted memory leaks in PL/Python (Andreas Freund, Tom) @@ -3089,12 +3089,12 @@ when the data directory is not specified (Magnus) together in function result type declarations Recover properly if error occurs during argument passing -in PL/python (Neil) +in PL/Python (Neil) Fix memory leak in plperl_return_next (Neil) -Fix PL/perl's handling of locales on +Fix PL/Perl's handling of locales on Win32 to match the backend (Andrew) Various optimizer fixes (Tom) diff --git a/doc/src/sgml/release-8.2.sgml b/doc/src/sgml/release-8.2.sgml index 2d21728cf7..acb7b810c6 100644 --- a/doc/src/sgml/release-8.2.sgml +++ b/doc/src/sgml/release-8.2.sgml @@ -1442,14 +1442,14 @@ - Update pl/perl's ppport.h for modern Perl versions + Update PL/Perl's ppport.h for modern Perl versions (Andrew) - Fix assorted memory leaks in pl/python (Andreas Freund, Tom) + Fix assorted memory leaks in PL/Python (Andreas Freund, Tom) diff --git a/doc/src/sgml/release-8.3.sgml b/doc/src/sgml/release-8.3.sgml index b1b5d4875c..76ab36e39c 100644 --- a/doc/src/sgml/release-8.3.sgml +++ b/doc/src/sgml/release-8.3.sgml @@ -3022,14 +3022,14 @@ - Update pl/perl's ppport.h for modern Perl versions + Update PL/Perl's ppport.h for modern Perl versions (Andrew) - Fix assorted memory leaks in pl/python (Andreas Freund, Tom) + Fix assorted memory leaks in PL/Python (Andreas Freund, Tom) diff --git a/doc/src/sgml/release-8.4.sgml b/doc/src/sgml/release-8.4.sgml index f43b0958bc..dfe4787757 100644 --- a/doc/src/sgml/release-8.4.sgml +++ b/doc/src/sgml/release-8.4.sgml @@ -5260,21 +5260,21 @@ - Fix pl/pgsql's CASE statement to not fail when the + Fix PL/pgSQL's CASE statement to not fail when the case expression is a query that returns no rows (Tom) - Update pl/perl's ppport.h for modern Perl versions + Update PL/Perl's ppport.h for modern Perl versions (Andrew) - Fix assorted memory leaks in pl/python (Andreas Freund, Tom) + Fix assorted memory leaks in PL/Python (Andreas Freund, Tom) diff --git a/doc/src/sgml/release-9.4.sgml b/doc/src/sgml/release-9.4.sgml index 3d5e4cf52a..5f0280a18a 100644 --- a/doc/src/sgml/release-9.4.sgml +++ b/doc/src/sgml/release-9.4.sgml @@ -8828,7 +8828,7 @@ Branch: REL9_4_STABLE [c2b06ab17] 2015-01-30 22:45:58 -0500 - Add ability to retrieve the current PL/PgSQL call stack + Add ability to retrieve the current PL/pgSQL call stack using GET DIAGNOSTICS (Pavel Stehule, Stephen Frost) diff --git a/doc/src/sgml/release-old.sgml b/doc/src/sgml/release-old.sgml index a8622188eb..037b1e8e0c 100644 --- a/doc/src/sgml/release-old.sgml +++ b/doc/src/sgml/release-old.sgml @@ -3145,7 +3145,7 @@ Fix rare cursor crash when using hash join (Tom) Fix for DROP TABLE/INDEX in rolled-back transaction (Hiroshi) Fix psql crash from \l+ if MULTIBYTE enabled (Peter E) Fix truncation of rule names during CREATE VIEW (Ross Reedstrom) -Fix PL/perl (Alex Kapranoff) +Fix PL/Perl (Alex Kapranoff) Disallow LOCK on views (Mark Hollomon) Disallow INSERT/UPDATE/DELETE on views (Mark Hollomon) Disallow DROP RULE, CREATE INDEX, TRUNCATE on views (Mark Hollomon) @@ -3171,7 +3171,7 @@ Fix OVERLAPS operators conform to SQL92 spec regarding NULLs (Tom) Fix lpad() and rpad() to handle length less than input string (Tom) Fix use of NOTIFY in some rules (Tom) Overhaul btree code (Tom) -Fix NOT NULL use in Pl/pgSQL variables (Tom) +Fix NOT NULL use in PL/pgSQL variables (Tom) Overhaul GIST code (Oleg) Fix CLUSTER to preserve constraints and column default (Tom) Improved deadlock detection handling (Tom) @@ -4341,7 +4341,7 @@ Fix for lo_import crash(Tatsuo) Adjust handling of data type names to suppress double quotes(Thomas) Use type coercion for matching columns and DEFAULT(Thomas) Fix deadlock so it only checks once after one second of sleep(Bruce) -Fixes for aggregates and PL/pgsql(Hiroshi) +Fixes for aggregates and PL/pgSQL(Hiroshi) Fix for subquery crash(Vadim) Fix for libpq function PQfnumber and case-insensitive names(Bahman Rafatjoo) Fix for large object write-in-middle, no extra block, memory consumption(Tatsuo) diff --git a/doc/src/sgml/xfunc.sgml b/doc/src/sgml/xfunc.sgml index 0a12231396..10e3c0cef9 100644 --- a/doc/src/sgml/xfunc.sgml +++ b/doc/src/sgml/xfunc.sgml @@ -153,7 +153,7 @@ SELECT clean_emp(); CREATE TABLE foo (...); INSERT INTO foo VALUES(...); will not work as desired if packaged up into a single SQL function, since foo won't exist yet when the INSERT - command is parsed. It's recommended to use PL/PgSQL + command is parsed. It's recommended to use PL/pgSQL instead of a SQL function in this type of situation. diff --git a/src/backend/access/transam/README.parallel b/src/backend/access/transam/README.parallel index db9ac3d504..4d98f4e65d 100644 --- a/src/backend/access/transam/README.parallel +++ b/src/backend/access/transam/README.parallel @@ -154,7 +154,7 @@ parallelism was started before all parallel workers have exited; and it's even more clearly crazy for a parallel worker to try to subcommit or subabort the current subtransaction and execute in some other transaction context than was present in the initiating backend. It might be practical to allow internal -sub-transactions (e.g. to implement a PL/pgsql EXCEPTION block) to be used in +sub-transactions (e.g. to implement a PL/pgSQL EXCEPTION block) to be used in parallel mode, provided that they are XID-less, because other backends wouldn't really need to know about those transactions or do anything differently because of them. Right now, we don't even allow that. diff --git a/src/backend/executor/spi.c b/src/backend/executor/spi.c index 3a89ccd913..ca547dc6d9 100644 --- a/src/backend/executor/spi.c +++ b/src/backend/executor/spi.c @@ -1582,7 +1582,7 @@ SPI_result_code_string(int code) * SPI_plan_get_plan_sources --- get a SPI plan's underlying list of * CachedPlanSources. * - * This is exported so that pl/pgsql can use it (this beats letting pl/pgsql + * This is exported so that PL/pgSQL can use it (this beats letting PL/pgSQL * look directly into the SPIPlan for itself). It's not documented in * spi.sgml because we'd just as soon not have too many places using this. */ @@ -1598,7 +1598,7 @@ SPI_plan_get_plan_sources(SPIPlanPtr plan) * if the SPI plan contains exactly one CachedPlanSource. If not, * return NULL. Caller is responsible for doing ReleaseCachedPlan(). * - * This is exported so that pl/pgsql can use it (this beats letting pl/pgsql + * This is exported so that PL/pgSQL can use it (this beats letting PL/pgSQL * look directly into the SPIPlan for itself). It's not documented in * spi.sgml because we'd just as soon not have too many places using this. */ diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 9d53a29ad2..5ecb6997b3 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -584,11 +584,11 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); /* * Non-keyword token types. These are hard-wired into the "flex" lexer. * They must be listed first so that their numeric codes do not depend on - * the set of keywords. PL/pgsql depends on this so that it can share the - * same lexer. If you add/change tokens here, fix PL/pgsql to match! + * the set of keywords. PL/pgSQL depends on this so that it can share the + * same lexer. If you add/change tokens here, fix PL/pgSQL to match! * * DOT_DOT is unused in the core SQL grammar, and so will always provoke - * parse errors. It is needed by PL/pgsql. + * parse errors. It is needed by PL/pgSQL. */ %token IDENT FCONST SCONST BCONST XCONST Op %token ICONST PARAM diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index 8dde1e8f9d..d40ed412fc 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -1898,7 +1898,7 @@ setup_schema(FILE *cmdfd) } /* - * load PL/pgsql server-side language + * load PL/pgSQL server-side language */ static void load_plpgsql(FILE *cmdfd) diff --git a/src/include/parser/scanner.h b/src/include/parser/scanner.h index 7ca924cff3..74f1cc897f 100644 --- a/src/include/parser/scanner.h +++ b/src/include/parser/scanner.h @@ -3,7 +3,7 @@ * scanner.h * API for the core scanner (flex machine) * - * The core scanner is also used by PL/pgsql, so we provide a public API + * The core scanner is also used by PL/pgSQL, so we provide a public API * for it. However, the rest of the backend is only expected to use the * higher-level API provided by parser.h. * diff --git a/src/pl/plpgsql/src/Makefile b/src/pl/plpgsql/src/Makefile index e073b2abd0..95348179ac 100644 --- a/src/pl/plpgsql/src/Makefile +++ b/src/pl/plpgsql/src/Makefile @@ -1,6 +1,6 @@ #------------------------------------------------------------------------- # -# Makefile for the pl/pgsql procedural language +# Makefile for the PL/pgSQL procedural language # # src/pl/plpgsql/src/Makefile # diff --git a/src/pl/plpgsql/src/pl_gram.y b/src/pl/plpgsql/src/pl_gram.y index 29729df550..94f1f58593 100644 --- a/src/pl/plpgsql/src/pl_gram.y +++ b/src/pl/plpgsql/src/pl_gram.y @@ -2862,11 +2862,11 @@ make_execsql_stmt(int firsttoken, int location) * clause lurking within it, and parse that via read_into_target(). * * Because INTO is sometimes used in the main SQL grammar, we have to be - * careful not to take any such usage of INTO as a pl/pgsql INTO clause. + * careful not to take any such usage of INTO as a PL/pgSQL INTO clause. * There are currently three such cases: * * 1. SELECT ... INTO. We don't care, we just override that with the - * pl/pgsql definition. + * PL/pgSQL definition. * * 2. INSERT INTO. This is relatively easy to recognize since the words * must appear adjacently; but we can't assume INSERT starts the command, diff --git a/src/pl/tcl/Makefile b/src/pl/tcl/Makefile index 0275c125b1..b8971d3cc8 100644 --- a/src/pl/tcl/Makefile +++ b/src/pl/tcl/Makefile @@ -1,6 +1,6 @@ #------------------------------------------------------------------------- # -# Makefile for the pl/tcl procedural language +# Makefile for the PL/Tcl procedural language # # src/pl/tcl/Makefile # diff --git a/src/pl/tcl/pltcl.c b/src/pl/tcl/pltcl.c index 5a48e5d175..89bb46fb4a 100644 --- a/src/pl/tcl/pltcl.c +++ b/src/pl/tcl/pltcl.c @@ -2535,7 +2535,7 @@ pltcl_SPI_prepare(ClientData cdata, Tcl_Interp *interp, * occur. FIXME someday. ************************************************************/ plan_cxt = AllocSetContextCreate(TopMemoryContext, - "PL/TCL spi_prepare query", + "PL/Tcl spi_prepare query", ALLOCSET_SMALL_SIZES); MemoryContextSwitchTo(plan_cxt); qdesc = (pltcl_query_desc *) palloc0(sizeof(pltcl_query_desc)); -- cgit v1.2.3 From 63e5d02af33414ab31d0d59e02333083ac696d37 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 5 Apr 2017 13:18:32 -0400 Subject: doc: Remove remark elements They were not rendered with DSSSL, but now they show up by default using XSLT. Just remove them, since they are not useful. --- doc/src/sgml/dml.sgml | 4 ---- doc/src/sgml/ecpg.sgml | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/dml.sgml b/doc/src/sgml/dml.sgml index 0c65578b59..071cdb610f 100644 --- a/doc/src/sgml/dml.sgml +++ b/doc/src/sgml/dml.sgml @@ -3,10 +3,6 @@ Data Manipulation - - This chapter is still quite incomplete. - - The previous chapter discussed how to create tables and other structures to hold your data. Now it is time to fill the tables diff --git a/doc/src/sgml/ecpg.sgml b/doc/src/sgml/ecpg.sgml index 89d4c57da7..564bd32144 100644 --- a/doc/src/sgml/ecpg.sgml +++ b/doc/src/sgml/ecpg.sgml @@ -3592,7 +3592,7 @@ void PGTYPESdecimal_free(decimal *var); EXEC SQL ALLOCATE DESCRIPTOR identifier; The identifier serves as the variable name of the - descriptor area. The scope of the allocated descriptor is WHAT?. + descriptor area. When you don't need the descriptor anymore, you should deallocate it: -- cgit v1.2.3 From 00b6b6feb12cef53737287b67ecef6aff1f1d8ab Mon Sep 17 00:00:00 2001 From: Simon Riggs Date: Wed, 5 Apr 2017 15:38:17 -0400 Subject: Allow --with-wal-segsize=n up to n=1024MB Other part of Beena Emerson's patch to allow testing --- configure | 6 +++++- configure.in | 6 +++++- doc/src/sgml/installation.sgml | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) (limited to 'doc/src') diff --git a/configure b/configure index 56e3f8f242..de8660d00d 100755 --- a/configure +++ b/configure @@ -3725,7 +3725,11 @@ case ${wal_segsize} in 16) ;; 32) ;; 64) ;; - *) as_fn_error $? "Invalid WAL segment size. Allowed values are 1,2,4,8,16,32,64." "$LINENO" 5 + 128) ;; + 256) ;; + 512) ;; + 1024) ;; + *) as_fn_error $? "Invalid WAL segment size. Allowed values are 1,2,4,8,16,32,64,128,256,512,1024." "$LINENO" 5 esac { $as_echo "$as_me:${as_lineno-$LINENO}: result: ${wal_segsize}MB" >&5 $as_echo "${wal_segsize}MB" >&6; } diff --git a/configure.in b/configure.in index d7c501af6a..b965f47669 100644 --- a/configure.in +++ b/configure.in @@ -358,7 +358,11 @@ case ${wal_segsize} in 16) ;; 32) ;; 64) ;; - *) AC_MSG_ERROR([Invalid WAL segment size. Allowed values are 1,2,4,8,16,32,64.]) + 128) ;; + 256) ;; + 512) ;; + 1024) ;; + *) AC_MSG_ERROR([Invalid WAL segment size. Allowed values are 1,2,4,8,16,32,64,128,256,512,1024.]) esac AC_MSG_RESULT([${wal_segsize}MB]) diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml index 39774a058a..26bfab1ea5 100644 --- a/doc/src/sgml/installation.sgml +++ b/doc/src/sgml/installation.sgml @@ -1049,7 +1049,7 @@ su - postgres the size of each individual file in the WAL log. It may be useful to adjust this size to control the granularity of WAL log shipping. The default size is 16 megabytes. - The value must be a power of 2 between 1 and 64 (megabytes). + The value must be a power of 2 between 1 and 1024 (megabytes). Note that changing this value requires an initdb. -- cgit v1.2.3 From 2686ee1b7ccfb9214064d4d2a98ea77382880306 Mon Sep 17 00:00:00 2001 From: Simon Riggs Date: Wed, 5 Apr 2017 18:00:42 -0400 Subject: Collect and use multi-column dependency stats Follow on patch in the multi-variate statistics patch series. CREATE STATISTICS s1 WITH (dependencies) ON (a, b) FROM t; ANALYZE; will collect dependency stats on (a, b) and then use the measured dependency in subsequent query planning. Commit 7b504eb282ca2f5104b5c00b4f05a3ef6bb1385b added CREATE STATISTICS with n-distinct coefficients. These are now specified using the mutually exclusive option WITH (ndistinct). Author: Tomas Vondra, David Rowley Reviewed-by: Kyotaro HORIGUCHI, Álvaro Herrera, Dean Rasheed, Robert Haas and many other comments and contributions Discussion: https://postgr.es/m/56f40b20-c464-fad2-ff39-06b668fac47c@2ndquadrant.com --- contrib/file_fdw/file_fdw.c | 1 + contrib/postgres_fdw/postgres_fdw.c | 5 +- doc/src/sgml/catalogs.sgml | 9 + doc/src/sgml/planstats.sgml | 154 +++ doc/src/sgml/ref/create_statistics.sgml | 42 +- src/backend/catalog/system_views.sql | 3 +- src/backend/commands/statscmds.c | 17 +- src/backend/optimizer/path/clausesel.c | 113 ++- src/backend/optimizer/path/costsize.c | 25 +- src/backend/optimizer/util/orclauses.c | 4 +- src/backend/optimizer/util/plancat.c | 12 + src/backend/statistics/Makefile | 2 +- src/backend/statistics/README | 68 +- src/backend/statistics/README.dependencies | 119 +++ src/backend/statistics/dependencies.c | 1079 ++++++++++++++++++++++ src/backend/statistics/extended_stats.c | 105 ++- src/backend/utils/adt/ruleutils.c | 54 +- src/backend/utils/adt/selfuncs.c | 20 +- src/bin/psql/describe.c | 12 +- src/include/catalog/pg_cast.h | 4 + src/include/catalog/pg_proc.h | 9 + src/include/catalog/pg_statistic_ext.h | 7 +- src/include/catalog/pg_type.h | 4 + src/include/optimizer/cost.h | 6 +- src/include/statistics/extended_stats_internal.h | 5 + src/include/statistics/statistics.h | 44 + src/test/regress/expected/opr_sanity.out | 3 +- src/test/regress/expected/rules.out | 3 +- src/test/regress/expected/stats_ext.out | 110 ++- src/test/regress/expected/type_sanity.out | 7 +- src/test/regress/sql/stats_ext.sql | 68 ++ 31 files changed, 2035 insertions(+), 79 deletions(-) create mode 100644 src/backend/statistics/README.dependencies create mode 100644 src/backend/statistics/dependencies.c (limited to 'doc/src') diff --git a/contrib/file_fdw/file_fdw.c b/contrib/file_fdw/file_fdw.c index 277639f6e9..7414c16128 100644 --- a/contrib/file_fdw/file_fdw.c +++ b/contrib/file_fdw/file_fdw.c @@ -1013,6 +1013,7 @@ estimate_size(PlannerInfo *root, RelOptInfo *baserel, baserel->baserestrictinfo, 0, JOIN_INNER, + NULL, NULL); nrows = clamp_row_est(nrows); diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c index 2851869932..a73d1a0bed 100644 --- a/contrib/postgres_fdw/postgres_fdw.c +++ b/contrib/postgres_fdw/postgres_fdw.c @@ -591,6 +591,7 @@ postgresGetForeignRelSize(PlannerInfo *root, fpinfo->local_conds, baserel->relid, JOIN_INNER, + NULL, NULL); cost_qual_eval(&fpinfo->local_conds_cost, fpinfo->local_conds, root); @@ -2572,6 +2573,7 @@ estimate_path_cost_size(PlannerInfo *root, local_param_join_conds, foreignrel->relid, JOIN_INNER, + NULL, NULL); local_sel *= fpinfo->local_conds_sel; @@ -4455,6 +4457,7 @@ postgresGetForeignJoinPaths(PlannerInfo *root, fpinfo->local_conds, 0, JOIN_INNER, + NULL, NULL); cost_qual_eval(&fpinfo->local_conds_cost, fpinfo->local_conds, root); @@ -4465,7 +4468,7 @@ postgresGetForeignJoinPaths(PlannerInfo *root, if (!fpinfo->use_remote_estimate) fpinfo->joinclause_sel = clauselist_selectivity(root, fpinfo->joinclauses, 0, fpinfo->jointype, - extra->sjinfo); + extra->sjinfo, NULL); /* Estimate costs for bare join relation */ estimate_path_cost_size(root, joinrel, NIL, NIL, &rows, diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 65ba919e7b..5c1930c745 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -4339,6 +4339,15 @@ + + stadependencies + pg_dependencies + + + Functional dependencies, serialized as pg_dependencies type. + + + diff --git a/doc/src/sgml/planstats.sgml b/doc/src/sgml/planstats.sgml index b73c66bde2..f8af42f394 100644 --- a/doc/src/sgml/planstats.sgml +++ b/doc/src/sgml/planstats.sgml @@ -446,6 +446,160 @@ rows = (outer_cardinality * inner_cardinality) * selectivity in src/backend/utils/adt/selfuncs.c. + + Functional Dependencies + + + The simplest type of extended statistics are functional dependencies, + used in definitions of database normal forms. When simplified, saying that + b is functionally dependent on a means that + knowledge of value of a is sufficient to determine value of + b. + + + + In normalized databases, only functional dependencies on primary keys + and superkeys are allowed. However, in practice, many data sets are not + fully normalized, for example, due to intentional denormalization for + performance reasons. + + + + Functional dependencies directly affect accuracy of the estimates, as + conditions on the dependent column(s) do not restrict the result set, + resulting in underestimates. + + + + To inform the planner about the functional dependencies, we collect + measurements of dependency during ANALYZE. Assessing + dependency between all sets of columns would be prohibitively + expensive, so we limit our search to potential dependencies defined + using the CREATE STATISTICS command. + + +CREATE TABLE t (a INT, b INT); +INSERT INTO t SELECT i/100, i/100 FROM generate_series(1,10000) s(i); +CREATE STATISTICS s1 WITH (dependencies) ON (a, b) FROM t; +ANALYZE t; +EXPLAIN ANALYZE SELECT * FROM t WHERE a = 1 AND b = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Seq Scan on t (cost=0.00..195.00 rows=100 width=8) (actual time=0.095..3.118 rows=100 loops=1) + Filter: ((a = 1) AND (b = 1)) + Rows Removed by Filter: 9900 + Planning time: 0.367 ms + Execution time: 3.380 ms +(5 rows) + + + The planner is now aware of the functional dependencies and considers + them when computing the selectivity of the second condition. Running + the query without the statistics would lead to quite different estimates. + + +DROP STATISTICS s1; +EXPLAIN ANALYZE SELECT * FROM t WHERE a = 1 AND b = 1; + QUERY PLAN +----------------------------------------------------------------------------------------------- + Seq Scan on t (cost=0.00..195.00 rows=1 width=8) (actual time=0.000..6.379 rows=100 loops=1) + Filter: ((a = 1) AND (b = 1)) + Rows Removed by Filter: 9900 + Planning time: 0.000 ms + Execution time: 6.379 ms +(5 rows) + + + + + If no dependency exists, the collected statistics do not influence the + query plan. The only effect is to slow down ANALYZE. Should + partial dependencies exist these will also be stored and applied + during planning. + + + + Similarly to per-column statistics, extended statistics are stored in + a system catalog called pg_statistic_ext, but + there is also a more convenient view pg_stats_ext. + To inspect the statistics s1 defined above, + you may do this: + + +SELECT tablename, staname, attnums, depsbytes + FROM pg_stats_ext WHERE staname = 's1'; + tablename | staname | attnums | depsbytes +-----------+---------+---------+----------- + t | s1 | 1 2 | 40 +(1 row) + + + This shows that the statistics are defined on table t, + attnums lists attribute numbers of columns + (references pg_attribute). It also shows + the length in bytes of the functional dependencies, as found by + ANALYZE when serialized into a bytea column. + + + + When computing the selectivity, the planner inspects all conditions and + attempts to identify which conditions are already implied by other + conditions. The selectivity estimates from any redundant conditions are + ignored from a selectivity point of view. In the example query above, + the selectivity estimates for either of the conditions may be eliminated, + thus improving the overall estimate. + + + + Limitations of functional dependencies + + + Functional dependencies are a very simple type of statistics, and + as such have several limitations. The first limitation is that they + only work with simple equality conditions, comparing columns and constant + values. It's not possible to use them to eliminate equality conditions + comparing two columns or a column to an expression, range clauses, + LIKE or any other type of conditions. + + + + When eliminating the implied conditions, the planner assumes that the + conditions are compatible. Consider the following example, violating + this assumption: + + +EXPLAIN ANALYZE SELECT * FROM t WHERE a = 1 AND b = 10; + QUERY PLAN +----------------------------------------------------------------------------------------------- + Seq Scan on t (cost=0.00..195.00 rows=100 width=8) (actual time=2.992..2.992 rows=0 loops=1) + Filter: ((a = 1) AND (b = 10)) + Rows Removed by Filter: 10000 + Planning time: 0.232 ms + Execution time: 3.033 ms +(5 rows) + + + While there are no rows with such combination of values, the planner + is unable to verify whether the values match - it only knows that + the columns are functionally dependent. + + + + This assumption is more about queries executed on the database - in many + cases, it's actually satisfied (e.g. when the GUI only allows selecting + compatible values). But if that's not the case, functional dependencies + may not be a viable option. + + + + For additional information about functional dependencies, see + src/backend/statistics/README.dependencies. + + + + + + diff --git a/doc/src/sgml/ref/create_statistics.sgml b/doc/src/sgml/ref/create_statistics.sgml index 60184a347b..163d43f2d4 100644 --- a/doc/src/sgml/ref/create_statistics.sgml +++ b/doc/src/sgml/ref/create_statistics.sgml @@ -21,8 +21,9 @@ PostgreSQL documentation -CREATE STATISTICS [ IF NOT EXISTS ] statistics_name ON ( - column_name, column_name [, ...]) +CREATE STATISTICS [ IF NOT EXISTS ] statistics_name + WITH ( option [= value] [, ... ] ) + ON ( column_name, column_name [, ...]) FROM table_name @@ -94,6 +95,41 @@ CREATE STATISTICS [ IF NOT EXISTS ] statistics_na + + Parameters + + + statistics parameters + + + + The WITH clause can specify options + for the statistics. Available options are listed below. + + + + + + dependencies (boolean) + + + Enables functional dependencies for the statistics. + + + + + + ndistinct (boolean) + + + Enables ndistinct coefficients for the statistics. + + + + + + + @@ -122,7 +158,7 @@ CREATE TABLE t1 ( INSERT INTO t1 SELECT i/100, i/500 FROM generate_series(1,1000000) s(i); -CREATE STATISTICS s1 ON (a, b) FROM t1; +CREATE STATISTICS s1 WITH (dependencies) ON (a, b) FROM t1; ANALYZE t1; diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 0217f3992f..500221ae77 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -192,7 +192,8 @@ CREATE VIEW pg_stats_ext AS C.relname AS tablename, S.staname AS staname, S.stakeys AS attnums, - length(s.standistinct) AS ndistbytes + length(s.standistinct::bytea) AS ndistbytes, + length(S.stadependencies::bytea) AS depsbytes FROM (pg_statistic_ext S JOIN pg_class C ON (C.oid = S.starelid)) LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace); diff --git a/src/backend/commands/statscmds.c b/src/backend/commands/statscmds.c index 0750329961..8d483dbb3a 100644 --- a/src/backend/commands/statscmds.c +++ b/src/backend/commands/statscmds.c @@ -62,10 +62,11 @@ CreateStatistics(CreateStatsStmt *stmt) Oid relid; ObjectAddress parentobject, childobject; - Datum types[1]; /* only ndistinct defined now */ + Datum types[2]; /* one for each possible type of statistics */ int ntypes; ArrayType *staenabled; bool build_ndistinct; + bool build_dependencies; bool requested_type = false; Assert(IsA(stmt, CreateStatsStmt)); @@ -159,7 +160,7 @@ CreateStatistics(CreateStatsStmt *stmt) errmsg("statistics require at least 2 columns"))); /* - * Sort the attnums, which makes detecting duplicies somewhat easier, and + * Sort the attnums, which makes detecting duplicities somewhat easier, and * it does not hurt (it does not affect the efficiency, unlike for * indexes, for example). */ @@ -182,6 +183,7 @@ CreateStatistics(CreateStatsStmt *stmt) * recognized. */ build_ndistinct = false; + build_dependencies = false; foreach(l, stmt->options) { DefElem *opt = (DefElem *) lfirst(l); @@ -191,6 +193,11 @@ CreateStatistics(CreateStatsStmt *stmt) build_ndistinct = defGetBoolean(opt); requested_type = true; } + else if (strcmp(opt->defname, "dependencies") == 0) + { + build_dependencies = defGetBoolean(opt); + requested_type = true; + } else ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), @@ -199,12 +206,17 @@ CreateStatistics(CreateStatsStmt *stmt) } /* If no statistic type was specified, build them all. */ if (!requested_type) + { build_ndistinct = true; + build_dependencies = true; + } /* construct the char array of enabled statistic types */ ntypes = 0; if (build_ndistinct) types[ntypes++] = CharGetDatum(STATS_EXT_NDISTINCT); + if (build_dependencies) + types[ntypes++] = CharGetDatum(STATS_EXT_DEPENDENCIES); Assert(ntypes > 0); staenabled = construct_array(types, ntypes, CHAROID, 1, true, 'c'); @@ -222,6 +234,7 @@ CreateStatistics(CreateStatsStmt *stmt) /* no statistics build yet */ nulls[Anum_pg_statistic_ext_standistinct - 1] = true; + nulls[Anum_pg_statistic_ext_stadependencies - 1] = true; /* insert it into pg_statistic_ext */ statrel = heap_open(StatisticExtRelationId, RowExclusiveLock); diff --git a/src/backend/optimizer/path/clausesel.c b/src/backend/optimizer/path/clausesel.c index af2934a721..1ba578130e 100644 --- a/src/backend/optimizer/path/clausesel.c +++ b/src/backend/optimizer/path/clausesel.c @@ -22,6 +22,7 @@ #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "utils/selfuncs.h" +#include "statistics/statistics.h" /* @@ -60,23 +61,30 @@ static void addRangeClause(RangeQueryClause **rqlist, Node *clause, * subclauses. However, that's only right if the subclauses have independent * probabilities, and in reality they are often NOT independent. So, * we want to be smarter where we can. - - * Currently, the only extra smarts we have is to recognize "range queries", - * such as "x > 34 AND x < 42". Clauses are recognized as possible range - * query components if they are restriction opclauses whose operators have - * scalarltsel() or scalargtsel() as their restriction selectivity estimator. - * We pair up clauses of this form that refer to the same variable. An - * unpairable clause of this kind is simply multiplied into the selectivity - * product in the normal way. But when we find a pair, we know that the - * selectivities represent the relative positions of the low and high bounds - * within the column's range, so instead of figuring the selectivity as - * hisel * losel, we can figure it as hisel + losel - 1. (To visualize this, - * see that hisel is the fraction of the range below the high bound, while - * losel is the fraction above the low bound; so hisel can be interpreted - * directly as a 0..1 value but we need to convert losel to 1-losel before - * interpreting it as a value. Then the available range is 1-losel to hisel. - * However, this calculation double-excludes nulls, so really we need - * hisel + losel + null_frac - 1.) + * + * When 'rel' is not null and rtekind = RTE_RELATION, we'll try to apply + * selectivity estimates using any extended statistcs on 'rel'. + * + * If we identify such extended statistics exist, we try to apply them. + * Currently we only have (soft) functional dependencies, so apply these in as + * many cases as possible, and fall back on normal estimates for remaining + * clauses. + * + * We also recognize "range queries", such as "x > 34 AND x < 42". Clauses + * are recognized as possible range query components if they are restriction + * opclauses whose operators have scalarltsel() or scalargtsel() as their + * restriction selectivity estimator. We pair up clauses of this form that + * refer to the same variable. An unpairable clause of this kind is simply + * multiplied into the selectivity product in the normal way. But when we + * find a pair, we know that the selectivities represent the relative + * positions of the low and high bounds within the column's range, so instead + * of figuring the selectivity as hisel * losel, we can figure it as hisel + + * losel - 1. (To visualize this, see that hisel is the fraction of the range + * below the high bound, while losel is the fraction above the low bound; so + * hisel can be interpreted directly as a 0..1 value but we need to convert + * losel to 1-losel before interpreting it as a value. Then the available + * range is 1-losel to hisel. However, this calculation double-excludes + * nulls, so really we need hisel + losel + null_frac - 1.) * * If either selectivity is exactly DEFAULT_INEQ_SEL, we forget this equation * and instead use DEFAULT_RANGE_INEQ_SEL. The same applies if the equation @@ -93,33 +101,70 @@ clauselist_selectivity(PlannerInfo *root, List *clauses, int varRelid, JoinType jointype, - SpecialJoinInfo *sjinfo) + SpecialJoinInfo *sjinfo, + RelOptInfo *rel) { Selectivity s1 = 1.0; RangeQueryClause *rqlist = NULL; ListCell *l; + Bitmapset *estimatedclauses = NULL; + int listidx; /* - * If there's exactly one clause, then no use in trying to match up pairs, - * so just go directly to clause_selectivity(). + * If there's exactly one clause, then extended statistics is futile at + * this level (we might be able to apply them later if it's AND/OR + * clause). So just go directly to clause_selectivity(). */ if (list_length(clauses) == 1) return clause_selectivity(root, (Node *) linitial(clauses), - varRelid, jointype, sjinfo); + varRelid, jointype, sjinfo, rel); + + /* + * When a relation of RTE_RELATION is given as 'rel', we'll try to + * perform selectivity estimation using extended statistics. + */ + if (rel && rel->rtekind == RTE_RELATION && rel->statlist != NIL) + { + /* + * Perform selectivity estimations on any clauses found applicable by + * dependencies_clauselist_selectivity. The 0-based list position of + * estimated clauses will be populated in 'estimatedclauses'. + */ + s1 *= dependencies_clauselist_selectivity(root, clauses, varRelid, + jointype, sjinfo, rel, &estimatedclauses); + + /* + * This would be the place to apply any other types of extended + * statistics selectivity estimations for remaining clauses. + */ + } /* - * Initial scan over clauses. Anything that doesn't look like a potential - * rangequery clause gets multiplied into s1 and forgotten. Anything that - * does gets inserted into an rqlist entry. + * Apply normal selectivity estimates for remaining clauses. We'll be + * careful to skip any clauses which were already estimated above. + * + * Anything that doesn't look like a potential rangequery clause gets + * multiplied into s1 and forgotten. Anything that does gets inserted into + * an rqlist entry. */ + listidx = -1; foreach(l, clauses) { Node *clause = (Node *) lfirst(l); RestrictInfo *rinfo; Selectivity s2; + listidx++; + + /* + * Skip this clause if it's already been estimated by some other + * statistics above. + */ + if (bms_is_member(listidx, estimatedclauses)) + continue; + /* Always compute the selectivity using clause_selectivity */ - s2 = clause_selectivity(root, clause, varRelid, jointype, sjinfo); + s2 = clause_selectivity(root, clause, varRelid, jointype, sjinfo, rel); /* * Check for being passed a RestrictInfo. @@ -484,7 +529,8 @@ clause_selectivity(PlannerInfo *root, Node *clause, int varRelid, JoinType jointype, - SpecialJoinInfo *sjinfo) + SpecialJoinInfo *sjinfo, + RelOptInfo *rel) { Selectivity s1 = 0.5; /* default for any unhandled clause type */ RestrictInfo *rinfo = NULL; @@ -604,7 +650,8 @@ clause_selectivity(PlannerInfo *root, (Node *) get_notclausearg((Expr *) clause), varRelid, jointype, - sjinfo); + sjinfo, + rel); } else if (and_clause(clause)) { @@ -613,7 +660,8 @@ clause_selectivity(PlannerInfo *root, ((BoolExpr *) clause)->args, varRelid, jointype, - sjinfo); + sjinfo, + rel); } else if (or_clause(clause)) { @@ -632,7 +680,8 @@ clause_selectivity(PlannerInfo *root, (Node *) lfirst(arg), varRelid, jointype, - sjinfo); + sjinfo, + rel); s1 = s1 + s2 - s1 * s2; } @@ -725,7 +774,8 @@ clause_selectivity(PlannerInfo *root, (Node *) ((RelabelType *) clause)->arg, varRelid, jointype, - sjinfo); + sjinfo, + rel); } else if (IsA(clause, CoerceToDomain)) { @@ -734,7 +784,8 @@ clause_selectivity(PlannerInfo *root, (Node *) ((CoerceToDomain *) clause)->arg, varRelid, jointype, - sjinfo); + sjinfo, + rel); } else { diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index ed07e2f655..bf0fb56ab0 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -3750,7 +3750,8 @@ compute_semi_anti_join_factors(PlannerInfo *root, joinquals, 0, jointype, - sjinfo); + sjinfo, + NULL); /* * Also get the normal inner-join selectivity of the join clauses. @@ -3773,7 +3774,8 @@ compute_semi_anti_join_factors(PlannerInfo *root, joinquals, 0, JOIN_INNER, - &norm_sjinfo); + &norm_sjinfo, + NULL); /* Avoid leaking a lot of ListCells */ if (jointype == JOIN_ANTI) @@ -3940,7 +3942,7 @@ approx_tuple_count(PlannerInfo *root, JoinPath *path, List *quals) Node *qual = (Node *) lfirst(l); /* Note that clause_selectivity will be able to cache its result */ - selec *= clause_selectivity(root, qual, 0, JOIN_INNER, &sjinfo); + selec *= clause_selectivity(root, qual, 0, JOIN_INNER, &sjinfo, NULL); } /* Apply it to the input relation sizes */ @@ -3976,7 +3978,8 @@ set_baserel_size_estimates(PlannerInfo *root, RelOptInfo *rel) rel->baserestrictinfo, 0, JOIN_INNER, - NULL); + NULL, + rel); rel->rows = clamp_row_est(nrows); @@ -4013,7 +4016,8 @@ get_parameterized_baserel_size(PlannerInfo *root, RelOptInfo *rel, allclauses, rel->relid, /* do not use 0! */ JOIN_INNER, - NULL); + NULL, + rel); nrows = clamp_row_est(nrows); /* For safety, make sure result is not more than the base estimate */ if (nrows > rel->rows) @@ -4179,12 +4183,14 @@ calc_joinrel_size_estimate(PlannerInfo *root, joinquals, 0, jointype, - sjinfo); + sjinfo, + NULL); pselec = clauselist_selectivity(root, pushedquals, 0, jointype, - sjinfo); + sjinfo, + NULL); /* Avoid leaking a lot of ListCells */ list_free(joinquals); @@ -4196,7 +4202,8 @@ calc_joinrel_size_estimate(PlannerInfo *root, restrictlist, 0, jointype, - sjinfo); + sjinfo, + NULL); pselec = 0.0; /* not used, keep compiler quiet */ } @@ -4491,7 +4498,7 @@ get_foreign_key_join_selectivity(PlannerInfo *root, Selectivity csel; csel = clause_selectivity(root, (Node *) rinfo, - 0, jointype, sjinfo); + 0, jointype, sjinfo, NULL); thisfksel = Min(thisfksel, csel); } fkselec *= thisfksel; diff --git a/src/backend/optimizer/util/orclauses.c b/src/backend/optimizer/util/orclauses.c index 9cbcaedb75..735697d2f3 100644 --- a/src/backend/optimizer/util/orclauses.c +++ b/src/backend/optimizer/util/orclauses.c @@ -280,7 +280,7 @@ consider_new_or_clause(PlannerInfo *root, RelOptInfo *rel, * saving work later.) */ or_selec = clause_selectivity(root, (Node *) or_rinfo, - 0, JOIN_INNER, NULL); + 0, JOIN_INNER, NULL, rel); /* * The clause is only worth adding to the query if it rejects a useful @@ -344,7 +344,7 @@ consider_new_or_clause(PlannerInfo *root, RelOptInfo *rel, /* Compute inner-join size */ orig_selec = clause_selectivity(root, (Node *) join_or_rinfo, - 0, JOIN_INNER, &sjinfo); + 0, JOIN_INNER, &sjinfo, NULL); /* And hack cached selectivity so join size remains the same */ join_or_rinfo->norm_selec = orig_selec / or_selec; diff --git a/src/backend/optimizer/util/plancat.c b/src/backend/optimizer/util/plancat.c index 19904b54bd..faebfeddb2 100644 --- a/src/backend/optimizer/util/plancat.c +++ b/src/backend/optimizer/util/plancat.c @@ -1308,6 +1308,18 @@ get_relation_statistics(RelOptInfo *rel, Relation relation) stainfos = lcons(info, stainfos); } + if (statext_is_kind_built(htup, STATS_EXT_DEPENDENCIES)) + { + StatisticExtInfo *info = makeNode(StatisticExtInfo); + + info->statOid = statOid; + info->rel = rel; + info->kind = STATS_EXT_DEPENDENCIES; + info->keys = bms_copy(keys); + + stainfos = lcons(info, stainfos); + } + ReleaseSysCache(htup); bms_free(keys); } diff --git a/src/backend/statistics/Makefile b/src/backend/statistics/Makefile index b3615bd69a..3404e4554a 100644 --- a/src/backend/statistics/Makefile +++ b/src/backend/statistics/Makefile @@ -12,6 +12,6 @@ subdir = src/backend/statistics top_builddir = ../../.. include $(top_builddir)/src/Makefile.global -OBJS = extended_stats.o mvdistinct.o +OBJS = extended_stats.o dependencies.o mvdistinct.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/statistics/README b/src/backend/statistics/README index beb7c2485f..af7651127e 100644 --- a/src/backend/statistics/README +++ b/src/backend/statistics/README @@ -8,10 +8,72 @@ not true, resulting in estimation errors. Extended statistics track different types of dependencies between the columns, hopefully improving the estimates and producing better plans. -Currently we only have one type of extended statistics - ndistinct -coefficients, and we use it to improve estimates of grouping queries. See -README.ndistinct for details. +Types of statistics +------------------- + +There are two kinds of extended statistics: + + (a) ndistinct coefficients + + (b) soft functional dependencies (README.dependencies) + + +Compatible clause types +----------------------- + +Each type of statistics may be used to estimate some subset of clause types. + + (a) functional dependencies - equality clauses (AND), possibly IS NULL + +Currently, only OpExprs in the form Var op Const, or Const op Var are +supported, however it's feasible to expand the code later to also estimate the +selectivities on clauses such as Var op Var. + + +Complex clauses +--------------- + +We also support estimating more complex clauses - essentially AND/OR clauses +with (Var op Const) as leaves, as long as all the referenced attributes are +covered by a single statistics. + +For example this condition + + (a=1) AND ((b=2) OR ((c=3) AND (d=4))) + +may be estimated using statistics on (a,b,c,d). If we only have statistics on +(b,c,d) we may estimate the second part, and estimate (a=1) using simple stats. + +If we only have statistics on (a,b,c) we can't apply it at all at this point, +but it's worth pointing out clauselist_selectivity() works recursively and when +handling the second part (the OR-clause), we'll be able to apply the statistics. + +Note: The multi-statistics estimation patch also makes it possible to pass some +clauses as 'conditions' into the deeper parts of the expression tree. + + +Selectivity estimation +---------------------- + +Throughout the planner clauselist_selectivity() still remains in charge of +most selectivity estimate requests. clauselist_selectivity() can be instructed +to try to make use of any extended statistics on the given RelOptInfo, which +it will do, if: + + (a) An actual valid RelOptInfo was given. Join relations are passed in as + NULL, therefore are invalid. + + (b) The relation given actually has any extended statistics defined which + are actually built. + +When the above conditions are met, clauselist_selectivity() first attempts to +pass the clause list off to the extended statistics selectivity estimation +function. This functions may not find any clauses which is can perform any +estimations on. In such cases these clauses are simply ignored. When actual +estimation work is performed in these functions they're expected to mark which +clauses they've performed estimations for so that any other function +performing estimations knows which clauses are to be skipped. Size of sample in ANALYZE ------------------------- diff --git a/src/backend/statistics/README.dependencies b/src/backend/statistics/README.dependencies new file mode 100644 index 0000000000..7bc2533dc6 --- /dev/null +++ b/src/backend/statistics/README.dependencies @@ -0,0 +1,119 @@ +Soft functional dependencies +============================ + +Functional dependencies are a concept well described in relational theory, +particularly in the definition of normalization and "normal forms". Wikipedia +has a nice definition of a functional dependency [1]: + + In a given table, an attribute Y is said to have a functional dependency + on a set of attributes X (written X -> Y) if and only if each X value is + associated with precisely one Y value. For example, in an "Employee" + table that includes the attributes "Employee ID" and "Employee Date of + Birth", the functional dependency + + {Employee ID} -> {Employee Date of Birth} + + would hold. It follows from the previous two sentences that each + {Employee ID} is associated with precisely one {Employee Date of Birth}. + + [1] https://en.wikipedia.org/wiki/Functional_dependency + +In practical terms, functional dependencies mean that a value in one column +determines values in some other column. Consider for example this trivial +table with two integer columns: + + CREATE TABLE t (a INT, b INT) + AS SELECT i, i/10 FROM generate_series(1,100000) s(i); + +Clearly, knowledge of the value in column 'a' is sufficient to determine the +value in column 'b', as it's simply (a/10). A more practical example may be +addresses, where the knowledge of a ZIP code (usually) determines city. Larger +cities may have multiple ZIP codes, so the dependency can't be reversed. + +Many datasets might be normalized not to contain such dependencies, but often +it's not practical for various reasons. In some cases, it's actually a conscious +design choice to model the dataset in a denormalized way, either because of +performance or to make querying easier. + + +Soft dependencies +----------------- + +Real-world data sets often contain data errors, either because of data entry +mistakes (user mistyping the ZIP code) or perhaps issues in generating the +data (e.g. a ZIP code mistakenly assigned to two cities in different states). + +A strict implementation would either ignore dependencies in such cases, +rendering the approach mostly useless even for slightly noisy data sets, or +result in sudden changes in behavior depending on minor differences between +samples provided to ANALYZE. + +For this reason, the statistics implements "soft" functional dependencies, +associating each functional dependency with a degree of validity (a number +between 0 and 1). This degree is then used to combine selectivities in a +smooth manner. + + +Mining dependencies (ANALYZE) +----------------------------- + +The current algorithm is fairly simple - generate all possible functional +dependencies, and for each one count the number of rows consistent with it. +Then use the fraction of rows (supporting/total) as the degree. + +To count the rows consistent with the dependency (a => b): + + (a) Sort the data lexicographically, i.e. first by 'a' then 'b'. + + (b) For each group of rows with the same 'a' value, count the number of + distinct values in 'b'. + + (c) If there's a single distinct value in 'b', the rows are consistent with + the functional dependency, otherwise they contradict it. + +The algorithm also requires a minimum size of the group to consider it +consistent (currently 3 rows in the sample). Small groups make it less likely +to break the consistency. + + +Clause reduction (planner/optimizer) +------------------------------------ + +Applying the functional dependencies is fairly simple - given a list of +equality clauses, we compute selectivities of each clause and then use the +degree to combine them using this formula + + P(a=?,b=?) = P(a=?) * (d + (1-d) * P(b=?)) + +Where 'd' is the degree of functional dependence (a=>b). + +With more than two equality clauses, this process happens recursively. For +example for (a,b,c) we first use (a,b=>c) to break the computation into + + P(a=?,b=?,c=?) = P(a=?,b=?) * (d + (1-d)*P(b=?)) + +and then apply (a=>b) the same way on P(a=?,b=?). + + +Consistency of clauses +---------------------- + +Functional dependencies only express general dependencies between columns, +without referencing particular values. This assumes that the equality clauses +are in fact consistent with the functional dependency, i.e. that given a +dependency (a=>b), the value in (b=?) clause is the value determined by (a=?). +If that's not the case, the clauses are "inconsistent" with the functional +dependency and the result will be over-estimation. + +This may happen, for example, when using conditions on the ZIP code and city +name with mismatching values (ZIP code for a different city), etc. In such a +case, the result set will be empty, but we'll estimate the selectivity using +the ZIP code condition. + +In this case, the default estimation based on AVIA principle happens to work +better, but mostly by chance. + +This issue is the price for the simplicity of functional dependencies. If the +application frequently constructs queries with clauses inconsistent with +functional dependencies present in the data, the best solution is not to +use functional dependencies, but one of the more complex types of statistics. diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c new file mode 100644 index 0000000000..fb958e1b0a --- /dev/null +++ b/src/backend/statistics/dependencies.c @@ -0,0 +1,1079 @@ +/*------------------------------------------------------------------------- + * + * dependencies.c + * POSTGRES functional dependencies + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/statistics/dependencies.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/htup_details.h" +#include "access/sysattr.h" +#include "catalog/pg_operator.h" +#include "catalog/pg_statistic_ext.h" +#include "lib/stringinfo.h" +#include "optimizer/clauses.h" +#include "optimizer/cost.h" +#include "optimizer/var.h" +#include "nodes/nodes.h" +#include "nodes/relation.h" +#include "statistics/extended_stats_internal.h" +#include "statistics/statistics.h" +#include "utils/bytea.h" +#include "utils/fmgroids.h" +#include "utils/fmgrprotos.h" +#include "utils/lsyscache.h" +#include "utils/syscache.h" +#include "utils/typcache.h" + +/* + * Internal state for DependencyGenerator of dependencies. Dependencies are similar to + * k-permutations of n elements, except that the order does not matter for the + * first (k-1) elements. That is, (a,b=>c) and (b,a=>c) are equivalent. + */ +typedef struct DependencyGeneratorData +{ + int k; /* size of the dependency */ + int n; /* number of possible attributes */ + int current; /* next dependency to return (index) */ + AttrNumber ndependencies; /* number of dependencies generated */ + AttrNumber *dependencies; /* array of pre-generated dependencies */ +} DependencyGeneratorData; + +typedef DependencyGeneratorData *DependencyGenerator; + +static void generate_dependencies_recurse(DependencyGenerator state, + int index, AttrNumber start, AttrNumber *current); +static void generate_dependencies(DependencyGenerator state); +static DependencyGenerator DependencyGenerator_init(int n, int k); +static void DependencyGenerator_free(DependencyGenerator state); +static AttrNumber *DependencyGenerator_next(DependencyGenerator state); +static double dependency_degree(int numrows, HeapTuple *rows, int k, + AttrNumber *dependency, VacAttrStats **stats, Bitmapset *attrs); +static bool dependency_is_fully_matched(MVDependency *dependency, + Bitmapset *attnums); +static bool dependency_implies_attribute(MVDependency *dependency, + AttrNumber attnum); +static bool dependency_is_compatible_clause(Node *clause, Index relid, + AttrNumber *attnum); +static MVDependency *find_strongest_dependency(StatisticExtInfo *stats, + MVDependencies *dependencies, + Bitmapset *attnums); + +static void +generate_dependencies_recurse(DependencyGenerator state, int index, + AttrNumber start, AttrNumber *current) +{ + /* + * The generator handles the first (k-1) elements differently from the + * last element. + */ + if (index < (state->k - 1)) + { + AttrNumber i; + + /* + * The first (k-1) values have to be in ascending order, which we + * generate recursively. + */ + + for (i = start; i < state->n; i++) + { + current[index] = i; + generate_dependencies_recurse(state, (index + 1), (i + 1), current); + } + } + else + { + int i; + + /* + * the last element is the implied value, which does not respect the + * ascending order. We just need to check that the value is not in the + * first (k-1) elements. + */ + + for (i = 0; i < state->n; i++) + { + int j; + bool match = false; + + current[index] = i; + + for (j = 0; j < index; j++) + { + if (current[j] == i) + { + match = true; + break; + } + } + + /* + * If the value is not found in the first part of the dependency, + * we're done. + */ + if (!match) + { + state->dependencies = (AttrNumber *) repalloc(state->dependencies, + state->k * (state->ndependencies + 1) * sizeof(AttrNumber)); + memcpy(&state->dependencies[(state->k * state->ndependencies)], + current, state->k * sizeof(AttrNumber)); + state->ndependencies++; + } + } + } +} + +/* generate all dependencies (k-permutations of n elements) */ +static void +generate_dependencies(DependencyGenerator state) +{ + AttrNumber *current = (AttrNumber *) palloc0(sizeof(AttrNumber) * state->k); + + generate_dependencies_recurse(state, 0, 0, current); + + pfree(current); +} + +/* + * initialize the DependencyGenerator of variations, and prebuild the variations + * + * This pre-builds all the variations. We could also generate them in + * DependencyGenerator_next(), but this seems simpler. + */ +static DependencyGenerator +DependencyGenerator_init(int n, int k) +{ + DependencyGenerator state; + + Assert((n >= k) && (k > 0)); + + /* allocate the DependencyGenerator state */ + state = (DependencyGenerator) palloc0(sizeof(DependencyGeneratorData)); + state->dependencies = (AttrNumber *) palloc(k * sizeof(AttrNumber)); + + state->ndependencies = 0; + state->current = 0; + state->k = k; + state->n = n; + + /* now actually pre-generate all the variations */ + generate_dependencies(state); + + return state; +} + +/* free the DependencyGenerator state */ +static void +DependencyGenerator_free(DependencyGenerator state) +{ + pfree(state->dependencies); + pfree(state); + +} + +/* generate next combination */ +static AttrNumber * +DependencyGenerator_next(DependencyGenerator state) +{ + if (state->current == state->ndependencies) + return NULL; + + return &state->dependencies[state->k * state->current++]; +} + + +/* + * validates functional dependency on the data + * + * An actual work horse of detecting functional dependencies. Given a variation + * of k attributes, it checks that the first (k-1) are sufficient to determine + * the last one. + */ +static double +dependency_degree(int numrows, HeapTuple *rows, int k, AttrNumber *dependency, + VacAttrStats **stats, Bitmapset *attrs) +{ + int i, + j; + int nvalues = numrows * k; + MultiSortSupport mss; + SortItem *items; + Datum *values; + bool *isnull; + int *attnums; + + /* counters valid within a group */ + int group_size = 0; + int n_violations = 0; + + /* total number of rows supporting (consistent with) the dependency */ + int n_supporting_rows = 0; + + /* Make sure we have at least two input attributes. */ + Assert(k >= 2); + + /* sort info for all attributes columns */ + mss = multi_sort_init(k); + + /* data for the sort */ + items = (SortItem *) palloc(numrows * sizeof(SortItem)); + values = (Datum *) palloc(sizeof(Datum) * nvalues); + isnull = (bool *) palloc(sizeof(bool) * nvalues); + + /* fix the pointers to values/isnull */ + for (i = 0; i < numrows; i++) + { + items[i].values = &values[i * k]; + items[i].isnull = &isnull[i * k]; + } + + /* + * Transform the bms into an array, to make accessing i-th member easier. + */ + attnums = (int *) palloc(sizeof(int) * bms_num_members(attrs)); + i = 0; + j = -1; + while ((j = bms_next_member(attrs, j)) >= 0) + attnums[i++] = j; + + /* + * Verify the dependency (a,b,...)->z, using a rather simple algorithm: + * + * (a) sort the data lexicographically + * + * (b) split the data into groups by first (k-1) columns + * + * (c) for each group count different values in the last column + */ + + /* prepare the sort function for the first dimension, and SortItem array */ + for (i = 0; i < k; i++) + { + VacAttrStats *colstat = stats[dependency[i]]; + TypeCacheEntry *type; + + type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR); + if (type->lt_opr == InvalidOid) /* shouldn't happen */ + elog(ERROR, "cache lookup failed for ordering operator for type %u", + colstat->attrtypid); + + /* prepare the sort function for this dimension */ + multi_sort_add_dimension(mss, i, type->lt_opr); + + /* accumulate all the data for both columns into an array and sort it */ + for (j = 0; j < numrows; j++) + { + items[j].values[i] = + heap_getattr(rows[j], attnums[dependency[i]], + stats[i]->tupDesc, &items[j].isnull[i]); + } + } + + /* sort the items so that we can detect the groups */ + qsort_arg((void *) items, numrows, sizeof(SortItem), + multi_sort_compare, mss); + + /* + * Walk through the sorted array, split it into rows according to the + * first (k-1) columns. If there's a single value in the last column, we + * count the group as 'supporting' the functional dependency. Otherwise we + * count it as contradicting. + * + * We also require a group to have a minimum number of rows to be + * considered useful for supporting the dependency. Contradicting groups + * may be of any size, though. + * + * XXX The minimum size requirement makes it impossible to identify case + * when both columns are unique (or nearly unique), and therefore + * trivially functionally dependent. + */ + + /* start with the first row forming a group */ + group_size = 1; + + /* loop 1 beyond the end of the array so that we count the final group */ + for (i = 1; i <= numrows; i++) + { + /* + * Check if the group ended, which may be either because we processed + * all the items (i==numrows), or because the i-th item is not equal + * to the preceding one. + */ + if (i == numrows || + multi_sort_compare_dims(0, k - 2, &items[i - 1], &items[i], mss) != 0) + { + /* + * If no violations were found in the group then track the rows of + * the group as supporting the functional dependency. + */ + if (n_violations == 0) + n_supporting_rows += group_size; + + /* Reset counters for the new group */ + n_violations = 0; + group_size = 1; + continue; + } + /* first columns match, but the last one does not (so contradicting) */ + else if (multi_sort_compare_dim(k - 1, &items[i - 1], &items[i], mss) != 0) + n_violations++; + + group_size++; + } + + pfree(items); + pfree(values); + pfree(isnull); + pfree(mss); + + /* Compute the 'degree of validity' as (supporting/total). */ + return (n_supporting_rows * 1.0 / numrows); +} + +/* + * detects functional dependencies between groups of columns + * + * Generates all possible subsets of columns (variations) and computes + * the degree of validity for each one. For example with a statistic on + * three columns (a,b,c) there are 9 possible dependencies + * + * two columns three columns + * ----------- ------------- + * (a) -> b (a,b) -> c + * (a) -> c (a,c) -> b + * (b) -> a (b,c) -> a + * (b) -> c + * (c) -> a + * (c) -> b + */ +MVDependencies * +statext_dependencies_build(int numrows, HeapTuple *rows, Bitmapset *attrs, + VacAttrStats **stats) +{ + int i, + j, + k; + int numattrs; + int *attnums; + + /* result */ + MVDependencies *dependencies = NULL; + + numattrs = bms_num_members(attrs); + + /* + * Transform the bms into an array, to make accessing i-th member easier. + */ + attnums = palloc(sizeof(int) * bms_num_members(attrs)); + i = 0; + j = -1; + while ((j = bms_next_member(attrs, j)) >= 0) + attnums[i++] = j; + + Assert(numattrs >= 2); + + /* + * We'll try build functional dependencies starting from the smallest ones + * covering just 2 columns, to the largest ones, covering all columns + * included in the statistics. We start from the smallest ones because we + * want to be able to skip already implied ones. + */ + for (k = 2; k <= numattrs; k++) + { + AttrNumber *dependency; /* array with k elements */ + + /* prepare a DependencyGenerator of variation */ + DependencyGenerator DependencyGenerator = DependencyGenerator_init(numattrs, k); + + /* generate all possible variations of k values (out of n) */ + while ((dependency = DependencyGenerator_next(DependencyGenerator))) + { + double degree; + MVDependency *d; + + /* compute how valid the dependency seems */ + degree = dependency_degree(numrows, rows, k, dependency, stats, attrs); + + /* + * if the dependency seems entirely invalid, don't store it it + */ + if (degree == 0.0) + continue; + + d = (MVDependency *) palloc0(offsetof(MVDependency, attributes) + + k * sizeof(AttrNumber)); + + /* copy the dependency (and keep the indexes into stakeys) */ + d->degree = degree; + d->nattributes = k; + for (i = 0; i < k; i++) + d->attributes[i] = attnums[dependency[i]]; + + /* initialize the list of dependencies */ + if (dependencies == NULL) + { + dependencies + = (MVDependencies *) palloc0(sizeof(MVDependencies)); + + dependencies->magic = STATS_DEPS_MAGIC; + dependencies->type = STATS_DEPS_TYPE_BASIC; + dependencies->ndeps = 0; + } + + dependencies->ndeps++; + dependencies = (MVDependencies *) repalloc(dependencies, + offsetof(MVDependencies, deps) + + dependencies->ndeps * sizeof(MVDependency)); + + dependencies->deps[dependencies->ndeps - 1] = d; + } + + /* + * we're done with variations of k elements, so free the + * DependencyGenerator + */ + DependencyGenerator_free(DependencyGenerator); + } + + return dependencies; +} + + +/* + * Serialize list of dependencies into a bytea value. + */ +bytea * +statext_dependencies_serialize(MVDependencies * dependencies) +{ + int i; + bytea *output; + char *tmp; + Size len; + + /* we need to store ndeps, with a number of attributes for each one */ + len = VARHDRSZ + SizeOfDependencies + + dependencies->ndeps * SizeOfDependency; + + /* and also include space for the actual attribute numbers and degrees */ + for (i = 0; i < dependencies->ndeps; i++) + len += (sizeof(AttrNumber) * dependencies->deps[i]->nattributes); + + output = (bytea *) palloc0(len); + SET_VARSIZE(output, len); + + tmp = VARDATA(output); + + /* Store the base struct values (magic, type, ndeps) */ + memcpy(tmp, &dependencies->magic, sizeof(uint32)); + tmp += sizeof(uint32); + memcpy(tmp, &dependencies->type, sizeof(uint32)); + tmp += sizeof(uint32); + memcpy(tmp, &dependencies->ndeps, sizeof(uint32)); + tmp += sizeof(uint32); + + /* store number of attributes and attribute numbers for each dependency */ + for (i = 0; i < dependencies->ndeps; i++) + { + MVDependency *d = dependencies->deps[i]; + + memcpy(tmp, d, SizeOfDependency); + tmp += SizeOfDependency; + + memcpy(tmp, d->attributes, sizeof(AttrNumber) * d->nattributes); + tmp += sizeof(AttrNumber) * d->nattributes; + + Assert(tmp <= ((char *) output + len)); + } + + return output; +} + +/* + * Reads serialized dependencies into MVDependencies structure. + */ +MVDependencies * +statext_dependencies_deserialize(bytea *data) +{ + int i; + Size min_expected_size; + MVDependencies *dependencies; + char *tmp; + + if (data == NULL) + return NULL; + + if (VARSIZE_ANY_EXHDR(data) < SizeOfDependencies) + elog(ERROR, "invalid MVDependencies size %ld (expected at least %ld)", + VARSIZE_ANY_EXHDR(data), SizeOfDependencies); + + /* read the MVDependencies header */ + dependencies = (MVDependencies *) palloc0(sizeof(MVDependencies)); + + /* initialize pointer to the data part (skip the varlena header) */ + tmp = VARDATA_ANY(data); + + /* read the header fields and perform basic sanity checks */ + memcpy(&dependencies->magic, tmp, sizeof(uint32)); + tmp += sizeof(uint32); + memcpy(&dependencies->type, tmp, sizeof(uint32)); + tmp += sizeof(uint32); + memcpy(&dependencies->ndeps, tmp, sizeof(uint32)); + tmp += sizeof(uint32); + + if (dependencies->magic != STATS_DEPS_MAGIC) + elog(ERROR, "invalid dependency magic %d (expected %d)", + dependencies->magic, STATS_DEPS_MAGIC); + + if (dependencies->type != STATS_DEPS_TYPE_BASIC) + elog(ERROR, "invalid dependency type %d (expected %d)", + dependencies->type, STATS_DEPS_TYPE_BASIC); + + if (dependencies->ndeps == 0) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("invalid zero-length item array in MVDependencies"))); + + /* what minimum bytea size do we expect for those parameters */ + min_expected_size = SizeOfDependencies + + dependencies->ndeps * (SizeOfDependency + + sizeof(AttrNumber) * 2); + + if (VARSIZE_ANY_EXHDR(data) < min_expected_size) + elog(ERROR, "invalid dependencies size %ld (expected at least %ld)", + VARSIZE_ANY_EXHDR(data), min_expected_size); + + /* allocate space for the MCV items */ + dependencies = repalloc(dependencies, offsetof(MVDependencies, deps) + + (dependencies->ndeps * sizeof(MVDependency *))); + + for (i = 0; i < dependencies->ndeps; i++) + { + double degree; + AttrNumber k; + MVDependency *d; + + /* degree of validity */ + memcpy(°ree, tmp, sizeof(double)); + tmp += sizeof(double); + + /* number of attributes */ + memcpy(&k, tmp, sizeof(AttrNumber)); + tmp += sizeof(AttrNumber); + + /* is the number of attributes valid? */ + Assert((k >= 2) && (k <= STATS_MAX_DIMENSIONS)); + + /* now that we know the number of attributes, allocate the dependency */ + d = (MVDependency *) palloc0(offsetof(MVDependency, attributes) + + (k * sizeof(AttrNumber))); + + d->degree = degree; + d->nattributes = k; + + /* copy attribute numbers */ + memcpy(d->attributes, tmp, sizeof(AttrNumber) * d->nattributes); + tmp += sizeof(AttrNumber) * d->nattributes; + + dependencies->deps[i] = d; + + /* still within the bytea */ + Assert(tmp <= ((char *) data + VARSIZE_ANY(data))); + } + + /* we should have consumed the whole bytea exactly */ + Assert(tmp == ((char *) data + VARSIZE_ANY(data))); + + return dependencies; +} + +/* + * dependency_is_fully_matched + * checks that a functional dependency is fully matched given clauses on + * attributes (assuming the clauses are suitable equality clauses) + */ +static bool +dependency_is_fully_matched(MVDependency * dependency, Bitmapset *attnums) +{ + int j; + + /* + * Check that the dependency actually is fully covered by clauses. We have + * to translate all attribute numbers, as those are referenced + */ + for (j = 0; j < dependency->nattributes; j++) + { + int attnum = dependency->attributes[j]; + + if (!bms_is_member(attnum, attnums)) + return false; + } + + return true; +} + +/* + * dependency_implies_attribute + * check that the attnum matches is implied by the functional dependency + */ +static bool +dependency_implies_attribute(MVDependency * dependency, AttrNumber attnum) +{ + if (attnum == dependency->attributes[dependency->nattributes - 1]) + return true; + + return false; +} + +/* + * staext_dependencies_load + * Load the functional dependencies for the indicated pg_statistic_ext tuple + */ +MVDependencies * +staext_dependencies_load(Oid mvoid) +{ + bool isnull; + Datum deps; + + /* + * Prepare to scan pg_statistic_ext for entries having indrelid = this + * rel. + */ + HeapTuple htup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(mvoid)); + + if (!HeapTupleIsValid(htup)) + elog(ERROR, "cache lookup failed for extended statistics %u", mvoid); + + deps = SysCacheGetAttr(STATEXTOID, htup, + Anum_pg_statistic_ext_stadependencies, &isnull); + + Assert(!isnull); + + ReleaseSysCache(htup); + + return statext_dependencies_deserialize(DatumGetByteaP(deps)); +} + +/* + * pg_dependencies_in - input routine for type pg_dependencies. + * + * pg_dependencies is real enough to be a table column, but it has no operations + * of its own, and disallows input too + */ +Datum +pg_dependencies_in(PG_FUNCTION_ARGS) +{ + /* + * pg_node_list stores the data in binary form and parsing text input is + * not needed, so disallow this. + */ + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot accept a value of type %s", "pg_dependencies"))); + + PG_RETURN_VOID(); /* keep compiler quiet */ +} + +/* + * pg_dependencies - output routine for type pg_dependencies. + */ +Datum +pg_dependencies_out(PG_FUNCTION_ARGS) +{ + int i, + j; + StringInfoData str; + + bytea *data = PG_GETARG_BYTEA_PP(0); + + MVDependencies *dependencies = statext_dependencies_deserialize(data); + + initStringInfo(&str); + appendStringInfoChar(&str, '['); + + for (i = 0; i < dependencies->ndeps; i++) + { + MVDependency *dependency = dependencies->deps[i]; + + if (i > 0) + appendStringInfoString(&str, ", "); + + appendStringInfoChar(&str, '{'); + for (j = 0; j < dependency->nattributes; j++) + { + if (j == dependency->nattributes - 1) + appendStringInfoString(&str, " => "); + else if (j > 0) + appendStringInfoString(&str, ", "); + + appendStringInfo(&str, "%d", dependency->attributes[j]); + } + appendStringInfo(&str, " : %f", dependency->degree); + appendStringInfoChar(&str, '}'); + } + + appendStringInfoChar(&str, ']'); + + PG_RETURN_CSTRING(str.data); +} + +/* + * pg_dependencies_recv - binary input routine for type pg_dependencies. + */ +Datum +pg_dependencies_recv(PG_FUNCTION_ARGS) +{ + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot accept a value of type %s", "pg_dependencies"))); + + PG_RETURN_VOID(); /* keep compiler quiet */ +} + +/* + * pg_dependencies_send - binary output routine for type pg_dependencies. + * + * Functional dependencies are serialized in a bytea value (although the type + * is named differently), so let's just send that. + */ +Datum +pg_dependencies_send(PG_FUNCTION_ARGS) +{ + return byteasend(fcinfo); +} + +/* + * dependency_is_compatible_clause + * Determines if the clause is compatible with functional dependencies + * + * Only OpExprs with two arguments using an equality operator are supported. + * When returning True attnum is set to the attribute number of the Var within + * the supported clause. + * + * Currently we only support Var = Const, or Const = Var. It may be possible + * to expand on this later. + */ +static bool +dependency_is_compatible_clause(Node *clause, Index relid, AttrNumber *attnum) +{ + RestrictInfo *rinfo = (RestrictInfo *) clause; + + if (!IsA(rinfo, RestrictInfo)) + return false; + + /* Pseudoconstants are not really interesting here. */ + if (rinfo->pseudoconstant) + return false; + + /* clauses referencing multiple varnos are incompatible */ + if (bms_membership(rinfo->clause_relids) != BMS_SINGLETON) + return false; + + if (is_opclause(rinfo->clause)) + { + OpExpr *expr = (OpExpr *) rinfo->clause; + Var *var; + bool varonleft = true; + bool ok; + + /* Only expressions with two arguments are considered compatible. */ + if (list_length(expr->args) != 2) + return false; + + /* see if it actually has the right */ + ok = (NumRelids((Node *) expr) == 1) && + (is_pseudo_constant_clause(lsecond(expr->args)) || + (varonleft = false, + is_pseudo_constant_clause(linitial(expr->args)))); + + /* unsupported structure (two variables or so) */ + if (!ok) + return false; + + /* + * If it's not "=" operator, just ignore the clause, as it's not + * compatible with functional dependencies. + * + * This uses the function for estimating selectivity, not the operator + * directly (a bit awkward, but well ...). + */ + if (get_oprrest(expr->opno) != F_EQSEL) + return false; + + var = (varonleft) ? linitial(expr->args) : lsecond(expr->args); + + /* We only support plain Vars for now */ + if (!IsA(var, Var)) + return false; + + /* Ensure var is from the correct relation */ + if (var->varno != relid) + return false; + + /* we also better ensure the Var is from the current level */ + if (var->varlevelsup > 0) + return false; + + /* Also skip system attributes (we don't allow stats on those). */ + if (!AttrNumberIsForUserDefinedAttr(var->varattno)) + return false; + + *attnum = var->varattno; + return true; + } + + return false; +} + +/* + * find_strongest_dependency + * find the strongest dependency on the attributes + * + * When applying functional dependencies, we start with the strongest + * dependencies. That is, we select the dependency that: + * + * (a) has all attributes covered by equality clauses + * + * (b) has the most attributes + * + * (c) has the highest degree of validity + * + * This guarantees that we eliminate the most redundant conditions first + * (see the comment in dependencies_clauselist_selectivity). + */ +static MVDependency * +find_strongest_dependency(StatisticExtInfo * stats, MVDependencies * dependencies, + Bitmapset *attnums) +{ + int i; + MVDependency *strongest = NULL; + + /* number of attnums in clauses */ + int nattnums = bms_num_members(attnums); + + /* + * Iterate over the MVDependency items and find the strongest one from the + * fully-matched dependencies. We do the cheap checks first, before + * matching it against the attnums. + */ + for (i = 0; i < dependencies->ndeps; i++) + { + MVDependency *dependency = dependencies->deps[i]; + + /* + * Skip dependencies referencing more attributes than available + * clauses, as those can't be fully matched. + */ + if (dependency->nattributes > nattnums) + continue; + + if (strongest) + { + /* skip dependencies on fewer attributes than the strongest. */ + if (dependency->nattributes < strongest->nattributes) + continue; + + /* also skip weaker dependencies when attribute count matches */ + if (strongest->nattributes == dependency->nattributes && + strongest->degree > dependency->degree) + continue; + } + + /* + * this dependency is stronger, but we must still check that it's + * fully matched to these attnums. We perform this check last as it's + * slightly more expensive than the previous checks. + */ + if (dependency_is_fully_matched(dependency, attnums)) + strongest = dependency; /* save new best match */ + } + + return strongest; +} + +/* + * dependencies_clauselist_selectivity + * Attempt to estimate selectivity using functional dependency statistics + * + * Given equality clauses on attributes (a,b) we find the strongest dependency + * between them, i.e. either (a=>b) or (b=>a). Assuming (a=>b) is the selected + * dependency, we then combine the per-clause selectivities using the formula + * + * P(a,b) = P(a) * [f + (1-f)*P(b)] + * + * where 'f' is the degree of the dependency. + * + * With clauses on more than two attributes, the dependencies are applied + * recursively, starting with the widest/strongest dependencies. For example + * P(a,b,c) is first split like this: + * + * P(a,b,c) = P(a,b) * [f + (1-f)*P(c)] + * + * assuming (a,b=>c) is the strongest dependency. + */ +Selectivity +dependencies_clauselist_selectivity(PlannerInfo *root, + List *clauses, + int varRelid, + JoinType jointype, + SpecialJoinInfo *sjinfo, + RelOptInfo *rel, + Bitmapset **estimatedclauses) +{ + Selectivity s1 = 1.0; + ListCell *l; + Bitmapset *clauses_attnums = NULL; + StatisticExtInfo *stat; + MVDependencies *dependencies; + AttrNumber *list_attnums; + int listidx; + + + /* check if there's any stats that might be useful for us. */ + if (!has_stats_of_kind(rel->statlist, STATS_EXT_DEPENDENCIES)) + return 1.0; + + list_attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * + list_length(clauses)); + + /* + * Pre-process the clauses list to extract the attnums seen in each item. + * We need to determine if there's any clauses which will be useful for + * dependency selectivity estimations. Along the way we'll record all of + * the attnums for each clause in a list which we'll reference later so we + * don't need to repeat the same work again. We'll also keep track of all + * attnums seen. + */ + listidx = 0; + foreach(l, clauses) + { + Node *clause = (Node *) lfirst(l); + AttrNumber attnum; + + if (dependency_is_compatible_clause(clause, rel->relid, &attnum)) + { + list_attnums[listidx] = attnum; + clauses_attnums = bms_add_member(clauses_attnums, attnum); + } + else + list_attnums[listidx] = InvalidAttrNumber; + + listidx++; + } + + /* + * If there's not at least two distinct attnums then reject the whole list + * of clauses. We must return 1.0 so the calling function's selectivity is + * unaffected. + */ + if (bms_num_members(clauses_attnums) < 2) + { + pfree(list_attnums); + return 1.0; + } + + /* find the best suited statistics for these attnums */ + stat = choose_best_statistics(rel->statlist, clauses_attnums, + STATS_EXT_DEPENDENCIES); + + /* if no matching stats could be found then we've nothing to do */ + if (!stat) + { + pfree(list_attnums); + return 1.0; + } + + /* load the dependency items stored in the statistics */ + dependencies = staext_dependencies_load(stat->statOid); + + /* + * Apply the dependencies recursively, starting with the widest/strongest + * ones, and proceeding to the smaller/weaker ones. At the end of each + * round we factor in the selectivity of clauses on the implied attribute, + * and remove the clauses from the list. + */ + while (true) + { + Selectivity s2 = 1.0; + MVDependency *dependency; + + /* the widest/strongest dependency, fully matched by clauses */ + dependency = find_strongest_dependency(stat, dependencies, + clauses_attnums); + + /* if no suitable dependency was found, we're done */ + if (!dependency) + break; + + /* + * We found an applicable dependency, so find all the clauses on the + * implied attribute - with dependency (a,b => c) we look for clauses + * on 'c'. + */ + listidx = -1; + foreach(l, clauses) + { + Node *clause; + + listidx++; + + /* + * Skip incompatible clauses, and ones we've already estimated on. + */ + if (list_attnums[listidx] == InvalidAttrNumber || + bms_is_member(listidx, *estimatedclauses)) + continue; + + /* + * Technically we could find more than one clause for a given + * attnum. Since these clauses must be equality clauses, we choose + * to only take the selectivity estimate from the final clause in + * the list for this attnum. If the attnum happens to be compared + * to a different Const in another clause then no rows will match + * anyway. If it happens to be compared to the same Const, then + * ignoring the additional clause is just the thing to do. + */ + if (dependency_implies_attribute(dependency, + list_attnums[listidx])) + { + clause = (Node *) lfirst(l); + + s2 = clause_selectivity(root, clause, varRelid, jointype, sjinfo, + NULL); /* don't try to use ext stats */ + + /* mark this one as done, so we don't touch it again. */ + *estimatedclauses = bms_add_member(*estimatedclauses, listidx); + + /* + * Mark that we've got and used the dependency on this clause. + * We'll want to ignore this when looking for the next + * strongest dependency above. + */ + clauses_attnums = bms_del_member(clauses_attnums, + list_attnums[listidx]); + } + } + + /* + * Now factor in the selectivity for all the "implied" clauses into + * the final one, using this formula: + * + * P(a,b) = P(a) * (f + (1-f) * P(b)) + * + * where 'f' is the degree of validity of the dependency. + */ + s1 *= (dependency->degree + (1 - dependency->degree) * s2); + } + + pfree(dependencies); + pfree(list_attnums); + + return s1; +} diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c index d2b9f6a7c7..006bb897c4 100644 --- a/src/backend/statistics/extended_stats.c +++ b/src/backend/statistics/extended_stats.c @@ -47,7 +47,7 @@ static List *fetch_statentries_for_relation(Relation pg_statext, Oid relid); static VacAttrStats **lookup_var_attr_stats(Relation rel, Bitmapset *attrs, int natts, VacAttrStats **vacattrstats); static void statext_store(Relation pg_stext, Oid relid, - MVNDistinct *ndistinct, + MVNDistinct *ndistinct, MVDependencies *dependencies, VacAttrStats **stats); @@ -74,6 +74,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows, { StatExtEntry *stat = (StatExtEntry *) lfirst(lc); MVNDistinct *ndistinct = NULL; + MVDependencies *dependencies = NULL; VacAttrStats **stats; ListCell *lc2; @@ -93,10 +94,13 @@ BuildRelationExtStatistics(Relation onerel, double totalrows, if (t == STATS_EXT_NDISTINCT) ndistinct = statext_ndistinct_build(totalrows, numrows, rows, stat->columns, stats); + else if (t == STATS_EXT_DEPENDENCIES) + dependencies = statext_dependencies_build(numrows, rows, + stat->columns, stats); } /* store the statistics in the catalog */ - statext_store(pg_stext, stat->statOid, ndistinct, stats); + statext_store(pg_stext, stat->statOid, ndistinct, dependencies, stats); } heap_close(pg_stext, RowExclusiveLock); @@ -117,6 +121,10 @@ statext_is_kind_built(HeapTuple htup, char type) attnum = Anum_pg_statistic_ext_standistinct; break; + case STATS_EXT_DEPENDENCIES: + attnum = Anum_pg_statistic_ext_stadependencies; + break; + default: elog(ERROR, "unexpected statistics type requested: %d", type); } @@ -178,7 +186,8 @@ fetch_statentries_for_relation(Relation pg_statext, Oid relid) enabled = (char *) ARR_DATA_PTR(arr); for (i = 0; i < ARR_DIMS(arr)[0]; i++) { - Assert(enabled[i] == STATS_EXT_NDISTINCT); + Assert((enabled[i] == STATS_EXT_NDISTINCT) || + (enabled[i] == STATS_EXT_DEPENDENCIES)); entry->types = lappend_int(entry->types, (int) enabled[i]); } @@ -256,7 +265,7 @@ lookup_var_attr_stats(Relation rel, Bitmapset *attrs, int natts, */ static void statext_store(Relation pg_stext, Oid statOid, - MVNDistinct *ndistinct, + MVNDistinct *ndistinct, MVDependencies *dependencies, VacAttrStats **stats) { HeapTuple stup, @@ -280,8 +289,17 @@ statext_store(Relation pg_stext, Oid statOid, values[Anum_pg_statistic_ext_standistinct - 1] = PointerGetDatum(data); } + if (dependencies != NULL) + { + bytea *data = statext_dependencies_serialize(dependencies); + + nulls[Anum_pg_statistic_ext_stadependencies - 1] = (data == NULL); + values[Anum_pg_statistic_ext_stadependencies - 1] = PointerGetDatum(data); + } + /* always replace the value (either by bytea or NULL) */ replaces[Anum_pg_statistic_ext_standistinct - 1] = true; + replaces[Anum_pg_statistic_ext_stadependencies - 1] = true; /* there should already be a pg_statistic_ext tuple */ oldtup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(statOid)); @@ -387,3 +405,82 @@ multi_sort_compare_dims(int start, int end, return 0; } + +/* + * has_stats_of_kind + * Check that the list contains statistic of a given kind + */ +bool +has_stats_of_kind(List *stats, char requiredkind) +{ + ListCell *l; + + foreach(l, stats) + { + StatisticExtInfo *stat = (StatisticExtInfo *) lfirst(l); + + if (stat->kind == requiredkind) + return true; + } + + return false; +} + +/* + * choose_best_statistics + * Look for statistics with the specified 'requiredkind' which have keys + * that match at least two attnums. + * + * The current selection criteria is very simple - we choose the statistics + * referencing the most attributes with the least keys. + * + * XXX if multiple statistics exists of the same size matching the same number + * of keys, then the statistics which are chosen depend on the order that they + * appear in the stats list. Perhaps this needs to be more definitive. + */ +StatisticExtInfo * +choose_best_statistics(List *stats, Bitmapset *attnums, char requiredkind) +{ + ListCell *lc; + StatisticExtInfo *best_match = NULL; + int best_num_matched = 2; /* goal #1: maximize */ + int best_match_keys = (STATS_MAX_DIMENSIONS + 1); /* goal #2: minimize */ + + foreach(lc, stats) + { + StatisticExtInfo *info = (StatisticExtInfo *) lfirst(lc); + int num_matched; + int numkeys; + Bitmapset *matched; + + /* skip statistics that are not the correct type */ + if (info->kind != requiredkind) + continue; + + /* determine how many attributes of these stats can be matched to */ + matched = bms_intersect(attnums, info->keys); + num_matched = bms_num_members(matched); + bms_free(matched); + + /* + * save the actual number of keys in the stats so that we can choose + * the narrowest stats with the most matching keys. + */ + numkeys = bms_num_members(info->keys); + + /* + * Use these statistics when it increases the number of matched + * clauses or when it matches the same number of attributes but these + * stats have fewer keys than any previous match. + */ + if (num_matched > best_num_matched || + (num_matched == best_num_matched && numkeys < best_match_keys)) + { + best_match = info; + best_num_matched = num_matched; + best_match_keys = numkeys; + } + } + + return best_match; +} diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c index 0c1a201ecb..342e52bcf7 100644 --- a/src/backend/utils/adt/ruleutils.c +++ b/src/backend/utils/adt/ruleutils.c @@ -1452,6 +1452,13 @@ pg_get_statisticsext_worker(Oid statextid, bool missing_ok) StringInfoData buf; int colno; char *nsp; + ArrayType *arr; + char *enabled; + Datum datum; + bool isnull; + bool ndistinct_enabled; + bool dependencies_enabled; + int i; statexttup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(statextid)); @@ -1467,10 +1474,55 @@ pg_get_statisticsext_worker(Oid statextid, bool missing_ok) initStringInfo(&buf); nsp = get_namespace_name(statextrec->stanamespace); - appendStringInfo(&buf, "CREATE STATISTICS %s ON (", + appendStringInfo(&buf, "CREATE STATISTICS %s", quote_qualified_identifier(nsp, NameStr(statextrec->staname))); + /* + * Lookup the staenabled column so that we know how to handle the WITH + * clause. + */ + datum = SysCacheGetAttr(STATEXTOID, statexttup, + Anum_pg_statistic_ext_staenabled, &isnull); + Assert(!isnull); + arr = DatumGetArrayTypeP(datum); + if (ARR_NDIM(arr) != 1 || + ARR_HASNULL(arr) || + ARR_ELEMTYPE(arr) != CHAROID) + elog(ERROR, "staenabled is not a 1-D char array"); + enabled = (char *) ARR_DATA_PTR(arr); + + ndistinct_enabled = false; + dependencies_enabled = false; + + for (i = 0; i < ARR_DIMS(arr)[0]; i++) + { + if (enabled[i] == STATS_EXT_NDISTINCT) + ndistinct_enabled = true; + if (enabled[i] == STATS_EXT_DEPENDENCIES) + dependencies_enabled = true; + } + + /* + * If any option is disabled, then we'll need to append a WITH clause to + * show which options are enabled. We omit the WITH clause on purpose + * when all options are enabled, so a pg_dump/pg_restore will create all + * statistics types on a newer postgres version, if the statistics had all + * options enabled on the original version. + */ + if (!ndistinct_enabled || !dependencies_enabled) + { + appendStringInfoString(&buf, " WITH ("); + if (ndistinct_enabled) + appendStringInfoString(&buf, "ndistinct"); + else if (dependencies_enabled) + appendStringInfoString(&buf, "dependencies"); + + appendStringInfoChar(&buf, ')'); + } + + appendStringInfoString(&buf, " ON ("); + for (colno = 0; colno < statextrec->stakeys.dim1; colno++) { AttrNumber attnum = statextrec->stakeys.values[colno]; diff --git a/src/backend/utils/adt/selfuncs.c b/src/backend/utils/adt/selfuncs.c index 68b54235b3..f5cffcb2ea 100644 --- a/src/backend/utils/adt/selfuncs.c +++ b/src/backend/utils/adt/selfuncs.c @@ -1633,13 +1633,17 @@ booltestsel(PlannerInfo *root, BoolTestType booltesttype, Node *arg, case IS_NOT_FALSE: selec = (double) clause_selectivity(root, arg, varRelid, - jointype, sjinfo); + jointype, + sjinfo, + NULL); break; case IS_FALSE: case IS_NOT_TRUE: selec = 1.0 - (double) clause_selectivity(root, arg, varRelid, - jointype, sjinfo); + jointype, + sjinfo, + NULL); break; default: elog(ERROR, "unrecognized booltesttype: %d", @@ -6436,7 +6440,8 @@ genericcostestimate(PlannerInfo *root, indexSelectivity = clauselist_selectivity(root, selectivityQuals, index->rel->relid, JOIN_INNER, - NULL); + NULL, + index->rel); /* * If caller didn't give us an estimate, estimate the number of index @@ -6757,7 +6762,8 @@ btcostestimate(PlannerInfo *root, IndexPath *path, double loop_count, btreeSelectivity = clauselist_selectivity(root, selectivityQuals, index->rel->relid, JOIN_INNER, - NULL); + NULL, + index->rel); numIndexTuples = btreeSelectivity * index->rel->tuples; /* @@ -7516,7 +7522,8 @@ gincostestimate(PlannerInfo *root, IndexPath *path, double loop_count, *indexSelectivity = clauselist_selectivity(root, selectivityQuals, index->rel->relid, JOIN_INNER, - NULL); + NULL, + index->rel); /* fetch estimated page cost for tablespace containing index */ get_tablespace_page_costs(index->reltablespace, @@ -7748,7 +7755,8 @@ brincostestimate(PlannerInfo *root, IndexPath *path, double loop_count, *indexSelectivity = clauselist_selectivity(root, indexQuals, path->indexinfo->rel->relid, - JOIN_INNER, NULL); + JOIN_INNER, NULL, + path->indexinfo->rel); *indexCorrelation = 1; /* diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index b0f3e5e347..2ef06261e6 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -2331,7 +2331,8 @@ describeOneTableDetails(const char *schemaname, " FROM ((SELECT pg_catalog.unnest(stakeys) AS attnum) s\n" " JOIN pg_catalog.pg_attribute a ON (starelid = a.attrelid AND\n" "a.attnum = s.attnum AND not attisdropped))) AS columns,\n" - " (staenabled::char[] @> '{d}'::char[]) AS ndist_enabled\n" + " (staenabled::char[] @> '{d}'::char[]) AS ndist_enabled,\n" + " (staenabled::char[] @> '{f}'::char[]) AS deps_enabled\n" "FROM pg_catalog.pg_statistic_ext stat WHERE starelid = '%s'\n" "ORDER BY 1;", oid); @@ -2348,7 +2349,7 @@ describeOneTableDetails(const char *schemaname, for (i = 0; i < tuples; i++) { - int cnt = 0; + bool gotone = false; printfPQExpBuffer(&buf, " "); @@ -2361,7 +2362,12 @@ describeOneTableDetails(const char *schemaname, if (strcmp(PQgetvalue(result, i, 5), "t") == 0) { appendPQExpBufferStr(&buf, "ndistinct"); - cnt++; + gotone = true; + } + + if (strcmp(PQgetvalue(result, i, 6), "t") == 0) + { + appendPQExpBuffer(&buf, "%sdependencies", gotone ? ", " : ""); } appendPQExpBuffer(&buf, ") ON (%s)", diff --git a/src/include/catalog/pg_cast.h b/src/include/catalog/pg_cast.h index bc5d28a4fa..ccc6fb35a8 100644 --- a/src/include/catalog/pg_cast.h +++ b/src/include/catalog/pg_cast.h @@ -258,6 +258,10 @@ DATA(insert ( 194 25 0 i b )); DATA(insert ( 3361 17 0 i b )); DATA(insert ( 3361 25 0 i i )); +/* pg_dependencies can be coerced to, but not from, bytea and text */ +DATA(insert ( 3402 17 0 i b )); +DATA(insert ( 3402 25 0 i i )); + /* * Datetime category */ diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index 711211d2e6..643838bb05 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -2775,6 +2775,15 @@ DESCR("I/O"); DATA(insert OID = 3358 ( pg_ndistinct_send PGNSP PGUID 12 1 0 0 0 f f f f t f s s 1 0 17 "3361" _null_ _null_ _null_ _null_ _null_ pg_ndistinct_send _null_ _null_ _null_ )); DESCR("I/O"); +DATA(insert OID = 3404 ( pg_dependencies_in PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 3402 "2275" _null_ _null_ _null_ _null_ _null_ pg_dependencies_in _null_ _null_ _null_ )); +DESCR("I/O"); +DATA(insert OID = 3405 ( pg_dependencies_out PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 2275 "3402" _null_ _null_ _null_ _null_ _null_ pg_dependencies_out _null_ _null_ _null_ )); +DESCR("I/O"); +DATA(insert OID = 3406 ( pg_dependencies_recv PGNSP PGUID 12 1 0 0 0 f f f f t f s s 1 0 3402 "2281" _null_ _null_ _null_ _null_ _null_ pg_dependencies_recv _null_ _null_ _null_ )); +DESCR("I/O"); +DATA(insert OID = 3407 ( pg_dependencies_send PGNSP PGUID 12 1 0 0 0 f f f f t f s s 1 0 17 "3402" _null_ _null_ _null_ _null_ _null_ pg_dependencies_send _null_ _null_ _null_ )); +DESCR("I/O"); + DATA(insert OID = 1928 ( pg_stat_get_numscans PGNSP PGUID 12 1 0 0 0 f f f f t f s r 1 0 20 "26" _null_ _null_ _null_ _null_ _null_ pg_stat_get_numscans _null_ _null_ _null_ )); DESCR("statistics: number of scans done for table/index"); DATA(insert OID = 1929 ( pg_stat_get_tuples_returned PGNSP PGUID 12 1 0 0 0 f f f f t f s r 1 0 20 "26" _null_ _null_ _null_ _null_ _null_ pg_stat_get_tuples_returned _null_ _null_ _null_ )); diff --git a/src/include/catalog/pg_statistic_ext.h b/src/include/catalog/pg_statistic_ext.h index 5f67fe7fc8..0a1cc0446e 100644 --- a/src/include/catalog/pg_statistic_ext.h +++ b/src/include/catalog/pg_statistic_ext.h @@ -46,6 +46,7 @@ CATALOG(pg_statistic_ext,3381) char staenabled[1] BKI_FORCE_NOT_NULL; /* statistic types * requested to build */ pg_ndistinct standistinct; /* ndistinct coefficients (serialized) */ + pg_dependencies stadependencies; /* dependencies (serialized) */ #endif } FormData_pg_statistic_ext; @@ -61,7 +62,7 @@ typedef FormData_pg_statistic_ext *Form_pg_statistic_ext; * compiler constants for pg_statistic_ext * ---------------- */ -#define Natts_pg_statistic_ext 7 +#define Natts_pg_statistic_ext 8 #define Anum_pg_statistic_ext_starelid 1 #define Anum_pg_statistic_ext_staname 2 #define Anum_pg_statistic_ext_stanamespace 3 @@ -69,7 +70,9 @@ typedef FormData_pg_statistic_ext *Form_pg_statistic_ext; #define Anum_pg_statistic_ext_stakeys 5 #define Anum_pg_statistic_ext_staenabled 6 #define Anum_pg_statistic_ext_standistinct 7 +#define Anum_pg_statistic_ext_stadependencies 8 -#define STATS_EXT_NDISTINCT 'd' +#define STATS_EXT_NDISTINCT 'd' +#define STATS_EXT_DEPENDENCIES 'f' #endif /* PG_STATISTIC_EXT_H */ diff --git a/src/include/catalog/pg_type.h b/src/include/catalog/pg_type.h index 9ad67258fe..345e916406 100644 --- a/src/include/catalog/pg_type.h +++ b/src/include/catalog/pg_type.h @@ -368,6 +368,10 @@ DATA(insert OID = 3361 ( pg_ndistinct PGNSP PGUID -1 f b S f t \054 0 0 0 pg_nd DESCR("multivariate ndistinct coefficients"); #define PGNDISTINCTOID 3361 +DATA(insert OID = 3402 ( pg_dependencies PGNSP PGUID -1 f b S f t \054 0 0 0 pg_dependencies_in pg_dependencies_out pg_dependencies_recv pg_dependencies_send - - - i x f 0 -1 0 100 _null_ _null_ _null_ )); +DESCR("multivariate dependencies"); +#define PGDEPENDENCIESOID 3402 + DATA(insert OID = 32 ( pg_ddl_command PGNSP PGUID SIZEOF_POINTER t p P f t \054 0 0 0 pg_ddl_command_in pg_ddl_command_out pg_ddl_command_recv pg_ddl_command_send - - - ALIGNOF_POINTER p f 0 -1 0 0 _null_ _null_ _null_ )); DESCR("internal type for passing CollectedCommand"); #define PGDDLCOMMANDOID 32 diff --git a/src/include/optimizer/cost.h b/src/include/optimizer/cost.h index 6909359bcf..81a84b5494 100644 --- a/src/include/optimizer/cost.h +++ b/src/include/optimizer/cost.h @@ -203,12 +203,14 @@ extern Selectivity clauselist_selectivity(PlannerInfo *root, List *clauses, int varRelid, JoinType jointype, - SpecialJoinInfo *sjinfo); + SpecialJoinInfo *sjinfo, + RelOptInfo *rel); extern Selectivity clause_selectivity(PlannerInfo *root, Node *clause, int varRelid, JoinType jointype, - SpecialJoinInfo *sjinfo); + SpecialJoinInfo *sjinfo, + RelOptInfo *rel); extern void cost_gather_merge(GatherMergePath *path, PlannerInfo *root, RelOptInfo *rel, ParamPathInfo *param_info, Cost input_startup_cost, Cost input_total_cost, diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h index 961f1f78c0..0c40b86716 100644 --- a/src/include/statistics/extended_stats_internal.h +++ b/src/include/statistics/extended_stats_internal.h @@ -52,6 +52,11 @@ extern MVNDistinct *statext_ndistinct_build(double totalrows, extern bytea *statext_ndistinct_serialize(MVNDistinct *ndistinct); extern MVNDistinct *statext_ndistinct_deserialize(bytea *data); +extern MVDependencies *statext_dependencies_build(int numrows, HeapTuple *rows, + Bitmapset *attrs, VacAttrStats **stats); +extern bytea *statext_dependencies_serialize(MVDependencies *dependencies); +extern MVDependencies *statext_dependencies_deserialize(bytea *data); + extern MultiSortSupport multi_sort_init(int ndims); extern void multi_sort_add_dimension(MultiSortSupport mss, int sortdim, Oid oper); diff --git a/src/include/statistics/statistics.h b/src/include/statistics/statistics.h index 91645bff4b..a3f0d90195 100644 --- a/src/include/statistics/statistics.h +++ b/src/include/statistics/statistics.h @@ -14,6 +14,7 @@ #define STATISTICS_H #include "commands/vacuum.h" +#include "nodes/relation.h" #define STATS_MAX_DIMENSIONS 8 /* max number of attributes */ @@ -44,11 +45,54 @@ typedef struct MVNDistinct #define SizeOfMVNDistinct (offsetof(MVNDistinct, nitems) + sizeof(uint32)) +/* size of the struct excluding the items array */ +#define SizeOfMVNDistinct (offsetof(MVNDistinct, nitems) + sizeof(uint32)) + +#define STATS_DEPS_MAGIC 0xB4549A2C /* marks serialized bytea */ +#define STATS_DEPS_TYPE_BASIC 1 /* basic dependencies type */ + +/* + * Functional dependencies, tracking column-level relationships (values + * in one column determine values in another one). + */ +typedef struct MVDependency +{ + double degree; /* degree of validity (0-1) */ + AttrNumber nattributes; /* number of attributes */ + AttrNumber attributes[FLEXIBLE_ARRAY_MEMBER]; /* attribute numbers */ +} MVDependency; + +/* size of the struct excluding the deps array */ +#define SizeOfDependency \ + (offsetof(MVDependency, nattributes) + sizeof(AttrNumber)) + +typedef struct MVDependencies +{ + uint32 magic; /* magic constant marker */ + uint32 type; /* type of MV Dependencies (BASIC) */ + uint32 ndeps; /* number of dependencies */ + MVDependency *deps[FLEXIBLE_ARRAY_MEMBER]; /* dependencies */ +} MVDependencies; + +/* size of the struct excluding the deps array */ +#define SizeOfDependencies (offsetof(MVDependencies, ndeps) + sizeof(uint32)) + extern MVNDistinct *statext_ndistinct_load(Oid mvoid); +extern MVDependencies *staext_dependencies_load(Oid mvoid); extern void BuildRelationExtStatistics(Relation onerel, double totalrows, int numrows, HeapTuple *rows, int natts, VacAttrStats **vacattrstats); extern bool statext_is_kind_built(HeapTuple htup, char kind); +extern Selectivity dependencies_clauselist_selectivity(PlannerInfo *root, + List *clauses, + int varRelid, + JoinType jointype, + SpecialJoinInfo *sjinfo, + RelOptInfo *rel, + Bitmapset **estimatedclauses); +extern bool has_stats_of_kind(List *stats, char requiredkind); +extern StatisticExtInfo *choose_best_statistics(List *stats, + Bitmapset *attnums, char requiredkind); #endif /* STATISTICS_H */ diff --git a/src/test/regress/expected/opr_sanity.out b/src/test/regress/expected/opr_sanity.out index 262036ac4f..d23f8764ba 100644 --- a/src/test/regress/expected/opr_sanity.out +++ b/src/test/regress/expected/opr_sanity.out @@ -824,11 +824,12 @@ WHERE c.castmethod = 'b' AND character varying | character | 0 | i pg_node_tree | text | 0 | i pg_ndistinct | bytea | 0 | i + pg_dependencies | bytea | 0 | i cidr | inet | 0 | i xml | text | 0 | a xml | character varying | 0 | a xml | character | 0 | a -(8 rows) +(9 rows) -- **************** pg_conversion **************** -- Look for illegal values in pg_conversion fields. diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index d706f42b2d..cba82bb114 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2192,7 +2192,8 @@ pg_stats_ext| SELECT n.nspname AS schemaname, c.relname AS tablename, s.staname, s.stakeys AS attnums, - length((s.standistinct)::text) AS ndistbytes + length((s.standistinct)::bytea) AS ndistbytes, + length((s.stadependencies)::bytea) AS depsbytes FROM ((pg_statistic_ext s JOIN pg_class c ON ((c.oid = s.starelid))) LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace))); diff --git a/src/test/regress/expected/stats_ext.out b/src/test/regress/expected/stats_ext.out index 8fe96d6878..b43208d7d8 100644 --- a/src/test/regress/expected/stats_ext.out +++ b/src/test/regress/expected/stats_ext.out @@ -31,7 +31,7 @@ ALTER TABLE ab1 DROP COLUMN a; b | integer | | | c | integer | | | Statistics: - "public.ab1_b_c_stats" WITH (ndistinct) ON (b, c) + "public.ab1_b_c_stats" WITH (ndistinct, dependencies) ON (b, c) DROP TABLE ab1; -- Ensure things work sanely with SET STATISTICS 0 @@ -135,7 +135,7 @@ SELECT staenabled, standistinct FROM pg_statistic_ext WHERE starelid = 'ndistinct'::regclass; staenabled | standistinct ------------+------------------------------------------------------------------------------------------------ - {d} | [{(b 3 4), 301.000000}, {(b 3 6), 301.000000}, {(b 4 6), 301.000000}, {(b 3 4 6), 301.000000}] + {d,f} | [{(b 3 4), 301.000000}, {(b 3 6), 301.000000}, {(b 4 6), 301.000000}, {(b 3 4 6), 301.000000}] (1 row) -- Hash Aggregate, thanks to estimates improved by the statistic @@ -201,7 +201,7 @@ SELECT staenabled, standistinct FROM pg_statistic_ext WHERE starelid = 'ndistinct'::regclass; staenabled | standistinct ------------+---------------------------------------------------------------------------------------------------- - {d} | [{(b 3 4), 2550.000000}, {(b 3 6), 800.000000}, {(b 4 6), 1632.000000}, {(b 3 4 6), 10000.000000}] + {d,f} | [{(b 3 4), 2550.000000}, {(b 3 6), 800.000000}, {(b 4 6), 1632.000000}, {(b 3 4 6), 10000.000000}] (1 row) -- plans using Group Aggregate, thanks to using correct esimates @@ -311,3 +311,107 @@ EXPLAIN (COSTS off) (3 rows) DROP TABLE ndistinct; +-- functional dependencies tests +CREATE TABLE functional_dependencies ( + filler1 TEXT, + filler2 NUMERIC, + a INT, + b TEXT, + filler3 DATE, + c INT, + d TEXT +); +SET random_page_cost = 1.2; +CREATE INDEX fdeps_ab_idx ON functional_dependencies (a, b); +CREATE INDEX fdeps_abc_idx ON functional_dependencies (a, b, c); +-- random data (no functional dependencies) +INSERT INTO functional_dependencies (a, b, c, filler1) + SELECT mod(i, 23), mod(i, 29), mod(i, 31), i FROM generate_series(1,5000) s(i); +ANALYZE functional_dependencies; +EXPLAIN (COSTS OFF) + SELECT * FROM functional_dependencies WHERE a = 1 AND b = '1'; + QUERY PLAN +--------------------------------------------------- + Bitmap Heap Scan on functional_dependencies + Recheck Cond: ((a = 1) AND (b = '1'::text)) + -> Bitmap Index Scan on fdeps_abc_idx + Index Cond: ((a = 1) AND (b = '1'::text)) +(4 rows) + +EXPLAIN (COSTS OFF) + SELECT * FROM functional_dependencies WHERE a = 1 AND b = '1' AND c = 1; + QUERY PLAN +----------------------------------------------------------- + Index Scan using fdeps_abc_idx on functional_dependencies + Index Cond: ((a = 1) AND (b = '1'::text) AND (c = 1)) +(2 rows) + +-- create statistics +CREATE STATISTICS func_deps_stat WITH (dependencies) ON (a, b, c) FROM functional_dependencies; +ANALYZE functional_dependencies; +EXPLAIN (COSTS OFF) + SELECT * FROM functional_dependencies WHERE a = 1 AND b = '1'; + QUERY PLAN +--------------------------------------------------- + Bitmap Heap Scan on functional_dependencies + Recheck Cond: ((a = 1) AND (b = '1'::text)) + -> Bitmap Index Scan on fdeps_abc_idx + Index Cond: ((a = 1) AND (b = '1'::text)) +(4 rows) + +EXPLAIN (COSTS OFF) + SELECT * FROM functional_dependencies WHERE a = 1 AND b = '1' AND c = 1; + QUERY PLAN +----------------------------------------------------------- + Index Scan using fdeps_abc_idx on functional_dependencies + Index Cond: ((a = 1) AND (b = '1'::text) AND (c = 1)) +(2 rows) + +-- a => b, a => c, b => c +TRUNCATE functional_dependencies; +DROP STATISTICS func_deps_stat; +INSERT INTO functional_dependencies (a, b, c, filler1) + SELECT mod(i,100), mod(i,50), mod(i,25), i FROM generate_series(1,5000) s(i); +ANALYZE functional_dependencies; +EXPLAIN (COSTS OFF) + SELECT * FROM functional_dependencies WHERE a = 1 AND b = '1'; + QUERY PLAN +----------------------------------------------------------- + Index Scan using fdeps_abc_idx on functional_dependencies + Index Cond: ((a = 1) AND (b = '1'::text)) +(2 rows) + +EXPLAIN (COSTS OFF) + SELECT * FROM functional_dependencies WHERE a = 1 AND b = '1' AND c = 1; + QUERY PLAN +----------------------------------------------------------- + Index Scan using fdeps_abc_idx on functional_dependencies + Index Cond: ((a = 1) AND (b = '1'::text) AND (c = 1)) +(2 rows) + +-- create statistics +CREATE STATISTICS func_deps_stat WITH (dependencies) ON (a, b, c) FROM functional_dependencies; +ANALYZE functional_dependencies; +EXPLAIN (COSTS OFF) + SELECT * FROM functional_dependencies WHERE a = 1 AND b = '1'; + QUERY PLAN +--------------------------------------------------- + Bitmap Heap Scan on functional_dependencies + Recheck Cond: ((a = 1) AND (b = '1'::text)) + -> Bitmap Index Scan on fdeps_abc_idx + Index Cond: ((a = 1) AND (b = '1'::text)) +(4 rows) + +EXPLAIN (COSTS OFF) + SELECT * FROM functional_dependencies WHERE a = 1 AND b = '1' AND c = 1; + QUERY PLAN +--------------------------------------------------- + Bitmap Heap Scan on functional_dependencies + Recheck Cond: ((a = 1) AND (b = '1'::text)) + Filter: (c = 1) + -> Bitmap Index Scan on fdeps_ab_idx + Index Cond: ((a = 1) AND (b = '1'::text)) +(5 rows) + +RESET random_page_cost; +DROP TABLE functional_dependencies; diff --git a/src/test/regress/expected/type_sanity.out b/src/test/regress/expected/type_sanity.out index 84022f6a29..7b200baef8 100644 --- a/src/test/regress/expected/type_sanity.out +++ b/src/test/regress/expected/type_sanity.out @@ -67,12 +67,13 @@ WHERE p1.typtype not in ('c','d','p') AND p1.typname NOT LIKE E'\\_%' (SELECT 1 FROM pg_type as p2 WHERE p2.typname = ('_' || p1.typname)::name AND p2.typelem = p1.oid and p1.typarray = p2.oid); - oid | typname -------+-------------- + oid | typname +------+----------------- 194 | pg_node_tree 3361 | pg_ndistinct + 3402 | pg_dependencies 210 | smgr -(3 rows) +(4 rows) -- Make sure typarray points to a varlena array type of our own base SELECT p1.oid, p1.typname as basetype, p2.typname as arraytype, diff --git a/src/test/regress/sql/stats_ext.sql b/src/test/regress/sql/stats_ext.sql index 4faaf88e06..1b0018dcea 100644 --- a/src/test/regress/sql/stats_ext.sql +++ b/src/test/regress/sql/stats_ext.sql @@ -163,3 +163,71 @@ EXPLAIN (COSTS off) SELECT COUNT(*) FROM ndistinct GROUP BY a, d; DROP TABLE ndistinct; + +-- functional dependencies tests +CREATE TABLE functional_dependencies ( + filler1 TEXT, + filler2 NUMERIC, + a INT, + b TEXT, + filler3 DATE, + c INT, + d TEXT +); + +SET random_page_cost = 1.2; + +CREATE INDEX fdeps_ab_idx ON functional_dependencies (a, b); +CREATE INDEX fdeps_abc_idx ON functional_dependencies (a, b, c); + +-- random data (no functional dependencies) +INSERT INTO functional_dependencies (a, b, c, filler1) + SELECT mod(i, 23), mod(i, 29), mod(i, 31), i FROM generate_series(1,5000) s(i); + +ANALYZE functional_dependencies; + +EXPLAIN (COSTS OFF) + SELECT * FROM functional_dependencies WHERE a = 1 AND b = '1'; + +EXPLAIN (COSTS OFF) + SELECT * FROM functional_dependencies WHERE a = 1 AND b = '1' AND c = 1; + +-- create statistics +CREATE STATISTICS func_deps_stat WITH (dependencies) ON (a, b, c) FROM functional_dependencies; + +ANALYZE functional_dependencies; + +EXPLAIN (COSTS OFF) + SELECT * FROM functional_dependencies WHERE a = 1 AND b = '1'; + +EXPLAIN (COSTS OFF) + SELECT * FROM functional_dependencies WHERE a = 1 AND b = '1' AND c = 1; + +-- a => b, a => c, b => c +TRUNCATE functional_dependencies; +DROP STATISTICS func_deps_stat; + +INSERT INTO functional_dependencies (a, b, c, filler1) + SELECT mod(i,100), mod(i,50), mod(i,25), i FROM generate_series(1,5000) s(i); + +ANALYZE functional_dependencies; + +EXPLAIN (COSTS OFF) + SELECT * FROM functional_dependencies WHERE a = 1 AND b = '1'; + +EXPLAIN (COSTS OFF) + SELECT * FROM functional_dependencies WHERE a = 1 AND b = '1' AND c = 1; + +-- create statistics +CREATE STATISTICS func_deps_stat WITH (dependencies) ON (a, b, c) FROM functional_dependencies; + +ANALYZE functional_dependencies; + +EXPLAIN (COSTS OFF) + SELECT * FROM functional_dependencies WHERE a = 1 AND b = '1'; + +EXPLAIN (COSTS OFF) + SELECT * FROM functional_dependencies WHERE a = 1 AND b = '1' AND c = 1; + +RESET random_page_cost; +DROP TABLE functional_dependencies; -- cgit v1.2.3 From 68ea2b7f9b52d35b5fcd9c8d44d88de5a64be3ba Mon Sep 17 00:00:00 2001 From: Simon Riggs Date: Wed, 5 Apr 2017 18:22:32 -0400 Subject: Reduce lock level for CREATE STATISTICS In line with other lock reductions related to planning. Simon Riggs --- doc/src/sgml/mvcc.sgml | 3 ++- src/backend/commands/statscmds.c | 8 +++++++- 2 files changed, 9 insertions(+), 2 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/mvcc.sgml b/doc/src/sgml/mvcc.sgml index 82e69fe2d2..7aa32932fa 100644 --- a/doc/src/sgml/mvcc.sgml +++ b/doc/src/sgml/mvcc.sgml @@ -923,7 +923,8 @@ ERROR: could not serialize access due to read/write dependencies among transact Acquired by VACUUM (without ), - ANALYZE, CREATE INDEX CONCURRENTLY, and + ANALYZE, CREATE INDEX CONCURRENTLY, + CREATE STATISTICS and ALTER TABLE VALIDATE and other ALTER TABLE variants (for full details see ). diff --git a/src/backend/commands/statscmds.c b/src/backend/commands/statscmds.c index 8d483dbb3a..46abadcc81 100644 --- a/src/backend/commands/statscmds.c +++ b/src/backend/commands/statscmds.c @@ -96,7 +96,13 @@ CreateStatistics(CreateStatsStmt *stmt) errmsg("statistics \"%s\" already exist", namestr))); } - rel = heap_openrv(stmt->relation, AccessExclusiveLock); + /* + * CREATE STATISTICS will influence future execution plans but does + * not interfere with currently executing plans so it is safe to + * take only ShareUpdateExclusiveLock on relation, conflicting with + * ANALYZE and other DDL that sets statistical information. + */ + rel = heap_openrv(stmt->relation, ShareUpdateExclusiveLock); relid = RelationGetRelid(rel); if (rel->rd_rel->relkind != RELKIND_RELATION && -- cgit v1.2.3 From 3217327053638085d24dd4d276e7c1f7ac2c4c6b Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 6 Apr 2017 08:33:16 -0400 Subject: Identity columns This is the SQL standard-conforming variant of PostgreSQL's serial columns. It fixes a few usability issues that serial columns have: - CREATE TABLE / LIKE copies default but refers to same sequence - cannot add/drop serialness with ALTER TABLE - dropping default does not drop sequence - need to grant separate privileges to sequence - other slight weirdnesses because serial is some kind of special macro Reviewed-by: Vitaly Burovoy --- doc/src/sgml/catalogs.sgml | 11 + doc/src/sgml/information_schema.sgml | 11 +- doc/src/sgml/ref/alter_table.sgml | 47 +++- doc/src/sgml/ref/copy.sgml | 7 + doc/src/sgml/ref/create_table.sgml | 65 ++++- doc/src/sgml/ref/insert.sgml | 41 +++ src/backend/access/common/tupdesc.c | 6 + src/backend/catalog/dependency.c | 7 + src/backend/catalog/genbki.pl | 7 +- src/backend/catalog/heap.c | 15 +- src/backend/catalog/index.c | 1 + src/backend/catalog/information_schema.sql | 17 +- src/backend/catalog/pg_depend.c | 52 ++-- src/backend/catalog/sql_features.txt | 12 +- src/backend/commands/sequence.c | 101 +++++-- src/backend/commands/tablecmds.c | 295 ++++++++++++++++++- src/backend/executor/execExpr.c | 12 + src/backend/executor/execExprInterp.c | 23 ++ src/backend/nodes/copyfuncs.c | 23 ++ src/backend/nodes/equalfuncs.c | 18 ++ src/backend/nodes/nodeFuncs.c | 11 + src/backend/nodes/outfuncs.c | 9 + src/backend/nodes/readfuncs.c | 1 + src/backend/parser/analyze.c | 2 + src/backend/parser/gram.y | 134 ++++++++- src/backend/parser/parse_utilcmd.c | 360 +++++++++++++++++++----- src/backend/rewrite/rewriteHandler.c | 56 +++- src/backend/utils/adt/ruleutils.c | 8 + src/backend/utils/cache/lsyscache.c | 32 +++ src/backend/utils/cache/relcache.c | 1 + src/backend/utils/errcodes.txt | 1 + src/bin/pg_dump/pg_dump.c | 95 ++++++- src/bin/pg_dump/pg_dump.h | 3 + src/bin/pg_dump/t/002_pg_dump.pl | 87 ++++++ src/bin/psql/describe.c | 27 +- src/bin/psql/tab-complete.c | 18 +- src/include/catalog/catversion.h | 2 +- src/include/catalog/dependency.h | 8 +- src/include/catalog/pg_attribute.h | 24 +- src/include/catalog/pg_class.h | 2 +- src/include/commands/sequence.h | 2 + src/include/executor/execExpr.h | 8 + src/include/nodes/nodes.h | 1 + src/include/nodes/parsenodes.h | 27 +- src/include/nodes/primnodes.h | 14 + src/include/parser/kwlist.h | 2 + src/include/utils/lsyscache.h | 1 + src/test/regress/expected/create_table_like.out | 47 ++++ src/test/regress/expected/identity.out | 322 +++++++++++++++++++++ src/test/regress/expected/sequence.out | 4 +- src/test/regress/expected/truncate.out | 30 ++ src/test/regress/parallel_schedule | 5 + src/test/regress/serial_schedule | 1 + src/test/regress/sql/create_table_like.sql | 14 + src/test/regress/sql/identity.sql | 192 +++++++++++++ src/test/regress/sql/sequence.sql | 2 +- src/test/regress/sql/truncate.sql | 18 ++ 57 files changed, 2140 insertions(+), 202 deletions(-) create mode 100644 src/test/regress/expected/identity.out create mode 100644 src/test/regress/sql/identity.sql (limited to 'doc/src') diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 5c1930c745..5883673448 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -1129,6 +1129,17 @@ + + attidentity + char + + + If a zero byte (''), then not an identity column. + Otherwise, a = generated + always, d = generated by default. + + + attisdropped bool diff --git a/doc/src/sgml/information_schema.sgml b/doc/src/sgml/information_schema.sgml index a3a19ce8ce..02f7927436 100644 --- a/doc/src/sgml/information_schema.sgml +++ b/doc/src/sgml/information_schema.sgml @@ -1583,13 +1583,20 @@ is_identity yes_or_no - Applies to a feature not available in PostgreSQL + + If the column is an identity column, then YES, + else NO. + identity_generation character_data - Applies to a feature not available in PostgreSQL + + If the column is an identity column, then ALWAYS + or BY DEFAULT, reflecting the definition of the + column. + diff --git a/doc/src/sgml/ref/alter_table.sgml b/doc/src/sgml/ref/alter_table.sgml index 7829f378bb..56ea830d41 100644 --- a/doc/src/sgml/ref/alter_table.sgml +++ b/doc/src/sgml/ref/alter_table.sgml @@ -46,6 +46,9 @@ ALTER TABLE [ IF EXISTS ] name ALTER [ COLUMN ] column_name SET DEFAULT expression ALTER [ COLUMN ] column_name DROP DEFAULT ALTER [ COLUMN ] column_name { SET | DROP } NOT NULL + ALTER [ COLUMN ] column_name ADD GENERATED { ALWAYS | BY DEFAULT } AS IDENTITY [ ( sequence_options ) ] + ALTER [ COLUMN ] column_name { SET GENERATED { ALWAYS | BY DEFAULT } | SET sequence_option | RESTART [ [ WITH ] restart ] } [...] + ALTER [ COLUMN ] column_name DROP IDENTITY [ IF EXISTS ] ALTER [ COLUMN ] column_name SET STATISTICS integer ALTER [ COLUMN ] column_name SET ( attribute_option = value [, ... ] ) ALTER [ COLUMN ] column_name RESET ( attribute_option [, ... ] ) @@ -187,6 +190,38 @@ ALTER TABLE [ IF EXISTS ] name + + ADD GENERATED { ALWAYS | BY DEFAULT } AS IDENTITY + SET GENERATED { ALWAYS | BY DEFAULT } + DROP IDENTITY [ IF EXISTS ] + + + These forms change whether a column is an identity column or change the + generation attribute of an existing identity column. + See for details. + + + + If DROP IDENTITY IF EXISTS is specified and the + column is not an identity column, no error is thrown. In this case a + notice is issued instead. + + + + + + SET sequence_option + RESTART + + + These forms alter the sequence that underlies an existing identity + column. sequence_option is an option + supported by such + as INCREMENT BY. + + + + SET STATISTICS @@ -1160,8 +1195,11 @@ ALTER TABLE [ IF EXISTS ] name - The TRIGGER, CLUSTER, OWNER, - and TABLESPACE actions never recurse to descendant tables; + The actions for identity columns (ADD + GENERATED, SET etc., DROP + IDENTITY), as well as the actions + TRIGGER, CLUSTER, OWNER, + and TABLESPACE never recurse to descendant tables; that is, they always act as though ONLY were specified. Adding a constraint recurses only for CHECK constraints that are not marked NO INHERIT. @@ -1371,8 +1409,9 @@ ALTER TABLE cities The forms ADD (without USING INDEX), - DROP, SET DEFAULT, - and SET DATA TYPE (without USING) + DROP [COLUMN], DROP IDENTITY, RESTART, + SET DEFAULT, SET DATA TYPE (without USING), + SET GENERATED, and SET sequence_option conform with the SQL standard. The other forms are PostgreSQL extensions of the SQL standard. Also, the ability to specify more than one manipulation in a single diff --git a/doc/src/sgml/ref/copy.sgml b/doc/src/sgml/ref/copy.sgml index 7ff62f2a82..215efcd69d 100644 --- a/doc/src/sgml/ref/copy.sgml +++ b/doc/src/sgml/ref/copy.sgml @@ -479,6 +479,13 @@ COPY count constraints on the destination table. However, it will not invoke rules. + + For identity columns, the COPY FROM command will always + write the column values provided in the input data, like + the INPUT option OVERRIDING SYSTEM + VALUE. + + COPY input and output is affected by DateStyle. To ensure portability to other diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml index 121418b6ca..1881d9257a 100644 --- a/doc/src/sgml/ref/create_table.sgml +++ b/doc/src/sgml/ref/create_table.sgml @@ -62,6 +62,7 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI NULL | CHECK ( expression ) [ NO INHERIT ] | DEFAULT default_expr | + GENERATED { ALWAYS | BY DEFAULT } AS IDENTITY [ ( sequence_options ) ] | UNIQUE index_parameters | PRIMARY KEY index_parameters | REFERENCES reftable [ ( refcolumn ) ] [ MATCH FULL | MATCH PARTIAL | MATCH SIMPLE ] @@ -81,7 +82,7 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI and like_option is: -{ INCLUDING | EXCLUDING } { DEFAULTS | CONSTRAINTS | INDEXES | STORAGE | COMMENTS | ALL } +{ INCLUDING | EXCLUDING } { DEFAULTS | CONSTRAINTS | IDENTITY | INDEXES | STORAGE | COMMENTS | ALL } and partition_bound_spec is: @@ -412,6 +413,11 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI Column STORAGE settings are also copied from parent tables. + + If a column in the parent table is an identity column, that property is + not inherited. A column in the child table can be declared identity + column if desired. + @@ -480,6 +486,12 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI such as nextval, may create a functional linkage between the original and new tables. + + Any identity specifications of copied column definitions will only be + copied if INCLUDING IDENTITY is specified. A new + sequence is created for each identity column of the new table, separate + from the sequences associated with the old table. + Not-null constraints are always copied to the new table. CHECK constraints will be copied only if @@ -512,7 +524,7 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI INCLUDING ALL is an abbreviated form of - INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES INCLUDING STORAGE INCLUDING COMMENTS. + INCLUDING DEFAULTS INCLUDING IDENTITY INCLUDING CONSTRAINTS INCLUDING INDEXES INCLUDING STORAGE INCLUDING COMMENTS. Note that unlike INHERITS, columns and @@ -626,6 +638,37 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI + + GENERATED { ALWAYS | BY DEFAULT } AS IDENTITY [ ( sequence_options ) ] + + + This clause creates the column as an identity + column. It will have an implicit sequence attached to it + and the column in new rows will automatically have values from the + sequence assigned to it. + + + + The clauses ALWAYS and BY DEFAULT + determine how the sequence value is given precedence over a + user-specified value in an INSERT statement. + If ALWAYS is specified, a user-specified value is + only accepted if the INSERT statement + specifies OVERRIDING SYSTEM VALUE. If BY + DEFAULT is specified, then the user-specified value takes + precedence. See for details. (In + the COPY command, user-specified values are always + used regardless of this setting.) + + + + The optional sequence_options clause can be + used to override the options of the sequence. + See for details. + + + + UNIQUE (column constraint) UNIQUE ( column_name [, ... ] ) (table constraint) @@ -1263,7 +1306,7 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI Using OIDs in new applications is not recommended: where - possible, using a SERIAL or other sequence + possible, using an identity column or other sequence generator as the table's primary key is preferred. However, if your application does make use of OIDs to identify specific rows of a table, it is recommended to create a unique constraint @@ -1323,7 +1366,7 @@ CREATE TABLE films ( ); CREATE TABLE distributors ( - did integer PRIMARY KEY DEFAULT nextval('serial'), + did integer PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, name varchar(40) NOT NULL CHECK (name <> '') ); @@ -1737,6 +1780,20 @@ CREATE TABLE cities_ab_10000_to_100000 + + Multiple Identity Columns + + + PostgreSQL allows a table to have more than one + identity column. The standard specifies that a table can have at most one + identity column. This is relaxed mainly to give more flexibility for + doing schema changes or migrations. Note that + the INSERT command supports only one override clause + that applies to the entire statement, so having multiple identity columns + with different behaviors is not well supported. + + + <literal>LIKE</> Clause diff --git a/doc/src/sgml/ref/insert.sgml b/doc/src/sgml/ref/insert.sgml index 521216b5d5..95aa77b907 100644 --- a/doc/src/sgml/ref/insert.sgml +++ b/doc/src/sgml/ref/insert.sgml @@ -23,6 +23,7 @@ PostgreSQL documentation [ WITH [ RECURSIVE ] with_query [, ...] ] INSERT INTO table_name [ AS alias ] [ ( column_name [, ...] ) ] + [ OVERRIDING { SYSTEM | USER} VALUE ] { DEFAULT VALUES | VALUES ( { expression | DEFAULT } [, ...] ) [, ...] | query } [ ON CONFLICT [ conflict_target ] conflict_action ] [ RETURNING * | output_expression [ [ AS ] output_name ] [, ...] ] @@ -201,11 +202,44 @@ INSERT INTO table_name [ AS + + OVERRIDING SYSTEM VALUE + + + Without this clause, it is an error to specify an explicit value + (other than DEFAULT) for an identity column defined + as GENERATED ALWAYS. This clause overrides that + restriction. + + + + + + OVERRIDING USER VALUE + + + If this clause is specified, then any values supplied for identity + columns defined as GENERATED BY DEFAULT are ignored + and the default sequence-generated values are applied. + + + + This clause is useful for example when copying values between tables. + Writing INSERT INTO tbl2 OVERRIDING USER VALUE SELECT * FROM + tbl1 will copy from tbl1 all columns that + are not identity columns in tbl2 but will continue + the sequence counters for any identity columns. + + + + DEFAULT VALUES All columns will be filled with their default values. + (An OVERRIDING clause is not permitted in this + form.) @@ -710,6 +744,13 @@ INSERT INTO distributors (did, dname) VALUES (10, 'Conrad International') is disallowed by the standard. + + The SQL standard specifies that OVERRIDING SYSTEM VALUE + can only be specified if an identity column that is generated always + exists. PostgreSQL allows the clause in any case and ignores it if it is + not applicable. + + Possible limitations of the query clause are documented under diff --git a/src/backend/access/common/tupdesc.c b/src/backend/access/common/tupdesc.c index 4e2ebe1ae7..9fd7b4e019 100644 --- a/src/backend/access/common/tupdesc.c +++ b/src/backend/access/common/tupdesc.c @@ -150,6 +150,7 @@ CreateTupleDescCopy(TupleDesc tupdesc) memcpy(desc->attrs[i], tupdesc->attrs[i], ATTRIBUTE_FIXED_PART_SIZE); desc->attrs[i]->attnotnull = false; desc->attrs[i]->atthasdef = false; + desc->attrs[i]->attidentity = '\0'; } desc->tdtypeid = tupdesc->tdtypeid; @@ -257,6 +258,7 @@ TupleDescCopyEntry(TupleDesc dst, AttrNumber dstAttno, /* since we're not copying constraints or defaults, clear these */ dst->attrs[dstAttno - 1]->attnotnull = false; dst->attrs[dstAttno - 1]->atthasdef = false; + dst->attrs[dstAttno - 1]->attidentity = '\0'; } /* @@ -401,6 +403,8 @@ equalTupleDescs(TupleDesc tupdesc1, TupleDesc tupdesc2) return false; if (attr1->atthasdef != attr2->atthasdef) return false; + if (attr1->attidentity != attr2->attidentity) + return false; if (attr1->attisdropped != attr2->attisdropped) return false; if (attr1->attislocal != attr2->attislocal) @@ -534,6 +538,7 @@ TupleDescInitEntry(TupleDesc desc, att->attnotnull = false; att->atthasdef = false; + att->attidentity = '\0'; att->attisdropped = false; att->attislocal = true; att->attinhcount = 0; @@ -591,6 +596,7 @@ TupleDescInitBuiltinEntry(TupleDesc desc, att->attnotnull = false; att->atthasdef = false; + att->attidentity = '\0'; att->attisdropped = false; att->attislocal = true; att->attinhcount = 0; diff --git a/src/backend/catalog/dependency.c b/src/backend/catalog/dependency.c index ee27cae7df..cdf453562f 100644 --- a/src/backend/catalog/dependency.c +++ b/src/backend/catalog/dependency.c @@ -1929,6 +1929,13 @@ find_expr_references_walker(Node *node, context->addrs); /* fall through to examine arguments */ } + else if (IsA(node, NextValueExpr)) + { + NextValueExpr *nve = (NextValueExpr *) node; + + add_object_address(OCLASS_CLASS, nve->seqid, 0, + context->addrs); + } return expression_tree_walker(node, find_expr_references_walker, (void *) context); diff --git a/src/backend/catalog/genbki.pl b/src/backend/catalog/genbki.pl index f9ecb02548..6e9d57aa8d 100644 --- a/src/backend/catalog/genbki.pl +++ b/src/backend/catalog/genbki.pl @@ -409,6 +409,7 @@ sub emit_pgattr_row attcacheoff => '-1', atttypmod => '-1', atthasdef => 'f', + attidentity => '', attisdropped => 'f', attislocal => 't', attinhcount => '0', @@ -424,7 +425,7 @@ sub bki_insert my $row = shift; my @attnames = @_; my $oid = $row->{oid} ? "OID = $row->{oid} " : ''; - my $bki_values = join ' ', map $row->{$_}, @attnames; + my $bki_values = join ' ', map { $_ eq '' ? '""' : $_ } map $row->{$_}, @attnames; printf $bki "insert %s( %s)\n", $oid, $bki_values; } @@ -435,10 +436,14 @@ sub emit_schemapg_row my $row = shift; my @bool_attrs = @_; + # Replace empty string by zero char constant + $row->{attidentity} ||= '\0'; + # Supply appropriate quoting for these fields. $row->{attname} = q|{"| . $row->{attname} . q|"}|; $row->{attstorage} = q|'| . $row->{attstorage} . q|'|; $row->{attalign} = q|'| . $row->{attalign} . q|'|; + $row->{attidentity} = q|'| . $row->{attidentity} . q|'|; # We don't emit initializers for the variable length fields at all. # Only the fixed-size portions of the descriptors are ever used. diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index 1cbe7f907f..a264f1b9eb 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -144,37 +144,37 @@ static List *insert_ordered_unique_oid(List *list, Oid datum); static FormData_pg_attribute a1 = { 0, {"ctid"}, TIDOID, 0, sizeof(ItemPointerData), SelfItemPointerAttributeNumber, 0, -1, -1, - false, 'p', 's', true, false, false, true, 0 + false, 'p', 's', true, false, '\0', false, true, 0 }; static FormData_pg_attribute a2 = { 0, {"oid"}, OIDOID, 0, sizeof(Oid), ObjectIdAttributeNumber, 0, -1, -1, - true, 'p', 'i', true, false, false, true, 0 + true, 'p', 'i', true, false, '\0', false, true, 0 }; static FormData_pg_attribute a3 = { 0, {"xmin"}, XIDOID, 0, sizeof(TransactionId), MinTransactionIdAttributeNumber, 0, -1, -1, - true, 'p', 'i', true, false, false, true, 0 + true, 'p', 'i', true, false, '\0', false, true, 0 }; static FormData_pg_attribute a4 = { 0, {"cmin"}, CIDOID, 0, sizeof(CommandId), MinCommandIdAttributeNumber, 0, -1, -1, - true, 'p', 'i', true, false, false, true, 0 + true, 'p', 'i', true, false, '\0', false, true, 0 }; static FormData_pg_attribute a5 = { 0, {"xmax"}, XIDOID, 0, sizeof(TransactionId), MaxTransactionIdAttributeNumber, 0, -1, -1, - true, 'p', 'i', true, false, false, true, 0 + true, 'p', 'i', true, false, '\0', false, true, 0 }; static FormData_pg_attribute a6 = { 0, {"cmax"}, CIDOID, 0, sizeof(CommandId), MaxCommandIdAttributeNumber, 0, -1, -1, - true, 'p', 'i', true, false, false, true, 0 + true, 'p', 'i', true, false, '\0', false, true, 0 }; /* @@ -186,7 +186,7 @@ static FormData_pg_attribute a6 = { static FormData_pg_attribute a7 = { 0, {"tableoid"}, OIDOID, 0, sizeof(Oid), TableOidAttributeNumber, 0, -1, -1, - true, 'p', 'i', true, false, false, true, 0 + true, 'p', 'i', true, false, '\0', false, true, 0 }; static const Form_pg_attribute SysAtt[] = {&a1, &a2, &a3, &a4, &a5, &a6, &a7}; @@ -621,6 +621,7 @@ InsertPgAttributeTuple(Relation pg_attribute_rel, values[Anum_pg_attribute_attalign - 1] = CharGetDatum(new_attribute->attalign); values[Anum_pg_attribute_attnotnull - 1] = BoolGetDatum(new_attribute->attnotnull); values[Anum_pg_attribute_atthasdef - 1] = BoolGetDatum(new_attribute->atthasdef); + values[Anum_pg_attribute_attidentity - 1] = CharGetDatum(new_attribute->attidentity); values[Anum_pg_attribute_attisdropped - 1] = BoolGetDatum(new_attribute->attisdropped); values[Anum_pg_attribute_attislocal - 1] = BoolGetDatum(new_attribute->attislocal); values[Anum_pg_attribute_attinhcount - 1] = Int32GetDatum(new_attribute->attinhcount); diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index 1eb163f539..2328b92b4e 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -353,6 +353,7 @@ ConstructTupleDescriptor(Relation heapRelation, to->attcacheoff = -1; to->attnotnull = false; to->atthasdef = false; + to->attidentity = '\0'; to->attislocal = true; to->attinhcount = 0; to->attcollation = collationObjectId[i]; diff --git a/src/backend/catalog/information_schema.sql b/src/backend/catalog/information_schema.sql index fa2a88fc5c..2185734b48 100644 --- a/src/backend/catalog/information_schema.sql +++ b/src/backend/catalog/information_schema.sql @@ -728,13 +728,13 @@ CREATE VIEW columns AS CAST(a.attnum AS sql_identifier) AS dtd_identifier, CAST('NO' AS yes_or_no) AS is_self_referencing, - CAST('NO' AS yes_or_no) AS is_identity, - CAST(null AS character_data) AS identity_generation, - CAST(null AS character_data) AS identity_start, - CAST(null AS character_data) AS identity_increment, - CAST(null AS character_data) AS identity_maximum, - CAST(null AS character_data) AS identity_minimum, - CAST(null AS yes_or_no) AS identity_cycle, + CAST(CASE WHEN a.attidentity IN ('a', 'd') THEN 'YES' ELSE 'NO' END AS yes_or_no) AS is_identity, + CAST(CASE a.attidentity WHEN 'a' THEN 'ALWAYS' WHEN 'd' THEN 'BY DEFAULT' END AS character_data) AS identity_generation, + CAST(seq.seqstart AS character_data) AS identity_start, + CAST(seq.seqincrement AS character_data) AS identity_increment, + CAST(seq.seqmax AS character_data) AS identity_maximum, + CAST(seq.seqmin AS character_data) AS identity_minimum, + CAST(CASE WHEN seq.seqcycle THEN 'YES' ELSE 'NO' END AS yes_or_no) AS identity_cycle, CAST('NEVER' AS character_data) AS is_generated, CAST(null AS character_data) AS generation_expression, @@ -751,6 +751,8 @@ CREATE VIEW columns AS ON (t.typtype = 'd' AND t.typbasetype = bt.oid) LEFT JOIN (pg_collation co JOIN pg_namespace nco ON (co.collnamespace = nco.oid)) ON a.attcollation = co.oid AND (nco.nspname, co.collname) <> ('pg_catalog', 'default') + LEFT JOIN (pg_depend dep JOIN pg_sequence seq ON (dep.classid = 'pg_class'::regclass AND dep.objid = seq.seqrelid AND dep.deptype = 'i')) + ON (dep.refclassid = 'pg_class'::regclass AND dep.refobjid = c.oid AND dep.refobjsubid = a.attnum) WHERE (NOT pg_is_other_temp_schema(nc.oid)) @@ -1545,6 +1547,7 @@ CREATE VIEW sequences AS FROM pg_namespace nc, pg_class c, pg_sequence s WHERE c.relnamespace = nc.oid AND c.relkind = 'S' + AND NOT EXISTS (SELECT 1 FROM pg_depend WHERE classid = 'pg_class'::regclass AND objid = c.oid AND deptype = 'i') AND (NOT pg_is_other_temp_schema(nc.oid)) AND c.oid = s.seqrelid AND (pg_has_role(c.relowner, 'USAGE') diff --git a/src/backend/catalog/pg_depend.c b/src/backend/catalog/pg_depend.c index d0ee851215..aae879e355 100644 --- a/src/backend/catalog/pg_depend.c +++ b/src/backend/catalog/pg_depend.c @@ -488,7 +488,7 @@ getExtensionOfObject(Oid classId, Oid objectId) /* * Detect whether a sequence is marked as "owned" by a column * - * An ownership marker is an AUTO dependency from the sequence to the + * An ownership marker is an AUTO or INTERNAL dependency from the sequence to the * column. If we find one, store the identity of the owning column * into *tableId and *colId and return TRUE; else return FALSE. * @@ -497,7 +497,7 @@ getExtensionOfObject(Oid classId, Oid objectId) * not happen, though. */ bool -sequenceIsOwned(Oid seqId, Oid *tableId, int32 *colId) +sequenceIsOwned(Oid seqId, char deptype, Oid *tableId, int32 *colId) { bool ret = false; Relation depRel; @@ -524,7 +524,7 @@ sequenceIsOwned(Oid seqId, Oid *tableId, int32 *colId) Form_pg_depend depform = (Form_pg_depend) GETSTRUCT(tup); if (depform->refclassid == RelationRelationId && - depform->deptype == DEPENDENCY_AUTO) + depform->deptype == deptype) { *tableId = depform->refobjid; *colId = depform->refobjsubid; @@ -541,27 +541,15 @@ sequenceIsOwned(Oid seqId, Oid *tableId, int32 *colId) } /* - * Remove any existing "owned" markers for the specified sequence. - * - * Note: we don't provide a special function to install an "owned" - * marker; just use recordDependencyOn(). - */ -void -markSequenceUnowned(Oid seqId) -{ - deleteDependencyRecordsForClass(RelationRelationId, seqId, - RelationRelationId, DEPENDENCY_AUTO); -} - -/* - * Collect a list of OIDs of all sequences owned by the specified relation. + * Collect a list of OIDs of all sequences owned by the specified relation, + * and column if specified. */ List * -getOwnedSequences(Oid relid) +getOwnedSequences(Oid relid, AttrNumber attnum) { List *result = NIL; Relation depRel; - ScanKeyData key[2]; + ScanKeyData key[3]; SysScanDesc scan; HeapTuple tup; @@ -575,23 +563,28 @@ getOwnedSequences(Oid relid) Anum_pg_depend_refobjid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relid)); + if (attnum) + ScanKeyInit(&key[2], + Anum_pg_depend_refobjsubid, + BTEqualStrategyNumber, F_INT4EQ, + Int32GetDatum(attnum)); scan = systable_beginscan(depRel, DependReferenceIndexId, true, - NULL, 2, key); + NULL, attnum ? 3 : 2, key); while (HeapTupleIsValid(tup = systable_getnext(scan))) { Form_pg_depend deprec = (Form_pg_depend) GETSTRUCT(tup); /* - * We assume any auto dependency of a sequence on a column must be + * We assume any auto or internal dependency of a sequence on a column must be * what we are looking for. (We need the relkind test because indexes * can also have auto dependencies on columns.) */ if (deprec->classid == RelationRelationId && deprec->objsubid == 0 && deprec->refobjsubid != 0 && - deprec->deptype == DEPENDENCY_AUTO && + (deprec->deptype == DEPENDENCY_AUTO || deprec->deptype == DEPENDENCY_INTERNAL) && get_rel_relkind(deprec->objid) == RELKIND_SEQUENCE) { result = lappend_oid(result, deprec->objid); @@ -605,6 +598,21 @@ getOwnedSequences(Oid relid) return result; } +/* + * Get owned sequence, error if not exactly one. + */ +Oid +getOwnedSequence(Oid relid, AttrNumber attnum) +{ + List *seqlist = getOwnedSequences(relid, attnum); + + if (list_length(seqlist) > 1) + elog(ERROR, "more than one owned sequence found"); + else if (list_length(seqlist) < 1) + elog(ERROR, "no owned sequence found"); + else + return linitial_oid(seqlist); +} /* * get_constraint_index diff --git a/src/backend/catalog/sql_features.txt b/src/backend/catalog/sql_features.txt index 8956ba9304..2821b9b702 100644 --- a/src/backend/catalog/sql_features.txt +++ b/src/backend/catalog/sql_features.txt @@ -200,7 +200,7 @@ F181 Multiple module support NO F191 Referential delete actions YES F200 TRUNCATE TABLE statement YES F201 CAST function YES -F202 TRUNCATE TABLE: identity column restart option NO +F202 TRUNCATE TABLE: identity column restart option YES F221 Explicit defaults YES F222 INSERT statement: DEFAULT VALUES clause YES F231 Privilege tables YES @@ -241,9 +241,9 @@ F381 Extended schema manipulation 02 ALTER TABLE statement: ADD CONSTRAINT claus F381 Extended schema manipulation 03 ALTER TABLE statement: DROP CONSTRAINT clause YES F382 Alter column data type YES F383 Set column not null clause YES -F384 Drop identity property clause NO +F384 Drop identity property clause YES F385 Drop column generation expression clause NO -F386 Set identity column generation clause NO +F386 Set identity column generation clause YES F391 Long identifiers YES F392 Unicode escapes in identifiers YES F393 Unicode escapes in literals YES @@ -420,11 +420,11 @@ T152 DISTINCT predicate with negation YES T171 LIKE clause in table definition YES T172 AS subquery clause in table definition YES T173 Extended LIKE clause in table definition YES -T174 Identity columns NO +T174 Identity columns YES T175 Generated columns NO T176 Sequence generator support NO -T177 Sequence generator support: simple restart option NO -T178 Identity columns: simple restart option NO +T177 Sequence generator support: simple restart option YES +T178 Identity columns: simple restart option YES T180 System-versioned tables NO T181 Application-time period tables NO T191 Referential action RESTRICT YES diff --git a/src/backend/commands/sequence.c b/src/backend/commands/sequence.c index 89b810bbb7..ad28225b36 100644 --- a/src/backend/commands/sequence.c +++ b/src/backend/commands/sequence.c @@ -93,17 +93,17 @@ static HTAB *seqhashtab = NULL; /* hash table for SeqTable items */ static SeqTableData *last_used_seq = NULL; static void fill_seq_with_data(Relation rel, HeapTuple tuple); -static int64 nextval_internal(Oid relid); static Relation open_share_lock(SeqTable seq); static void create_seq_hashtable(void); static void init_sequence(Oid relid, SeqTable *p_elm, Relation *p_rel); static Form_pg_sequence_data read_seq_tuple(Relation rel, Buffer *buf, HeapTuple seqdatatuple); -static void init_params(ParseState *pstate, List *options, bool isInit, +static void init_params(ParseState *pstate, List *options, bool for_identity, + bool isInit, Form_pg_sequence seqform, Form_pg_sequence_data seqdataform, List **owned_by); static void do_setval(Oid relid, int64 next, bool iscalled); -static void process_owned_by(Relation seqrel, List *owned_by); +static void process_owned_by(Relation seqrel, List *owned_by, bool for_identity); /* @@ -153,7 +153,7 @@ DefineSequence(ParseState *pstate, CreateSeqStmt *seq) } /* Check and set all option values */ - init_params(pstate, seq->options, true, &seqform, &seqdataform, &owned_by); + init_params(pstate, seq->options, seq->for_identity, true, &seqform, &seqdataform, &owned_by); /* * Create relation (and fill value[] and null[] for the tuple) @@ -219,7 +219,7 @@ DefineSequence(ParseState *pstate, CreateSeqStmt *seq) /* process OWNED BY if given */ if (owned_by) - process_owned_by(rel, owned_by); + process_owned_by(rel, owned_by, seq->for_identity); heap_close(rel, NoLock); @@ -455,7 +455,7 @@ AlterSequence(ParseState *pstate, AlterSeqStmt *stmt) seqform = (Form_pg_sequence) GETSTRUCT(tuple); /* Check and set new values */ - init_params(pstate, stmt->options, false, seqform, &newseqdata, &owned_by); + init_params(pstate, stmt->options, stmt->for_identity, false, seqform, &newseqdata, &owned_by); /* Clear local cache so that we don't think we have cached numbers */ /* Note that we do not change the currval() state */ @@ -498,7 +498,7 @@ AlterSequence(ParseState *pstate, AlterSeqStmt *stmt) /* process OWNED BY if given */ if (owned_by) - process_owned_by(seqrel, owned_by); + process_owned_by(seqrel, owned_by, stmt->for_identity); InvokeObjectPostAlterHook(RelationRelationId, relid, 0); @@ -554,7 +554,7 @@ nextval(PG_FUNCTION_ARGS) */ relid = RangeVarGetRelid(sequence, NoLock, false); - PG_RETURN_INT64(nextval_internal(relid)); + PG_RETURN_INT64(nextval_internal(relid, true)); } Datum @@ -562,11 +562,11 @@ nextval_oid(PG_FUNCTION_ARGS) { Oid relid = PG_GETARG_OID(0); - PG_RETURN_INT64(nextval_internal(relid)); + PG_RETURN_INT64(nextval_internal(relid, true)); } -static int64 -nextval_internal(Oid relid) +int64 +nextval_internal(Oid relid, bool check_permissions) { SeqTable elm; Relation seqrel; @@ -592,7 +592,8 @@ nextval_internal(Oid relid) /* open and AccessShareLock sequence */ init_sequence(relid, &elm, &seqrel); - if (pg_class_aclcheck(elm->relid, GetUserId(), + if (check_permissions && + pg_class_aclcheck(elm->relid, GetUserId(), ACL_USAGE | ACL_UPDATE) != ACLCHECK_OK) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), @@ -1219,7 +1220,8 @@ read_seq_tuple(Relation rel, Buffer *buf, HeapTuple seqdatatuple) * otherwise, do not change existing options that aren't explicitly overridden. */ static void -init_params(ParseState *pstate, List *options, bool isInit, +init_params(ParseState *pstate, List *options, bool for_identity, + bool isInit, Form_pg_sequence seqform, Form_pg_sequence_data seqdataform, List **owned_by) { @@ -1322,6 +1324,18 @@ init_params(ParseState *pstate, List *options, bool isInit, parser_errposition(pstate, defel->location))); *owned_by = defGetQualifiedName(defel); } + else if (strcmp(defel->defname, "sequence_name") == 0) + { + /* + * The parser allows this, but it is only for identity columns, in + * which case it is filtered out in parse_utilcmd.c. We only get + * here if someone puts it into a CREATE SEQUENCE. + */ + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("invalid sequence option SEQUENCE NAME"), + parser_errposition(pstate, defel->location))); + } else elog(ERROR, "option \"%s\" not recognized", defel->defname); @@ -1344,7 +1358,9 @@ init_params(ParseState *pstate, List *options, bool isInit, newtypid != INT8OID) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("sequence type must be smallint, integer, or bigint"))); + for_identity + ? errmsg("identity column type must be smallint, integer, or bigint") + : errmsg("sequence type must be smallint, integer, or bigint"))); if (!isInit) { @@ -1588,12 +1604,15 @@ init_params(ParseState *pstate, List *options, bool isInit, * as the sequence. */ static void -process_owned_by(Relation seqrel, List *owned_by) +process_owned_by(Relation seqrel, List *owned_by, bool for_identity) { + DependencyType deptype; int nnames; Relation tablerel; AttrNumber attnum; + deptype = for_identity ? DEPENDENCY_INTERNAL : DEPENDENCY_AUTO; + nnames = list_length(owned_by); Assert(nnames > 0); if (nnames == 1) @@ -1624,6 +1643,7 @@ process_owned_by(Relation seqrel, List *owned_by) /* Must be a regular or foreign table */ if (!(tablerel->rd_rel->relkind == RELKIND_RELATION || tablerel->rd_rel->relkind == RELKIND_FOREIGN_TABLE || + tablerel->rd_rel->relkind == RELKIND_VIEW || tablerel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), @@ -1650,10 +1670,28 @@ process_owned_by(Relation seqrel, List *owned_by) } /* - * OK, we are ready to update pg_depend. First remove any existing AUTO + * Catch user explicitly running OWNED BY on identity sequence. + */ + if (deptype == DEPENDENCY_AUTO) + { + Oid tableId; + int32 colId; + + if (sequenceIsOwned(RelationGetRelid(seqrel), DEPENDENCY_INTERNAL, &tableId, &colId)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot change ownership of identity sequence"), + errdetail("Sequence \"%s\" is linked to table \"%s\".", + RelationGetRelationName(seqrel), + get_rel_name(tableId)))); + } + + /* + * OK, we are ready to update pg_depend. First remove any existing * dependencies for the sequence, then optionally add a new one. */ - markSequenceUnowned(RelationGetRelid(seqrel)); + deleteDependencyRecordsForClass(RelationRelationId, RelationGetRelid(seqrel), + RelationRelationId, deptype); if (tablerel) { @@ -1666,7 +1704,7 @@ process_owned_by(Relation seqrel, List *owned_by) depobject.classId = RelationRelationId; depobject.objectId = RelationGetRelid(seqrel); depobject.objectSubId = 0; - recordDependencyOn(&depobject, &refobject, DEPENDENCY_AUTO); + recordDependencyOn(&depobject, &refobject, deptype); } /* Done, but hold lock until commit */ @@ -1675,6 +1713,33 @@ process_owned_by(Relation seqrel, List *owned_by) } +/* + * Return sequence parameters in a list of the form created by the parser. + */ +List * +sequence_options(Oid relid) +{ + HeapTuple pgstuple; + Form_pg_sequence pgsform; + List *options = NIL; + + pgstuple = SearchSysCache1(SEQRELID, relid); + if (!HeapTupleIsValid(pgstuple)) + elog(ERROR, "cache lookup failed for sequence %u", relid); + pgsform = (Form_pg_sequence) GETSTRUCT(pgstuple); + + options = lappend(options, makeDefElem("cache", (Node *) makeInteger(pgsform->seqcache), -1)); + options = lappend(options, makeDefElem("cycle", (Node *) makeInteger(pgsform->seqcycle), -1)); + options = lappend(options, makeDefElem("increment", (Node *) makeInteger(pgsform->seqincrement), -1)); + options = lappend(options, makeDefElem("maxvalue", (Node *) makeInteger(pgsform->seqmax), -1)); + options = lappend(options, makeDefElem("minvalue", (Node *) makeInteger(pgsform->seqmin), -1)); + options = lappend(options, makeDefElem("start", (Node *) makeInteger(pgsform->seqstart), -1)); + + ReleaseSysCache(pgstuple); + + return options; +} + /* * Return sequence parameters (formerly for use by information schema) */ diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index d418d56b54..49a73707bc 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -361,6 +361,11 @@ static ObjectAddress ATExecSetNotNull(AlteredTableInfo *tab, Relation rel, const char *colName, LOCKMODE lockmode); static ObjectAddress ATExecColumnDefault(Relation rel, const char *colName, Node *newDefault, LOCKMODE lockmode); +static ObjectAddress ATExecAddIdentity(Relation rel, const char *colName, + Node *def, LOCKMODE lockmode); +static ObjectAddress ATExecSetIdentity(Relation rel, const char *colName, + Node *def, LOCKMODE lockmode); +static ObjectAddress ATExecDropIdentity(Relation rel, const char *colName, bool missing_ok, LOCKMODE lockmode); static void ATPrepSetStatistics(Relation rel, const char *colName, Node *newValue, LOCKMODE lockmode); static ObjectAddress ATExecSetStatistics(Relation rel, const char *colName, @@ -696,6 +701,9 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, cookedDefaults = lappend(cookedDefaults, cooked); descriptor->attrs[attnum - 1]->atthasdef = true; } + + if (colDef->identity) + descriptor->attrs[attnum - 1]->attidentity = colDef->identity; } /* @@ -1281,7 +1289,7 @@ ExecuteTruncate(TruncateStmt *stmt) foreach(cell, rels) { Relation rel = (Relation) lfirst(cell); - List *seqlist = getOwnedSequences(RelationGetRelid(rel)); + List *seqlist = getOwnedSequences(RelationGetRelid(rel), 0); ListCell *seqcell; foreach(seqcell, seqlist) @@ -2078,6 +2086,12 @@ MergeAttributes(List *schema, List *supers, char relpersistence, get_collation_name(defcollid), get_collation_name(newcollid)))); + /* + * Identity is never inherited. The new column can have an + * identity definition, so we always just take that one. + */ + def->identity = newdef->identity; + /* Copy storage parameter */ if (def->storage == 0) def->storage = newdef->storage; @@ -3217,6 +3231,9 @@ AlterTableGetLockLevel(List *cmds) case AT_DisableRowSecurity: case AT_ForceRowSecurity: case AT_NoForceRowSecurity: + case AT_AddIdentity: + case AT_DropIdentity: + case AT_SetIdentity: cmd_lockmode = AccessExclusiveLock; break; @@ -3447,6 +3464,18 @@ ATPrepCmd(List **wqueue, Relation rel, AlterTableCmd *cmd, /* No command-specific prep needed */ pass = cmd->def ? AT_PASS_ADD_CONSTR : AT_PASS_DROP; break; + case AT_AddIdentity: + ATSimplePermissions(rel, ATT_TABLE | ATT_VIEW | ATT_FOREIGN_TABLE); + pass = AT_PASS_ADD_CONSTR; + break; + case AT_DropIdentity: + ATSimplePermissions(rel, ATT_TABLE | ATT_VIEW | ATT_FOREIGN_TABLE); + pass = AT_PASS_DROP; + break; + case AT_SetIdentity: + ATSimplePermissions(rel, ATT_TABLE | ATT_VIEW | ATT_FOREIGN_TABLE); + pass = AT_PASS_COL_ATTRS; + break; case AT_DropNotNull: /* ALTER COLUMN DROP NOT NULL */ ATSimplePermissions(rel, ATT_TABLE | ATT_FOREIGN_TABLE); ATPrepDropNotNull(rel, recurse, recursing); @@ -3772,6 +3801,15 @@ ATExecCmd(List **wqueue, AlteredTableInfo *tab, Relation rel, case AT_ColumnDefault: /* ALTER COLUMN DEFAULT */ address = ATExecColumnDefault(rel, cmd->name, cmd->def, lockmode); break; + case AT_AddIdentity: + address = ATExecAddIdentity(rel, cmd->name, cmd->def, lockmode); + break; + case AT_SetIdentity: + address = ATExecSetIdentity(rel, cmd->name, cmd->def, lockmode); + break; + case AT_DropIdentity: + address = ATExecDropIdentity(rel, cmd->name, cmd->missing_ok, lockmode); + break; case AT_DropNotNull: /* ALTER COLUMN DROP NOT NULL */ address = ATExecDropNotNull(rel, cmd->name, lockmode); break; @@ -5120,6 +5158,17 @@ ATExecAddColumn(List **wqueue, AlteredTableInfo *tab, Relation rel, elog(ERROR, "cache lookup failed for relation %u", myrelid); relkind = ((Form_pg_class) GETSTRUCT(reltup))->relkind; + /* + * Cannot add identity column if table has children, because identity does + * not inherit. (Adding column and identity separately will work.) + */ + if (colDef->identity && + recurse && + find_inheritance_children(myrelid, NoLock) != NIL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("cannot recursively add identity column to table that has child tables"))); + /* skip if the name already exists and if_not_exists is true */ if (!check_for_column_name_collision(rel, colDef->colname, if_not_exists)) { @@ -5172,6 +5221,7 @@ ATExecAddColumn(List **wqueue, AlteredTableInfo *tab, Relation rel, attribute.attalign = tform->typalign; attribute.attnotnull = colDef->is_not_null; attribute.atthasdef = false; + attribute.attidentity = colDef->identity; attribute.attisdropped = false; attribute.attislocal = colDef->is_local; attribute.attinhcount = colDef->inhcount; @@ -5539,6 +5589,12 @@ ATExecDropNotNull(Relation rel, const char *colName, LOCKMODE lockmode) errmsg("cannot alter system column \"%s\"", colName))); + if (get_attidentity(RelationGetRelid(rel), attnum)) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("column \"%s\" of relation \"%s\" is an identity column", + colName, RelationGetRelationName(rel)))); + /* * Check that the attribute is not in a primary key * @@ -5755,6 +5811,13 @@ ATExecColumnDefault(Relation rel, const char *colName, errmsg("cannot alter system column \"%s\"", colName))); + if (get_attidentity(RelationGetRelid(rel), attnum)) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("column \"%s\" of relation \"%s\" is an identity column", + colName, RelationGetRelationName(rel)), + newDefault ? 0 : errhint("Use ALTER TABLE ... ALTER COLUMN ... DROP IDENTITY instead."))); + /* * Remove any old default for the column. We use RESTRICT here for * safety, but at present we do not expect anything to depend on the @@ -5789,6 +5852,224 @@ ATExecColumnDefault(Relation rel, const char *colName, return address; } +/* + * ALTER TABLE ALTER COLUMN ADD IDENTITY + * + * Return the address of the affected column. + */ +static ObjectAddress +ATExecAddIdentity(Relation rel, const char *colName, + Node *def, LOCKMODE lockmode) +{ + Relation attrelation; + HeapTuple tuple; + Form_pg_attribute attTup; + AttrNumber attnum; + ObjectAddress address; + ColumnDef *cdef = castNode(ColumnDef, def); + + attrelation = heap_open(AttributeRelationId, RowExclusiveLock); + + tuple = SearchSysCacheCopyAttName(RelationGetRelid(rel), colName); + if (!HeapTupleIsValid(tuple)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_COLUMN), + errmsg("column \"%s\" of relation \"%s\" does not exist", + colName, RelationGetRelationName(rel)))); + attTup = (Form_pg_attribute) GETSTRUCT(tuple); + attnum = attTup->attnum; + + /* Can't alter a system attribute */ + if (attnum <= 0) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot alter system column \"%s\"", + colName))); + + /* + * Creating a column as identity implies NOT NULL, so adding the identity + * to an existing column that is not NOT NULL would create a state that + * cannot be reproduced without contortions. + */ + if (!attTup->attnotnull) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("column \"%s\" of relation \"%s\" must be declared NOT NULL before identity can be added", + colName, RelationGetRelationName(rel)))); + + if (attTup->attidentity) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("column \"%s\" of relation \"%s\" is already an identity column", + colName, RelationGetRelationName(rel)))); + + if (attTup->atthasdef) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("column \"%s\" of relation \"%s\" already has a default value", + colName, RelationGetRelationName(rel)))); + + attTup->attidentity = cdef->identity; + CatalogTupleUpdate(attrelation, &tuple->t_self, tuple); + + InvokeObjectPostAlterHook(RelationRelationId, + RelationGetRelid(rel), + attTup->attnum); + ObjectAddressSubSet(address, RelationRelationId, + RelationGetRelid(rel), attnum); + heap_freetuple(tuple); + + heap_close(attrelation, RowExclusiveLock); + + return address; +} + +static ObjectAddress +ATExecSetIdentity(Relation rel, const char *colName, Node *def, LOCKMODE lockmode) +{ + ListCell *option; + DefElem *generatedEl = NULL; + HeapTuple tuple; + Form_pg_attribute attTup; + AttrNumber attnum; + Relation attrelation; + ObjectAddress address; + + foreach(option, castNode(List, def)) + { + DefElem *defel = castNode(DefElem, lfirst(option)); + + if (strcmp(defel->defname, "generated") == 0) + { + if (generatedEl) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"))); + generatedEl = defel; + } + else + elog(ERROR, "option \"%s\" not recognized", + defel->defname); + } + + /* + * Even if there is nothing to change here, we run all the checks. There + * will be a subsequent ALTER SEQUENCE that relies on everything being + * there. + */ + + attrelation = heap_open(AttributeRelationId, RowExclusiveLock); + tuple = SearchSysCacheCopyAttName(RelationGetRelid(rel), colName); + if (!HeapTupleIsValid(tuple)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_COLUMN), + errmsg("column \"%s\" of relation \"%s\" does not exist", + colName, RelationGetRelationName(rel)))); + + attTup = (Form_pg_attribute) GETSTRUCT(tuple); + attnum = attTup->attnum; + + if (attnum <= 0) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot alter system column \"%s\"", + colName))); + + if (!attTup->attidentity) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("column \"%s\" of relation \"%s\" is not an identity column", + colName, RelationGetRelationName(rel)))); + + if (generatedEl) + { + attTup->attidentity = defGetInt32(generatedEl); + CatalogTupleUpdate(attrelation, &tuple->t_self, tuple); + + InvokeObjectPostAlterHook(RelationRelationId, + RelationGetRelid(rel), + attTup->attnum); + ObjectAddressSubSet(address, RelationRelationId, + RelationGetRelid(rel), attnum); + } + + heap_freetuple(tuple); + heap_close(attrelation, RowExclusiveLock); + + return address; +} + +static ObjectAddress +ATExecDropIdentity(Relation rel, const char *colName, bool missing_ok, LOCKMODE lockmode) +{ + HeapTuple tuple; + Form_pg_attribute attTup; + AttrNumber attnum; + Relation attrelation; + ObjectAddress address; + Oid seqid; + ObjectAddress seqaddress; + + attrelation = heap_open(AttributeRelationId, RowExclusiveLock); + tuple = SearchSysCacheCopyAttName(RelationGetRelid(rel), colName); + if (!HeapTupleIsValid(tuple)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_COLUMN), + errmsg("column \"%s\" of relation \"%s\" does not exist", + colName, RelationGetRelationName(rel)))); + + attTup = (Form_pg_attribute) GETSTRUCT(tuple); + attnum = attTup->attnum; + + if (attnum <= 0) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot alter system column \"%s\"", + colName))); + + if (!attTup->attidentity) + { + if (!missing_ok) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("column \"%s\" of relation \"%s\" is not an identity column", + colName, RelationGetRelationName(rel)))); + else + { + ereport(NOTICE, + (errmsg("column \"%s\" of relation \"%s\" is not an identity column, skipping", + colName, RelationGetRelationName(rel)))); + heap_freetuple(tuple); + heap_close(attrelation, RowExclusiveLock); + return InvalidObjectAddress; + } + } + + attTup->attidentity = '\0'; + CatalogTupleUpdate(attrelation, &tuple->t_self, tuple); + + InvokeObjectPostAlterHook(RelationRelationId, + RelationGetRelid(rel), + attTup->attnum); + ObjectAddressSubSet(address, RelationRelationId, + RelationGetRelid(rel), attnum); + heap_freetuple(tuple); + + heap_close(attrelation, RowExclusiveLock); + + /* drop the internal sequence */ + seqid = getOwnedSequence(RelationGetRelid(rel), attnum); + deleteDependencyRecordsForClass(RelationRelationId, seqid, + RelationRelationId, DEPENDENCY_INTERNAL); + CommandCounterIncrement(); + seqaddress.classId = RelationRelationId; + seqaddress.objectId = seqid; + seqaddress.objectSubId = 0; + performDeletion(&seqaddress, DROP_RESTRICT, PERFORM_DELETION_INTERNAL); + + return address; +} + /* * ALTER TABLE ALTER COLUMN SET STATISTICS */ @@ -9539,7 +9820,8 @@ ATExecChangeOwner(Oid relationOid, Oid newOwnerId, bool recursing, LOCKMODE lock Oid tableId; int32 colId; - if (sequenceIsOwned(relationOid, &tableId, &colId)) + if (sequenceIsOwned(relationOid, DEPENDENCY_AUTO, &tableId, &colId) || + sequenceIsOwned(relationOid, DEPENDENCY_INTERNAL, &tableId, &colId)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot change owner of sequence \"%s\"", @@ -9810,7 +10092,7 @@ change_owner_recurse_to_sequences(Oid relationOid, Oid newOwnerId, LOCKMODE lock if (depForm->refobjsubid == 0 || depForm->classid != RelationRelationId || depForm->objsubid != 0 || - depForm->deptype != DEPENDENCY_AUTO) + !(depForm->deptype == DEPENDENCY_AUTO || depForm->deptype == DEPENDENCY_INTERNAL)) continue; /* Use relation_open just in case it's an index */ @@ -12115,7 +12397,8 @@ AlterTableNamespace(AlterObjectSchemaStmt *stmt, Oid *oldschema) Oid tableId; int32 colId; - if (sequenceIsOwned(relid, &tableId, &colId)) + if (sequenceIsOwned(relid, DEPENDENCY_AUTO, &tableId, &colId) || + sequenceIsOwned(relid, DEPENDENCY_INTERNAL, &tableId, &colId)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot move an owned sequence into another schema"), @@ -12298,7 +12581,7 @@ AlterIndexNamespaces(Relation classRel, Relation rel, } /* - * Move all SERIAL-column sequences of the specified relation to another + * Move all identity and SERIAL-column sequences of the specified relation to another * namespace. * * Note: we assume adequate permission checking was done by the caller, @@ -12342,7 +12625,7 @@ AlterSeqNamespaces(Relation classRel, Relation rel, if (depForm->refobjsubid == 0 || depForm->classid != RelationRelationId || depForm->objsubid != 0 || - depForm->deptype != DEPENDENCY_AUTO) + !(depForm->deptype == DEPENDENCY_AUTO || depForm->deptype == DEPENDENCY_INTERNAL)) continue; /* Use relation_open just in case it's an index */ diff --git a/src/backend/executor/execExpr.c b/src/backend/executor/execExpr.c index 5a84742d14..cd0dce150d 100644 --- a/src/backend/executor/execExpr.c +++ b/src/backend/executor/execExpr.c @@ -1993,6 +1993,18 @@ ExecInitExprRec(Expr *node, PlanState *parent, ExprState *state, break; } + case T_NextValueExpr: + { + NextValueExpr *nve = (NextValueExpr *) node; + + scratch.opcode = EEOP_NEXTVALUEEXPR; + scratch.d.nextvalueexpr.seqid = nve->seqid; + scratch.d.nextvalueexpr.seqtypid = nve->typeId; + + ExprEvalPushStep(state, &scratch); + break; + } + default: elog(ERROR, "unrecognized node type: %d", (int) nodeTag(node)); diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c index 4fbb5c1e74..22eb81edad 100644 --- a/src/backend/executor/execExprInterp.c +++ b/src/backend/executor/execExprInterp.c @@ -60,6 +60,7 @@ #include "access/tuptoaster.h" #include "catalog/pg_type.h" +#include "commands/sequence.h" #include "executor/execExpr.h" #include "executor/nodeSubplan.h" #include "funcapi.h" @@ -337,6 +338,7 @@ ExecInterpExpr(ExprState *state, ExprContext *econtext, bool *isnull) &&CASE_EEOP_NULLIF, &&CASE_EEOP_SQLVALUEFUNCTION, &&CASE_EEOP_CURRENTOFEXPR, + &&CASE_EEOP_NEXTVALUEEXPR, &&CASE_EEOP_ARRAYEXPR, &&CASE_EEOP_ARRAYCOERCE, &&CASE_EEOP_ROW, @@ -1228,6 +1230,27 @@ ExecInterpExpr(ExprState *state, ExprContext *econtext, bool *isnull) EEO_NEXT(); } + EEO_CASE(EEOP_NEXTVALUEEXPR) + { + switch (op->d.nextvalueexpr.seqtypid) + { + case INT2OID: + *op->resvalue = Int16GetDatum((int16) nextval_internal(op->d.nextvalueexpr.seqid, false)); + break; + case INT4OID: + *op->resvalue = Int32GetDatum((int32) nextval_internal(op->d.nextvalueexpr.seqid, false)); + break; + case INT8OID: + *op->resvalue = Int64GetDatum((int64) nextval_internal(op->d.nextvalueexpr.seqid, false)); + break; + default: + elog(ERROR, "unsupported sequence type %u", op->d.nextvalueexpr.seqtypid); + } + *op->resnull = false; + + EEO_NEXT(); + } + EEO_CASE(EEOP_ARRAYEXPR) { /* too complex for an inline implementation */ diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index 61bc5025e2..96619da8a9 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -2003,6 +2003,20 @@ _copyCurrentOfExpr(const CurrentOfExpr *from) return newnode; } + /* + * _copyNextValueExpr + */ +static NextValueExpr * +_copyNextValueExpr(const NextValueExpr *from) +{ + NextValueExpr *newnode = makeNode(NextValueExpr); + + COPY_SCALAR_FIELD(seqid); + COPY_SCALAR_FIELD(typeId); + + return newnode; +} + /* * _copyInferenceElem */ @@ -2790,6 +2804,7 @@ _copyColumnDef(const ColumnDef *from) COPY_SCALAR_FIELD(storage); COPY_NODE_FIELD(raw_default); COPY_NODE_FIELD(cooked_default); + COPY_SCALAR_FIELD(identity); COPY_NODE_FIELD(collClause); COPY_SCALAR_FIELD(collOid); COPY_NODE_FIELD(constraints); @@ -2812,6 +2827,7 @@ _copyConstraint(const Constraint *from) COPY_SCALAR_FIELD(is_no_inherit); COPY_NODE_FIELD(raw_expr); COPY_STRING_FIELD(cooked_expr); + COPY_SCALAR_FIELD(generated_when); COPY_NODE_FIELD(keys); COPY_NODE_FIELD(exclusions); COPY_NODE_FIELD(options); @@ -2920,6 +2936,7 @@ _copyQuery(const Query *from) COPY_NODE_FIELD(rtable); COPY_NODE_FIELD(jointree); COPY_NODE_FIELD(targetList); + COPY_SCALAR_FIELD(override); COPY_NODE_FIELD(onConflict); COPY_NODE_FIELD(returningList); COPY_NODE_FIELD(groupClause); @@ -2963,6 +2980,7 @@ _copyInsertStmt(const InsertStmt *from) COPY_NODE_FIELD(onConflictClause); COPY_NODE_FIELD(returningList); COPY_NODE_FIELD(withClause); + COPY_SCALAR_FIELD(override); return newnode; } @@ -3811,6 +3829,7 @@ _copyCreateSeqStmt(const CreateSeqStmt *from) COPY_NODE_FIELD(sequence); COPY_NODE_FIELD(options); COPY_SCALAR_FIELD(ownerId); + COPY_SCALAR_FIELD(for_identity); COPY_SCALAR_FIELD(if_not_exists); return newnode; @@ -3823,6 +3842,7 @@ _copyAlterSeqStmt(const AlterSeqStmt *from) COPY_NODE_FIELD(sequence); COPY_NODE_FIELD(options); + COPY_SCALAR_FIELD(for_identity); COPY_SCALAR_FIELD(missing_ok); return newnode; @@ -4927,6 +4947,9 @@ copyObjectImpl(const void *from) case T_CurrentOfExpr: retval = _copyCurrentOfExpr(from); break; + case T_NextValueExpr: + retval = _copyNextValueExpr(from); + break; case T_InferenceElem: retval = _copyInferenceElem(from); break; diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index 5941b7a2bf..46573ae767 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -733,6 +733,15 @@ _equalCurrentOfExpr(const CurrentOfExpr *a, const CurrentOfExpr *b) return true; } +static bool +_equalNextValueExpr(const NextValueExpr *a, const NextValueExpr *b) +{ + COMPARE_SCALAR_FIELD(seqid); + COMPARE_SCALAR_FIELD(typeId); + + return true; +} + static bool _equalInferenceElem(const InferenceElem *a, const InferenceElem *b) { @@ -963,6 +972,7 @@ _equalQuery(const Query *a, const Query *b) COMPARE_NODE_FIELD(rtable); COMPARE_NODE_FIELD(jointree); COMPARE_NODE_FIELD(targetList); + COMPARE_SCALAR_FIELD(override); COMPARE_NODE_FIELD(onConflict); COMPARE_NODE_FIELD(returningList); COMPARE_NODE_FIELD(groupClause); @@ -1002,6 +1012,7 @@ _equalInsertStmt(const InsertStmt *a, const InsertStmt *b) COMPARE_NODE_FIELD(onConflictClause); COMPARE_NODE_FIELD(returningList); COMPARE_NODE_FIELD(withClause); + COMPARE_SCALAR_FIELD(override); return true; } @@ -1713,6 +1724,7 @@ _equalCreateSeqStmt(const CreateSeqStmt *a, const CreateSeqStmt *b) COMPARE_NODE_FIELD(sequence); COMPARE_NODE_FIELD(options); COMPARE_SCALAR_FIELD(ownerId); + COMPARE_SCALAR_FIELD(for_identity); COMPARE_SCALAR_FIELD(if_not_exists); return true; @@ -1723,6 +1735,7 @@ _equalAlterSeqStmt(const AlterSeqStmt *a, const AlterSeqStmt *b) { COMPARE_NODE_FIELD(sequence); COMPARE_NODE_FIELD(options); + COMPARE_SCALAR_FIELD(for_identity); COMPARE_SCALAR_FIELD(missing_ok); return true; @@ -2530,6 +2543,7 @@ _equalColumnDef(const ColumnDef *a, const ColumnDef *b) COMPARE_SCALAR_FIELD(storage); COMPARE_NODE_FIELD(raw_default); COMPARE_NODE_FIELD(cooked_default); + COMPARE_SCALAR_FIELD(identity); COMPARE_NODE_FIELD(collClause); COMPARE_SCALAR_FIELD(collOid); COMPARE_NODE_FIELD(constraints); @@ -2550,6 +2564,7 @@ _equalConstraint(const Constraint *a, const Constraint *b) COMPARE_SCALAR_FIELD(is_no_inherit); COMPARE_NODE_FIELD(raw_expr); COMPARE_STRING_FIELD(cooked_expr); + COMPARE_SCALAR_FIELD(generated_when); COMPARE_NODE_FIELD(keys); COMPARE_NODE_FIELD(exclusions); COMPARE_NODE_FIELD(options); @@ -3099,6 +3114,9 @@ equal(const void *a, const void *b) case T_CurrentOfExpr: retval = _equalCurrentOfExpr(a, b); break; + case T_NextValueExpr: + retval = _equalNextValueExpr(a, b); + break; case T_InferenceElem: retval = _equalInferenceElem(a, b); break; diff --git a/src/backend/nodes/nodeFuncs.c b/src/backend/nodes/nodeFuncs.c index d5293a1a78..4149e9fd1c 100644 --- a/src/backend/nodes/nodeFuncs.c +++ b/src/backend/nodes/nodeFuncs.c @@ -246,6 +246,9 @@ exprType(const Node *expr) case T_CurrentOfExpr: type = BOOLOID; break; + case T_NextValueExpr: + type = ((const NextValueExpr *) expr)->typeId; + break; case T_InferenceElem: { const InferenceElem *n = (const InferenceElem *) expr; @@ -919,6 +922,9 @@ exprCollation(const Node *expr) case T_CurrentOfExpr: coll = InvalidOid; /* result is always boolean */ break; + case T_NextValueExpr: + coll = InvalidOid; /* result is always an integer type */ + break; case T_InferenceElem: coll = exprCollation((Node *) ((const InferenceElem *) expr)->expr); break; @@ -1122,6 +1128,9 @@ exprSetCollation(Node *expr, Oid collation) case T_CurrentOfExpr: Assert(!OidIsValid(collation)); /* result is always boolean */ break; + case T_NextValueExpr: + Assert(!OidIsValid(collation)); /* result is always an integer type */ + break; default: elog(ERROR, "unrecognized node type: %d", (int) nodeTag(expr)); break; @@ -1881,6 +1890,7 @@ expression_tree_walker(Node *node, case T_CaseTestExpr: case T_SetToDefault: case T_CurrentOfExpr: + case T_NextValueExpr: case T_SQLValueFunction: case T_RangeTblRef: case T_SortGroupClause: @@ -2476,6 +2486,7 @@ expression_tree_mutator(Node *node, case T_CaseTestExpr: case T_SetToDefault: case T_CurrentOfExpr: + case T_NextValueExpr: case T_SQLValueFunction: case T_RangeTblRef: case T_SortGroupClause: diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 83fb39fe18..13672979b4 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -2763,6 +2763,7 @@ _outColumnDef(StringInfo str, const ColumnDef *node) WRITE_CHAR_FIELD(storage); WRITE_NODE_FIELD(raw_default); WRITE_NODE_FIELD(cooked_default); + WRITE_CHAR_FIELD(identity); WRITE_NODE_FIELD(collClause); WRITE_OID_FIELD(collOid); WRITE_NODE_FIELD(constraints); @@ -2868,6 +2869,7 @@ _outQuery(StringInfo str, const Query *node) WRITE_NODE_FIELD(rtable); WRITE_NODE_FIELD(jointree); WRITE_NODE_FIELD(targetList); + WRITE_ENUM_FIELD(override, OverridingKind); WRITE_NODE_FIELD(onConflict); WRITE_NODE_FIELD(returningList); WRITE_NODE_FIELD(groupClause); @@ -3405,6 +3407,13 @@ _outConstraint(StringInfo str, const Constraint *node) WRITE_STRING_FIELD(cooked_expr); break; + case CONSTR_IDENTITY: + appendStringInfoString(str, "IDENTITY"); + WRITE_NODE_FIELD(raw_expr); + WRITE_STRING_FIELD(cooked_expr); + WRITE_CHAR_FIELD(generated_when); + break; + case CONSTR_CHECK: appendStringInfoString(str, "CHECK"); WRITE_BOOL_FIELD(is_no_inherit); diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index 766f2d8db1..e027154e67 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -247,6 +247,7 @@ _readQuery(void) READ_NODE_FIELD(rtable); READ_NODE_FIELD(jointree); READ_NODE_FIELD(targetList); + READ_ENUM_FIELD(override, OverridingKind); READ_NODE_FIELD(onConflict); READ_NODE_FIELD(returningList); READ_NODE_FIELD(groupClause); diff --git a/src/backend/parser/analyze.c b/src/backend/parser/analyze.c index 811fccaec9..c4140a65d2 100644 --- a/src/backend/parser/analyze.c +++ b/src/backend/parser/analyze.c @@ -487,6 +487,8 @@ transformInsertStmt(ParseState *pstate, InsertStmt *stmt) qry->hasModifyingCTE = pstate->p_hasModifyingCTE; } + qry->override = stmt->override; + isOnConflictUpdate = (stmt->onConflictClause && stmt->onConflictClause->action == ONCONFLICT_UPDATE); diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 5ecb6997b3..29ca5f13ea 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -292,6 +292,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); %type alter_table_cmd alter_type_cmd opt_collate_clause replica_identity partition_cmd %type alter_table_cmds alter_type_cmds +%type alter_identity_column_option_list +%type alter_identity_column_option %type opt_drop_behavior @@ -449,7 +451,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); select_offset_value2 opt_select_fetch_first_value %type row_or_rows first_or_next -%type OptSeqOptList SeqOptList +%type OptSeqOptList SeqOptList OptParenthesizedSeqOptList %type SeqOptElem %type insert_rest @@ -569,6 +571,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); opt_frame_clause frame_extent frame_bound %type opt_existing_window_name %type opt_if_not_exists +%type generated_when override_kind %type PartitionSpec OptPartitionSpec %type part_strategy %type part_elem @@ -631,7 +634,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); FALSE_P FAMILY FETCH FILTER FIRST_P FLOAT_P FOLLOWING FOR FORCE FOREIGN FORWARD FREEZE FROM FULL FUNCTION FUNCTIONS - GLOBAL GRANT GRANTED GREATEST GROUP_P GROUPING + GENERATED GLOBAL GRANT GRANTED GREATEST GROUP_P GROUPING HANDLER HAVING HEADER_P HOLD HOUR_P @@ -655,7 +658,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); NULLS_P NUMERIC OBJECT_P OF OFF OFFSET OIDS OLD ON ONLY OPERATOR OPTION OPTIONS OR - ORDER ORDINALITY OUT_P OUTER_P OVER OVERLAPS OVERLAY OWNED OWNER + ORDER ORDINALITY OUT_P OUTER_P OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER PARALLEL PARSER PARTIAL PARTITION PASSING PASSWORD PLACING PLANS POLICY POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY @@ -726,6 +729,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); * same as if they weren't keywords). We need to do this for PARTITION, * RANGE, ROWS to support opt_existing_window_name; and for RANGE, ROWS * so that they can follow a_expr without creating postfix-operator problems; + * for GENERATED so that it can follow b_expr; * and for NULL so that it can follow b_expr in ColQualList without creating * postfix-operator problems. * @@ -744,7 +748,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); * blame any funny behavior of UNBOUNDED on the SQL standard, though. */ %nonassoc UNBOUNDED /* ideally should have same precedence as IDENT */ -%nonassoc IDENT NULL_P PARTITION RANGE ROWS PRECEDING FOLLOWING CUBE ROLLUP +%nonassoc IDENT GENERATED NULL_P PARTITION RANGE ROWS PRECEDING FOLLOWING CUBE ROLLUP %left Op OPERATOR /* multi-character ops and user-defined operators */ %left '+' '-' %left '*' '/' '%' @@ -2128,6 +2132,50 @@ alter_table_cmd: n->def = (Node *) makeString($6); $$ = (Node *)n; } + /* ALTER TABLE ALTER [COLUMN] ADD GENERATED ... AS IDENTITY ... */ + | ALTER opt_column ColId ADD_P GENERATED generated_when AS IDENTITY_P OptParenthesizedSeqOptList + { + AlterTableCmd *n = makeNode(AlterTableCmd); + Constraint *c = makeNode(Constraint); + + c->contype = CONSTR_IDENTITY; + c->generated_when = $6; + c->options = $9; + c->location = @5; + + n->subtype = AT_AddIdentity; + n->name = $3; + n->def = (Node *) c; + + $$ = (Node *)n; + } + /* ALTER TABLE ALTER [COLUMN] SET /RESET */ + | ALTER opt_column ColId alter_identity_column_option_list + { + AlterTableCmd *n = makeNode(AlterTableCmd); + n->subtype = AT_SetIdentity; + n->name = $3; + n->def = (Node *) $4; + $$ = (Node *)n; + } + /* ALTER TABLE ALTER [COLUMN] DROP IDENTITY */ + | ALTER opt_column ColId DROP IDENTITY_P + { + AlterTableCmd *n = makeNode(AlterTableCmd); + n->subtype = AT_DropIdentity; + n->name = $3; + n->missing_ok = false; + $$ = (Node *)n; + } + /* ALTER TABLE ALTER [COLUMN] DROP IDENTITY IF EXISTS */ + | ALTER opt_column ColId DROP IDENTITY_P IF_P EXISTS + { + AlterTableCmd *n = makeNode(AlterTableCmd); + n->subtype = AT_DropIdentity; + n->name = $3; + n->missing_ok = true; + $$ = (Node *)n; + } /* ALTER TABLE DROP [COLUMN] IF EXISTS [RESTRICT|CASCADE] */ | DROP opt_column IF_P EXISTS ColId opt_drop_behavior { @@ -2565,6 +2613,39 @@ reloption_elem: } ; +alter_identity_column_option_list: + alter_identity_column_option + { $$ = list_make1($1); } + | alter_identity_column_option_list alter_identity_column_option + { $$ = lappend($1, $2); } + ; + +alter_identity_column_option: + RESTART + { + $$ = makeDefElem("restart", NULL, @1); + } + | RESTART opt_with NumericOnly + { + $$ = makeDefElem("restart", (Node *)$3, @1); + } + | SET SeqOptElem + { + if (strcmp($2->defname, "as") == 0 || + strcmp($2->defname, "restart") == 0 || + strcmp($2->defname, "owned_by") == 0) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("sequence option \"%s\" not supported here", $2->defname), + parser_errposition(@2))); + $$ = $2; + } + | SET GENERATED generated_when + { + $$ = makeDefElem("generated", (Node *) makeInteger($3), @1); + } + ; + ForValues: /* a LIST partition */ FOR VALUES IN_P '(' partbound_datum_list ')' @@ -3347,6 +3428,15 @@ ColConstraintElem: n->cooked_expr = NULL; $$ = (Node *)n; } + | GENERATED generated_when AS IDENTITY_P OptParenthesizedSeqOptList + { + Constraint *n = makeNode(Constraint); + n->contype = CONSTR_IDENTITY; + n->generated_when = $2; + n->options = $5; + n->location = @1; + $$ = (Node *)n; + } | REFERENCES qualified_name opt_column_list key_match key_actions { Constraint *n = makeNode(Constraint); @@ -3364,6 +3454,11 @@ ColConstraintElem: } ; +generated_when: + ALWAYS { $$ = ATTRIBUTE_IDENTITY_ALWAYS; } + | BY DEFAULT { $$ = ATTRIBUTE_IDENTITY_BY_DEFAULT; } + ; + /* * ConstraintAttr represents constraint attributes, which we parse as if * they were independent constraint clauses, in order to avoid shift/reduce @@ -3430,6 +3525,7 @@ TableLikeOptionList: TableLikeOption: DEFAULTS { $$ = CREATE_TABLE_LIKE_DEFAULTS; } | CONSTRAINTS { $$ = CREATE_TABLE_LIKE_CONSTRAINTS; } + | IDENTITY_P { $$ = CREATE_TABLE_LIKE_IDENTITY; } | INDEXES { $$ = CREATE_TABLE_LIKE_INDEXES; } | STORAGE { $$ = CREATE_TABLE_LIKE_STORAGE; } | COMMENTS { $$ = CREATE_TABLE_LIKE_COMMENTS; } @@ -3967,6 +4063,10 @@ OptSeqOptList: SeqOptList { $$ = $1; } | /*EMPTY*/ { $$ = NIL; } ; +OptParenthesizedSeqOptList: '(' SeqOptList ')' { $$ = $2; } + | /*EMPTY*/ { $$ = NIL; } + ; + SeqOptList: SeqOptElem { $$ = list_make1($1); } | SeqOptList SeqOptElem { $$ = lappend($1, $2); } ; @@ -4011,6 +4111,11 @@ SeqOptElem: AS SimpleTypename { $$ = makeDefElem("owned_by", (Node *)$3, @1); } + | SEQUENCE NAME_P any_name + { + /* not documented, only used by pg_dump */ + $$ = makeDefElem("sequence_name", (Node *)$3, @1); + } | START opt_with NumericOnly { $$ = makeDefElem("start", (Node *)$3, @1); @@ -10412,12 +10517,26 @@ insert_rest: $$->cols = NIL; $$->selectStmt = $1; } + | OVERRIDING override_kind VALUE_P SelectStmt + { + $$ = makeNode(InsertStmt); + $$->cols = NIL; + $$->override = $2; + $$->selectStmt = $4; + } | '(' insert_column_list ')' SelectStmt { $$ = makeNode(InsertStmt); $$->cols = $2; $$->selectStmt = $4; } + | '(' insert_column_list ')' OVERRIDING override_kind VALUE_P SelectStmt + { + $$ = makeNode(InsertStmt); + $$->cols = $2; + $$->override = $5; + $$->selectStmt = $7; + } | DEFAULT VALUES { $$ = makeNode(InsertStmt); @@ -10426,6 +10545,11 @@ insert_rest: } ; +override_kind: + USER { $$ = OVERRIDING_USER_VALUE; } + | SYSTEM_P { $$ = OVERRIDING_SYSTEM_VALUE; } + ; + insert_column_list: insert_column_item { $$ = list_make1($1); } @@ -14597,6 +14721,7 @@ unreserved_keyword: | FORWARD | FUNCTION | FUNCTIONS + | GENERATED | GLOBAL | GRANTED | HANDLER @@ -14666,6 +14791,7 @@ unreserved_keyword: | OPTIONS | ORDINALITY | OVER + | OVERRIDING | OWNED | OWNER | PARALLEL diff --git a/src/backend/parser/parse_utilcmd.c b/src/backend/parser/parse_utilcmd.c index 1ae43dc25d..926699608b 100644 --- a/src/backend/parser/parse_utilcmd.c +++ b/src/backend/parser/parse_utilcmd.c @@ -42,6 +42,7 @@ #include "catalog/pg_type.h" #include "commands/comment.h" #include "commands/defrem.h" +#include "commands/sequence.h" #include "commands/tablecmds.h" #include "commands/tablespace.h" #include "miscadmin.h" @@ -356,6 +357,132 @@ transformCreateStmt(CreateStmt *stmt, const char *queryString) return result; } +static void +generateSerialExtraStmts(CreateStmtContext *cxt, ColumnDef *column, + Oid seqtypid, List *seqoptions, bool for_identity, + char **snamespace_p, char **sname_p) +{ + ListCell *option; + DefElem *nameEl = NULL; + Oid snamespaceid; + char *snamespace; + char *sname; + CreateSeqStmt *seqstmt; + AlterSeqStmt *altseqstmt; + List *attnamelist; + + /* + * Determine namespace and name to use for the sequence. + * + * First, check if a sequence name was passed in as an option. This is + * used by pg_dump. Else, generate a name. + * + * Although we use ChooseRelationName, it's not guaranteed that the + * selected sequence name won't conflict; given sufficiently long + * field names, two different serial columns in the same table could + * be assigned the same sequence name, and we'd not notice since we + * aren't creating the sequence quite yet. In practice this seems + * quite unlikely to be a problem, especially since few people would + * need two serial columns in one table. + */ + + foreach(option, seqoptions) + { + DefElem *defel = castNode(DefElem, lfirst(option)); + + if (strcmp(defel->defname, "sequence_name") == 0) + { + if (nameEl) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"))); + nameEl = defel; + } + } + + if (nameEl) + { + RangeVar *rv = makeRangeVarFromNameList(castNode(List, nameEl->arg)); + snamespace = rv->schemaname; + sname = rv->relname; + seqoptions = list_delete_ptr(seqoptions, nameEl); + } + else + { + if (cxt->rel) + snamespaceid = RelationGetNamespace(cxt->rel); + else + { + snamespaceid = RangeVarGetCreationNamespace(cxt->relation); + RangeVarAdjustRelationPersistence(cxt->relation, snamespaceid); + } + snamespace = get_namespace_name(snamespaceid); + sname = ChooseRelationName(cxt->relation->relname, + column->colname, + "seq", + snamespaceid); + } + + ereport(DEBUG1, + (errmsg("%s will create implicit sequence \"%s\" for serial column \"%s.%s\"", + cxt->stmtType, sname, + cxt->relation->relname, column->colname))); + + /* + * Build a CREATE SEQUENCE command to create the sequence object, and + * add it to the list of things to be done before this CREATE/ALTER + * TABLE. + */ + seqstmt = makeNode(CreateSeqStmt); + seqstmt->for_identity = for_identity; + seqstmt->sequence = makeRangeVar(snamespace, sname, -1); + seqstmt->options = seqoptions; + /* + * If a sequence data type was specified, add it to the options. Prepend + * to the list rather than append; in case a user supplied their own AS + * clause, the "redundant options" error will point to their occurrence, + * not our synthetic one. + */ + if (seqtypid) + seqstmt->options = lcons(makeDefElem("as", (Node *) makeTypeNameFromOid(seqtypid, -1), -1), + seqstmt->options); + + /* + * If this is ALTER ADD COLUMN, make sure the sequence will be owned + * by the table's owner. The current user might be someone else + * (perhaps a superuser, or someone who's only a member of the owning + * role), but the SEQUENCE OWNED BY mechanisms will bleat unless table + * and sequence have exactly the same owning role. + */ + if (cxt->rel) + seqstmt->ownerId = cxt->rel->rd_rel->relowner; + else + seqstmt->ownerId = InvalidOid; + + cxt->blist = lappend(cxt->blist, seqstmt); + + /* + * Build an ALTER SEQUENCE ... OWNED BY command to mark the sequence + * as owned by this column, and add it to the list of things to be + * done after this CREATE/ALTER TABLE. + */ + altseqstmt = makeNode(AlterSeqStmt); + altseqstmt->sequence = makeRangeVar(snamespace, sname, -1); + attnamelist = list_make3(makeString(snamespace), + makeString(cxt->relation->relname), + makeString(column->colname)); + altseqstmt->options = list_make1(makeDefElem("owned_by", + (Node *) attnamelist, -1)); + altseqstmt->for_identity = for_identity; + + cxt->alist = lappend(cxt->alist, altseqstmt); + + if (snamespace_p) + *snamespace_p = snamespace; + if (sname_p) + *sname_p = sname; +} + /* * transformColumnDefinition - * transform a single ColumnDef within CREATE TABLE @@ -367,7 +494,7 @@ transformColumnDefinition(CreateStmtContext *cxt, ColumnDef *column) bool is_serial; bool saw_nullable; bool saw_default; - Constraint *constraint; + bool saw_identity; ListCell *clist; cxt->columns = lappend(cxt->columns, column); @@ -422,83 +549,17 @@ transformColumnDefinition(CreateStmtContext *cxt, ColumnDef *column) /* Special actions for SERIAL pseudo-types */ if (is_serial) { - Oid snamespaceid; char *snamespace; char *sname; char *qstring; A_Const *snamenode; TypeCast *castnode; FuncCall *funccallnode; - CreateSeqStmt *seqstmt; - AlterSeqStmt *altseqstmt; - List *attnamelist; + Constraint *constraint; - /* - * Determine namespace and name to use for the sequence. - * - * Although we use ChooseRelationName, it's not guaranteed that the - * selected sequence name won't conflict; given sufficiently long - * field names, two different serial columns in the same table could - * be assigned the same sequence name, and we'd not notice since we - * aren't creating the sequence quite yet. In practice this seems - * quite unlikely to be a problem, especially since few people would - * need two serial columns in one table. - */ - if (cxt->rel) - snamespaceid = RelationGetNamespace(cxt->rel); - else - { - snamespaceid = RangeVarGetCreationNamespace(cxt->relation); - RangeVarAdjustRelationPersistence(cxt->relation, snamespaceid); - } - snamespace = get_namespace_name(snamespaceid); - sname = ChooseRelationName(cxt->relation->relname, - column->colname, - "seq", - snamespaceid); - - ereport(DEBUG1, - (errmsg("%s will create implicit sequence \"%s\" for serial column \"%s.%s\"", - cxt->stmtType, sname, - cxt->relation->relname, column->colname))); - - /* - * Build a CREATE SEQUENCE command to create the sequence object, and - * add it to the list of things to be done before this CREATE/ALTER - * TABLE. - */ - seqstmt = makeNode(CreateSeqStmt); - seqstmt->sequence = makeRangeVar(snamespace, sname, -1); - seqstmt->options = list_make1(makeDefElem("as", (Node *) makeTypeNameFromOid(column->typeName->typeOid, -1), -1)); - - /* - * If this is ALTER ADD COLUMN, make sure the sequence will be owned - * by the table's owner. The current user might be someone else - * (perhaps a superuser, or someone who's only a member of the owning - * role), but the SEQUENCE OWNED BY mechanisms will bleat unless table - * and sequence have exactly the same owning role. - */ - if (cxt->rel) - seqstmt->ownerId = cxt->rel->rd_rel->relowner; - else - seqstmt->ownerId = InvalidOid; - - cxt->blist = lappend(cxt->blist, seqstmt); - - /* - * Build an ALTER SEQUENCE ... OWNED BY command to mark the sequence - * as owned by this column, and add it to the list of things to be - * done after this CREATE/ALTER TABLE. - */ - altseqstmt = makeNode(AlterSeqStmt); - altseqstmt->sequence = makeRangeVar(snamespace, sname, -1); - attnamelist = list_make3(makeString(snamespace), - makeString(cxt->relation->relname), - makeString(column->colname)); - altseqstmt->options = list_make1(makeDefElem("owned_by", - (Node *) attnamelist, -1)); - - cxt->alist = lappend(cxt->alist, altseqstmt); + generateSerialExtraStmts(cxt, column, + column->typeName->typeOid, NIL, false, + &snamespace, &sname); /* * Create appropriate constraints for SERIAL. We do this in full, @@ -540,10 +601,11 @@ transformColumnDefinition(CreateStmtContext *cxt, ColumnDef *column) saw_nullable = false; saw_default = false; + saw_identity = false; foreach(clist, column->constraints) { - constraint = castNode(Constraint, lfirst(clist)); + Constraint *constraint = castNode(Constraint, lfirst(clist)); switch (constraint->contype) { @@ -584,6 +646,33 @@ transformColumnDefinition(CreateStmtContext *cxt, ColumnDef *column) saw_default = true; break; + case CONSTR_IDENTITY: + { + Type ctype; + Oid typeOid; + + ctype = typenameType(cxt->pstate, column->typeName, NULL); + typeOid = HeapTupleGetOid(ctype); + ReleaseSysCache(ctype); + + if (saw_identity) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("multiple identity specifications for column \"%s\" of table \"%s\"", + column->colname, cxt->relation->relname), + parser_errposition(cxt->pstate, + constraint->location))); + + generateSerialExtraStmts(cxt, column, + typeOid, constraint->options, true, + NULL, NULL); + + column->identity = constraint->generated_when; + saw_identity = true; + column->is_not_null = TRUE; + break; + } + case CONSTR_CHECK: cxt->ckconstraints = lappend(cxt->ckconstraints, constraint); break; @@ -660,6 +749,14 @@ transformColumnDefinition(CreateStmtContext *cxt, ColumnDef *column) constraint->contype); break; } + + if (saw_default && saw_identity) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("both default and identity specified for column \"%s\" of table \"%s\"", + column->colname, cxt->relation->relname), + parser_errposition(cxt->pstate, + constraint->location))); } /* @@ -932,6 +1029,27 @@ transformTableLikeClause(CreateStmtContext *cxt, TableLikeClause *table_like_cla def->cooked_default = this_default; } + /* + * Copy identity if requested + */ + if (attribute->attidentity && + (table_like_clause->options & CREATE_TABLE_LIKE_IDENTITY)) + { + Oid seq_relid; + List *seq_options; + + /* + * find sequence owned by old column; extract sequence parameters; + * build new create sequence command + */ + seq_relid = getOwnedSequence(RelationGetRelid(relation), attribute->attnum); + seq_options = sequence_options(seq_relid); + generateSerialExtraStmts(cxt, def, + InvalidOid, seq_options, true, + NULL, NULL); + def->identity = attribute->attidentity; + } + /* Likewise, copy storage if requested */ if (table_like_clause->options & CREATE_TABLE_LIKE_STORAGE) def->storage = attribute->attstorage; @@ -2628,6 +2746,7 @@ transformAlterTableStmt(Oid relid, AlterTableStmt *stmt, case AT_AlterColumnType: { ColumnDef *def = (ColumnDef *) cmd->def; + AttrNumber attnum; /* * For ALTER COLUMN TYPE, transform the USING clause if @@ -2640,6 +2759,103 @@ transformAlterTableStmt(Oid relid, AlterTableStmt *stmt, EXPR_KIND_ALTER_COL_TRANSFORM); } + /* + * For identity column, create ALTER SEQUENCE command to + * change the data type of the sequence. + */ + attnum = get_attnum(relid, cmd->name); + /* if attribute not found, something will error about it later */ + if (attnum != InvalidAttrNumber && get_attidentity(relid, attnum)) + { + Oid seq_relid = getOwnedSequence(relid, attnum); + Oid typeOid = typenameTypeId(pstate, def->typeName); + AlterSeqStmt *altseqstmt = makeNode(AlterSeqStmt); + + altseqstmt->sequence = makeRangeVar(get_namespace_name(get_rel_namespace(seq_relid)), + get_rel_name(seq_relid), + -1); + altseqstmt->options = list_make1(makeDefElem("as", (Node *) makeTypeNameFromOid(typeOid, -1), -1)); + altseqstmt->for_identity = true; + cxt.blist = lappend(cxt.blist, altseqstmt); + } + + newcmds = lappend(newcmds, cmd); + break; + } + + case AT_AddIdentity: + { + Constraint *def = castNode(Constraint, cmd->def); + ColumnDef *newdef = makeNode(ColumnDef); + AttrNumber attnum; + + newdef->colname = cmd->name; + newdef->identity = def->generated_when; + cmd->def = (Node *) newdef; + + attnum = get_attnum(relid, cmd->name); + /* if attribute not found, something will error about it later */ + if (attnum != InvalidAttrNumber) + generateSerialExtraStmts(&cxt, newdef, + get_atttype(relid, attnum), + def->options, true, + NULL, NULL); + + newcmds = lappend(newcmds, cmd); + break; + } + + case AT_SetIdentity: + { + /* + * Create an ALTER SEQUENCE statement for the internal + * sequence of the identity column. + */ + ListCell *lc; + List *newseqopts = NIL; + List *newdef = NIL; + List *seqlist; + AttrNumber attnum; + + /* + * Split options into those handled by ALTER SEQUENCE and + * those for ALTER TABLE proper. + */ + foreach(lc, castNode(List, cmd->def)) + { + DefElem *def = castNode(DefElem, lfirst(lc)); + + if (strcmp(def->defname, "generated") == 0) + newdef = lappend(newdef, def); + else + newseqopts = lappend(newseqopts, def); + } + + attnum = get_attnum(relid, cmd->name); + + if (attnum) + { + seqlist = getOwnedSequences(relid, attnum); + if (seqlist) + { + AlterSeqStmt *seqstmt; + Oid seq_relid; + + seqstmt = makeNode(AlterSeqStmt); + seq_relid = linitial_oid(seqlist); + seqstmt->sequence = makeRangeVar(get_namespace_name(get_rel_namespace(seq_relid)), + get_rel_name(seq_relid), -1); + seqstmt->options = newseqopts; + seqstmt->for_identity = true; + seqstmt->missing_ok = false; + + cxt.alist = lappend(cxt.alist, seqstmt); + } + } + /* If column was not found or was not an identity column, we + * just let the ALTER TABLE command error out later. */ + + cmd->def = (Node *) newdef; newcmds = lappend(newcmds, cmd); break; } diff --git a/src/backend/rewrite/rewriteHandler.c b/src/backend/rewrite/rewriteHandler.c index 424be0c768..cb860ec4e5 100644 --- a/src/backend/rewrite/rewriteHandler.c +++ b/src/backend/rewrite/rewriteHandler.c @@ -21,6 +21,7 @@ #include "postgres.h" #include "access/sysattr.h" +#include "catalog/dependency.h" #include "catalog/pg_type.h" #include "commands/trigger.h" #include "foreign/fdwapi.h" @@ -61,6 +62,7 @@ static Query *rewriteRuleAction(Query *parsetree, static List *adjustJoinTreeList(Query *parsetree, bool removert, int rt_index); static List *rewriteTargetListIU(List *targetList, CmdType commandType, + OverridingKind override, Relation target_relation, int result_rti, List **attrno_list); @@ -709,6 +711,7 @@ adjustJoinTreeList(Query *parsetree, bool removert, int rt_index) static List * rewriteTargetListIU(List *targetList, CmdType commandType, + OverridingKind override, Relation target_relation, int result_rti, List **attrno_list) @@ -789,6 +792,7 @@ rewriteTargetListIU(List *targetList, for (attrno = 1; attrno <= numattrs; attrno++) { TargetEntry *new_tle = new_tles[attrno - 1]; + bool apply_default; att_tup = target_relation->rd_att->attrs[attrno - 1]; @@ -801,12 +805,51 @@ rewriteTargetListIU(List *targetList, * it's an INSERT and there's no tlist entry for the column, or the * tlist entry is a DEFAULT placeholder node. */ - if ((new_tle == NULL && commandType == CMD_INSERT) || - (new_tle && new_tle->expr && IsA(new_tle->expr, SetToDefault))) + apply_default = ((new_tle == NULL && commandType == CMD_INSERT) || + (new_tle && new_tle->expr && IsA(new_tle->expr, SetToDefault))); + + if (commandType == CMD_INSERT) + { + if (att_tup->attidentity == ATTRIBUTE_IDENTITY_ALWAYS && !apply_default) + { + if (override != OVERRIDING_SYSTEM_VALUE) + ereport(ERROR, + (errcode(ERRCODE_GENERATED_ALWAYS), + errmsg("cannot insert into column \"%s\"", NameStr(att_tup->attname)), + errdetail("Column \"%s\" is an identity column defined as GENERATED ALWAYS.", + NameStr(att_tup->attname)), + errhint("Use OVERRIDING SYSTEM VALUE to override."))); + } + + if (att_tup->attidentity == ATTRIBUTE_IDENTITY_BY_DEFAULT && override == OVERRIDING_USER_VALUE) + apply_default = true; + } + + if (commandType == CMD_UPDATE) + { + if (att_tup->attidentity == ATTRIBUTE_IDENTITY_ALWAYS && !apply_default) + ereport(ERROR, + (errcode(ERRCODE_GENERATED_ALWAYS), + errmsg("column \"%s\" can only be updated to DEFAULT", NameStr(att_tup->attname)), + errdetail("Column \"%s\" is an identity column defined as GENERATED ALWAYS.", + NameStr(att_tup->attname)))); + } + + if (apply_default) { Node *new_expr; - new_expr = build_column_default(target_relation, attrno); + if (att_tup->attidentity) + { + NextValueExpr *nve = makeNode(NextValueExpr); + + nve->seqid = getOwnedSequence(RelationGetRelid(target_relation), attrno); + nve->typeId = att_tup->atttypid; + + new_expr = (Node *) nve; + } + else + new_expr = build_column_default(target_relation, attrno); /* * If there is no default (ie, default is effectively NULL), we @@ -3232,6 +3275,7 @@ RewriteQuery(Query *parsetree, List *rewrite_events) /* Process the main targetlist ... */ parsetree->targetList = rewriteTargetListIU(parsetree->targetList, parsetree->commandType, + parsetree->override, rt_entry_relation, parsetree->resultRelation, &attrnos); @@ -3244,6 +3288,7 @@ RewriteQuery(Query *parsetree, List *rewrite_events) parsetree->targetList = rewriteTargetListIU(parsetree->targetList, parsetree->commandType, + parsetree->override, rt_entry_relation, parsetree->resultRelation, NULL); } @@ -3254,6 +3299,7 @@ RewriteQuery(Query *parsetree, List *rewrite_events) parsetree->onConflict->onConflictSet = rewriteTargetListIU(parsetree->onConflict->onConflictSet, CMD_UPDATE, + parsetree->override, rt_entry_relation, parsetree->resultRelation, NULL); @@ -3263,7 +3309,9 @@ RewriteQuery(Query *parsetree, List *rewrite_events) { parsetree->targetList = rewriteTargetListIU(parsetree->targetList, - parsetree->commandType, rt_entry_relation, + parsetree->commandType, + parsetree->override, + rt_entry_relation, parsetree->resultRelation, NULL); rewriteTargetListUD(parsetree, rt_entry, rt_entry_relation); } diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c index 342e52bcf7..241b81a48f 100644 --- a/src/backend/utils/adt/ruleutils.c +++ b/src/backend/utils/adt/ruleutils.c @@ -5936,6 +5936,14 @@ get_insert_query_def(Query *query, deparse_context *context) if (query->targetList) appendStringInfoString(buf, ") "); + if (query->override) + { + if (query->override == OVERRIDING_SYSTEM_VALUE) + appendStringInfoString(buf, "OVERRIDING SYSTEM VALUE "); + else if (query->override == OVERRIDING_USER_VALUE) + appendStringInfoString(buf, "OVERRIDING USER VALUE "); + } + if (select_rte) { /* Add the SELECT */ diff --git a/src/backend/utils/cache/lsyscache.c b/src/backend/utils/cache/lsyscache.c index b891f388e5..0667ef5a81 100644 --- a/src/backend/utils/cache/lsyscache.c +++ b/src/backend/utils/cache/lsyscache.c @@ -836,6 +836,38 @@ get_attnum(Oid relid, const char *attname) return InvalidAttrNumber; } +/* + * get_attidentity + * + * Given the relation id and the attribute name, + * return the "attidentity" field from the attribute relation. + * + * Returns '\0' if not found. + * + * Since no identity is represented by '\0', this can also be used as a + * Boolean test. + */ +char +get_attidentity(Oid relid, AttrNumber attnum) +{ + HeapTuple tp; + + tp = SearchSysCache2(ATTNUM, + ObjectIdGetDatum(relid), + Int16GetDatum(attnum)); + if (HeapTupleIsValid(tp)) + { + Form_pg_attribute att_tup = (Form_pg_attribute) GETSTRUCT(tp); + char result; + + result = att_tup->attidentity; + ReleaseSysCache(tp); + return result; + } + else + return '\0'; +} + /* * get_atttype * diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c index bc220989a1..24ffea8f40 100644 --- a/src/backend/utils/cache/relcache.c +++ b/src/backend/utils/cache/relcache.c @@ -3268,6 +3268,7 @@ RelationBuildLocalRelation(const char *relname, has_not_null = false; for (i = 0; i < natts; i++) { + rel->rd_att->attrs[i]->attidentity = tupDesc->attrs[i]->attidentity; rel->rd_att->attrs[i]->attnotnull = tupDesc->attrs[i]->attnotnull; has_not_null |= tupDesc->attrs[i]->attnotnull; } diff --git a/src/backend/utils/errcodes.txt b/src/backend/utils/errcodes.txt index b6e0e987a8..4f35471762 100644 --- a/src/backend/utils/errcodes.txt +++ b/src/backend/utils/errcodes.txt @@ -327,6 +327,7 @@ Section: Class 42 - Syntax Error or Access Rule Violation 42P21 E ERRCODE_COLLATION_MISMATCH collation_mismatch 42P22 E ERRCODE_INDETERMINATE_COLLATION indeterminate_collation 42809 E ERRCODE_WRONG_OBJECT_TYPE wrong_object_type +428C9 E ERRCODE_GENERATED_ALWAYS generated_always # Note: for ERRCODE purposes, we divide namable objects into these categories: # databases, schemas, prepared statements, cursors, tables, columns, diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 262f5539bc..65a2f2307a 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -43,6 +43,7 @@ #include "access/sysattr.h" #include "access/transam.h" #include "catalog/pg_am.h" +#include "catalog/pg_attribute.h" #include "catalog/pg_cast.h" #include "catalog/pg_class.h" #include "catalog/pg_default_acl.h" @@ -1925,6 +1926,9 @@ dumpTableData_insert(Archive *fout, void *dcontext) appendPQExpBufferStr(insertStmt, ") "); } + if (tbinfo->needs_override) + appendPQExpBufferStr(insertStmt, "OVERRIDING SYSTEM VALUE "); + appendPQExpBufferStr(insertStmt, "VALUES ("); } } @@ -5451,6 +5455,7 @@ getTables(Archive *fout, int *numTables) int i_toastreloptions; int i_reloftype; int i_relpages; + int i_is_identity_sequence; int i_changed_acl; /* Make sure we are in proper schema */ @@ -5528,6 +5533,7 @@ getTables(Archive *fout, int *numTables) "CASE WHEN 'check_option=local' = ANY (c.reloptions) THEN 'LOCAL'::text " "WHEN 'check_option=cascaded' = ANY (c.reloptions) THEN 'CASCADED'::text ELSE NULL END AS checkoption, " "tc.reloptions AS toast_reloptions, " + "c.relkind = '%c' AND EXISTS (SELECT 1 FROM pg_depend WHERE classid = 'pg_class'::regclass AND objid = c.oid AND objsubid = 0 AND refclassid = 'pg_class'::regclass AND deptype = 'i') AS is_identity_sequence, " "EXISTS (SELECT 1 FROM pg_attribute at LEFT JOIN pg_init_privs pip ON " "(c.oid = pip.objoid " "AND pip.classoid = 'pg_class'::regclass " @@ -5544,7 +5550,7 @@ getTables(Archive *fout, int *numTables) "(c.relkind = '%c' AND " "d.classid = c.tableoid AND d.objid = c.oid AND " "d.objsubid = 0 AND " - "d.refclassid = c.tableoid AND d.deptype = 'a') " + "d.refclassid = c.tableoid AND d.deptype IN ('a', 'i')) " "LEFT JOIN pg_class tc ON (c.reltoastrelid = tc.oid) " "LEFT JOIN pg_init_privs pip ON " "(c.oid = pip.objoid " @@ -5557,6 +5563,7 @@ getTables(Archive *fout, int *numTables) initacl_subquery->data, initracl_subquery->data, username_subquery, + RELKIND_SEQUENCE, attacl_subquery->data, attracl_subquery->data, attinitacl_subquery->data, @@ -5979,6 +5986,7 @@ getTables(Archive *fout, int *numTables) i_checkoption = PQfnumber(res, "checkoption"); i_toastreloptions = PQfnumber(res, "toast_reloptions"); i_reloftype = PQfnumber(res, "reloftype"); + i_is_identity_sequence = PQfnumber(res, "is_identity_sequence"); i_changed_acl = PQfnumber(res, "changed_acl"); if (dopt->lockWaitTimeout) @@ -6079,6 +6087,9 @@ getTables(Archive *fout, int *numTables) tblinfo[i].dummy_view = false; /* might get set during sort */ tblinfo[i].postponed_def = false; /* might get set during sort */ + tblinfo[i].is_identity_sequence = (i_is_identity_sequence >= 0 && + strcmp(PQgetvalue(res, i, i_is_identity_sequence), "t") == 0); + /* * Read-lock target tables to make sure they aren't DROPPED or altered * in schema before we get around to dumping them. @@ -7735,6 +7746,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) int i_typstorage; int i_attnotnull; int i_atthasdef; + int i_attidentity; int i_attisdropped; int i_attlen; int i_attalign; @@ -7777,7 +7789,34 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) resetPQExpBuffer(q); - if (fout->remoteVersion >= 90200) + if (fout->remoteVersion >= 100000) + { + /* + * attidentity is new in version 10. + */ + appendPQExpBuffer(q, "SELECT a.attnum, a.attname, a.atttypmod, " + "a.attstattarget, a.attstorage, t.typstorage, " + "a.attnotnull, a.atthasdef, a.attisdropped, " + "a.attlen, a.attalign, a.attislocal, " + "pg_catalog.format_type(t.oid,a.atttypmod) AS atttypname, " + "array_to_string(a.attoptions, ', ') AS attoptions, " + "CASE WHEN a.attcollation <> t.typcollation " + "THEN a.attcollation ELSE 0 END AS attcollation, " + "a.attidentity, " + "pg_catalog.array_to_string(ARRAY(" + "SELECT pg_catalog.quote_ident(option_name) || " + "' ' || pg_catalog.quote_literal(option_value) " + "FROM pg_catalog.pg_options_to_table(attfdwoptions) " + "ORDER BY option_name" + "), E',\n ') AS attfdwoptions " + "FROM pg_catalog.pg_attribute a LEFT JOIN pg_catalog.pg_type t " + "ON a.atttypid = t.oid " + "WHERE a.attrelid = '%u'::pg_catalog.oid " + "AND a.attnum > 0::pg_catalog.int2 " + "ORDER BY a.attnum", + tbinfo->dobj.catId.oid); + } + else if (fout->remoteVersion >= 90200) { /* * attfdwoptions is new in 9.2. @@ -7876,6 +7915,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) i_typstorage = PQfnumber(res, "typstorage"); i_attnotnull = PQfnumber(res, "attnotnull"); i_atthasdef = PQfnumber(res, "atthasdef"); + i_attidentity = PQfnumber(res, "attidentity"); i_attisdropped = PQfnumber(res, "attisdropped"); i_attlen = PQfnumber(res, "attlen"); i_attalign = PQfnumber(res, "attalign"); @@ -7891,6 +7931,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) tbinfo->attstattarget = (int *) pg_malloc(ntups * sizeof(int)); tbinfo->attstorage = (char *) pg_malloc(ntups * sizeof(char)); tbinfo->typstorage = (char *) pg_malloc(ntups * sizeof(char)); + tbinfo->attidentity = (char *) pg_malloc(ntups * sizeof(bool)); tbinfo->attisdropped = (bool *) pg_malloc(ntups * sizeof(bool)); tbinfo->attlen = (int *) pg_malloc(ntups * sizeof(int)); tbinfo->attalign = (char *) pg_malloc(ntups * sizeof(char)); @@ -7915,6 +7956,8 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) tbinfo->attstattarget[j] = atoi(PQgetvalue(res, j, i_attstattarget)); tbinfo->attstorage[j] = *(PQgetvalue(res, j, i_attstorage)); tbinfo->typstorage[j] = *(PQgetvalue(res, j, i_typstorage)); + tbinfo->attidentity[j] = (i_attidentity >= 0 ? *(PQgetvalue(res, j, i_attidentity)) : '\0'); + tbinfo->needs_override = tbinfo->needs_override || (tbinfo->attidentity[j] == ATTRIBUTE_IDENTITY_ALWAYS); tbinfo->attisdropped[j] = (PQgetvalue(res, j, i_attisdropped)[0] == 't'); tbinfo->attlen[j] = atoi(PQgetvalue(res, j, i_attlen)); tbinfo->attalign[j] = *(PQgetvalue(res, j, i_attalign)); @@ -16307,10 +16350,13 @@ dumpSequence(Archive *fout, TableInfo *tbinfo) /* * DROP must be fully qualified in case same name appears in pg_catalog */ - appendPQExpBuffer(delqry, "DROP SEQUENCE %s.", - fmtId(tbinfo->dobj.namespace->dobj.name)); - appendPQExpBuffer(delqry, "%s;\n", - fmtId(tbinfo->dobj.name)); + if (!tbinfo->is_identity_sequence) + { + appendPQExpBuffer(delqry, "DROP SEQUENCE %s.", + fmtId(tbinfo->dobj.namespace->dobj.name)); + appendPQExpBuffer(delqry, "%s;\n", + fmtId(tbinfo->dobj.name)); + } resetPQExpBuffer(query); @@ -16322,12 +16368,32 @@ dumpSequence(Archive *fout, TableInfo *tbinfo) tbinfo->dobj.catId.oid); } - appendPQExpBuffer(query, - "CREATE SEQUENCE %s\n", - fmtId(tbinfo->dobj.name)); + if (tbinfo->is_identity_sequence) + { + TableInfo *owning_tab = findTableByOid(tbinfo->owning_tab); - if (strcmp(seqtype, "bigint") != 0) - appendPQExpBuffer(query, " AS %s\n", seqtype); + appendPQExpBuffer(query, + "ALTER TABLE %s ", + fmtId(owning_tab->dobj.name)); + appendPQExpBuffer(query, + "ALTER COLUMN %s ADD GENERATED ", + fmtId(owning_tab->attnames[tbinfo->owning_col - 1])); + if (owning_tab->attidentity[tbinfo->owning_col - 1] == ATTRIBUTE_IDENTITY_ALWAYS) + appendPQExpBuffer(query, "ALWAYS"); + else if (owning_tab->attidentity[tbinfo->owning_col - 1] == ATTRIBUTE_IDENTITY_BY_DEFAULT) + appendPQExpBuffer(query, "BY DEFAULT"); + appendPQExpBuffer(query, " AS IDENTITY (\n SEQUENCE NAME %s\n", + fmtId(tbinfo->dobj.name)); + } + else + { + appendPQExpBuffer(query, + "CREATE SEQUENCE %s\n", + fmtId(tbinfo->dobj.name)); + + if (strcmp(seqtype, "bigint") != 0) + appendPQExpBuffer(query, " AS %s\n", seqtype); + } if (fout->remoteVersion >= 80400) appendPQExpBuffer(query, " START WITH %s\n", startv); @@ -16348,7 +16414,10 @@ dumpSequence(Archive *fout, TableInfo *tbinfo) " CACHE %s%s", cache, (cycled ? "\n CYCLE" : "")); - appendPQExpBufferStr(query, ";\n"); + if (tbinfo->is_identity_sequence) + appendPQExpBufferStr(query, "\n);\n"); + else + appendPQExpBufferStr(query, ";\n"); appendPQExpBuffer(labelq, "SEQUENCE %s", fmtId(tbinfo->dobj.name)); @@ -16381,7 +16450,7 @@ dumpSequence(Archive *fout, TableInfo *tbinfo) * We need not schema-qualify the table reference because both sequence * and table must be in the same schema. */ - if (OidIsValid(tbinfo->owning_tab)) + if (OidIsValid(tbinfo->owning_tab) && !tbinfo->is_identity_sequence) { TableInfo *owning_tab = findTableByOid(tbinfo->owning_tab); diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h index cb22f63bd6..61097e6d99 100644 --- a/src/bin/pg_dump/pg_dump.h +++ b/src/bin/pg_dump/pg_dump.h @@ -288,6 +288,7 @@ typedef struct _tableInfo /* these two are set only if table is a sequence owned by a column: */ Oid owning_tab; /* OID of table owning sequence */ int owning_col; /* attr # of column owning sequence */ + bool is_identity_sequence; int relpages; /* table's size in pages (from pg_class) */ bool interesting; /* true if need to collect more data */ @@ -306,6 +307,7 @@ typedef struct _tableInfo char *attstorage; /* attribute storage scheme */ char *typstorage; /* type storage scheme */ bool *attisdropped; /* true if attr is dropped; don't dump it */ + char *attidentity; int *attlen; /* attribute length, used by binary_upgrade */ char *attalign; /* attribute align, used by binary_upgrade */ bool *attislocal; /* true if attr has local definition */ @@ -317,6 +319,7 @@ typedef struct _tableInfo struct _attrDefInfo **attrdefs; /* DEFAULT expressions */ struct _constraintInfo *checkexprs; /* CHECK constraints */ char *partkeydef; /* partition key definition */ + bool needs_override; /* has GENERATED ALWAYS AS IDENTITY */ /* * Stuff computed only for dumpable tables. diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl index 366737440c..5030bc204c 100644 --- a/src/bin/pg_dump/t/002_pg_dump.pl +++ b/src/bin/pg_dump/t/002_pg_dump.pl @@ -2263,6 +2263,37 @@ my %tests = ( only_dump_test_table => 1, role => 1, }, }, + 'COPY test_table_identity' => { + all_runs => 1, + catch_all => 'COPY ... commands', + create_order => 54, + create_sql => +'INSERT INTO dump_test.test_table_identity (col2) VALUES (\'test\');', + regexp => qr/^ + \QCOPY test_table_identity (col1, col2) FROM stdin;\E + \n1\ttest\n\\\.\n + /xm, + like => { + clean => 1, + clean_if_exists => 1, + createdb => 1, + data_only => 1, + defaults => 1, + exclude_test_table => 1, + exclude_test_table_data => 1, + no_blobs => 1, + no_privs => 1, + no_owner => 1, + only_dump_test_schema => 1, + pg_dumpall_dbprivs => 1, + section_data => 1, + test_schema_plus_blobs => 1, + with_oids => 1, }, + unlike => { + exclude_dump_test_schema => 1, + only_dump_test_table => 1, + role => 1, }, }, + 'COPY ... commands' => { # catch-all for COPY all_runs => 0, # catch-all regexp => qr/^COPY /m, @@ -2318,6 +2349,14 @@ qr/^\QINSERT INTO test_fifth_table (col1, col2, col3, col4, col5) VALUES (NULL, like => { column_inserts => 1, }, unlike => {}, }, + 'INSERT INTO test_table_identity' => { + all_runs => 1, + catch_all => 'INSERT INTO ...', + regexp => +qr/^\QINSERT INTO test_table_identity (col1, col2) OVERRIDING SYSTEM VALUE VALUES (1, 'test');\E/m, + like => { column_inserts => 1, }, + unlike => {}, }, + # INSERT INTO catch-all 'INSERT INTO ...' => { all_runs => 0, # catch-all @@ -4709,6 +4748,54 @@ qr/CREATE TRANSFORM FOR integer LANGUAGE sql \(FROM SQL WITH FUNCTION pg_catalog role => 1, section_post_data => 1, }, }, + 'CREATE TABLE test_table_identity' => { + all_runs => 1, + catch_all => 'CREATE ... commands', + create_order => 3, + create_sql => 'CREATE TABLE dump_test.test_table_identity ( + col1 int generated always as identity primary key, + col2 text + );', + regexp => qr/^ + \QCREATE TABLE test_table_identity (\E\n + \s+\Qcol1 integer NOT NULL,\E\n + \s+\Qcol2 text\E\n + \); + .* + \QALTER TABLE test_table_identity ALTER COLUMN col1 ADD GENERATED ALWAYS AS IDENTITY (\E\n + \s+\QSEQUENCE NAME test_table_identity_col1_seq\E\n + \s+\QSTART WITH 1\E\n + \s+\QINCREMENT BY 1\E\n + \s+\QNO MINVALUE\E\n + \s+\QNO MAXVALUE\E\n + \s+\QCACHE 1\E\n + \); + /xms, + like => { + binary_upgrade => 1, + clean => 1, + clean_if_exists => 1, + createdb => 1, + defaults => 1, + exclude_test_table => 1, + exclude_test_table_data => 1, + no_blobs => 1, + no_privs => 1, + no_owner => 1, + only_dump_test_schema => 1, + pg_dumpall_dbprivs => 1, + schema_only => 1, + section_pre_data => 1, + test_schema_plus_blobs => 1, + with_oids => 1, }, + unlike => { + exclude_dump_test_schema => 1, + only_dump_test_table => 1, + pg_dumpall_globals => 1, + pg_dumpall_globals_clean => 1, + role => 1, + section_post_data => 1, }, }, + 'CREATE SEQUENCE test_table_col1_seq' => { all_runs => 1, catch_all => 'CREATE ... commands', diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index 2ef06261e6..ddb3942e95 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -14,6 +14,7 @@ #include +#include "catalog/pg_attribute.h" #include "catalog/pg_class.h" #include "catalog/pg_default_acl.h" #include "fe_utils/string_utils.h" @@ -1592,6 +1593,10 @@ describeOneTableDetails(const char *schemaname, " WHERE c.oid = a.attcollation AND t.oid = a.atttypid AND a.attcollation <> t.typcollation) AS attcollation"); else appendPQExpBufferStr(&buf, "\n NULL AS attcollation"); + if (pset.sversion >= 100000) + appendPQExpBufferStr(&buf, ", a.attidentity"); + else + appendPQExpBufferStr(&buf, ", ''::\"char\" AS attidentity"); if (tableinfo.relkind == RELKIND_INDEX) appendPQExpBufferStr(&buf, ",\n pg_catalog.pg_get_indexdef(a.attrelid, a.attnum, TRUE) AS indexdef"); else @@ -1778,12 +1783,24 @@ describeOneTableDetails(const char *schemaname, /* Collation, Nullable, Default */ if (show_column_details) { + char *identity; + char *default_str = ""; + printTableAddCell(&cont, PQgetvalue(res, i, 5), false, false); printTableAddCell(&cont, strcmp(PQgetvalue(res, i, 3), "t") == 0 ? "not null" : "", false, false); - /* (note: above we cut off the 'default' string at 128) */ - printTableAddCell(&cont, PQgetvalue(res, i, 2), false, false); + identity = PQgetvalue(res, i, 6); + + if (!identity[0]) + /* (note: above we cut off the 'default' string at 128) */ + default_str = PQgetvalue(res, i, 2); + else if (identity[0] == ATTRIBUTE_IDENTITY_ALWAYS) + default_str = "generated always as identity"; + else if (identity[0] == ATTRIBUTE_IDENTITY_BY_DEFAULT) + default_str = "generated by default as identity"; + + printTableAddCell(&cont, default_str, false, false); } /* Value: for sequences only */ @@ -1792,16 +1809,16 @@ describeOneTableDetails(const char *schemaname, /* Expression for index column */ if (tableinfo.relkind == RELKIND_INDEX) - printTableAddCell(&cont, PQgetvalue(res, i, 6), false, false); + printTableAddCell(&cont, PQgetvalue(res, i, 7), false, false); /* FDW options for foreign table column, only for 9.2 or later */ if (tableinfo.relkind == RELKIND_FOREIGN_TABLE && pset.sversion >= 90200) - printTableAddCell(&cont, PQgetvalue(res, i, 7), false, false); + printTableAddCell(&cont, PQgetvalue(res, i, 8), false, false); /* Storage and Description */ if (verbose) { - int firstvcol = 8; + int firstvcol = 9; char *storage = PQgetvalue(res, i, firstvcol); /* these strings are literal in our syntax, so not translated. */ diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index 813f719b9b..9dbb4a0074 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -1895,7 +1895,7 @@ psql_completion(const char *text, int start, int end) /* ALTER TABLE ALTER [COLUMN] */ else if (Matches6("ALTER", "TABLE", MatchAny, "ALTER", "COLUMN", MatchAny) || Matches5("ALTER", "TABLE", MatchAny, "ALTER", MatchAny)) - COMPLETE_WITH_LIST4("TYPE", "SET", "RESET", "DROP"); + COMPLETE_WITH_LIST6("TYPE", "SET", "RESET", "RESTART", "ADD", "DROP"); /* ALTER TABLE ALTER [COLUMN] SET */ else if (Matches7("ALTER", "TABLE", MatchAny, "ALTER", "COLUMN", MatchAny, "SET") || Matches6("ALTER", "TABLE", MatchAny, "ALTER", MatchAny, "SET")) @@ -1911,7 +1911,7 @@ psql_completion(const char *text, int start, int end) /* ALTER TABLE ALTER [COLUMN] DROP */ else if (Matches7("ALTER", "TABLE", MatchAny, "ALTER", "COLUMN", MatchAny, "DROP") || Matches6("ALTER", "TABLE", MatchAny, "ALTER", MatchAny, "DROP")) - COMPLETE_WITH_LIST2("DEFAULT", "NOT NULL"); + COMPLETE_WITH_LIST3("DEFAULT", "IDENTITY", "NOT NULL"); else if (Matches4("ALTER", "TABLE", MatchAny, "CLUSTER")) COMPLETE_WITH_CONST("ON"); else if (Matches5("ALTER", "TABLE", MatchAny, "CLUSTER", "ON")) @@ -2920,17 +2920,25 @@ psql_completion(const char *text, int start, int end) /* * Complete INSERT INTO with "(" or "VALUES" or "SELECT" or - * "TABLE" or "DEFAULT VALUES" + * "TABLE" or "DEFAULT VALUES" or "OVERRIDING" */ else if (TailMatches3("INSERT", "INTO", MatchAny)) - COMPLETE_WITH_LIST5("(", "DEFAULT VALUES", "SELECT", "TABLE", "VALUES"); + COMPLETE_WITH_LIST6("(", "DEFAULT VALUES", "SELECT", "TABLE", "VALUES", "OVERRIDING"); /* * Complete INSERT INTO
    (attribs) with "VALUES" or "SELECT" or - * "TABLE" + * "TABLE" or "OVERRIDING" */ else if (TailMatches4("INSERT", "INTO", MatchAny, MatchAny) && ends_with(prev_wd, ')')) + COMPLETE_WITH_LIST4("SELECT", "TABLE", "VALUES", "OVERRIDING"); + + /* Complete OVERRIDING */ + else if (TailMatches1("OVERRIDING")) + COMPLETE_WITH_LIST2("SYSTEM VALUE", "USER VALUE"); + + /* Complete after OVERRIDING clause */ + else if (TailMatches3("OVERRIDING", MatchAny, "VALUE")) COMPLETE_WITH_LIST3("SELECT", "TABLE", "VALUES"); /* Insert an open parenthesis after "VALUES" */ diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 1db7a4d715..4dd571a2e2 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201704012 +#define CATALOG_VERSION_NO 201704061 #endif diff --git a/src/include/catalog/dependency.h b/src/include/catalog/dependency.h index 9effbce2f1..33361ffce9 100644 --- a/src/include/catalog/dependency.h +++ b/src/include/catalog/dependency.h @@ -238,11 +238,9 @@ extern long changeDependencyFor(Oid classId, Oid objectId, extern Oid getExtensionOfObject(Oid classId, Oid objectId); -extern bool sequenceIsOwned(Oid seqId, Oid *tableId, int32 *colId); - -extern void markSequenceUnowned(Oid seqId); - -extern List *getOwnedSequences(Oid relid); +extern bool sequenceIsOwned(Oid seqId, char deptype, Oid *tableId, int32 *colId); +extern List *getOwnedSequences(Oid relid, AttrNumber attnum); +extern Oid getOwnedSequence(Oid relid, AttrNumber attnum); extern Oid get_constraint_index(Oid constraintId); diff --git a/src/include/catalog/pg_attribute.h b/src/include/catalog/pg_attribute.h index e861f4c589..753d45f2d1 100644 --- a/src/include/catalog/pg_attribute.h +++ b/src/include/catalog/pg_attribute.h @@ -133,6 +133,9 @@ CATALOG(pg_attribute,1249) BKI_BOOTSTRAP BKI_WITHOUT_OIDS BKI_ROWTYPE_OID(75) BK /* Has DEFAULT value or not */ bool atthasdef; + /* One of the ATTRIBUTE_IDENTITY_* constants below, or '\0' */ + char attidentity; + /* Is dropped (ie, logically invisible) or not */ bool attisdropped; @@ -188,7 +191,7 @@ typedef FormData_pg_attribute *Form_pg_attribute; * ---------------- */ -#define Natts_pg_attribute 21 +#define Natts_pg_attribute 22 #define Anum_pg_attribute_attrelid 1 #define Anum_pg_attribute_attname 2 #define Anum_pg_attribute_atttypid 3 @@ -203,13 +206,14 @@ typedef FormData_pg_attribute *Form_pg_attribute; #define Anum_pg_attribute_attalign 12 #define Anum_pg_attribute_attnotnull 13 #define Anum_pg_attribute_atthasdef 14 -#define Anum_pg_attribute_attisdropped 15 -#define Anum_pg_attribute_attislocal 16 -#define Anum_pg_attribute_attinhcount 17 -#define Anum_pg_attribute_attcollation 18 -#define Anum_pg_attribute_attacl 19 -#define Anum_pg_attribute_attoptions 20 -#define Anum_pg_attribute_attfdwoptions 21 +#define Anum_pg_attribute_attidentity 15 +#define Anum_pg_attribute_attisdropped 16 +#define Anum_pg_attribute_attislocal 17 +#define Anum_pg_attribute_attinhcount 18 +#define Anum_pg_attribute_attcollation 19 +#define Anum_pg_attribute_attacl 20 +#define Anum_pg_attribute_attoptions 21 +#define Anum_pg_attribute_attfdwoptions 22 /* ---------------- @@ -220,4 +224,8 @@ typedef FormData_pg_attribute *Form_pg_attribute; * ---------------- */ + +#define ATTRIBUTE_IDENTITY_ALWAYS 'a' +#define ATTRIBUTE_IDENTITY_BY_DEFAULT 'd' + #endif /* PG_ATTRIBUTE_H */ diff --git a/src/include/catalog/pg_class.h b/src/include/catalog/pg_class.h index d1d493ee05..5a288830e8 100644 --- a/src/include/catalog/pg_class.h +++ b/src/include/catalog/pg_class.h @@ -149,7 +149,7 @@ typedef FormData_pg_class *Form_pg_class; */ DATA(insert OID = 1247 ( pg_type PGNSP 71 0 PGUID 0 0 0 0 0 0 0 f f p r 30 0 t f f f f f f t n f 3 1 _null_ _null_ _null_)); DESCR(""); -DATA(insert OID = 1249 ( pg_attribute PGNSP 75 0 PGUID 0 0 0 0 0 0 0 f f p r 21 0 f f f f f f f t n f 3 1 _null_ _null_ _null_)); +DATA(insert OID = 1249 ( pg_attribute PGNSP 75 0 PGUID 0 0 0 0 0 0 0 f f p r 22 0 f f f f f f f t n f 3 1 _null_ _null_ _null_)); DESCR(""); DATA(insert OID = 1255 ( pg_proc PGNSP 81 0 PGUID 0 0 0 0 0 0 0 f f p r 29 0 t f f f f f f t n f 3 1 _null_ _null_ _null_)); DESCR(""); diff --git a/src/include/commands/sequence.h b/src/include/commands/sequence.h index 49a77c42fc..304586e48e 100644 --- a/src/include/commands/sequence.h +++ b/src/include/commands/sequence.h @@ -51,7 +51,9 @@ typedef struct xl_seq_rec /* SEQUENCE TUPLE DATA FOLLOWS AT THE END */ } xl_seq_rec; +extern int64 nextval_internal(Oid relid, bool check_permissions); extern Datum nextval(PG_FUNCTION_ARGS); +extern List *sequence_options(Oid relid); extern ObjectAddress DefineSequence(ParseState *pstate, CreateSeqStmt *stmt); extern ObjectAddress AlterSequence(ParseState *pstate, AlterSeqStmt *stmt); diff --git a/src/include/executor/execExpr.h b/src/include/executor/execExpr.h index a665388232..86fdb33cd3 100644 --- a/src/include/executor/execExpr.h +++ b/src/include/executor/execExpr.h @@ -144,6 +144,7 @@ typedef enum ExprEvalOp EEOP_NULLIF, EEOP_SQLVALUEFUNCTION, EEOP_CURRENTOFEXPR, + EEOP_NEXTVALUEEXPR, EEOP_ARRAYEXPR, EEOP_ARRAYCOERCE, EEOP_ROW, @@ -361,6 +362,13 @@ typedef struct ExprEvalStep SQLValueFunction *svf; } sqlvaluefunction; + /* for EEOP_NEXTVALUEXPR */ + struct + { + Oid seqid; + Oid seqtypid; + } nextvalueexpr; + /* for EEOP_ARRAYEXPR */ struct { diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index 177853b3bf..f59d719923 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -190,6 +190,7 @@ typedef enum NodeTag T_FromExpr, T_OnConflictExpr, T_IntoClause, + T_NextValueExpr, /* * TAGS FOR EXPRESSION STATE NODES (execnodes.h) diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index b2afd50818..9f57388781 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -27,6 +27,13 @@ #include "nodes/primnodes.h" #include "nodes/value.h" +typedef enum OverridingKind +{ + OVERRIDING_NOT_SET = 0, + OVERRIDING_USER_VALUE, + OVERRIDING_SYSTEM_VALUE +} OverridingKind; + /* Possible sources of a Query */ typedef enum QuerySource { @@ -130,6 +137,8 @@ typedef struct Query List *targetList; /* target list (of TargetEntry) */ + OverridingKind override; /* OVERRIDING clause */ + OnConflictExpr *onConflict; /* ON CONFLICT DO [NOTHING | UPDATE] */ List *returningList; /* return-values list (of TargetEntry) */ @@ -637,6 +646,7 @@ typedef struct ColumnDef char storage; /* attstorage setting, or 0 for default */ Node *raw_default; /* default value (untransformed parse tree) */ Node *cooked_default; /* default value (transformed expr tree) */ + char identity; /* attidentity setting */ CollateClause *collClause; /* untransformed COLLATE spec, if any */ Oid collOid; /* collation OID (InvalidOid if not set) */ List *constraints; /* other constraints on column */ @@ -658,9 +668,10 @@ typedef enum TableLikeOption { CREATE_TABLE_LIKE_DEFAULTS = 1 << 0, CREATE_TABLE_LIKE_CONSTRAINTS = 1 << 1, - CREATE_TABLE_LIKE_INDEXES = 1 << 2, - CREATE_TABLE_LIKE_STORAGE = 1 << 3, - CREATE_TABLE_LIKE_COMMENTS = 1 << 4, + CREATE_TABLE_LIKE_IDENTITY = 1 << 2, + CREATE_TABLE_LIKE_INDEXES = 1 << 3, + CREATE_TABLE_LIKE_STORAGE = 1 << 4, + CREATE_TABLE_LIKE_COMMENTS = 1 << 5, CREATE_TABLE_LIKE_ALL = PG_INT32_MAX } TableLikeOption; @@ -1403,6 +1414,7 @@ typedef struct InsertStmt OnConflictClause *onConflictClause; /* ON CONFLICT clause */ List *returningList; /* list of expressions to return */ WithClause *withClause; /* WITH clause */ + OverridingKind override; /* OVERRIDING clause */ } InsertStmt; /* ---------------------- @@ -1713,7 +1725,10 @@ typedef enum AlterTableType AT_NoForceRowSecurity, /* NO FORCE ROW SECURITY */ AT_GenericOptions, /* OPTIONS (...) */ AT_AttachPartition, /* ATTACH PARTITION */ - AT_DetachPartition /* DETACH PARTITION */ + AT_DetachPartition, /* DETACH PARTITION */ + AT_AddIdentity, /* ADD IDENTITY */ + AT_SetIdentity, /* SET identity column options */ + AT_DropIdentity /* DROP IDENTITY */ } AlterTableType; typedef struct ReplicaIdentityStmt @@ -2000,6 +2015,7 @@ typedef enum ConstrType /* types of constraints */ * expect it */ CONSTR_NOTNULL, CONSTR_DEFAULT, + CONSTR_IDENTITY, CONSTR_CHECK, CONSTR_PRIMARY, CONSTR_UNIQUE, @@ -2038,6 +2054,7 @@ typedef struct Constraint bool is_no_inherit; /* is constraint non-inheritable? */ Node *raw_expr; /* expr, as untransformed parse tree */ char *cooked_expr; /* expr, as nodeToString representation */ + char generated_when; /* Fields used for unique constraints (UNIQUE and PRIMARY KEY): */ List *keys; /* String nodes naming referenced column(s) */ @@ -2416,6 +2433,7 @@ typedef struct CreateSeqStmt RangeVar *sequence; /* the sequence to create */ List *options; Oid ownerId; /* ID of owner, or InvalidOid for default */ + bool for_identity; bool if_not_exists; /* just do nothing if it already exists? */ } CreateSeqStmt; @@ -2424,6 +2442,7 @@ typedef struct AlterSeqStmt NodeTag type; RangeVar *sequence; /* the sequence to alter */ List *options; + bool for_identity; bool missing_ok; /* skip error if a role is missing? */ } AlterSeqStmt; diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h index d57b4fab3d..b87fe84545 100644 --- a/src/include/nodes/primnodes.h +++ b/src/include/nodes/primnodes.h @@ -1292,6 +1292,20 @@ typedef struct InferenceElem Oid inferopclass; /* OID of att opclass, or InvalidOid */ } InferenceElem; +/* + * NextValueExpr - get next value from sequence + * + * This has the same effect as calling the nextval() function, but it does not + * check permissions on the sequence. This is used for identity columns, + * where the sequence is an implicit dependency without its own permissions. + */ +typedef struct NextValueExpr +{ + Expr xpr; + Oid seqid; + Oid typeId; +} NextValueExpr; + /*-------------------- * TargetEntry - * a target entry (used in query target lists) diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h index cd21a789d5..37542aaee4 100644 --- a/src/include/parser/kwlist.h +++ b/src/include/parser/kwlist.h @@ -174,6 +174,7 @@ PG_KEYWORD("from", FROM, RESERVED_KEYWORD) PG_KEYWORD("full", FULL, TYPE_FUNC_NAME_KEYWORD) PG_KEYWORD("function", FUNCTION, UNRESERVED_KEYWORD) PG_KEYWORD("functions", FUNCTIONS, UNRESERVED_KEYWORD) +PG_KEYWORD("generated", GENERATED, UNRESERVED_KEYWORD) PG_KEYWORD("global", GLOBAL, UNRESERVED_KEYWORD) PG_KEYWORD("grant", GRANT, RESERVED_KEYWORD) PG_KEYWORD("granted", GRANTED, UNRESERVED_KEYWORD) @@ -287,6 +288,7 @@ PG_KEYWORD("outer", OUTER_P, TYPE_FUNC_NAME_KEYWORD) PG_KEYWORD("over", OVER, UNRESERVED_KEYWORD) PG_KEYWORD("overlaps", OVERLAPS, TYPE_FUNC_NAME_KEYWORD) PG_KEYWORD("overlay", OVERLAY, COL_NAME_KEYWORD) +PG_KEYWORD("overriding", OVERRIDING, UNRESERVED_KEYWORD) PG_KEYWORD("owned", OWNED, UNRESERVED_KEYWORD) PG_KEYWORD("owner", OWNER, UNRESERVED_KEYWORD) PG_KEYWORD("parallel", PARALLEL, UNRESERVED_KEYWORD) diff --git a/src/include/utils/lsyscache.h b/src/include/utils/lsyscache.h index b6d1fca2fa..88629d99aa 100644 --- a/src/include/utils/lsyscache.h +++ b/src/include/utils/lsyscache.h @@ -64,6 +64,7 @@ extern Oid get_opfamily_proc(Oid opfamily, Oid lefttype, Oid righttype, extern char *get_attname(Oid relid, AttrNumber attnum); extern char *get_relid_attribute_name(Oid relid, AttrNumber attnum); extern AttrNumber get_attnum(Oid relid, const char *attname); +extern char get_attidentity(Oid relid, AttrNumber attnum); extern Oid get_atttype(Oid relid, AttrNumber attnum); extern int32 get_atttypmod(Oid relid, AttrNumber attnum); extern void get_atttypetypmodcoll(Oid relid, AttrNumber attnum, diff --git a/src/test/regress/expected/create_table_like.out b/src/test/regress/expected/create_table_like.out index a25b221703..3f405c94ce 100644 --- a/src/test/regress/expected/create_table_like.out +++ b/src/test/regress/expected/create_table_like.out @@ -66,6 +66,53 @@ SELECT * FROM inhg; /* Two records with three columns in order x=x, xx=text, y=y (2 rows) DROP TABLE inhg; +CREATE TABLE test_like_id_1 (a int GENERATED ALWAYS AS IDENTITY, b text); +\d test_like_id_1 + Table "public.test_like_id_1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+------------------------------ + a | integer | | not null | generated always as identity + b | text | | | + +INSERT INTO test_like_id_1 (b) VALUES ('b1'); +SELECT * FROM test_like_id_1; + a | b +---+---- + 1 | b1 +(1 row) + +CREATE TABLE test_like_id_2 (LIKE test_like_id_1); +\d test_like_id_2 + Table "public.test_like_id_2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | not null | + b | text | | | + +INSERT INTO test_like_id_2 (b) VALUES ('b2'); +ERROR: null value in column "a" violates not-null constraint +DETAIL: Failing row contains (null, b2). +SELECT * FROM test_like_id_2; -- identity was not copied + a | b +---+--- +(0 rows) + +CREATE TABLE test_like_id_3 (LIKE test_like_id_1 INCLUDING IDENTITY); +\d test_like_id_3 + Table "public.test_like_id_3" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+------------------------------ + a | integer | | not null | generated always as identity + b | text | | | + +INSERT INTO test_like_id_3 (b) VALUES ('b3'); +SELECT * FROM test_like_id_3; -- identity was copied and applied + a | b +---+---- + 1 | b3 +(1 row) + +DROP TABLE test_like_id_1, test_like_id_2, test_like_id_3; CREATE TABLE inhg (x text, LIKE inhx INCLUDING INDEXES, y text); /* copies indexes */ INSERT INTO inhg VALUES (5, 10); INSERT INTO inhg VALUES (20, 10); -- should fail diff --git a/src/test/regress/expected/identity.out b/src/test/regress/expected/identity.out new file mode 100644 index 0000000000..88b56dad93 --- /dev/null +++ b/src/test/regress/expected/identity.out @@ -0,0 +1,322 @@ +-- sanity check of system catalog +SELECT attrelid, attname, attidentity FROM pg_attribute WHERE attidentity NOT IN ('', 'a', 'd'); + attrelid | attname | attidentity +----------+---------+------------- +(0 rows) + +CREATE TABLE itest1 (a int generated by default as identity, b text); +CREATE TABLE itest2 (a bigint generated always as identity, b text); +CREATE TABLE itest3 (a smallint generated by default as identity (start with 7 increment by 5), b text); +ALTER TABLE itest3 ALTER COLUMN a ADD GENERATED ALWAYS AS IDENTITY; -- error +ERROR: column "a" of relation "itest3" is already an identity column +SELECT table_name, column_name, column_default, is_nullable, is_identity, identity_generation, identity_start, identity_increment, identity_maximum, identity_minimum, identity_cycle FROM information_schema.columns WHERE table_name LIKE 'itest_' ORDER BY 1, 2; + table_name | column_name | column_default | is_nullable | is_identity | identity_generation | identity_start | identity_increment | identity_maximum | identity_minimum | identity_cycle +------------+-------------+----------------+-------------+-------------+---------------------+----------------+--------------------+---------------------+------------------+---------------- + itest1 | a | | NO | YES | BY DEFAULT | 1 | 1 | 2147483647 | 1 | NO + itest1 | b | | YES | NO | | | | | | NO + itest2 | a | | NO | YES | ALWAYS | 1 | 1 | 9223372036854775807 | 1 | NO + itest2 | b | | YES | NO | | | | | | NO + itest3 | a | | NO | YES | BY DEFAULT | 7 | 5 | 32767 | 1 | NO + itest3 | b | | YES | NO | | | | | | NO +(6 rows) + +-- internal sequences should not be shown here +SELECT sequence_name FROM information_schema.sequences WHERE sequence_name LIKE 'itest%'; + sequence_name +--------------- +(0 rows) + +CREATE TABLE itest4 (a int, b text); +ALTER TABLE itest4 ALTER COLUMN a ADD GENERATED ALWAYS AS IDENTITY; -- error, requires NOT NULL +ERROR: column "a" of relation "itest4" must be declared NOT NULL before identity can be added +ALTER TABLE itest4 ALTER COLUMN a SET NOT NULL; +ALTER TABLE itest4 ALTER COLUMN a ADD GENERATED ALWAYS AS IDENTITY; -- ok +ALTER TABLE itest4 ALTER COLUMN a DROP NOT NULL; -- error, disallowed +ERROR: column "a" of relation "itest4" is an identity column +ALTER TABLE itest4 ALTER COLUMN a ADD GENERATED ALWAYS AS IDENTITY; -- error, already set +ERROR: column "a" of relation "itest4" is already an identity column +ALTER TABLE itest4 ALTER COLUMN b ADD GENERATED ALWAYS AS IDENTITY; -- error, wrong data type +ERROR: identity column type must be smallint, integer, or bigint +-- for later +ALTER TABLE itest4 ALTER COLUMN b SET DEFAULT ''; +-- invalid column type +CREATE TABLE itest_err_1 (a text generated by default as identity); +ERROR: identity column type must be smallint, integer, or bigint +-- duplicate identity +CREATE TABLE itest_err_2 (a int generated always as identity generated by default as identity); +ERROR: multiple identity specifications for column "a" of table "itest_err_2" +LINE 1: ...E itest_err_2 (a int generated always as identity generated ... + ^ +-- cannot have default and identity +CREATE TABLE itest_err_3 (a int default 5 generated by default as identity); +ERROR: both default and identity specified for column "a" of table "itest_err_3" +LINE 1: CREATE TABLE itest_err_3 (a int default 5 generated by defau... + ^ +-- cannot combine serial and identity +CREATE TABLE itest_err_4 (a serial generated by default as identity); +ERROR: both default and identity specified for column "a" of table "itest_err_4" +INSERT INTO itest1 DEFAULT VALUES; +INSERT INTO itest1 DEFAULT VALUES; +INSERT INTO itest2 DEFAULT VALUES; +INSERT INTO itest2 DEFAULT VALUES; +INSERT INTO itest3 DEFAULT VALUES; +INSERT INTO itest3 DEFAULT VALUES; +INSERT INTO itest4 DEFAULT VALUES; +INSERT INTO itest4 DEFAULT VALUES; +SELECT * FROM itest1; + a | b +---+--- + 1 | + 2 | +(2 rows) + +SELECT * FROM itest2; + a | b +---+--- + 1 | + 2 | +(2 rows) + +SELECT * FROM itest3; + a | b +----+--- + 7 | + 12 | +(2 rows) + +SELECT * FROM itest4; + a | b +---+--- + 1 | + 2 | +(2 rows) + +-- OVERRIDING tests +INSERT INTO itest1 VALUES (10, 'xyz'); +INSERT INTO itest1 OVERRIDING USER VALUE VALUES (10, 'xyz'); +SELECT * FROM itest1; + a | b +----+----- + 1 | + 2 | + 10 | xyz + 3 | xyz +(4 rows) + +INSERT INTO itest2 VALUES (10, 'xyz'); +ERROR: cannot insert into column "a" +DETAIL: Column "a" is an identity column defined as GENERATED ALWAYS. +HINT: Use OVERRIDING SYSTEM VALUE to override. +INSERT INTO itest2 OVERRIDING SYSTEM VALUE VALUES (10, 'xyz'); +SELECT * FROM itest2; + a | b +----+----- + 1 | + 2 | + 10 | xyz +(3 rows) + +-- UPDATE tests +UPDATE itest1 SET a = 101 WHERE a = 1; +UPDATE itest1 SET a = DEFAULT WHERE a = 2; +SELECT * FROM itest1; + a | b +-----+----- + 10 | xyz + 3 | xyz + 101 | + 4 | +(4 rows) + +UPDATE itest2 SET a = 101 WHERE a = 1; +ERROR: column "a" can only be updated to DEFAULT +DETAIL: Column "a" is an identity column defined as GENERATED ALWAYS. +UPDATE itest2 SET a = DEFAULT WHERE a = 2; +SELECT * FROM itest2; + a | b +----+----- + 1 | + 10 | xyz + 3 | +(3 rows) + +-- DROP IDENTITY tests +ALTER TABLE itest4 ALTER COLUMN a DROP IDENTITY; +ALTER TABLE itest4 ALTER COLUMN a DROP IDENTITY; -- error +ERROR: column "a" of relation "itest4" is not an identity column +ALTER TABLE itest4 ALTER COLUMN a DROP IDENTITY IF EXISTS; -- noop +NOTICE: column "a" of relation "itest4" is not an identity column, skipping +INSERT INTO itest4 DEFAULT VALUES; -- fails because NOT NULL is not dropped +ERROR: null value in column "a" violates not-null constraint +DETAIL: Failing row contains (null, ). +ALTER TABLE itest4 ALTER COLUMN a DROP NOT NULL; +INSERT INTO itest4 DEFAULT VALUES; +SELECT * FROM itest4; + a | b +---+--- + 1 | + 2 | + | +(3 rows) + +-- check that sequence is removed +SELECT sequence_name FROM itest4_a_seq; +ERROR: relation "itest4_a_seq" does not exist +LINE 1: SELECT sequence_name FROM itest4_a_seq; + ^ +-- test views +CREATE TABLE itest10 (a int generated by default as identity, b text); +CREATE TABLE itest11 (a int generated always as identity, b text); +CREATE VIEW itestv10 AS SELECT * FROM itest10; +CREATE VIEW itestv11 AS SELECT * FROM itest11; +INSERT INTO itestv10 DEFAULT VALUES; +INSERT INTO itestv10 DEFAULT VALUES; +INSERT INTO itestv11 DEFAULT VALUES; +INSERT INTO itestv11 DEFAULT VALUES; +SELECT * FROM itestv10; + a | b +---+--- + 1 | + 2 | +(2 rows) + +SELECT * FROM itestv11; + a | b +---+--- + 1 | + 2 | +(2 rows) + +INSERT INTO itestv10 VALUES (10, 'xyz'); +INSERT INTO itestv10 OVERRIDING USER VALUE VALUES (11, 'xyz'); +SELECT * FROM itestv10; + a | b +----+----- + 1 | + 2 | + 10 | xyz + 3 | xyz +(4 rows) + +INSERT INTO itestv11 VALUES (10, 'xyz'); +ERROR: cannot insert into column "a" +DETAIL: Column "a" is an identity column defined as GENERATED ALWAYS. +HINT: Use OVERRIDING SYSTEM VALUE to override. +INSERT INTO itestv11 OVERRIDING SYSTEM VALUE VALUES (11, 'xyz'); +SELECT * FROM itestv11; + a | b +----+----- + 1 | + 2 | + 11 | xyz +(3 rows) + +-- various ALTER COLUMN tests +-- fail, not allowed for identity columns +ALTER TABLE itest1 ALTER COLUMN a SET DEFAULT 1; +ERROR: column "a" of relation "itest1" is an identity column +-- fail, not allowed, already has a default +CREATE TABLE itest5 (a serial, b text); +ALTER TABLE itest5 ALTER COLUMN a ADD GENERATED ALWAYS AS IDENTITY; +ERROR: column "a" of relation "itest5" already has a default value +ALTER TABLE itest3 ALTER COLUMN a TYPE int; +SELECT seqtypid::regtype FROM pg_sequence WHERE seqrelid = 'itest3_a_seq'::regclass; + seqtypid +---------- + integer +(1 row) + +\d itest3 + Table "public.itest3" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+---------------------------------- + a | integer | | not null | generated by default as identity + b | text | | | + +ALTER TABLE itest3 ALTER COLUMN a TYPE text; -- error +ERROR: identity column type must be smallint, integer, or bigint +-- ALTER COLUMN ... SET +CREATE TABLE itest6 (a int GENERATED ALWAYS AS IDENTITY, b text); +INSERT INTO itest6 DEFAULT VALUES; +ALTER TABLE itest6 ALTER COLUMN a SET GENERATED BY DEFAULT SET INCREMENT BY 2 SET START WITH 100 RESTART; +INSERT INTO itest6 DEFAULT VALUES; +INSERT INTO itest6 DEFAULT VALUES; +SELECT * FROM itest6; + a | b +-----+--- + 1 | + 100 | + 102 | +(3 rows) + +SELECT table_name, column_name, is_identity, identity_generation FROM information_schema.columns WHERE table_name = 'itest6'; + table_name | column_name | is_identity | identity_generation +------------+-------------+-------------+--------------------- + itest6 | a | YES | BY DEFAULT + itest6 | b | NO | +(2 rows) + +ALTER TABLE itest6 ALTER COLUMN b SET INCREMENT BY 2; -- fail, not identity +ERROR: column "b" of relation "itest6" is not an identity column +-- prohibited direct modification of sequence +ALTER SEQUENCE itest6_a_seq OWNED BY NONE; +ERROR: cannot change ownership of identity sequence +DETAIL: Sequence "itest6_a_seq" is linked to table "itest6". +-- inheritance +CREATE TABLE itest7 (a int GENERATED ALWAYS AS IDENTITY); +INSERT INTO itest7 DEFAULT VALUES; +SELECT * FROM itest7; + a +--- + 1 +(1 row) + +-- identity property is not inherited +CREATE TABLE itest7a (b text) INHERITS (itest7); +-- make column identity in child table +CREATE TABLE itest7b (a int); +CREATE TABLE itest7c (a int GENERATED ALWAYS AS IDENTITY) INHERITS (itest7b); +NOTICE: merging column "a" with inherited definition +INSERT INTO itest7c DEFAULT VALUES; +SELECT * FROM itest7c; + a +--- + 1 +(1 row) + +CREATE TABLE itest7d (a int not null); +CREATE TABLE itest7e () INHERITS (itest7d); +ALTER TABLE itest7d ALTER COLUMN a ADD GENERATED ALWAYS AS IDENTITY; +ALTER TABLE itest7d ADD COLUMN b int GENERATED ALWAYS AS IDENTITY; -- error +ERROR: cannot recursively add identity column to table that has child tables +SELECT table_name, column_name, is_nullable, is_identity, identity_generation FROM information_schema.columns WHERE table_name LIKE 'itest7%' ORDER BY 1, 2; + table_name | column_name | is_nullable | is_identity | identity_generation +------------+-------------+-------------+-------------+--------------------- + itest7 | a | NO | YES | ALWAYS + itest7a | a | NO | NO | + itest7a | b | YES | NO | + itest7b | a | YES | NO | + itest7c | a | NO | YES | ALWAYS + itest7d | a | NO | YES | ALWAYS + itest7e | a | NO | NO | +(7 rows) + +-- These ALTER TABLE variants will not recurse. +ALTER TABLE itest7 ALTER COLUMN a SET GENERATED BY DEFAULT; +ALTER TABLE itest7 ALTER COLUMN a RESTART; +ALTER TABLE itest7 ALTER COLUMN a DROP IDENTITY; +-- privileges +CREATE USER regress_user1; +CREATE TABLE itest8 (a int GENERATED ALWAYS AS IDENTITY, b text); +GRANT SELECT, INSERT ON itest8 TO regress_user1; +SET ROLE regress_user1; +INSERT INTO itest8 DEFAULT VALUES; +SELECT * FROM itest8; + a | b +---+--- + 1 | +(1 row) + +RESET ROLE; +DROP TABLE itest8; +DROP USER regress_user1; diff --git a/src/test/regress/expected/sequence.out b/src/test/regress/expected/sequence.out index 1d8d02b800..16c12f3434 100644 --- a/src/test/regress/expected/sequence.out +++ b/src/test/regress/expected/sequence.out @@ -20,8 +20,8 @@ ERROR: CACHE (0) must be greater than zero CREATE SEQUENCE sequence_testx OWNED BY nobody; -- nonsense word ERROR: invalid OWNED BY option HINT: Specify OWNED BY table.column or OWNED BY NONE. -CREATE SEQUENCE sequence_testx OWNED BY pg_tables.tablename; -- not a table -ERROR: referenced relation "pg_tables" is not a table or foreign table +CREATE SEQUENCE sequence_testx OWNED BY pg_class_oid_index.oid; -- not a table +ERROR: referenced relation "pg_class_oid_index" is not a table or foreign table CREATE SEQUENCE sequence_testx OWNED BY pg_class.relname; -- not same schema ERROR: sequence must be in same schema as table it is linked to CREATE TABLE sequence_test_table (a int); diff --git a/src/test/regress/expected/truncate.out b/src/test/regress/expected/truncate.out index 81612d8c88..b652562f5b 100644 --- a/src/test/regress/expected/truncate.out +++ b/src/test/regress/expected/truncate.out @@ -393,6 +393,36 @@ SELECT * FROM truncate_a; 2 | 34 (2 rows) +CREATE TABLE truncate_b (id int GENERATED ALWAYS AS IDENTITY (START WITH 44)); +INSERT INTO truncate_b DEFAULT VALUES; +INSERT INTO truncate_b DEFAULT VALUES; +SELECT * FROM truncate_b; + id +---- + 44 + 45 +(2 rows) + +TRUNCATE truncate_b; +INSERT INTO truncate_b DEFAULT VALUES; +INSERT INTO truncate_b DEFAULT VALUES; +SELECT * FROM truncate_b; + id +---- + 46 + 47 +(2 rows) + +TRUNCATE truncate_b RESTART IDENTITY; +INSERT INTO truncate_b DEFAULT VALUES; +INSERT INTO truncate_b DEFAULT VALUES; +SELECT * FROM truncate_b; + id +---- + 44 + 45 +(2 rows) + -- check rollback of a RESTART IDENTITY operation BEGIN; TRUNCATE truncate_a RESTART IDENTITY; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 9f95b016fd..1f8f0987e3 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -111,6 +111,11 @@ test: select_views portals_p2 foreign_key cluster dependency guc bitmapops combo # ---------- test: plancache limit plpgsql copy2 temp domain rangefuncs prepare without_oid conversion truncate alter_table sequence polymorphism rowtypes returning largeobject with xml +# ---------- +# Another group of parallel tests +# ---------- +test: identity + # event triggers cannot run concurrently with any test that runs DDL test: event_trigger diff --git a/src/test/regress/serial_schedule b/src/test/regress/serial_schedule index e026b7cc90..04206c3162 100644 --- a/src/test/regress/serial_schedule +++ b/src/test/regress/serial_schedule @@ -170,6 +170,7 @@ test: conversion test: truncate test: alter_table test: sequence +test: identity test: polymorphism test: rowtypes test: returning diff --git a/src/test/regress/sql/create_table_like.sql b/src/test/regress/sql/create_table_like.sql index 900ca804cb..557040bbe7 100644 --- a/src/test/regress/sql/create_table_like.sql +++ b/src/test/regress/sql/create_table_like.sql @@ -37,6 +37,20 @@ INSERT INTO inhg VALUES ('x', 'foo', 'y'); /* fails due to constraint */ SELECT * FROM inhg; /* Two records with three columns in order x=x, xx=text, y=y */ DROP TABLE inhg; +CREATE TABLE test_like_id_1 (a int GENERATED ALWAYS AS IDENTITY, b text); +\d test_like_id_1 +INSERT INTO test_like_id_1 (b) VALUES ('b1'); +SELECT * FROM test_like_id_1; +CREATE TABLE test_like_id_2 (LIKE test_like_id_1); +\d test_like_id_2 +INSERT INTO test_like_id_2 (b) VALUES ('b2'); +SELECT * FROM test_like_id_2; -- identity was not copied +CREATE TABLE test_like_id_3 (LIKE test_like_id_1 INCLUDING IDENTITY); +\d test_like_id_3 +INSERT INTO test_like_id_3 (b) VALUES ('b3'); +SELECT * FROM test_like_id_3; -- identity was copied and applied +DROP TABLE test_like_id_1, test_like_id_2, test_like_id_3; + CREATE TABLE inhg (x text, LIKE inhx INCLUDING INDEXES, y text); /* copies indexes */ INSERT INTO inhg VALUES (5, 10); INSERT INTO inhg VALUES (20, 10); -- should fail diff --git a/src/test/regress/sql/identity.sql b/src/test/regress/sql/identity.sql new file mode 100644 index 0000000000..a7e7b15737 --- /dev/null +++ b/src/test/regress/sql/identity.sql @@ -0,0 +1,192 @@ +-- sanity check of system catalog +SELECT attrelid, attname, attidentity FROM pg_attribute WHERE attidentity NOT IN ('', 'a', 'd'); + + +CREATE TABLE itest1 (a int generated by default as identity, b text); +CREATE TABLE itest2 (a bigint generated always as identity, b text); +CREATE TABLE itest3 (a smallint generated by default as identity (start with 7 increment by 5), b text); +ALTER TABLE itest3 ALTER COLUMN a ADD GENERATED ALWAYS AS IDENTITY; -- error + +SELECT table_name, column_name, column_default, is_nullable, is_identity, identity_generation, identity_start, identity_increment, identity_maximum, identity_minimum, identity_cycle FROM information_schema.columns WHERE table_name LIKE 'itest_' ORDER BY 1, 2; + +-- internal sequences should not be shown here +SELECT sequence_name FROM information_schema.sequences WHERE sequence_name LIKE 'itest%'; + +CREATE TABLE itest4 (a int, b text); +ALTER TABLE itest4 ALTER COLUMN a ADD GENERATED ALWAYS AS IDENTITY; -- error, requires NOT NULL +ALTER TABLE itest4 ALTER COLUMN a SET NOT NULL; +ALTER TABLE itest4 ALTER COLUMN a ADD GENERATED ALWAYS AS IDENTITY; -- ok +ALTER TABLE itest4 ALTER COLUMN a DROP NOT NULL; -- error, disallowed +ALTER TABLE itest4 ALTER COLUMN a ADD GENERATED ALWAYS AS IDENTITY; -- error, already set +ALTER TABLE itest4 ALTER COLUMN b ADD GENERATED ALWAYS AS IDENTITY; -- error, wrong data type + +-- for later +ALTER TABLE itest4 ALTER COLUMN b SET DEFAULT ''; + +-- invalid column type +CREATE TABLE itest_err_1 (a text generated by default as identity); + +-- duplicate identity +CREATE TABLE itest_err_2 (a int generated always as identity generated by default as identity); + +-- cannot have default and identity +CREATE TABLE itest_err_3 (a int default 5 generated by default as identity); + +-- cannot combine serial and identity +CREATE TABLE itest_err_4 (a serial generated by default as identity); + +INSERT INTO itest1 DEFAULT VALUES; +INSERT INTO itest1 DEFAULT VALUES; +INSERT INTO itest2 DEFAULT VALUES; +INSERT INTO itest2 DEFAULT VALUES; +INSERT INTO itest3 DEFAULT VALUES; +INSERT INTO itest3 DEFAULT VALUES; +INSERT INTO itest4 DEFAULT VALUES; +INSERT INTO itest4 DEFAULT VALUES; + +SELECT * FROM itest1; +SELECT * FROM itest2; +SELECT * FROM itest3; +SELECT * FROM itest4; + + +-- OVERRIDING tests + +INSERT INTO itest1 VALUES (10, 'xyz'); +INSERT INTO itest1 OVERRIDING USER VALUE VALUES (10, 'xyz'); + +SELECT * FROM itest1; + +INSERT INTO itest2 VALUES (10, 'xyz'); +INSERT INTO itest2 OVERRIDING SYSTEM VALUE VALUES (10, 'xyz'); + +SELECT * FROM itest2; + + +-- UPDATE tests + +UPDATE itest1 SET a = 101 WHERE a = 1; +UPDATE itest1 SET a = DEFAULT WHERE a = 2; +SELECT * FROM itest1; + +UPDATE itest2 SET a = 101 WHERE a = 1; +UPDATE itest2 SET a = DEFAULT WHERE a = 2; +SELECT * FROM itest2; + + +-- DROP IDENTITY tests + +ALTER TABLE itest4 ALTER COLUMN a DROP IDENTITY; +ALTER TABLE itest4 ALTER COLUMN a DROP IDENTITY; -- error +ALTER TABLE itest4 ALTER COLUMN a DROP IDENTITY IF EXISTS; -- noop + +INSERT INTO itest4 DEFAULT VALUES; -- fails because NOT NULL is not dropped +ALTER TABLE itest4 ALTER COLUMN a DROP NOT NULL; +INSERT INTO itest4 DEFAULT VALUES; +SELECT * FROM itest4; + +-- check that sequence is removed +SELECT sequence_name FROM itest4_a_seq; + + +-- test views + +CREATE TABLE itest10 (a int generated by default as identity, b text); +CREATE TABLE itest11 (a int generated always as identity, b text); + +CREATE VIEW itestv10 AS SELECT * FROM itest10; +CREATE VIEW itestv11 AS SELECT * FROM itest11; + +INSERT INTO itestv10 DEFAULT VALUES; +INSERT INTO itestv10 DEFAULT VALUES; + +INSERT INTO itestv11 DEFAULT VALUES; +INSERT INTO itestv11 DEFAULT VALUES; + +SELECT * FROM itestv10; +SELECT * FROM itestv11; + +INSERT INTO itestv10 VALUES (10, 'xyz'); +INSERT INTO itestv10 OVERRIDING USER VALUE VALUES (11, 'xyz'); + +SELECT * FROM itestv10; + +INSERT INTO itestv11 VALUES (10, 'xyz'); +INSERT INTO itestv11 OVERRIDING SYSTEM VALUE VALUES (11, 'xyz'); + +SELECT * FROM itestv11; + + +-- various ALTER COLUMN tests + +-- fail, not allowed for identity columns +ALTER TABLE itest1 ALTER COLUMN a SET DEFAULT 1; + +-- fail, not allowed, already has a default +CREATE TABLE itest5 (a serial, b text); +ALTER TABLE itest5 ALTER COLUMN a ADD GENERATED ALWAYS AS IDENTITY; + +ALTER TABLE itest3 ALTER COLUMN a TYPE int; +SELECT seqtypid::regtype FROM pg_sequence WHERE seqrelid = 'itest3_a_seq'::regclass; +\d itest3 + +ALTER TABLE itest3 ALTER COLUMN a TYPE text; -- error + + +-- ALTER COLUMN ... SET + +CREATE TABLE itest6 (a int GENERATED ALWAYS AS IDENTITY, b text); +INSERT INTO itest6 DEFAULT VALUES; + +ALTER TABLE itest6 ALTER COLUMN a SET GENERATED BY DEFAULT SET INCREMENT BY 2 SET START WITH 100 RESTART; +INSERT INTO itest6 DEFAULT VALUES; +INSERT INTO itest6 DEFAULT VALUES; +SELECT * FROM itest6; + +SELECT table_name, column_name, is_identity, identity_generation FROM information_schema.columns WHERE table_name = 'itest6'; + +ALTER TABLE itest6 ALTER COLUMN b SET INCREMENT BY 2; -- fail, not identity + + +-- prohibited direct modification of sequence + +ALTER SEQUENCE itest6_a_seq OWNED BY NONE; + + +-- inheritance + +CREATE TABLE itest7 (a int GENERATED ALWAYS AS IDENTITY); +INSERT INTO itest7 DEFAULT VALUES; +SELECT * FROM itest7; + +-- identity property is not inherited +CREATE TABLE itest7a (b text) INHERITS (itest7); + +-- make column identity in child table +CREATE TABLE itest7b (a int); +CREATE TABLE itest7c (a int GENERATED ALWAYS AS IDENTITY) INHERITS (itest7b); +INSERT INTO itest7c DEFAULT VALUES; +SELECT * FROM itest7c; + +CREATE TABLE itest7d (a int not null); +CREATE TABLE itest7e () INHERITS (itest7d); +ALTER TABLE itest7d ALTER COLUMN a ADD GENERATED ALWAYS AS IDENTITY; +ALTER TABLE itest7d ADD COLUMN b int GENERATED ALWAYS AS IDENTITY; -- error + +SELECT table_name, column_name, is_nullable, is_identity, identity_generation FROM information_schema.columns WHERE table_name LIKE 'itest7%' ORDER BY 1, 2; + +-- These ALTER TABLE variants will not recurse. +ALTER TABLE itest7 ALTER COLUMN a SET GENERATED BY DEFAULT; +ALTER TABLE itest7 ALTER COLUMN a RESTART; +ALTER TABLE itest7 ALTER COLUMN a DROP IDENTITY; + +-- privileges +CREATE USER regress_user1; +CREATE TABLE itest8 (a int GENERATED ALWAYS AS IDENTITY, b text); +GRANT SELECT, INSERT ON itest8 TO regress_user1; +SET ROLE regress_user1; +INSERT INTO itest8 DEFAULT VALUES; +SELECT * FROM itest8; +RESET ROLE; +DROP TABLE itest8; +DROP USER regress_user1; diff --git a/src/test/regress/sql/sequence.sql b/src/test/regress/sql/sequence.sql index 74663d7351..d53e33d779 100644 --- a/src/test/regress/sql/sequence.sql +++ b/src/test/regress/sql/sequence.sql @@ -13,7 +13,7 @@ CREATE SEQUENCE sequence_testx CACHE 0; -- OWNED BY errors CREATE SEQUENCE sequence_testx OWNED BY nobody; -- nonsense word -CREATE SEQUENCE sequence_testx OWNED BY pg_tables.tablename; -- not a table +CREATE SEQUENCE sequence_testx OWNED BY pg_class_oid_index.oid; -- not a table CREATE SEQUENCE sequence_testx OWNED BY pg_class.relname; -- not same schema CREATE TABLE sequence_test_table (a int); CREATE SEQUENCE sequence_testx OWNED BY sequence_test_table.b; -- wrong column diff --git a/src/test/regress/sql/truncate.sql b/src/test/regress/sql/truncate.sql index d61eea1a42..9d3d8de54a 100644 --- a/src/test/regress/sql/truncate.sql +++ b/src/test/regress/sql/truncate.sql @@ -202,6 +202,24 @@ INSERT INTO truncate_a DEFAULT VALUES; INSERT INTO truncate_a DEFAULT VALUES; SELECT * FROM truncate_a; +CREATE TABLE truncate_b (id int GENERATED ALWAYS AS IDENTITY (START WITH 44)); + +INSERT INTO truncate_b DEFAULT VALUES; +INSERT INTO truncate_b DEFAULT VALUES; +SELECT * FROM truncate_b; + +TRUNCATE truncate_b; + +INSERT INTO truncate_b DEFAULT VALUES; +INSERT INTO truncate_b DEFAULT VALUES; +SELECT * FROM truncate_b; + +TRUNCATE truncate_b RESTART IDENTITY; + +INSERT INTO truncate_b DEFAULT VALUES; +INSERT INTO truncate_b DEFAULT VALUES; +SELECT * FROM truncate_b; + -- check rollback of a RESTART IDENTITY operation BEGIN; TRUNCATE truncate_a RESTART IDENTITY; -- cgit v1.2.3 From a071fe87a21920e5c2e79d521d31b2ddaf83875b Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 6 Apr 2017 12:27:45 -0400 Subject: doc: Formatting fix for XSL-FO PDF build --- doc/src/sgml/stylesheet-fo.xsl | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'doc/src') diff --git a/doc/src/sgml/stylesheet-fo.xsl b/doc/src/sgml/stylesheet-fo.xsl index 8b555d12a7..7a0fbe9564 100644 --- a/doc/src/sgml/stylesheet-fo.xsl +++ b/doc/src/sgml/stylesheet-fo.xsl @@ -32,6 +32,14 @@ center + + + 1em + 0.8em + 1.2em + + -- cgit v1.2.3 From 6f1b9aaae35bfabe2654a8e44ce226c91e7d8bd9 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 5 Apr 2017 10:44:23 -0400 Subject: Fix logical replication between different encodings When sending a tuple attribute, the previous coding erroneously sent the length byte before encoding conversion, which would lead to protocol failures on the receiving side if the length did not match the following string. To fix that, use pq_sendcountedtext() for sending tuple attributes, which takes care of all of that internally. To match the API of pq_sendcountedtext(), send even text values without a trailing zero byte and have the receiving end put it in place instead. This matches how the standard FE/BE protocol behaves. Reported-by: Kyotaro HORIGUCHI --- doc/src/sgml/protocol.sgml | 7 +++-- src/backend/replication/logical/proto.c | 10 +++---- src/test/subscription/t/005_encoding.pl | 46 +++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 8 deletions(-) create mode 100644 src/test/subscription/t/005_encoding.pl (limited to 'doc/src') diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index 5f971412ae..9d46d74113 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -6107,11 +6107,14 @@ TupleData - String + Byten - The text value. + The value of the column, in text format. (A future release + might support additional formats.) + n is the above length. + diff --git a/src/backend/replication/logical/proto.c b/src/backend/replication/logical/proto.c index bc6e9b5a98..bb607b6147 100644 --- a/src/backend/replication/logical/proto.c +++ b/src/backend/replication/logical/proto.c @@ -417,7 +417,6 @@ logicalrep_write_tuple(StringInfo out, Relation rel, HeapTuple tuple) Form_pg_type typclass; Form_pg_attribute att = desc->attrs[i]; char *outputstr; - int len; /* skip dropped columns */ if (att->attisdropped) @@ -442,10 +441,7 @@ logicalrep_write_tuple(StringInfo out, Relation rel, HeapTuple tuple) pq_sendbyte(out, 't'); /* 'text' data follows */ outputstr = OidOutputFunctionCall(typclass->typoutput, values[i]); - len = strlen(outputstr) + 1; /* null terminated */ - pq_sendint(out, len, 4); /* length */ - pq_sendstring(out, outputstr); /* data */ - + pq_sendcountedtext(out, outputstr, strlen(outputstr), false); pfree(outputstr); ReleaseSysCache(typtup); @@ -492,7 +488,9 @@ logicalrep_read_tuple(StringInfo in, LogicalRepTupleData *tuple) len = pq_getmsgint(in, 4); /* read length */ /* and data */ - tuple->values[i] = (char *) pq_getmsgbytes(in, len); + tuple->values[i] = palloc(len + 1); + pq_copymsgbytes(in, tuple->values[i], len); + tuple->values[i][len] = '\0'; } break; default: diff --git a/src/test/subscription/t/005_encoding.pl b/src/test/subscription/t/005_encoding.pl new file mode 100644 index 0000000000..42a4eee5b4 --- /dev/null +++ b/src/test/subscription/t/005_encoding.pl @@ -0,0 +1,46 @@ +# Test replication between databases with different encodings +use strict; +use warnings; +use PostgresNode; +use TestLib; +use Test::More tests => 1; + +sub wait_for_caught_up +{ + my ($node, $appname) = @_; + + $node->poll_query_until('postgres', + "SELECT pg_current_wal_location() <= replay_location FROM pg_stat_replication WHERE application_name = '$appname';") + or die "Timed out while waiting for subscriber to catch up"; +} + +my $node_publisher = get_new_node('publisher'); +$node_publisher->init(allows_streaming => 'logical', extra => ['--locale=C', '--encoding=UTF8']); +$node_publisher->start; + +my $node_subscriber = get_new_node('subscriber'); +$node_subscriber->init(allows_streaming => 'logical', extra => ['--locale=C', '--encoding=LATIN1']); +$node_subscriber->start; + +my $ddl = "CREATE TABLE test1 (a int, b text);"; +$node_publisher->safe_psql('postgres', $ddl); +$node_subscriber->safe_psql('postgres', $ddl); + +my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; +my $appname = 'encoding_test'; + +$node_publisher->safe_psql('postgres', "CREATE PUBLICATION mypub FOR ALL TABLES;"); +$node_subscriber->safe_psql('postgres', "CREATE SUBSCRIPTION mysub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION mypub;"); + +wait_for_caught_up($node_publisher, $appname); + +$node_publisher->safe_psql('postgres', q{INSERT INTO test1 VALUES (1, E'Mot\xc3\xb6rhead')}); # hand-rolled UTF-8 + +wait_for_caught_up($node_publisher, $appname); + +is($node_subscriber->safe_psql('postgres', q{SELECT a FROM test1 WHERE b = E'Mot\xf6rhead'}), # LATIN1 + qq(1), + 'data replicated to subscriber'); + +$node_subscriber->stop; +$node_publisher->stop; -- cgit v1.2.3 From 510074f9f0131a04322d6a3d2a51c87e6db243f9 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 6 Apr 2017 22:03:52 -0400 Subject: Remove use of Jade and DSSSL All documentation is now built using XSLT. Remove all references to Jade, DSSSL, also JadeTex and some other outdated tooling. For chunked HTML builds, this changes nothing, but removes the transitional "oldhtml" target. The single-page HTML build is ported over to XSLT. For PDF builds, this removes the JadeTex builds and moves the FOP builds in their place. --- config/docbook.m4 | 62 --- configure | 180 ++----- configure.in | 4 +- doc/src/sgml/.gitignore | 8 - doc/src/sgml/Makefile | 174 ++----- doc/src/sgml/docguide.sgml | 619 ++++-------------------- doc/src/sgml/filelist.sgml | 12 - doc/src/sgml/fixrtf | 46 -- doc/src/sgml/install-windows.sgml | 10 - doc/src/sgml/jadetex.cfg | 89 ---- doc/src/sgml/postgres.sgml | 3 +- doc/src/sgml/stylesheet-html-common.xsl | 266 +++++++++++ doc/src/sgml/stylesheet-html-nochunk.xsl | 12 + doc/src/sgml/stylesheet.dsl | 798 ------------------------------- doc/src/sgml/stylesheet.xsl | 255 +--------- src/Makefile.global.in | 5 +- 16 files changed, 468 insertions(+), 2075 deletions(-) delete mode 100755 doc/src/sgml/fixrtf delete mode 100644 doc/src/sgml/jadetex.cfg create mode 100644 doc/src/sgml/stylesheet-html-common.xsl create mode 100644 doc/src/sgml/stylesheet-html-nochunk.xsl delete mode 100644 doc/src/sgml/stylesheet.dsl (limited to 'doc/src') diff --git a/config/docbook.m4 b/config/docbook.m4 index 4304fa7ea1..d550309353 100644 --- a/config/docbook.m4 +++ b/config/docbook.m4 @@ -1,11 +1,5 @@ # config/docbook.m4 -# PGAC_PROG_JADE -# -------------- -AC_DEFUN([PGAC_PROG_JADE], -[AC_CHECK_PROGS([JADE], [openjade jade])]) - - # PGAC_PROG_NSGMLS # ---------------- AC_DEFUN([PGAC_PROG_NSGMLS], @@ -44,59 +38,3 @@ rm -f conftest.sgml]) have_docbook=$pgac_cv_check_docbook AC_SUBST([have_docbook]) ])# PGAC_CHECK_DOCBOOK - - -# PGAC_PATH_DOCBOOK_STYLESHEETS -# ----------------------------- -AC_DEFUN([PGAC_PATH_DOCBOOK_STYLESHEETS], -[AC_ARG_VAR(DOCBOOKSTYLE, [location of DocBook stylesheets])dnl -AC_MSG_CHECKING([for DocBook stylesheets]) -AC_CACHE_VAL([pgac_cv_path_stylesheets], -[if test -n "$DOCBOOKSTYLE"; then - pgac_cv_path_stylesheets=$DOCBOOKSTYLE -else - for pgac_prefix in /usr /usr/local /opt /sw; do - for pgac_infix in share lib; do - for pgac_postfix in \ - sgml/stylesheets/nwalsh-modular \ - sgml/stylesheets/docbook \ - sgml/stylesheets/dsssl/docbook \ - sgml/docbook-dsssl \ - sgml/docbook/dsssl/modular \ - sgml/docbook/stylesheet/dsssl/modular \ - sgml/docbook/dsssl-stylesheets \ - sgml/dsssl/docbook-dsssl-nwalsh - do - pgac_candidate=$pgac_prefix/$pgac_infix/$pgac_postfix - if test -r "$pgac_candidate/html/docbook.dsl" \ - && test -r "$pgac_candidate/print/docbook.dsl" - then - pgac_cv_path_stylesheets=$pgac_candidate - break 3 - fi - done - done - done -fi]) -DOCBOOKSTYLE=$pgac_cv_path_stylesheets -AC_SUBST([DOCBOOKSTYLE]) -if test -n "$DOCBOOKSTYLE"; then - AC_MSG_RESULT([$DOCBOOKSTYLE]) -else - AC_MSG_RESULT(no) -fi])# PGAC_PATH_DOCBOOK_STYLESHEETS - - -# PGAC_PATH_COLLATEINDEX -# ---------------------- -# Some DocBook installations provide collateindex.pl in $DOCBOOKSTYLE/bin, -# but it's not necessarily marked executable, so we can't use AC_PATH_PROG -# to check for it there. Other installations just put it in the PATH. -AC_DEFUN([PGAC_PATH_COLLATEINDEX], -[AC_REQUIRE([PGAC_PATH_DOCBOOK_STYLESHEETS])dnl -if test -n "$DOCBOOKSTYLE" -a -r "$DOCBOOKSTYLE/bin/collateindex.pl"; then - COLLATEINDEX="$DOCBOOKSTYLE/bin/collateindex.pl" - AC_SUBST([COLLATEINDEX]) -else - AC_PATH_PROG(COLLATEINDEX, collateindex.pl) -fi])# PGAC_PATH_COLLATEINDEX diff --git a/configure b/configure index de8660d00d..99d05bf084 100755 --- a/configure +++ b/configure @@ -629,14 +629,12 @@ ac_subst_vars='LTLIBOBJS vpath_build PG_VERSION_NUM PROVE +FOP OSX XSLTPROC XMLLINT DBTOEPUB -COLLATEINDEX -DOCBOOKSTYLE have_docbook -JADE NSGMLS TCL_SHLIB_LD_LIBS TCL_SHARED_BUILD @@ -869,8 +867,7 @@ PKG_CONFIG_LIBDIR ICU_CFLAGS ICU_LIBS LDFLAGS_EX -LDFLAGS_SL -DOCBOOKSTYLE' +LDFLAGS_SL' # Initialize some variables set by options. @@ -1568,8 +1565,6 @@ Some influential environment variables: ICU_LIBS linker flags for ICU, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only - DOCBOOKSTYLE - location of DocBook stylesheets Use these variables to override the choices made by `configure' or to help it to find libraries and programs with nonstandard names/locations. @@ -15951,48 +15946,6 @@ fi test -n "$NSGMLS" && break done -for ac_prog in openjade jade -do - # Extract the first word of "$ac_prog", so it can be a program name with args. -set dummy $ac_prog; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_JADE+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$JADE"; then - ac_cv_prog_JADE="$JADE" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_JADE="$ac_prog" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -JADE=$ac_cv_prog_JADE -if test -n "$JADE"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $JADE" >&5 -$as_echo "$JADE" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - - test -n "$JADE" && break -done - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for DocBook V4.2" >&5 $as_echo_n "checking for DocBook V4.2... " >&6; } @@ -16029,93 +15982,6 @@ $as_echo "$pgac_cv_check_docbook" >&6; } have_docbook=$pgac_cv_check_docbook -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for DocBook stylesheets" >&5 -$as_echo_n "checking for DocBook stylesheets... " >&6; } -if ${pgac_cv_path_stylesheets+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$DOCBOOKSTYLE"; then - pgac_cv_path_stylesheets=$DOCBOOKSTYLE -else - for pgac_prefix in /usr /usr/local /opt /sw; do - for pgac_infix in share lib; do - for pgac_postfix in \ - sgml/stylesheets/nwalsh-modular \ - sgml/stylesheets/docbook \ - sgml/stylesheets/dsssl/docbook \ - sgml/docbook-dsssl \ - sgml/docbook/dsssl/modular \ - sgml/docbook/stylesheet/dsssl/modular \ - sgml/docbook/dsssl-stylesheets \ - sgml/dsssl/docbook-dsssl-nwalsh - do - pgac_candidate=$pgac_prefix/$pgac_infix/$pgac_postfix - if test -r "$pgac_candidate/html/docbook.dsl" \ - && test -r "$pgac_candidate/print/docbook.dsl" - then - pgac_cv_path_stylesheets=$pgac_candidate - break 3 - fi - done - done - done -fi -fi - -DOCBOOKSTYLE=$pgac_cv_path_stylesheets - -if test -n "$DOCBOOKSTYLE"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $DOCBOOKSTYLE" >&5 -$as_echo "$DOCBOOKSTYLE" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi -if test -n "$DOCBOOKSTYLE" -a -r "$DOCBOOKSTYLE/bin/collateindex.pl"; then - COLLATEINDEX="$DOCBOOKSTYLE/bin/collateindex.pl" - -else - # Extract the first word of "collateindex.pl", so it can be a program name with args. -set dummy collateindex.pl; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_path_COLLATEINDEX+:} false; then : - $as_echo_n "(cached) " >&6 -else - case $COLLATEINDEX in - [\\/]* | ?:[\\/]*) - ac_cv_path_COLLATEINDEX="$COLLATEINDEX" # Let the user override the test with a path. - ;; - *) - as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_path_COLLATEINDEX="$as_dir/$ac_word$ac_exec_ext" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - - ;; -esac -fi -COLLATEINDEX=$ac_cv_path_COLLATEINDEX -if test -n "$COLLATEINDEX"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $COLLATEINDEX" >&5 -$as_echo "$COLLATEINDEX" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - -fi for ac_prog in dbtoepub do # Extract the first word of "$ac_prog", so it can be a program name with args. @@ -16284,6 +16150,48 @@ fi test -n "$OSX" && break done +for ac_prog in fop +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_FOP+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$FOP"; then + ac_cv_prog_FOP="$FOP" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_prog_FOP="$ac_prog" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +FOP=$ac_cv_prog_FOP +if test -n "$FOP"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $FOP" >&5 +$as_echo "$FOP" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$FOP" && break +done + # # Check for test tools diff --git a/configure.in b/configure.in index b965f47669..c36c50384f 100644 --- a/configure.in +++ b/configure.in @@ -2117,14 +2117,12 @@ fi # Check for DocBook and tools # PGAC_PROG_NSGMLS -PGAC_PROG_JADE PGAC_CHECK_DOCBOOK(4.2) -PGAC_PATH_DOCBOOK_STYLESHEETS -PGAC_PATH_COLLATEINDEX AC_CHECK_PROGS(DBTOEPUB, dbtoepub) AC_CHECK_PROGS(XMLLINT, xmllint) AC_CHECK_PROGS(XSLTPROC, xsltproc) AC_CHECK_PROGS(OSX, [osx sgml2xml sx]) +AC_CHECK_PROGS(FOP, fop) # # Check for test tools diff --git a/doc/src/sgml/.gitignore b/doc/src/sgml/.gitignore index 8197c0140d..a74513837f 100644 --- a/doc/src/sgml/.gitignore +++ b/doc/src/sgml/.gitignore @@ -16,15 +16,7 @@ /features-unsupported.sgml /errcodes-table.sgml /version.sgml -/bookindex.sgml -/HTML.index # Assorted byproducts from building the above /postgres.xml /INSTALL.html /INSTALL.xml -/postgres-US.aux -/postgres-US.log -/postgres-US.out -/postgres-A4.aux -/postgres-A4.log -/postgres-A4.out diff --git a/doc/src/sgml/Makefile b/doc/src/sgml/Makefile index 774d35de20..6a2cbf26ee 100644 --- a/doc/src/sgml/Makefile +++ b/doc/src/sgml/Makefile @@ -33,9 +33,10 @@ ifndef DBTOEPUB DBTOEPUB = $(missing) dbtoepub endif -ifndef JADE -JADE = $(missing) jade +ifndef FOP +FOP = $(missing) fop endif + SGMLINCLUDE = -D . -D $(srcdir) ifndef NSGMLS @@ -57,18 +58,11 @@ endif override XSLTPROCFLAGS += --stringparam pg.version '$(VERSION)' -GENERATED_SGML = bookindex.sgml version.sgml \ +GENERATED_SGML = version.sgml \ features-supported.sgml features-unsupported.sgml errcodes-table.sgml ALLSGML := $(wildcard $(srcdir)/*.sgml $(srcdir)/ref/*.sgml) $(GENERATED_SGML) -# Sometimes we don't want this one. -ALMOSTALLSGML := $(filter-out %bookindex.sgml,$(ALLSGML)) - -ifdef DOCBOOKSTYLE -CATALOG = -c $(DOCBOOKSTYLE)/catalog -endif - # Enable some extra warnings # -wfully-tagged needed to throw a warning on missing tags # for older tool chains, 2007-08-31 @@ -77,6 +71,7 @@ endif # noisy to turn on by default, unfortunately. override SPFLAGS += -wall -wno-unused-param -wno-empty -wfully-tagged + ## ## Man pages ## @@ -90,50 +85,9 @@ man-stamp: stylesheet-man.xsl postgres.xml ## -## HTML +## common files ## -.PHONY: draft - -JADE.html.call = $(JADE) $(JADEFLAGS) $(SPFLAGS) $(SGMLINCLUDE) $(CATALOG) -d stylesheet.dsl -t sgml -i output-html -ifeq ($(STYLE),website) -JADE.html.call += -V website-stylesheet -endif - -# The draft target creates HTML output in draft mode, without index (for faster build). -draft: postgres.sgml $(ALMOSTALLSGML) stylesheet.dsl - $(MKDIR_P) html - $(JADE.html.call) -V draft-mode $< - cp $(srcdir)/stylesheet.css html/ - -oldhtml: oldhtml-stamp - -oldhtml-stamp: postgres.sgml $(ALLSGML) stylesheet.dsl - $(MAKE) check-tabs - $(MKDIR_P) html - $(JADE.html.call) -i include-index $< - cp $(srcdir)/stylesheet.css html/ - touch $@ - -# single-page HTML -postgres.html: postgres.sgml $(ALLSGML) stylesheet.dsl - $(JADE.html.call) -V nochunks -V rootchunk -V '(define %root-filename% #f)' -V '(define use-output-dir #f)' -i include-index $< - -# single-page text -postgres.txt: postgres.html - $(LYNX) -force_html -dump -nolist $< > $@ - -HTML.index: postgres.sgml $(ALMOSTALLSGML) stylesheet.dsl - @$(MKDIR_P) html - $(JADE.html.call) -V html-index $< - -bookindex.sgml: HTML.index -ifdef COLLATEINDEX - LC_ALL=C $(PERL) $(COLLATEINDEX) -f -g -i 'bookindex' -o $@ $< -else - @$(missing) collateindex.pl $< $@ -endif - # Technically, this should depend on Makefile.global, but then # version.sgml would need to be rebuilt after every configure run, # even in distribution tarballs. So this is cheating a bit, but it @@ -154,67 +108,6 @@ features-unsupported.sgml: $(top_srcdir)/src/backend/catalog/sql_feature_package errcodes-table.sgml: $(top_srcdir)/src/backend/utils/errcodes.txt generate-errcodes-table.pl $(PERL) $(srcdir)/generate-errcodes-table.pl $< > $@ -## -## Print -## - - -# RTF to allow minor editing for hardcopy -%.rtf: %.sgml $(ALLSGML) - $(JADE) $(JADEFLAGS) $(SGMLINCLUDE) $(CATALOG) -d stylesheet.dsl -t rtf -V rtf-backend -i output-print -i include-index postgres.sgml - -# TeX -# Regular TeX and pdfTeX have slightly differing requirements, so we -# need to distinguish the path we're taking. - -JADE.tex.call = $(JADE) $(JADEFLAGS) $(SGMLINCLUDE) $(CATALOG) -d $(srcdir)/stylesheet.dsl -t tex -V tex-backend -i output-print -i include-index - -%-A4.tex-ps: %.sgml $(ALLSGML) - $(JADE.tex.call) -V texdvi-output -V '%paper-type%'=A4 -o $@ $< - -%-US.tex-ps: %.sgml $(ALLSGML) - $(JADE.tex.call) -V texdvi-output -V '%paper-type%'=USletter -o $@ $< - -%-A4.tex-pdf: %.sgml $(ALLSGML) - $(JADE.tex.call) -V texpdf-output -V '%paper-type%'=A4 -o $@ $< - -%-US.tex-pdf: %.sgml $(ALLSGML) - $(JADE.tex.call) -V texpdf-output -V '%paper-type%'=USletter -o $@ $< - -%.dvi: %.tex-ps - @rm -f $*.aux $*.log -# multiple runs are necessary to create proper intra-document links - jadetex $< - jadetex $< - jadetex $< - -# PostScript from TeX -postgres.ps: - $(error Invalid target; use postgres-A4.ps or postgres-US.ps as targets) - -%.ps: %.dvi - dvips -o $@ $< - -postgres.pdf: - $(error Invalid target; use postgres-A4.pdf or postgres-US.pdf as targets) - -%.pdf: %.tex-pdf - @rm -f $*.aux $*.log $*.out -# multiple runs are necessary to create proper intra-document links - pdfjadetex $< - pdfjadetex $< - pdfjadetex $< - -# Cancel built-in suffix rules, interfering with PS building -.SUFFIXES: - - -# This generates an XML version of the flow-object tree. It's useful -# for debugging DSSSL code, and possibly to interface to some other -# tools that can make use of this. -%.fot: %.sgml $(ALLSGML) - $(JADE) $(JADEFLAGS) $(SGMLINCLUDE) $(CATALOG) -d stylesheet.dsl -t fot -i output-print -i include-index -o $@ $< - ## ## Generation of some text files. @@ -239,19 +132,19 @@ INSTALL.html: %.html : stylesheet-text.xsl %.xml $(XSLTPROC) $(XSLTPROCFLAGS) $(XSLTPROC_HTML_FLAGS) $^ >$@ INSTALL.xml: standalone-install.sgml installation.sgml version.sgml - $(OSX) -D. -x lower $(filter-out version.sgml,$^) >$@.tmp + $(OSX) $(SGMLINCLUDE) -x lower $(filter-out version.sgml,$^) >$@.tmp $(call mangle-xml,chapter) ## -## XSLT processing +## SGML->XML conversion ## # For obscure reasons, GNU make 3.81 complains about circular dependencies # if we try to do "make all" in a VPATH build without the explicit # $(srcdir) on the postgres.sgml dependency in this rule. GNU make bug? -postgres.xml: $(srcdir)/postgres.sgml $(ALMOSTALLSGML) - $(OSX) -D. -x lower -i include-xslt-index $< >$@.tmp +postgres.xml: $(srcdir)/postgres.sgml $(ALLSGML) + $(OSX) $(SGMLINCLUDE) -x lower $< >$@.tmp $(call mangle-xml,book) define mangle-xml @@ -261,6 +154,11 @@ $(PERL) -p -e 's/\[(aacute|acirc|aelig|agrave|amp|aring|atilde|auml|bull|copy|ea rm $@.tmp endef + +## +## HTML +## + ifeq ($(STYLE),website) XSLTPROC_HTML_FLAGS += --param website.stylesheet 1 endif @@ -277,6 +175,23 @@ htmlhelp: stylesheet-hh.xsl postgres.xml $(XMLLINT) --noout --valid postgres.xml $(XSLTPROC) $(XSLTPROCFLAGS) $^ +# single-page HTML +postgres.html: stylesheet-html-nochunk.xsl postgres.xml + $(XMLLINT) --noout --valid postgres.xml + $(XSLTPROC) $(XSLTPROCFLAGS) $(XSLTPROC_HTML_FLAGS) -o $@ $^ + +# single-page text +postgres.txt: postgres.html + $(LYNX) -force_html -dump -nolist $< > $@ + + +## +## Print +## + +postgres.pdf: + $(error Invalid target; use postgres-A4.pdf or postgres-US.pdf as targets) + %-A4.fo: stylesheet-fo.xsl %.xml $(XMLLINT) --noout --valid $*.xml $(XSLTPROC) $(XSLTPROCFLAGS) --stringparam paper.type A4 -o $@ $^ @@ -285,12 +200,13 @@ htmlhelp: stylesheet-hh.xsl postgres.xml $(XMLLINT) --noout --valid $*.xml $(XSLTPROC) $(XSLTPROCFLAGS) --stringparam paper.type USletter -o $@ $^ -FOP = fop +%.pdf: %.fo + $(FOP) -fo $< -pdf $@ -.SECONDARY: postgres-A4.fo postgres-US.fo -%-fop.pdf: %.fo - $(FOP) -fo $< -pdf $@ +## +## EPUB +## epub: postgres.epub postgres.epub: postgres.xml @@ -321,7 +237,7 @@ MAKEINFO = makeinfo ## # Quick syntax check without style processing -check: postgres.sgml $(ALMOSTALLSGML) check-tabs +check: postgres.sgml $(ALLSGML) check-tabs $(NSGMLS) $(SPFLAGS) $(SGMLINCLUDE) -s $< @@ -394,7 +310,7 @@ check-tabs: # keeping the dependencies satisfied. .SECONDARY: postgres.xml $(GENERATED_SGML) HTML.index .SECONDARY: INSTALL.html INSTALL.xml -.SECONDARY: %-A4.tex-ps %-US.tex-ps %-A4.tex-pdf %-US.tex-pdf +.SECONDARY: postgres-A4.fo postgres-US.fo clean: # text --- these are shipped, but not in this directory @@ -403,11 +319,13 @@ clean: # single-page output rm -f postgres.html postgres.txt # print - rm -f *.rtf *.tex-ps *.tex-pdf *.dvi *.aux *.log *.ps *.pdf *.out *.fot -# index - rm -f HTML.index $(GENERATED_SGML) -# XSLT - rm -f postgres.xml *.tmp htmlhelp.hhp toc.hhc index.hhk *.fo + rm -f *.fo *.pdf +# generated SGML files + rm -f $(GENERATED_SGML) +# SGML->XML conversion + rm -f postgres.xml *.tmp +# HTML Help + rm -f htmlhelp.hhp toc.hhc index.hhk # EPUB rm -f postgres.epub # Texinfo diff --git a/doc/src/sgml/docguide.sgml b/doc/src/sgml/docguide.sgml index 57b67137b1..810d4dac40 100644 --- a/doc/src/sgml/docguide.sgml +++ b/doc/src/sgml/docguide.sgml @@ -20,7 +20,7 @@ - PDF or PostScript, for printing + PDF, for printing @@ -37,8 +37,8 @@ HTML documentation and man pages are part of a - standard distribution and are installed by default. PDF and - PostScript format documentation is available separately for + standard distribution and are installed by default. PDF + format documentation is available separately for download. @@ -55,6 +55,15 @@ used, but technically they are not interchangeable. + + + The PostgreSQL documentation is currently being transitioned from DocBook + SGML and DSSSL style sheets to DocBook XML and XSLT style sheets. Be + careful to look at the instructions relating to the PostgreSQL version you + are dealing with, as the procedures and required tools will change. + + + DocBook allows an author to specify the structure and content of a technical document without worrying @@ -86,11 +95,11 @@ DocBook DTD - This is the definition of DocBook itself. We currently use - version 4.2; you cannot use later or earlier versions. You - need the SGML variant of the DocBook DTD, - but to build man pages you also need the XML - variant of the same version. + This is the definition of DocBook itself. We currently use version + 4.2; you cannot use later or earlier versions. You need + the SGML and the XML variant of + the DocBook DTD of the same version. These will usually be in separate + packages. @@ -99,51 +108,35 @@ ISO 8879 character entities - These are required by DocBook but are distributed separately + These are required by DocBook SGML but are distributed separately because they are maintained by ISO. - DocBook DSSSL Stylesheets + DocBook XSL Stylesheets These contain the processing instructions for converting the DocBook sources to other formats, such as HTML. - - - - - DocBook XSL Stylesheets - - - This is another stylesheet for converting DocBook to other - formats. We currently use this to produce man pages and - optionally HTMLHelp. You can also use this toolchain to - produce HTML or PDF output, but official PostgreSQL releases - use the DSSSL stylesheets for that. - - The minimum required version is currently 1.74.0. + The minimum required version is currently 1.77.0, but it is recommended + to use the latest available version for best results. - OpenJade + OpenSP - This is the base package of SGML processing. - It contains an SGML parser, a - DSSSL processor (that is, a program to - convert SGML to other formats using - DSSSL stylesheets), as well as a number of - related tools. Jade is now being - maintained by the OpenJade group, no longer by James Clark. + This is the base package of SGML processing. Note + that we no longer need OpenJade, the DSSSL + processor, only the OpenSP package for converting SGML to XML. @@ -166,31 +159,17 @@ Libxslt for xsltproc - This is the processing tool to use with the XSLT stylesheets - (like jade is the processing tool for DSSSL - stylesheets). + xsltproc is an XSLT processor, that is, a program to + convert XML to other formats using XSLT stylesheets. - JadeTeX + FOP - If you want to, you can also install - JadeTeX to use - TeX as a formatting backend for - Jade. - JadeTeX can create PostScript or - PDF files (the latter with bookmarks). - - - - However, the output from JadeTeX is - inferior to what you get from the RTF - backend. Particular problem areas are tables and various - artifacts of vertical and horizontal spacing. Also, there is - no opportunity to manually polish the results. + This is a program for converting, among other things, XML to PDF. @@ -206,13 +185,23 @@ here. + + You can get away with not installing DocBook XML and the DocBook XSLT + stylesheets locally, because the required files will be downloaded from the + Internet and cached locally. This may in fact be the preferred solution if + your operating system packages provide only an old version of especially + the stylesheets or if no packages are available at all. See + the option for xmllint + and xsltproc for more information. + + Installation on Fedora, RHEL, and Derivatives To install the required packages, use: -yum install docbook-dtds docbook-style-dsssl docbook-style-xsl libxslt openjade +yum install docbook-dtds docbook-style-xsl fop libxslt opensp @@ -243,21 +232,18 @@ yum install docbook-dtds docbook-style-dsssl docbook-style-xsl libxslt openjade textproc/libxslt - textproc/openjade + textproc/fop + + + textproc/opensp - - A number of things from /usr/ports/print - (tex, jadetex) might - also be of interest. - - To install the required packages with pkg, use: -pkg install docbook-sgml docbook-xml docbook-xsl dsssl-docbook-modular libxslt openjade +pkg install docbook-sgml docbook-xml docbook-xsl fop libxslt opensp @@ -282,7 +268,7 @@ pkg install docbook-sgml docbook-xml docbook-xsl dsssl-docbook-modular libxslt o available for Debian GNU/Linux. To install, simply use: -apt-get install docbook docbook-dsssl docbook-xsl libxml2-utils openjade1.3 opensp xsltproc +apt-get install docbook docbook-xml docbook-xsl fop libxml2-utils opensp xsltproc @@ -293,7 +279,7 @@ apt-get install docbook docbook-dsssl docbook-xsl libxml2-utils openjade1.3 open If you use MacPorts, the following will get you set up: -sudo port install docbook-dsssl docbook-sgml-4.2 docbook-xml-4.2 docbook-xsl libxslt openjade opensp +sudo port install docbook-sgml-4.2 docbook-xml-4.2 docbook-xsl fop libxslt opensp @@ -311,73 +297,24 @@ sudo port install docbook-dsssl docbook-sgml-4.2 docbook-xml-4.2 docbook-xsl lib - Installing OpenJade + Installing OpenSP - - - - The installation of OpenJade offers a GNU-style - ./configure; make; make install build - process. Details can be found in the OpenJade source - distribution. In a nutshell: + + The installation of OpenSP offers a GNU-style + ./configure; make; make install build process. + Details can be found in the OpenSP source distribution. In a nutshell: -./configure --enable-default-catalog=/usr/local/share/sgml/catalog +./configure --enable-default-catalog=/usr/local/etc/sgml/catalog make make install - Be sure to remember where you put the default - catalog; you will need it below. You can also leave - it off, but then you will have to set the environment variable - SGML_CATALOG_FILES to point to the file - whenever you use jade later on. - (This method is also an option if OpenJade is already - installed and you want to install the rest of the toolchain - locally.) - - - - - Some users have reported encountering a segmentation fault using - OpenJade 1.4devel to build the PDFs, with a message like: - -openjade:./stylesheet.dsl:664:2:E: flow object not accepted by port; only display flow objects accepted -make: *** [postgres-A4.tex-pdf] Segmentation fault - - Downgrading to OpenJade 1.3 should get rid of this error. - - - - - - - - Additionally, you should install the files - dsssl.dtd, fot.dtd, - style-sheet.dtd, and - catalog from the - dsssl directory somewhere, perhaps into - /usr/local/share/sgml/dsssl. It's - probably easiest to copy the entire directory: - -cp -R dsssl /usr/local/share/sgml - - - - - - - Finally, create the file - /usr/local/share/sgml/catalog and add - this line to it: - -CATALOG "dsssl/catalog" - - (This is a relative path reference to the file installed in - . Be sure to adjust it - if you chose your installation layout differently.) - - - + Be sure to remember where you put the default catalog; you + will need it below. You can also leave it off, but then you will have to + set the environment variable SGML_CATALOG_FILES to point + to the file whenever you use any programs from OpenSP later on. (This + method is also an option if OpenSP is already installed and you want to + install the rest of the toolchain locally.) + @@ -451,88 +388,6 @@ perl -pi -e 's/iso-(.*).gml/ISO\1/g' docbook.cat - - - Installing the DocBook <acronym>DSSSL</acronym> Style Sheets - - - To install the style sheets, unzip and untar the distribution and - move it to a suitable place, for example - /usr/local/share/sgml. (The archive will - automatically create a subdirectory.) - -$ gunzip docbook-dsssl-1.xx.tar.gz -$ tar -C /usr/local/share/sgml -xf docbook-dsssl-1.xx.tar - - - - - The usual catalog entry in - /usr/local/share/sgml/catalog can also be - made: - -CATALOG "docbook-dsssl-1.xx/catalog" - - Because stylesheets change rather often, and it's sometimes - beneficial to try out alternative versions, - PostgreSQL doesn't use this catalog - entry. See for - information about how to select the stylesheets instead. - - - - - Installing <productname>JadeTeX</productname> - - - To install and use JadeTeX, you will - need a working installation of TeX and - LaTeX2e, including the supported - tools and - graphics packages, - Babel, - AMS fonts and - AMS-LaTeX, the - PSNFSS extension - and companion kit of the 35 fonts, the - dvips program for generating - PostScript, the macro packages - fancyhdr, - hyperref, - minitoc, - url and - ot2enc. All of these can be found on - your friendly neighborhood - CTAN site. - The installation of the TeX base - system is far beyond the scope of this introduction. Binary - packages should be available for any system that can run - TeX. - - - - Before you can use JadeTeX with the - PostgreSQL documentation sources, you - will need to increase the size of - TeX's internal data structures. - Details on this can be found in the JadeTeX - installation instructions. - - - - Once that is finished you can install JadeTeX: - -$ gunzip jadetex-xxx.tar.gz -$ tar xf jadetex-xxx.tar -$ cd jadetex -$ make install -$ mktexlsr - - The last two need to be done as root. - - - - @@ -547,28 +402,24 @@ CATALOG "docbook-dsssl-1.xx/catalog" checking for onsgmls... onsgmls -checking for openjade... openjade checking for DocBook V4.2... yes -checking for DocBook stylesheets... /usr/share/sgml/docbook/stylesheet/dsssl/modular -checking for collateindex.pl... /usr/bin/collateindex.pl +checking for dbtoepub... dbtoepub +checking for xmllint... xmllint checking for xsltproc... xsltproc checking for osx... osx +checking for fop... fop If neither onsgmls nor nsgmls were found then some of the following tests - will be skipped. nsgmls is part of the Jade - package. You can pass the environment variables - JADE and NSGMLS to configure to point + will be skipped. nsgmls is part of the OpenSP + package. You can pass the environment variable + NSGMLS to configure to point to the programs if they are not found automatically. If DocBook V4.2 was not found then you did not install - the DocBook DTD kit in a place where Jade can find it, or you have + the DocBook DTD kit in a place where OpenSP can find it, or you have not set up the catalog files correctly. See the installation hints - above. The DocBook stylesheets are looked for in a number of - relatively standard places, but if you have them some other place - then you should set the environment variable - DOCBOOKSTYLE to the location and rerun - configure afterwards. + above. @@ -602,22 +453,6 @@ checking for osx... osx default simple style use: doc/src/sgml$ make STYLE=website html - - - - - To create a proper index, the build might process several identical - stages. If you do not care about the index, and just want to - proof-read the output, use draft: - -doc/src/sgml$ make draft - - - - - To build the documentation as a single HTML page, use: - -doc/src/sgml$ make postgres.html @@ -632,330 +467,68 @@ checking for osx... osx pages. The man pages are also distributed as a tar archive, similar to the HTML version. To create the man pages, use the commands: - -cd doc/src/sgml -make man - + +doc/src/sgml$ make man + - Print Output via <application>JadeTeX</application> + PDF - If you want to use JadeTex to produce a - printable rendition of the documentation, you can use one of the - following commands: + To produce a PDF rendition of the documentation + using FOP, you can use one of the following + commands, depending on the preferred paper format: - To generate PostScript via DVI in A4 format: - -doc/src/sgml$ make postgres-A4.ps - - In U.S. letter format: + For A4 format: -doc/src/sgml$ make postgres-US.ps +doc/src/sgml$ make postgres-A4.pdf - To make a PDF: - -doc/src/sgml$ make postgres-A4.pdf - - or: + For U.S. letter format: doc/src/sgml$ make postgres-US.pdf - (Of course you can also make a PDF version - from the PostScript, but if you generate PDF - directly, it will have hyperlinks and other enhanced features.) - When using JadeTeX to build the PostgreSQL documentation, you will - probably need to increase some of TeX's internal parameters. These - can be set in the file texmf.cnf. The following - settings worked at the time of this writing: + Because the PostgreSQL documentation is fairly + big, FOP will require a significant amount of + memory. Because of that, on some systems, the build will fail with a + memory-related error message. This can usually be fixed by configuring + Java heap settings in the configuration + file ~/.foprc, for example: -hash_extra.jadetex = 200000 -hash_extra.pdfjadetex = 200000 -pool_size.jadetex = 2000000 -pool_size.pdfjadetex = 2000000 -string_vacancies.jadetex = 150000 -string_vacancies.pdfjadetex = 150000 -max_strings.jadetex = 300000 -max_strings.pdfjadetex = 300000 -save_size.jadetex = 15000 -save_size.pdfjadetex = 15000 +# FOP binary distribution +FOP_OPTS='-Xmx1000m' +# Debian +JAVA_ARGS='-Xmx1000m' +# Red Hat +ADDITIONAL_FLAGS='-Xmx1000m' + There is a minimum amount of memory that is required, and to some extent + more memory appears to make things a bit faster. On systems with very + little memory (less than 1 GB), the build will either be very slow due to + swapping or will not work at all. - - - - Overflow Text - - Occasionally text is too wide for the printed margins, and in - extreme cases, too wide for the printed page, e.g. non-wrapped - text, wide tables. Overly wide text generates Overfull - hbox messages in the TeX log output file, e.g. - postgres-US.log or postgres-A4.log. - There are 72 points in an inch so anything reported as over 72 - points too wide will probably not fit on the printed page (assuming - one inch margins). To find the SGML text - causing the overflow, find the first page number mentioned above - the overflow message, e.g. [50 ###] (page 50), and - look at the page after that (e.g. page 51) in the PDF - file to see the overflow text and adjust the SGML - accordingly. + Other XSL-FO processors can also be used manually, but the automated build + process only supports FOP. - - Print Output via <acronym>RTF</acronym> - - - You can also create a printable version of the PostgreSQL - documentation by converting it to RTF and - applying minor formatting corrections using an office suite. - Depending on the capabilities of the particular office suite, you - can then convert the documentation to PostScript of - PDF. The procedure below illustrates this - process using Applixware. - - - - - It appears that current versions of the PostgreSQL documentation - trigger some bug in or exceed the size limit of OpenJade. If the - build process of the RTF version hangs for a - long time and the output file still has size 0, then you might have - hit that problem. (But keep in mind that a normal build takes 5 - to 10 minutes, so don't abort too soon.) - - - - - <productname>Applixware</productname> <acronym>RTF</acronym> Cleanup - - - OpenJade omits specifying a default - style for body text. In the past, this undiagnosed problem led to - a long process of table of contents generation. However, with - great help from the Applixware folks - the symptom was diagnosed and a workaround is available. - - - - - Generate the RTF version by typing: - -doc/src/sgml$ make postgres.rtf - - - - - - - Repair the RTF file to correctly specify all styles, in - particular the default style. If the document contains - refentry sections, one must also replace - formatting hints which tie a preceding paragraph to the current - paragraph, and instead tie the current paragraph to the - following one. A utility, fixrtf, is - available in doc/src/sgml to accomplish - these repairs: - -doc/src/sgml$ ./fixrtf --refentry postgres.rtf - - - - - The script adds {\s0 Normal;} as the zeroth - style in the document. According to - Applixware, the RTF standard would - prohibit adding an implicit zeroth style, though Microsoft Word - happens to handle this case. For repairing - refentry sections, the script replaces - \keepn tags with \keep. - - - - - - Open a new document in Applixware Words and - then import the RTF file. - - - - - - Generate a new table of contents (ToC) using - Applixware. - - - - - - Select the existing ToC lines, from the beginning of the first - character on the first line to the last character of the last - line. - - - - - - Build a new ToC using - ToolsBook - BuildingCreate Table of - Contents. Select the first three - levels of headers for inclusion in the ToC. This will replace - the existing lines imported in the RTF with a native - Applixware ToC. - - - - - - Adjust the ToC formatting by using - FormatStyle, - selecting each of the three ToC styles, and adjusting the - indents for First and - Left. Use the following values: - - - - - - Style - First Indent (inches) - Left Indent (inches) - - - - - - TOC-Heading 1 - 0.4 - 0.4 - - - - TOC-Heading 2 - 0.8 - 0.8 - - - - TOC-Heading 3 - 1.2 - 1.2 - - - - - - - - - - - - Work through the document to: - - - - - Adjust page breaks. - - - - - - Adjust table column widths. - - - - - - - - - Replace the right-justified page numbers in the Examples and - Figures portions of the ToC with correct values. This only takes - a few minutes. - - - - - - Delete the index section from the document if it is empty. - - - - - - Regenerate and adjust the table of contents. - - - - - - Select the ToC field. - - - - - - Select ToolsBook - BuildingCreate Table of - Contents. - - - - - - Unbind the ToC by selecting - ToolsField - EditingUnprotect. - - - - - - Delete the first line in the ToC, which is an entry for the - ToC itself. - - - - - - - - Save the document as native Applixware - Words format to allow easier last minute editing - later. - - - - - - Print the document - to a file in PostScript format. - - - - - Plain Text Files diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml index 6782f07aea..e2aa1d2a0e 100644 --- a/doc/src/sgml/filelist.sgml +++ b/doc/src/sgml/filelist.sgml @@ -193,7 +193,6 @@ - - - - - - - diff --git a/doc/src/sgml/fixrtf b/doc/src/sgml/fixrtf deleted file mode 100755 index 31cb5f85c0..0000000000 --- a/doc/src/sgml/fixrtf +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/sh -# fixrtf - -# doc/src/sgml/fixrtf - -# Repair (slightly) damaged RTF generated by jade -# Applixware wants the s0 stylesheet defined, whereas -# M$Word does not care about it. -# (c) 2001, Thomas Lockhart, PostgreSQL Inc. - -flist="" -RPAT="" -for i in $@ ; do - case "$i" in - -r|--refentry) - RPAT='-e s/\\\keepn/\\\keep/g' - ;; - -?|--help) - echo "$0 [--refentry] ..." - exit 0 - ;; - -*) - echo "Command $i not recognized" - $0 --help - exit 1 - ;; - *) - flist="$flist $i" - esac -done - -if [ "$flist" = "" ] ; then - flist=*.rtf -fi - -for f in $flist ; do - echo -n "Repairing '$f' ..." - if [ -r $f ] ; then - (sed -e 's/{\\stylesheet{\\s1/{\\stylesheet{\\s0 Normal 0;}{\\s1/g' $RPAT $f > $f.new \ - && mv -f $f.new $f \ - && echo " done") || echo " failed" - else - echo " file not found" - fi -done -exit diff --git a/doc/src/sgml/install-windows.sgml b/doc/src/sgml/install-windows.sgml index 7a9cf7f893..b95b04f5d8 100644 --- a/doc/src/sgml/install-windows.sgml +++ b/doc/src/sgml/install-windows.sgml @@ -517,16 +517,6 @@ $ENV{PERL5LIB}=$ENV{PERL5LIB} . ';c:\IPC-Run-0.94\lib'; - - DocBook DSSSL 1.79 - - Download from - - and uncompress in the subdirectory - docbook-dsssl-1.79. - - - ISO character entities diff --git a/doc/src/sgml/jadetex.cfg b/doc/src/sgml/jadetex.cfg deleted file mode 100644 index 875598f151..0000000000 --- a/doc/src/sgml/jadetex.cfg +++ /dev/null @@ -1,89 +0,0 @@ -% doc/src/sgml/jadetex.cfg -% -% This file redefines \FlowObjectSetup and some related macros to greatly -% reduce the number of control sequence names created, and also to avoid -% creation of many useless hyperlink anchors (bookmarks) in PDF files. -% -% The original coding of \FlowObjectSetup defined a control sequence x@LABEL -% for pretty nearly every flow object in the file, whether that object was -% cross-referenced or not. Worse yet, it created a hyperlink anchor for -% every such object, which not only bloated the output PDF with useless -% anchors but consumed an additional control sequence name per anchor. -% This results in overrunning TeX's limited-size string pool. -% -% To fix, extend \PageLabel's already-existing mechanism whereby a p@LABEL -% control sequence is filled in only for labels that are referenced by at -% least one \Pageref call. We now also fill in p@LABEL for labels that are -% referenced by a \Link. Then, we can drop x@LABEL entirely, and use p@LABEL -% to control emission of both a hyperlink anchor and a page-number label. -% Now, both of those things are emitted for all and only the flow objects -% that have either a hyperlink reference or a page-number reference. -% We consume about one control sequence name per flow object plus one per -% referenced object, which is a lot better than three per flow object. -% -% (With a more invasive patch, we could track the need for an anchor and a -% page-number label separately, but that would probably require two control -% sequences for every flow object. Besides, many objects that have one kind -% of reference will have the other one too; that's certainly true for objects -% referenced in either the TOC or the index, for example.) -% -% -% In addition to checking p@LABEL not x@LABEL, this version of \FlowObjectSetup -% is fixed to clear \Label and \Element whether or not it emits an anchor -% and page label. Failure to do that seems to explain some pre-existing bugs -% in which certain SGML constructs weren't correctly cross-referenced. -% -\def\FlowObjectSetup#1{% -\ifDoFOBSet - \ifLabelElements - \ifx\Label\@empty\let\Label\Element\fi - \fi - \ifx\Label\@empty\else - \expandafter\ifx\csname p@\Label\endcsname\relax - \else - \bgroup - \ifNestedLink - \else - \hyper@anchorstart{\Label}\hyper@anchorend - \PageLabel{\Label}% - \fi - \egroup - \fi - \let\Label\@empty - \let\Element\@empty - \fi -\fi -} -% -% Adjust \PageLabel so that the p@NAME control sequence acquires a correct -% value immediately; this seems to be needed to avoid scenarios wherein -% additional TeX runs are needed to reach a stable state of the .aux file. -% -\def\PageLabel#1{% - \@bsphack - \expandafter\ifx\csname p@#1\endcsname\relax - \else - \protected@write\@auxout{}% - {\string\pagelabel{#1}{\thepage}}% - % Ensure the p@NAME control sequence acquires correct value immediately - \expandafter\xdef\csname p@#1\endcsname{\thepage}% - \fi - \@esphack} -% -% In \Link, add code to emit an aux-file entry if the p@NAME sequence isn't -% defined. Much as in \@Setref, this ensures we'll process the referenced -% item correctly on the next TeX run. -% -\def\Link#1{% - \begingroup - \SetupICs{#1}% - \ifx\Label\@empty\let\Label\Element\fi -% \typeout{Made a Link at \the\inputlineno, to \Label}% - \hyper@linkstart{\LinkType}{\Label}% - \NestedLinktrue - % If p@NAME control sequence isn't defined, emit dummy def to aux file - % so it will get defined properly on next run, much as in \@Setref - \expandafter\ifx\csname p@\Label\endcsname\relax - \immediate\write\@mainaux{\string\pagelabel{\Label}{qqq}}% - \fi -} diff --git a/doc/src/sgml/postgres.sgml b/doc/src/sgml/postgres.sgml index 4e169d1b18..8a3bfc9b0d 100644 --- a/doc/src/sgml/postgres.sgml +++ b/doc/src/sgml/postgres.sgml @@ -277,7 +277,6 @@ &biblio; - - ]]> + diff --git a/doc/src/sgml/stylesheet-html-common.xsl b/doc/src/sgml/stylesheet-html-common.xsl new file mode 100644 index 0000000000..72fac1e806 --- /dev/null +++ b/doc/src/sgml/stylesheet-html-common.xsl @@ -0,0 +1,266 @@ + + +%common.entities; +]> + + + + + + + +pgsql-docs@postgresql.org +2 + + + + + + + + + + + + + + , + + + + + + + + + + ISBN + + + + + + + + + +appendix toc,title +article/appendix nop +article toc,title +book toc,title +chapter toc,title +part toc,title +preface toc,title +qandadiv toc +qandaset toc +reference toc,title +sect1 toc +sect2 toc +sect3 toc +sect4 toc +sect5 toc +section toc +set toc,title + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + +

    + + + + + + + + + + + + +

    + + + + + + +
    +

    + + + +

    +
    + + + + + + + +
    +
    +     + + + + + +
    +

    + + + +

    +
    + + + + + + + +
    +
    +     +     +     + + + + + + + + +
    +     + + + + + + + + + +
    + + + + + + +

    + +

    +     +
    + + + + + + + +
    +
    +     +     + + + + + + + + + + + + | + + + + + + + + + + +     diff --git a/doc/src/sgml/stylesheet-html-nochunk.xsl b/doc/src/sgml/stylesheet-html-nochunk.xsl new file mode 100644 index 0000000000..ffd2012e91 --- /dev/null +++ b/doc/src/sgml/stylesheet-html-nochunk.xsl @@ -0,0 +1,12 @@ + + + + + + + + + diff --git a/doc/src/sgml/stylesheet.dsl b/doc/src/sgml/stylesheet.dsl deleted file mode 100644 index 05cab9a5f3..0000000000 --- a/doc/src/sgml/stylesheet.dsl +++ /dev/null @@ -1,798 +0,0 @@ - - - - - - -]]> - - -]]> - -]> - - - - - - - - - -(define draft-mode #f) - -(define pgsql-docs-list "pgsql-docs@postgresql.org") - -;; Don't show manpage volume numbers -(define %refentry-xref-manvolnum% #f) - -;; Don't use graphics for callouts. (We could probably do that, but -;; it needs extra work.) -(define %callout-graphics% #f) - -;; Show comments during the development stage. -(define %show-comments% draft-mode) - -;; Don't append period if run-in title ends with any of these -;; characters. We had to add the colon here. This is fixed in -;; stylesheets version 1.71, so it can be removed sometime. -(define %content-title-end-punct% - '(#\. #\! #\? #\:)) - -;; No automatic punctuation after honorific name parts -(define %honorific-punctuation% "") - -;; Change display of some elements -(element command ($mono-seq$)) -(element envar ($mono-seq$)) -(element lineannotation ($italic-seq$)) -(element literal ($mono-seq$)) -(element option ($mono-seq$)) -(element parameter ($mono-seq$)) -(element structfield ($mono-seq$)) -(element structname ($mono-seq$)) -(element symbol ($mono-seq$)) -(element token ($mono-seq$)) -(element type ($mono-seq$)) -(element varname ($mono-seq$)) -(element (programlisting emphasis) ($bold-seq$)) ;; to highlight sections of code - -;; Special support for Tcl synopses -(element optional - (if (equal? (attribute-string (normalize "role")) "tcl") - (make sequence - (literal "?") - ($charseq$) - (literal "?")) - (make sequence - (literal %arg-choice-opt-open-str%) - ($charseq$) - (literal %arg-choice-opt-close-str%)))) - -;; Avoid excessive cross-reference labels -(define (auto-xref-indirect? target ancestor) - (cond -; ;; Always add indirect references to another book -; ((member (gi ancestor) (book-element-list)) -; #t) - ;; Add indirect references to the section or component a block - ;; is in iff chapters aren't autolabelled. (Otherwise "Figure 1-3" - ;; is sufficient) - ((and (member (gi target) (block-element-list)) - (not %chapter-autolabel%)) - #t) - ;; Add indirect references to the component a section is in if - ;; the sections are not autolabelled - ((and (member (gi target) (section-element-list)) - (member (gi ancestor) (component-element-list)) - (not %section-autolabel%)) - #t) - (else #f))) - - -;; Bibliography things - -;; Use the titles of bibliography entries in cross-references -(define biblio-xref-title #t) - -;; Process bibliography entry components in the order shown below, not -;; in the order they appear in the document. (I suppose this should -;; be made to fit some publishing standard.) -(define %biblioentry-in-entry-order% #f) - -(define (biblioentry-inline-elements) - (list - (normalize "author") - (normalize "authorgroup") - (normalize "title") - (normalize "subtitle") - (normalize "volumenum") - (normalize "edition") - (normalize "othercredit") - (normalize "contrib") - (normalize "editor") - (normalize "publishername") - (normalize "confgroup") - (normalize "publisher") - (normalize "isbn") - (normalize "issn") - (normalize "pubsnumber") - (normalize "date") - (normalize "pubdate") - (normalize "pagenums") - (normalize "bibliomisc"))) - -(mode biblioentry-inline-mode - - (element confgroup - (make sequence - (literal "Proc. ") - (next-match))) - - (element isbn - (make sequence - (literal "ISBN ") - (process-children))) - - (element issn - (make sequence - (literal "ISSN ") - (process-children)))) - - -;; The rules in the default stylesheet for productname format it as a -;; paragraph. This may be suitable for productname directly within -;; *info, but it's nonsense when productname is used inline, as we do. -(mode book-titlepage-recto-mode - (element (para productname) ($charseq$))) -(mode book-titlepage-verso-mode - (element (para productname) ($charseq$))) -;; Add more here if needed... - - -;; Replace a sequence of whitespace in a string by a single space -(define (normalize-whitespace str #!optional (whitespace '(#\space #\U-000D))) - (let loop ((characters (string->list str)) - (result '()) - (prev-was-space #f)) - (if (null? characters) - (list->string (reverse result)) - (let ((c (car characters)) - (rest (cdr characters))) - (if (member c whitespace) - (if prev-was-space - (loop rest result #t) - (loop rest (cons #\space result) #t)) - (loop rest (cons c result) #f)))))) - - - - -string (time) #t))))) - - -;; Block elements are allowed in PARA in DocBook, but not in P in -;; HTML. With %fix-para-wrappers% turned on, the stylesheets attempt -;; to avoid putting block elements in HTML P tags by outputting -;; additional end/begin P pairs around them. -(define %fix-para-wrappers% #t) - -;; ...but we need to do some extra work to make the above apply to PRE -;; as well. (mostly pasted from dbverb.dsl) -(define ($verbatim-display$ indent line-numbers?) - (let ((content (make element gi: "PRE" - attributes: (list - (list "CLASS" (gi))) - (if (or indent line-numbers?) - ($verbatim-line-by-line$ indent line-numbers?) - (process-children))))) - (if %shade-verbatim% - (make element gi: "TABLE" - attributes: ($shade-verbatim-attr$) - (make element gi: "TR" - (make element gi: "TD" - content))) - (make sequence - (para-check) - content - (para-check 'restart))))) - -;; ...and for notes. -(element note - (make sequence - (para-check) - ($admonition$) - (para-check 'restart))) - -;;; XXX The above is very ugly. It might be better to run 'tidy' on -;;; the resulting *.html files. - - -;; Format multiple terms in varlistentry vertically, instead -;; of comma-separated. -(element (varlistentry term) - (make sequence - (process-children-trim) - (if (not (last-sibling?)) - (make empty-element gi: "BR") - (empty-sosofo)))) - - -;; Customization of header -;; - make title a link to the home page -;; - add tool tips to Prev/Next links -;; - add Up link -;; (overrides dbnavig.dsl) -(define (default-header-nav-tbl-noff elemnode prev next prevsib nextsib) - (let* ((r1? (nav-banner? elemnode)) - (r1-sosofo (make element gi: "TR" - (make element gi: "TH" - attributes: (list - (list "COLSPAN" "4") - (list "ALIGN" "center") - (list "VALIGN" "bottom")) - (make element gi: "A" - attributes: (list - (list "HREF" (href-to (nav-home elemnode)))) - (nav-banner elemnode))))) - (r2? (or (not (node-list-empty? prev)) - (not (node-list-empty? next)) - (nav-context? elemnode))) - (r2-sosofo (make element gi: "TR" - (make element gi: "TD" - attributes: (list - (list "WIDTH" "10%") - (list "ALIGN" "left") - (list "VALIGN" "top")) - (if (node-list-empty? prev) - (make entity-ref name: "nbsp") - (make element gi: "A" - attributes: (list - (list "TITLE" (element-title-string prev)) - (list "HREF" - (href-to - prev)) - (list "ACCESSKEY" - "P")) - (gentext-nav-prev prev)))) - (make element gi: "TD" - attributes: (list - (list "WIDTH" "10%") - (list "ALIGN" "left") - (list "VALIGN" "top")) - (if (nav-up? elemnode) - (nav-up elemnode) - (nav-home-link elemnode))) - (make element gi: "TD" - attributes: (list - (list "WIDTH" "60%") - (list "ALIGN" "center") - (list "VALIGN" "bottom")) - (nav-context elemnode)) - (make element gi: "TD" - attributes: (list - (list "WIDTH" "20%") - (list "ALIGN" "right") - (list "VALIGN" "top")) - (if (node-list-empty? next) - (make entity-ref name: "nbsp") - (make element gi: "A" - attributes: (list - (list "TITLE" (element-title-string next)) - (list "HREF" - (href-to - next)) - (list "ACCESSKEY" - "N")) - (gentext-nav-next next))))))) - (if (or r1? r2?) - (make element gi: "DIV" - attributes: '(("CLASS" "NAVHEADER")) - (make element gi: "TABLE" - attributes: (list - (list "SUMMARY" "Header navigation table") - (list "WIDTH" %gentext-nav-tblwidth%) - (list "BORDER" "0") - (list "CELLPADDING" "0") - (list "CELLSPACING" "0")) - (if r1? r1-sosofo (empty-sosofo)) - (if r2? r2-sosofo (empty-sosofo))) - (make empty-element gi: "HR" - attributes: (list - (list "ALIGN" "LEFT") - (list "WIDTH" %gentext-nav-tblwidth%)))) - (empty-sosofo)))) - - -;; Put index "quicklinks" (A | B | C | ...) at the top of the bookindex page. - -(element index - (let ((preamble (node-list-filter-by-not-gi - (children (current-node)) - (list (normalize "indexentry")))) - (indexdivs (node-list-filter-by-gi - (children (current-node)) - (list (normalize "indexdiv")))) - (entries (node-list-filter-by-gi - (children (current-node)) - (list (normalize "indexentry"))))) - (html-document - (with-mode head-title-mode - (literal (element-title-string (current-node)))) - (make element gi: "DIV" - attributes: (list (list "CLASS" (gi))) - ($component-separator$) - ($component-title$) - (if (node-list-empty? indexdivs) - (empty-sosofo) - (make element gi: "P" - attributes: (list (list "CLASS" "INDEXDIV-QUICKLINKS")) - (with-mode indexdiv-quicklinks-mode - (process-node-list indexdivs)))) - (process-node-list preamble) - (if (node-list-empty? entries) - (empty-sosofo) - (make element gi: "DL" - (process-node-list entries))))))) - - -(mode indexdiv-quicklinks-mode - (element indexdiv - (make sequence - (make element gi: "A" - attributes: (list (list "HREF" (href-to (current-node)))) - (element-title-sosofo)) - (if (not (last-sibling?)) - (literal " | ") - (literal ""))))) - - -;; Changed to strip and normalize index term content (overrides -;; dbindex.dsl) -(define (htmlindexterm) - (let* ((attr (gi (current-node))) - (content (data (current-node))) - (string (strip (normalize-whitespace content))) ;; changed - (sortas (attribute-string (normalize "sortas")))) - (make sequence - (make formatting-instruction data: attr) - (if sortas - (make sequence - (make formatting-instruction data: "[") - (make formatting-instruction data: sortas) - (make formatting-instruction data: "]")) - (empty-sosofo)) - (make formatting-instruction data: " ") - (make formatting-instruction data: string) - (htmlnewline)))) - - -]]> - - - - - (string->number (attribute-string (normalize "columns"))) 0) - (string->number (attribute-string (normalize "columns"))) - 1) - 1)) - (members (select-elements (children (current-node)) (normalize "member")))) - (cond - ((equal? type (normalize "inline")) - (if (equal? (gi (parent (current-node))) - (normalize "para")) - (process-children) - (make paragraph - space-before: %para-sep% - space-after: %para-sep% - start-indent: (inherited-start-indent)))) - ((equal? type (normalize "vert")) - (my-simplelist-vert members)) - ((equal? type (normalize "horiz")) - (simplelist-table 'row cols members))))) - -(element member - (let ((type (inherited-attribute-string (normalize "type")))) - (cond - ((equal? type (normalize "inline")) - (make sequence - (process-children) - (if (not (last-sibling?)) - (literal ", ") - (literal "")))) - ((equal? type (normalize "vert")) - (make paragraph - space-before: 0pt - space-after: 0pt)) - ((equal? type (normalize "horiz")) - (make paragraph - quadding: 'start - (process-children)))))) - - -;; Jadetex doesn't handle links to the content of tables, so -;; indexterms that point to table entries will go nowhere. We fix -;; this by pointing the index entry to the table itself instead, which -;; should be equally useful in practice. - -(define (find-parent-table nd) - (let ((table (ancestor-member nd ($table-element-list$)))) - (if (node-list-empty? table) - nd - table))) - -;; (The function below overrides the one in print/dbindex.dsl.) - -(define (indexentry-link nd) - (let* ((id (attribute-string (normalize "role") nd)) - (prelim-target (find-indexterm id)) - (target (find-parent-table prelim-target)) - (preferred (not (node-list-empty? - (select-elements (children (current-node)) - (normalize "emphasis"))))) - (sosofo (if (node-list-empty? target) - (literal "?") - (make link - destination: (node-list-address target) - (with-mode toc-page-number-mode - (process-node-list target)))))) - (if preferred - (make sequence - font-weight: 'bold - sosofo) - sosofo))) - - -;; By default, the part and reference title pages get wrong page -;; numbers: The first title page gets roman numerals carried over from -;; preface/toc -- we want Arabic numerals. We also need to make sure -;; that page-number-restart is set of #f explicitly, because otherwise -;; it will carry over from the previous component, which is not good. -;; -;; (This looks worse than it is. It's copied from print/dbttlpg.dsl -;; and common/dbcommon.dsl and modified in minor detail.) - -(define (first-part?) - (let* ((book (ancestor (normalize "book"))) - (nd (ancestor-member (current-node) - (append - (component-element-list) - (division-element-list)))) - (bookch (children book))) - (let loop ((nl bookch)) - (if (node-list-empty? nl) - #f - (if (equal? (gi (node-list-first nl)) (normalize "part")) - (if (node-list=? (node-list-first nl) nd) - #t - #f) - (loop (node-list-rest nl))))))) - -(define (first-reference?) - (let* ((book (ancestor (normalize "book"))) - (nd (ancestor-member (current-node) - (append - (component-element-list) - (division-element-list)))) - (bookch (children book))) - (let loop ((nl bookch)) - (if (node-list-empty? nl) - #f - (if (equal? (gi (node-list-first nl)) (normalize "reference")) - (if (node-list=? (node-list-first nl) nd) - #t - #f) - (loop (node-list-rest nl))))))) - - -(define (part-titlepage elements #!optional (side 'recto)) - (let ((nodelist (titlepage-nodelist - (if (equal? side 'recto) - (reference-titlepage-recto-elements) - (reference-titlepage-verso-elements)) - elements)) - ;; partintro is a special case... - (partintro (node-list-first - (node-list-filter-by-gi elements (list (normalize "partintro")))))) - (if (part-titlepage-content? elements side) - (make simple-page-sequence - page-n-columns: %titlepage-n-columns% - ;; Make sure that page number format is correct. - page-number-format: ($page-number-format$) - ;; Make sure that the page number is set to 1 if this is the - ;; first part in the book - page-number-restart?: (first-part?) - input-whitespace-treatment: 'collapse - use: default-text-style - - ;; This hack is required for the RTF backend. If an external-graphic - ;; is the first thing on the page, RTF doesn't seem to do the right - ;; thing (the graphic winds up on the baseline of the first line - ;; of the page, left justified). This "one point rule" fixes - ;; that problem. - (make paragraph - line-spacing: 1pt - (literal "")) - - (let loop ((nl nodelist) (lastnode (empty-node-list))) - (if (node-list-empty? nl) - (empty-sosofo) - (make sequence - (if (or (node-list-empty? lastnode) - (not (equal? (gi (node-list-first nl)) - (gi lastnode)))) - (part-titlepage-before (node-list-first nl) side) - (empty-sosofo)) - (cond - ((equal? (gi (node-list-first nl)) (normalize "subtitle")) - (part-titlepage-subtitle (node-list-first nl) side)) - ((equal? (gi (node-list-first nl)) (normalize "title")) - (part-titlepage-title (node-list-first nl) side)) - (else - (part-titlepage-default (node-list-first nl) side))) - (loop (node-list-rest nl) (node-list-first nl))))) - - (if (and %generate-part-toc% - %generate-part-toc-on-titlepage% - (equal? side 'recto)) - (make display-group - (build-toc (current-node) - (toc-depth (current-node)))) - (empty-sosofo)) - - ;; PartIntro is a special case - (if (and (equal? side 'recto) - (not (node-list-empty? partintro)) - %generate-partintro-on-titlepage%) - ($process-partintro$ partintro #f) - (empty-sosofo))) - - (empty-sosofo)))) - - -(define (reference-titlepage elements #!optional (side 'recto)) - (let ((nodelist (titlepage-nodelist - (if (equal? side 'recto) - (reference-titlepage-recto-elements) - (reference-titlepage-verso-elements)) - elements)) - ;; partintro is a special case... - (partintro (node-list-first - (node-list-filter-by-gi elements (list (normalize "partintro")))))) - (if (reference-titlepage-content? elements side) - (make simple-page-sequence - page-n-columns: %titlepage-n-columns% - ;; Make sure that page number format is correct. - page-number-format: ($page-number-format$) - ;; Make sure that the page number is set to 1 if this is the - ;; first part in the book - page-number-restart?: (first-reference?) - input-whitespace-treatment: 'collapse - use: default-text-style - - ;; This hack is required for the RTF backend. If an external-graphic - ;; is the first thing on the page, RTF doesn't seem to do the right - ;; thing (the graphic winds up on the baseline of the first line - ;; of the page, left justified). This "one point rule" fixes - ;; that problem. - (make paragraph - line-spacing: 1pt - (literal "")) - - (let loop ((nl nodelist) (lastnode (empty-node-list))) - (if (node-list-empty? nl) - (empty-sosofo) - (make sequence - (if (or (node-list-empty? lastnode) - (not (equal? (gi (node-list-first nl)) - (gi lastnode)))) - (reference-titlepage-before (node-list-first nl) side) - (empty-sosofo)) - (cond - ((equal? (gi (node-list-first nl)) (normalize "author")) - (reference-titlepage-author (node-list-first nl) side)) - ((equal? (gi (node-list-first nl)) (normalize "authorgroup")) - (reference-titlepage-authorgroup (node-list-first nl) side)) - ((equal? (gi (node-list-first nl)) (normalize "corpauthor")) - (reference-titlepage-corpauthor (node-list-first nl) side)) - ((equal? (gi (node-list-first nl)) (normalize "editor")) - (reference-titlepage-editor (node-list-first nl) side)) - ((equal? (gi (node-list-first nl)) (normalize "subtitle")) - (reference-titlepage-subtitle (node-list-first nl) side)) - ((equal? (gi (node-list-first nl)) (normalize "title")) - (reference-titlepage-title (node-list-first nl) side)) - (else - (reference-titlepage-default (node-list-first nl) side))) - (loop (node-list-rest nl) (node-list-first nl))))) - - (if (and %generate-reference-toc% - %generate-reference-toc-on-titlepage% - (equal? side 'recto)) - (make display-group - (build-toc (current-node) - (toc-depth (current-node)))) - (empty-sosofo)) - - ;; PartIntro is a special case - (if (and (equal? side 'recto) - (not (node-list-empty? partintro)) - %generate-partintro-on-titlepage%) - ($process-partintro$ partintro #f) - (empty-sosofo))) - - (empty-sosofo)))) - -]]> - - - - - - diff --git a/doc/src/sgml/stylesheet.xsl b/doc/src/sgml/stylesheet.xsl index e36e8cc5cc..c9fb7b225c 100644 --- a/doc/src/sgml/stylesheet.xsl +++ b/doc/src/sgml/stylesheet.xsl @@ -1,8 +1,4 @@ - -%common.entities; -]> + - - -pgsql-docs@postgresql.org -2 @@ -34,252 +27,6 @@ - - - - - - - - - - - - , - - - - - - - - - - ISBN - - - - - - - - - -appendix toc,title -article/appendix nop -article toc,title -book toc,title -chapter toc,title -part toc,title -preface toc,title -qandadiv toc -qandaset toc -reference toc,title -sect1 toc -sect2 toc -sect3 toc -sect4 toc -sect5 toc -section toc -set toc,title - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - -

    - - - - - - - - - - - - -

    - - - - - - -
    -

    - - - -

    -
    - - - - - - - -
    -
    -     - - - - - -
    -

    - - - -

    -
    - - - - - - - -
    -
    -     -     -     - - - - - - - - -
    -     - - - - - - - - - -
    - - - - - - -

    - -

    -     -
    - - - - - - - -
    -
    -     -     - - - - - - - - - - - - | - - - - - - - - - - - -- cgit v1.2.3 From a6e7d591d0129bc7f1f186cb40a6ebd7963956ab Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Thu, 13 Apr 2017 11:29:53 +0900 Subject: Improve documentations for ALTER PUBLICATION and ALTER SUBSCRIPTION. Discussion: http://postgr.es/m/CAD21AoC32YgtateNqTFXzTJmHHe6hXs4cpJTND3n-Ts8f-aMqw@mail.gmail.com --- doc/src/sgml/ref/alter_publication.sgml | 19 +++++++++++++++++++ doc/src/sgml/ref/alter_subscription.sgml | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/alter_publication.sgml b/doc/src/sgml/ref/alter_publication.sgml index 776661bbeb..0a965b3bbf 100644 --- a/doc/src/sgml/ref/alter_publication.sgml +++ b/doc/src/sgml/ref/alter_publication.sgml @@ -30,6 +30,7 @@ ALTER PUBLICATION name WITH ( name OWNER TO { new_owner | CURRENT_USER | SESSION_USER } +ALTER PUBLICATION name RENAME TO new_name ALTER PUBLICATION name ADD TABLE table_name [, ...] ALTER PUBLICATION name SET TABLE table_name [, ...] ALTER PUBLICATION name DROP TABLE table_name [, ...] @@ -78,6 +79,24 @@ ALTER PUBLICATION name DROP TABLE <     + + new_owner + + + The user name of the new owner of the publication. + + + + + + new_name + + + The new name for the publication. + + + + PUBLISH INSERT NOPUBLISH INSERT diff --git a/doc/src/sgml/ref/alter_subscription.sgml b/doc/src/sgml/ref/alter_subscription.sgml index 0e332bc143..640fac0a15 100644 --- a/doc/src/sgml/ref/alter_subscription.sgml +++ b/doc/src/sgml/ref/alter_subscription.sgml @@ -35,6 +35,7 @@ ALTER SUBSCRIPTION name REFRESH PUB COPY DATA | NOCOPY DATA ALTER SUBSCRIPTION name OWNER TO { new_owner | CURRENT_USER | SESSION_USER } +ALTER SUBSCRIPTION name RENAME TO new_name ALTER SUBSCRIPTION name CONNECTION 'conninfo' ALTER SUBSCRIPTION name ENABLE ALTER SUBSCRIPTION name DISABLE @@ -69,6 +70,24 @@ ALTER SUBSCRIPTION name DISABLE + + new_owner + + + The user name of the new owner of the subscription. + + + + + + new_name + + + The new name for the subscription. + + + + CONNECTION 'conninfo' SLOT NAME = slot_name -- cgit v1.2.3 From 7a3e8d7b503c25e009b9f591554617a2434c72eb Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Thu, 13 Apr 2017 12:09:14 +0900 Subject: Move pg_stat_progress_vacuum to the table of Dynamic Statistics Views in doc. Previously the description about pg_stat_progress_vacuum was in the table of "Collected Statistics Views" in the doc. But since it repors dynamic information, i.e., the current progress of VACUUM, its description should be in the table of "Dynamic Statistics Views". Back-patch to 9.6 where pg_stat_progress_vacuum was added. Author: Amit Langote Discussion: http://postgr.es/m/7ab51b59-8d4d-6193-c60a-b75f222efb12@lab.ntt.co.jp --- doc/src/sgml/monitoring.sgml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index d42a461ad9..2a83671b53 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -324,6 +324,14 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser + + pg_stat_progress_vacuumpg_stat_progress_vacuum + One row for each backend (including autovacuum worker processes) running + VACUUM, showing current progress. + See . + + +
    @@ -515,12 +523,6 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser yet included in pg_stat_user_functions). - - pg_stat_progress_vacuumpg_stat_progress_vacuum - One row for each backend (including autovacuum worker processes) running - VACUUM, showing current progress. - See . - -- cgit v1.2.3 From 3d5facfd9ab66c819ed583b2614b0560405a6aa2 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Thu, 13 Apr 2017 11:35:22 -0300 Subject: Remove pg_stats_ext view It was created as equivalent of pg_stats, but since the code underlying pg_statistic_ext is more convenient than the one for pg_statistic, pg_stats_ext is no longer useful. Author: David Rowley Reviewed-by: Tomas Vondra Discussion: https://postgr.es/m/CAKJS1f9zAkPUf9nQrqpFBAsrOHvb5eYa2FVNsmCJy1wegcO_TQ@mail.gmail.com --- doc/src/sgml/planstats.sgml | 12 +++++------- src/backend/catalog/system_views.sql | 11 ----------- src/test/regress/expected/rules.out | 9 --------- 3 files changed, 5 insertions(+), 27 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/planstats.sgml b/doc/src/sgml/planstats.sgml index f8af42f394..a4f91c737a 100644 --- a/doc/src/sgml/planstats.sgml +++ b/doc/src/sgml/planstats.sgml @@ -520,17 +520,15 @@ EXPLAIN ANALYZE SELECT * FROM t WHERE a = 1 AND b = 1; Similarly to per-column statistics, extended statistics are stored in - a system catalog called pg_statistic_ext, but - there is also a more convenient view pg_stats_ext. + a system catalog called pg_statistic_ext. To inspect the statistics s1 defined above, you may do this: -SELECT tablename, staname, attnums, depsbytes - FROM pg_stats_ext WHERE staname = 's1'; - tablename | staname | attnums | depsbytes ------------+---------+---------+----------- - t | s1 | 1 2 | 40 +SELECT staname,stadependencies FROM pg_statistic_ext WHERE staname = 's1'; + staname | stadependencies +---------+-------------------------------------------- + s1 | [{1 => 2 : 1.000000}, {2 => 1 : 1.000000}] (1 row) diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 500221ae77..421d51db47 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -186,17 +186,6 @@ CREATE OR REPLACE VIEW pg_sequences AS WHERE NOT pg_is_other_temp_schema(N.oid) AND relkind = 'S'; -CREATE VIEW pg_stats_ext AS - SELECT - N.nspname AS schemaname, - C.relname AS tablename, - S.staname AS staname, - S.stakeys AS attnums, - length(s.standistinct::bytea) AS ndistbytes, - length(S.stadependencies::bytea) AS depsbytes - FROM (pg_statistic_ext S JOIN pg_class C ON (C.oid = S.starelid)) - LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace); - CREATE VIEW pg_stats WITH (security_barrier) AS SELECT nspname AS schemaname, diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index b2779d9698..409692d695 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2188,15 +2188,6 @@ pg_stats| SELECT n.nspname AS schemaname, JOIN pg_attribute a ON (((c.oid = a.attrelid) AND (a.attnum = s.staattnum)))) LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace))) WHERE ((NOT a.attisdropped) AND has_column_privilege(c.oid, a.attnum, 'select'::text) AND ((c.relrowsecurity = false) OR (NOT row_security_active(c.oid)))); -pg_stats_ext| SELECT n.nspname AS schemaname, - c.relname AS tablename, - s.staname, - s.stakeys AS attnums, - length((s.standistinct)::bytea) AS ndistbytes, - length((s.stadependencies)::bytea) AS depsbytes - FROM ((pg_statistic_ext s - JOIN pg_class c ON ((c.oid = s.starelid))) - LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace))); pg_tables| SELECT n.nspname AS schemaname, c.relname AS tablename, pg_get_userbyid(c.relowner) AS tableowner, -- cgit v1.2.3 From 73c1748d833617c6ba19750236f8e09beedb132a Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Thu, 13 Apr 2017 12:08:34 -0300 Subject: Fix XMLTABLE synopsis, add XMLNAMESPACES example Add a missing comma in the synopsis after the XMLNAMESPACES clause. Also, add an example illustrating the use of that clause. Author: Arjen Nienhuis and Pavel Stěhule --- doc/src/sgml/func.sgml | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index adab3030c4..f06d0a92c0 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -10525,7 +10525,7 @@ SELECT xpath_exists('/my:a/text()', 'test -xmltable( XMLNAMESPACES(namespace uri AS namespace name, ...) +xmltable( XMLNAMESPACES(namespace uri AS namespace name, ...), row_expression PASSING BY REF document_expression BY REF COLUMNS name { type PATH column_expression DEFAULT default_expression NOT NULL | NULL | FOR ORDINALITY } @@ -10708,6 +10708,36 @@ SELECT xmltable.* element ---------------------- Hello2a2 bbbCC +]]> + + + + The following example illustrates how + the XMLNAMESPACES clause can be used to specify + the default namespace, and a list of additional namespaces + used in the XML document as well as in the XPath expressions: + + + + + +'::xml) +) +SELECT xmltable.* + FROM XMLTABLE(XMLNAMESPACES('http://example.com/myns' AS x, + 'http://example.com/b' AS "B"), + '/x:example/x:item' + PASSING (SELECT data FROM xmldata) + COLUMNS foo int PATH '@foo', + bar int PATH '@B:bar'); + foo | bar +-----+----- + 1 | 2 + 3 | 4 + 4 | 5 +(3 rows) ]]> -- cgit v1.2.3 From c31671f9b5f6eee9b6726baad2db1795c94839d1 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 11 Apr 2017 22:02:59 -0400 Subject: pg_dump: Dump subscriptions by default Dump subscriptions if the current user is a superuser, otherwise write a warning and skip them. Remove the pg_dump option --include-subscriptions. Discussion: https://www.postgresql.org/message-id/e4fbfad5-c6ac-fd50-6777-18c84b34eb2f@2ndquadrant.com --- doc/src/sgml/logical-replication.sgml | 7 +++--- doc/src/sgml/ref/pg_dump.sgml | 9 -------- src/bin/pg_dump/pg_backup.h | 2 -- src/bin/pg_dump/pg_backup_archiver.c | 1 - src/bin/pg_dump/pg_dump.c | 43 ++++++++++++++++++++++++++++++----- src/bin/pg_dump/pg_restore.c | 3 --- src/bin/pg_dump/t/002_pg_dump.pl | 24 ++++++++----------- 7 files changed, 50 insertions(+), 39 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml index 48db9cd08b..8c70ce3b6e 100644 --- a/doc/src/sgml/logical-replication.sgml +++ b/doc/src/sgml/logical-replication.sgml @@ -167,9 +167,10 @@     - Subscriptions are not dumped by pg_dump by default, but - this can be requested using the command-line - option . + Subscriptions are dumped by pg_dump if the current user + is a superuser. Otherwise a warning is written and subscriptions are + skipped, because non-superusers cannot read all subscription information + from the pg_subscription catalog. diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml index 4f19b89232..53b5dd5239 100644 --- a/doc/src/sgml/ref/pg_dump.sgml +++ b/doc/src/sgml/ref/pg_dump.sgml @@ -755,15 +755,6 @@ PostgreSQL documentation - - - - - Include logical replication subscriptions in the dump. - - - - diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index d82938141e..1d14b68983 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -119,7 +119,6 @@ typedef struct _restoreOptions bool *idWanted; /* array showing which dump IDs to emit */ int enable_row_security; int sequence_data; /* dump sequence data even in schema-only mode */ - int include_subscriptions; int binary_upgrade; } RestoreOptions; @@ -154,7 +153,6 @@ typedef struct _dumpOptions int outputNoTablespaces; int use_setsessauth; int enable_row_security; - int include_subscriptions; int no_subscription_connect; /* default, if no "inclusion" switches appear, is to dump everything */ diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 79bfbdf1a1..b622506a00 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -171,7 +171,6 @@ dumpOptionsFromRestoreOptions(RestoreOptions *ropt) dopt->include_everything = ropt->include_everything; dopt->enable_row_security = ropt->enable_row_security; dopt->sequence_data = ropt->sequence_data; - dopt->include_subscriptions = ropt->include_subscriptions; return dopt; } diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 5bd4af22c9..dcefe975d8 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -342,7 +342,6 @@ main(int argc, char **argv) {"enable-row-security", no_argument, &dopt.enable_row_security, 1}, {"exclude-table-data", required_argument, NULL, 4}, {"if-exists", no_argument, &dopt.if_exists, 1}, - {"include-subscriptions", no_argument, &dopt.include_subscriptions, 1}, {"inserts", no_argument, &dopt.dump_inserts, 1}, {"lock-wait-timeout", required_argument, NULL, 2}, {"no-tablespaces", no_argument, &dopt.outputNoTablespaces, 1}, @@ -868,7 +867,6 @@ main(int argc, char **argv) ropt->include_everything = dopt.include_everything; ropt->enable_row_security = dopt.enable_row_security; ropt->sequence_data = dopt.sequence_data; - ropt->include_subscriptions = dopt.include_subscriptions; ropt->binary_upgrade = dopt.binary_upgrade; if (compressLevel == -1) @@ -951,7 +949,6 @@ help(const char *progname) " access to)\n")); printf(_(" --exclude-table-data=TABLE do NOT dump data for the named table(s)\n")); printf(_(" --if-exists use IF EXISTS when dropping objects\n")); - printf(_(" --include-subscriptions dump logical replication subscriptions\n")); printf(_(" --inserts dump data as INSERT commands, rather than COPY\n")); printf(_(" --no-security-labels do not dump security label assignments\n")); printf(_(" --no-subscription-connect dump subscriptions so they don't connect on restore\n")); @@ -3641,6 +3638,22 @@ dumpPublicationTable(Archive *fout, PublicationRelInfo *pubrinfo) destroyPQExpBuffer(query); } +/* + * Is the currently connected user a superuser? + */ +static bool +is_superuser(Archive *fout) +{ + ArchiveHandle *AH = (ArchiveHandle *) fout; + const char *val; + + val = PQparameterStatus(AH->connection, "is_superuser"); + + if (val && strcmp(val, "on") == 0) + return true; + + return false; +} /* * getSubscriptions @@ -3649,7 +3662,6 @@ dumpPublicationTable(Archive *fout, PublicationRelInfo *pubrinfo) void getSubscriptions(Archive *fout) { - DumpOptions *dopt = fout->dopt; PQExpBuffer query; PGresult *res; SubscriptionInfo *subinfo; @@ -3664,9 +3676,25 @@ getSubscriptions(Archive *fout) int i, ntups; - if (!dopt->include_subscriptions || fout->remoteVersion < 100000) + if (fout->remoteVersion < 100000) return; + if (!is_superuser(fout)) + { + int n; + + res = ExecuteSqlQuery(fout, + "SELECT count(*) FROM pg_subscription " + "WHERE subdbid = (SELECT oid FROM pg_catalog.pg_database" + " WHERE datname = current_database())", + PGRES_TUPLES_OK); + n = atoi(PQgetvalue(res, 0, 0)); + if (n > 0) + write_msg(NULL, "WARNING: subscriptions not dumped because current user is not a superuser\n"); + PQclear(res); + return; + } + query = createPQExpBuffer(); resetPQExpBuffer(query); @@ -3714,6 +3742,9 @@ getSubscriptions(Archive *fout) if (strlen(subinfo[i].rolname) == 0) write_msg(NULL, "WARNING: owner of subscription \"%s\" appears to be invalid\n", subinfo[i].dobj.name); + + /* Decide whether we want to dump it */ + selectDumpableObject(&(subinfo[i].dobj), fout); } PQclear(res); @@ -3735,7 +3766,7 @@ dumpSubscription(Archive *fout, SubscriptionInfo *subinfo) int npubnames = 0; int i; - if (dopt->dataOnly) + if (!(subinfo->dobj.dump & DUMP_COMPONENT_DEFINITION)) return; delq = createPQExpBuffer(); diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c index 5f61fb2764..ddd79429b4 100644 --- a/src/bin/pg_dump/pg_restore.c +++ b/src/bin/pg_dump/pg_restore.c @@ -67,7 +67,6 @@ main(int argc, char **argv) char *inputFileSpec; static int disable_triggers = 0; static int enable_row_security = 0; - static int include_subscriptions = 0; static int if_exists = 0; static int no_data_for_failed_tables = 0; static int outputNoTablespaces = 0; @@ -112,7 +111,6 @@ main(int argc, char **argv) {"disable-triggers", no_argument, &disable_triggers, 1}, {"enable-row-security", no_argument, &enable_row_security, 1}, {"if-exists", no_argument, &if_exists, 1}, - {"include-subscriptions", no_argument, &include_subscriptions, 1}, {"no-data-for-failed-tables", no_argument, &no_data_for_failed_tables, 1}, {"no-tablespaces", no_argument, &outputNoTablespaces, 1}, {"role", required_argument, NULL, 2}, @@ -353,7 +351,6 @@ main(int argc, char **argv) opts->disable_triggers = disable_triggers; opts->enable_row_security = enable_row_security; - opts->include_subscriptions = include_subscriptions; opts->noDataForFailedTables = no_data_for_failed_tables; opts->noTablespace = outputNoTablespaces; opts->use_setsessauth = use_setsessauth; diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl index cccad04ab6..1db3767f46 100644 --- a/src/bin/pg_dump/t/002_pg_dump.pl +++ b/src/bin/pg_dump/t/002_pg_dump.pl @@ -43,7 +43,6 @@ my %pgdump_runs = ( '--format=custom', "--file=$tempdir/binary_upgrade.dump", '-w', - '--include-subscriptions', # XXX Should not be necessary? '--schema-only', '--binary-upgrade', '-d', 'postgres', # alternative way to specify database @@ -58,7 +57,6 @@ my %pgdump_runs = ( 'pg_dump', '--no-sync', "--file=$tempdir/clean.sql", - '--include-subscriptions', '-c', '-d', 'postgres', # alternative way to specify database ], }, @@ -67,7 +65,6 @@ my %pgdump_runs = ( 'pg_dump', '--no-sync', "--file=$tempdir/clean_if_exists.sql", - '--include-subscriptions', '-c', '--if-exists', '--encoding=UTF8', # no-op, just tests that option is accepted @@ -85,7 +82,6 @@ my %pgdump_runs = ( 'pg_dump', '--no-sync', "--file=$tempdir/createdb.sql", - '--include-subscriptions', '-C', '-R', # no-op, just for testing '-v', @@ -95,7 +91,6 @@ my %pgdump_runs = ( 'pg_dump', '--no-sync', "--file=$tempdir/data_only.sql", - '--include-subscriptions', '-a', '--superuser=test_superuser', '--disable-triggers', @@ -253,7 +248,6 @@ my %pgdump_runs = ( section_pre_data => { dump_cmd => [ 'pg_dump', "--file=$tempdir/section_pre_data.sql", - '--include-subscriptions', '--section=pre-data', '--no-sync', 'postgres', ], }, section_data => { dump_cmd => [ @@ -271,7 +265,7 @@ my %pgdump_runs = ( with_oids => { dump_cmd => [ 'pg_dump', '--oids', - '--include-subscriptions', '--no-sync', + '--no-sync', "--file=$tempdir/with_oids.sql", 'postgres', ], },); ############################################################### @@ -1405,7 +1399,7 @@ my %tests = ( # catch-all for ALTER ... OWNER (except LARGE OBJECTs and PUBLICATIONs) 'ALTER ... OWNER commands (except LARGE OBJECTs and PUBLICATIONs)' => { all_runs => 0, # catch-all - regexp => qr/^ALTER (?!LARGE OBJECT|PUBLICATION)(.*) OWNER TO .*;/m, + regexp => qr/^ALTER (?!LARGE OBJECT|PUBLICATION|SUBSCRIPTION)(.*) OWNER TO .*;/m, like => {}, # use more-specific options above unlike => { column_inserts => 1, @@ -4318,25 +4312,25 @@ qr/CREATE TRANSFORM FOR integer LANGUAGE sql \(FROM SQL WITH FUNCTION pg_catalog clean => 1, clean_if_exists => 1, createdb => 1, - with_oids => 1, }, - unlike => { defaults => 1, exclude_test_table_data => 1, exclude_dump_test_schema => 1, exclude_test_table => 1, - section_pre_data => 1, no_blobs => 1, no_privs => 1, no_owner => 1, + pg_dumpall_dbprivs => 1, + schema_only => 1, + section_post_data => 1, + with_oids => 1, }, + unlike => { + section_pre_data => 1, only_dump_test_schema => 1, only_dump_test_table => 1, - pg_dumpall_dbprivs => 1, pg_dumpall_globals => 1, pg_dumpall_globals_clean => 1, - schema_only => 1, # XXX Should be like? role => 1, - section_pre_data => 1, # XXX Should be like? - section_post_data => 1, + section_pre_data => 1, test_schema_plus_blobs => 1, }, }, 'ALTER PUBLICATION pub1 ADD TABLE test_table' => { -- cgit v1.2.3 From a9254e675bde7dc2d976d207450c559d914c0dd6 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 12 Apr 2017 22:12:30 -0400 Subject: pg_dump: Always dump subscriptions NOCONNECT This removes the pg_dump option --no-subscription-connect and makes it the default. Dumping a subscription so that it activates right away when restored is not very useful, because the state of the publication server is unclear. Discussion: https://www.postgresql.org/message-id/e4fbfad5-c6ac-fd50-6777-18c84b34eb2f@2ndquadrant.com --- doc/src/sgml/ref/pg_dump.sgml | 26 +++++++++++++------------- src/bin/pg_dump/pg_backup.h | 1 - src/bin/pg_dump/pg_dump.c | 22 ++-------------------- src/bin/pg_dump/pg_dump.h | 1 - src/bin/pg_dump/t/002_pg_dump.pl | 4 ++-- 5 files changed, 17 insertions(+), 37 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml index 53b5dd5239..6cf7e570ef 100644 --- a/doc/src/sgml/ref/pg_dump.sgml +++ b/doc/src/sgml/ref/pg_dump.sgml @@ -798,19 +798,6 @@ PostgreSQL documentation - - - - - When dumping logical replication subscriptions, - generate CREATE SUBSCRIPTION commands that do not - make remote connections for creating replication slot or initial table - copy. That way, the dump can be restored without requiring network - access to the remote servers. - - - - @@ -1235,6 +1222,19 @@ CREATE DATABASE foo WITH TEMPLATE template0; in cross-version cases, as it can prevent problems arising from varying reserved-word lists in different PostgreSQL versions. + + + When dumping logical replication subscriptions, + pg_dump will generate CREATE + SUBSCRIPTION commands that use the NOCONNECT + option, so that restoring the subscription does not make remote connections + for creating a replication slot or for initial table copy. That way, the + dump can be restored without requiring network access to the remote + servers. It is then up to the user to reactivate the subscriptions in a + suitable way. If the involved hosts have changed, the connection + information might have to be changed. It might also be appropriate to + truncate the target tables before initiating a new full table copy. +     diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 1d14b68983..08b883efb0 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -153,7 +153,6 @@ typedef struct _dumpOptions int outputNoTablespaces; int use_setsessauth; int enable_row_security; - int no_subscription_connect; /* default, if no "inclusion" switches appear, is to dump everything */ bool include_everything; diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index dcefe975d8..14dc1b2423 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -353,7 +353,6 @@ main(int argc, char **argv) {"strict-names", no_argument, &strict_names, 1}, {"use-set-session-authorization", no_argument, &dopt.use_setsessauth, 1}, {"no-security-labels", no_argument, &dopt.no_security_labels, 1}, - {"no-subscription-connect", no_argument, &dopt.no_subscription_connect, 1}, {"no-synchronized-snapshots", no_argument, &dopt.no_synchronized_snapshots, 1}, {"no-unlogged-table-data", no_argument, &dopt.no_unlogged_table_data, 1}, {"no-sync", no_argument, NULL, 7}, @@ -951,7 +950,6 @@ help(const char *progname) printf(_(" --if-exists use IF EXISTS when dropping objects\n")); printf(_(" --inserts dump data as INSERT commands, rather than COPY\n")); printf(_(" --no-security-labels do not dump security label assignments\n")); - printf(_(" --no-subscription-connect dump subscriptions so they don't connect on restore\n")); printf(_(" --no-synchronized-snapshots do not use synchronized snapshots in parallel jobs\n")); printf(_(" --no-tablespaces do not dump tablespace assignments\n")); printf(_(" --no-unlogged-table-data do not dump unlogged table data\n")); @@ -3669,7 +3667,6 @@ getSubscriptions(Archive *fout) int i_oid; int i_subname; int i_rolname; - int i_subenabled; int i_subconninfo; int i_subslotname; int i_subpublications; @@ -3702,7 +3699,7 @@ getSubscriptions(Archive *fout) /* Get the subscriptions in current database. */ appendPQExpBuffer(query, "SELECT s.tableoid, s.oid, s.subname," - "(%s s.subowner) AS rolname, s.subenabled, " + "(%s s.subowner) AS rolname, " " s.subconninfo, s.subslotname, s.subpublications " "FROM pg_catalog.pg_subscription s " "WHERE s.subdbid = (SELECT oid FROM pg_catalog.pg_database" @@ -3716,7 +3713,6 @@ getSubscriptions(Archive *fout) i_oid = PQfnumber(res, "oid"); i_subname = PQfnumber(res, "subname"); i_rolname = PQfnumber(res, "rolname"); - i_subenabled = PQfnumber(res, "subenabled"); i_subconninfo = PQfnumber(res, "subconninfo"); i_subslotname = PQfnumber(res, "subslotname"); i_subpublications = PQfnumber(res, "subpublications"); @@ -3732,8 +3728,6 @@ getSubscriptions(Archive *fout) AssignDumpId(&subinfo[i].dobj); subinfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_subname)); subinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); - subinfo[i].subenabled = - (strcmp(PQgetvalue(res, i, i_subenabled), "t") == 0); subinfo[i].subconninfo = pg_strdup(PQgetvalue(res, i, i_subconninfo)); subinfo[i].subslotname = pg_strdup(PQgetvalue(res, i, i_subslotname)); subinfo[i].subpublications = @@ -3758,7 +3752,6 @@ getSubscriptions(Archive *fout) static void dumpSubscription(Archive *fout, SubscriptionInfo *subinfo) { - DumpOptions *dopt = fout->dopt; PQExpBuffer delq; PQExpBuffer query; PQExpBuffer publications; @@ -3799,19 +3792,8 @@ dumpSubscription(Archive *fout, SubscriptionInfo *subinfo) appendPQExpBufferStr(publications, fmtId(pubnames[i])); } - appendPQExpBuffer(query, " PUBLICATION %s WITH (", publications->data); - - if (subinfo->subenabled) - appendPQExpBufferStr(query, "ENABLED"); - else - appendPQExpBufferStr(query, "DISABLED"); - - appendPQExpBufferStr(query, ", SLOT NAME = "); + appendPQExpBuffer(query, " PUBLICATION %s WITH (NOCONNECT, SLOT NAME = ", publications->data); appendStringLiteralAH(query, subinfo->subslotname, fout); - - if (dopt->no_subscription_connect) - appendPQExpBufferStr(query, ", NOCONNECT"); - appendPQExpBufferStr(query, ");\n"); ArchiveEntry(fout, subinfo->dobj.catId, subinfo->dobj.dumpId, diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h index 61097e6d99..ba85392f11 100644 --- a/src/bin/pg_dump/pg_dump.h +++ b/src/bin/pg_dump/pg_dump.h @@ -614,7 +614,6 @@ typedef struct _SubscriptionInfo { DumpableObject dobj; char *rolname; - bool subenabled; char *subconninfo; char *subslotname; char *subpublications; diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl index 1db3767f46..e0d1ce6232 100644 --- a/src/bin/pg_dump/t/002_pg_dump.pl +++ b/src/bin/pg_dump/t/002_pg_dump.pl @@ -4303,9 +4303,9 @@ qr/CREATE TRANSFORM FOR integer LANGUAGE sql \(FROM SQL WITH FUNCTION pg_catalog create_order => 50, create_sql => 'CREATE SUBSCRIPTION sub1 CONNECTION \'dbname=doesnotexist\' PUBLICATION pub1 - WITH (DISABLED, NOCONNECT);', + WITH (NOCONNECT);', regexp => qr/^ - \QCREATE SUBSCRIPTION sub1 CONNECTION 'dbname=doesnotexist' PUBLICATION pub1 WITH (DISABLED, SLOT NAME = 'sub1');\E + \QCREATE SUBSCRIPTION sub1 CONNECTION 'dbname=doesnotexist' PUBLICATION pub1 WITH (NOCONNECT, SLOT NAME = 'sub1');\E /xm, like => { binary_upgrade => 1, -- cgit v1.2.3 From 4f3b87ab780b95c2cc8a591259baefaff4852037 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Thu, 13 Apr 2017 19:34:16 +0300 Subject: Improve the SASL authentication protocol. This contains some protocol changes to SASL authentiation (which is new in v10): * For future-proofing, in the AuthenticationSASL message that begins SASL authentication, provide a list of SASL mechanisms that the server supports, for the client to choose from. Currently, it's always just SCRAM-SHA-256. * Add a separate authentication message type for the final server->client SASL message, which the client doesn't need to respond to. This makes it unambiguous whether the client is supposed to send a response or not. The SASL mechanism should know that anyway, but better to be explicit. Also, in the server, support clients that don't send an Initial Client response in the first SASLInitialResponse message. The server is supposed to first send an empty request in that case, to which the client will respond with the data that usually comes in the Initial Client Response. libpq uses the Initial Client Response field and doesn't need this, and I would assume any other sensible implementation to use Initial Client Response, too, but let's follow the SASL spec. Improve the documentation on SASL authentication in protocol. Add a section describing the SASL message flow, and some details on our SCRAM-SHA-256 implementation. Document the different kinds of PasswordMessages that the frontend sends in different phases of SASL authentication, as well as GSS/SSPI authentication as separate message formats. Even though they're all 'p' messages, and the exact format depends on the context, describing them as separate message formats makes the documentation more clear. Reviewed by Michael Paquier and Álvaro Hernández Tortosa. Discussion: https://www.postgresql.org/message-id/CAB7nPqS-aFg0iM3AQOJwKDv_0WkAedRjs1W2X8EixSz+sKBXCQ@mail.gmail.com --- doc/src/sgml/protocol.sgml | 418 ++++++++++++++++++++++++++++++++++++++--- src/backend/libpq/auth-scram.c | 27 ++- src/backend/libpq/auth.c | 68 ++++++- src/include/libpq/pqcomm.h | 5 +- src/interfaces/libpq/fe-auth.c | 158 +++++++++++----- 5 files changed, 588 insertions(+), 88 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index 4e8bb32d33..bc7809544e 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -330,7 +330,7 @@ The frontend must now initiate a GSSAPI negotiation. The frontend - will send a PasswordMessage with the first part of the GSSAPI + will send a GSSResponse message with the first part of the GSSAPI data stream in response to this. If further messages are needed, the server will respond with AuthenticationGSSContinue. @@ -342,7 +342,7 @@ The frontend must now initiate a SSPI negotiation. The frontend - will send a PasswordMessage with the first part of the SSPI + will send a GSSResponse with the first part of the SSPI data stream in response to this. If further messages are needed, the server will respond with AuthenticationGSSContinue. @@ -358,7 +358,7 @@ or a previous AuthenticationGSSContinue). If the GSSAPI or SSPI data in this message indicates more data is needed to complete the authentication, - the frontend must send that data as another PasswordMessage. If + the frontend must send that data as another GSSResponse message. If GSSAPI or SSPI authentication is completed by this message, the server will next send AuthenticationOk to indicate successful authentication or ErrorResponse to indicate failure. @@ -370,27 +370,38 @@ AuthenticationSASL - The frontend must now initiate a SASL negotiation, using the SASL - mechanism specified in the message. The frontend will send a - PasswordMessage with the first part of the SASL data stream in - response to this. If further messages are needed, the server will - respond with AuthenticationSASLContinue. + The frontend must now initiate a SASL negotiation, using one of the + SASL mechanisms listed in the message. The frontend will send a + SASLInitialResponse with the name of the selected mechanism, and the + first part of the SASL data stream in response to this. If further + messages are needed, the server will respond with + AuthenticationSASLContinue. See + for details. - + AuthenticationSASLContinue - This message contains the response data from the previous step - of SASL negotiation (AuthenticationSASL, or a previous - AuthenticationSASLContinue). If the SASL data in this message - indicates more data is needed to complete the authentication, - the frontend must send that data as another PasswordMessage. If - SASL authentication is completed by this message, the server - will next send AuthenticationOk to indicate successful authentication - or ErrorResponse to indicate failure. + This message contains challenge data from the previous step of SASL + negotiation (AuthenticationSASL, or a previous + AuthenticationSASLContinue). The frontend must respond with a + SASLResponse message. + + + + + + AuthenticationSASLFinal + + + SASL authentication has completed with additional mechanism-specific + data for the client. The server will next send AuthenticationOk to + indicate successful authentication, or an ErrorResponse to indicate + failure. This message is sent only if the SASL mechanism specifies + additional data to be sent from server to client at completion. @@ -1326,6 +1337,141 @@ + +SASL Authentication + + +SASL is a framework for authentication in connection-oriented +protocols. At the moment, PostgreSQL implements only one SASL +authentication mechanism, SCRAM-SHA-256, but more might be added in the +future. The below steps illustrate how SASL authentication is performed in +general, while the next subsection gives more details on SCRAM-SHA-256. + + + +SASL Authentication Message Flow + + + + To begin a SASL authentication exchange, the server an AuthenticationSASL + message. It includes a list of SASL authentication mechanisms that the + server can accept, in the server's preferred order. + + + + + + The client selects one of the supported mechanisms from the list, and sends + a SASLInitialResponse message to the server. The message includes the name + of the selected mechanism, and an optional Initial Client Response, if the + selected mechanism uses that. + + + + + + One or more server-challenge and client-response message will follow. Each + server-challenge is sent in an AuthenticationSASLContinue message, followed + by a response from client in an SASLResponse message. The particulars of + the messages are mechanism specific. + + + + + + Finally, when the authentication exchange is completed successfully, the + server sends an AuthenticationSASLFinal message, followed + immediately by an AuthenticationOk message. The AuthenticationSASLFinal + contains additional server-to-client data, whose content is particular to the + selected authentication mechanism. If the authentication mechanism doesn't + use additional data that's sent at completion, the AuthenticationSASLFinal + message is not sent. + + + + + +On error, the server can abort the authentication at any stage, and send an +ErrorMessage. + + + + SCRAM-SHA-256 authentication + + + SCRAM-SHA-256 (called just SCRAM from now on) is + the only implemented SASL mechanism, at the moment. It is described in detail + in RFC 7677 and RFC 5741. + + + +When SCRAM-SHA-256 is used in PostgreSQL, the server will ignore the username +that the client sends in the client-first-message. The username +that was already sent in the startup message is used instead. +PostgreSQL supports multiple character encodings, while SCRAM +dictates UTF-8 to be used for the username, so it might be impossible to +represent the PostgreSQL username in UTF-8. To avoid confusion, the client +should use pg_same_as_startup_message as the username in the +client-first-message. + + + +The SCRAM specification dictates that the password is also in UTF-8, and is +processed with the SASLprep algorithm. +PostgreSQL, however, does not require UTF-8 to be used for +the password. When a user's password is set, it is processed with SASLprep +as if it was in UTF-8, regardless of the actual encoding used. However, if +it is not a legal UTF-8 byte sequence, or it contains UTF-8 byte sequences +that are prohibited by the SASLprep algorithm, the raw password will be used +without SASLprep processing, instead of throwing an error. This allows the +password to be normalized when it is in UTF-8, but still allows a non-UTF-8 +password to be used, and doesn't require the system to know which encoding +the password is in. + + + +Channel binding has not been implemented yet. + + + +Example + + + The server sends an AuthenticationSASL message. It includes a list of + SASL authentication mechanisms that the server can accept. + + + + + The client responds by sending a SASLInitialResponse message, which + indicates the chosen mechanism, SCRAM-SHA-256. In the Initial + Client response field, the message contains the SCRAM + client-first-message. + + + + + Server sends an AuthenticationSASLContinue message, with a SCRAM + server-first message as the content. + + + + + Client sends a SASLResponse message, with SCRAM + client-final-message as the content. + + + + + Server sends an AuthenticationSASLFinal message, with the SCRAM + server-final-message, followed immediately by + an AuthenticationOk message. + + + + + + Streaming Replication Protocol @@ -2802,6 +2948,8 @@ AuthenticationSSPI (B) + + AuthenticationGSSContinue (B) @@ -2856,6 +3004,7 @@ AuthenticationGSSContinue (B) + AuthenticationSASL (B) @@ -2890,10 +3039,16 @@ AuthenticationSASL (B) - Specifies that SASL authentication is started. + Specifies that SASL authentication is required. + +The message body is a list of SASL authentication mechanisms, in the +server's order of preference. A zero byte is required as terminator after +the last authentication mechanism name. For each mechanism, there is the +following: + String @@ -2910,6 +3065,7 @@ AuthenticationSASL (B) + AuthenticationSASLContinue (B) @@ -2944,8 +3100,7 @@ AuthenticationSASLContinue (B) - Specifies that this message contains SASL-mechanism specific - data. + Specifies that this message contains a SASL challenge. @@ -2965,6 +3120,63 @@ AuthenticationSASLContinue (B) + + + +AuthenticationSASLFinal (B) + + + + + + + + Byte1('R') + + + + Identifies the message as an authentication request. + + + + + + Int32 + + + + Length of message contents in bytes, including self. + + + + + + Int32(12) + + + + Specifies that SASL authentication has completed. + + + + + + Byten + + + + SASL outcome "additional data", specific to the SASL mechanism + being used. + + + + + + + + + + BackendKeyData (B) @@ -4314,6 +4526,52 @@ FunctionCallResponse (B) + + +GSSResponse (F) + + + + + + + + Byte1('p') + + + + Identifies the message as a GSSAPI or SSPI response. Note that + this is also used for SASL and password response messages. + The exact message type can be deduced from the context. + + + + + + Int32 + + + + Length of message contents in bytes, including self. + + + + + + Byten + + + + GSSAPI/SSPI specific message data. + + + + + + + + + NoData (B) @@ -4726,10 +4984,8 @@ PasswordMessage (F) Identifies the message as a password response. Note that - this is also used for GSSAPI, SSPI and SASL response messages - (which is really a design error, since the contained data - is not a null-terminated string in that case, but can be - arbitrary binary data). + this is also used for GSSAPI, SSPI and SASL response messages. + The exact message type can be deduced from the context. @@ -5016,6 +5272,120 @@ RowDescription (B) + + +SASLInitialresponse (F) + + + + + + + + Byte1('p') + + + + Identifies the message as an initial SASL response. Note that + this is also used for GSSAPI, SSPI and password response messages. + The exact message type is deduced from the context. + + + + + + Int32 + + + + Length of message contents in bytes, including self. + + + + + + String + + + + Name of the SASL authentication mechanism that the client + selected. + + + + + + Int32 + + + + Length of SASL mechanism specific "Initial Client Response" that + follows, or -1 if there is no Initial Response. + + + + + + Byten + + + + SASL mechanism specific "Initial Response". + + + + + + + + + + + +SASLResponse (F) + + + + + + + + Byte1('p') + + + + Identifies the message as a SASL response. Note that + this is also used for GSSAPI, SSPI and password response messages. + The exact message type can be deduced from the context. + + + + + + Int32 + + + + Length of message contents in bytes, including self. + + + + + + Byten + + + + SASL mechanism specific message data. + + + + + + + + + SSLRequest (F) diff --git a/src/backend/libpq/auth-scram.c b/src/backend/libpq/auth-scram.c index a47c48d980..338afede9d 100644 --- a/src/backend/libpq/auth-scram.c +++ b/src/backend/libpq/auth-scram.c @@ -254,8 +254,16 @@ pg_be_scram_init(const char *username, const char *shadow_pass) /* * Continue a SCRAM authentication exchange. * - * The next message to send to client is saved in "output", for a length - * of "outputlen". In the case of an error, optionally store a palloc'd + * 'input' is the SCRAM payload sent by the client. On the first call, + * 'input' contains the "Initial Client Response" that the client sent as + * part of the SASLInitialResponse message, or NULL if no Initial Client + * Response was given. (The SASL specification distinguishes between an + * empty response and non-existing one.) On subsequent calls, 'input' + * cannot be NULL. For convenience in this function, the caller must + * ensure that there is a null terminator at input[inputlen]. + * + * The next message to send to client is saved in 'output', for a length + * of 'outputlen'. In the case of an error, optionally store a palloc'd * string at *logdetail that will be sent to the postmaster log (but not * the client). */ @@ -268,6 +276,21 @@ pg_be_scram_exchange(void *opaq, char *input, int inputlen, *output = NULL; + /* + * If the client didn't include an "Initial Client Response" in the + * SASLInitialResponse message, send an empty challenge, to which the + * client will respond with the same data that usually comes in the + * Initial Client Response. + */ + if (input == NULL) + { + Assert(state->state == SCRAM_AUTH_INIT); + + *output = pstrdup(""); + *outputlen = 0; + return SASL_EXCHANGE_CONTINUE; + } + /* * Check that the input length agrees with the string length of the input. * We can ignore inputlen after this. diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c index b4c98c45c9..848561e188 100644 --- a/src/backend/libpq/auth.c +++ b/src/backend/libpq/auth.c @@ -620,10 +620,11 @@ sendAuthRequest(Port *port, AuthRequest areq, char *extradata, int extralen) pq_endmessage(&buf); /* - * Flush message so client will see it, except for AUTH_REQ_OK, which need - * not be sent until we are ready for queries. + * Flush message so client will see it, except for AUTH_REQ_OK and + * AUTH_REQ_SASL_FIN, which need not be sent until we are ready for + * queries. */ - if (areq != AUTH_REQ_OK) + if (areq != AUTH_REQ_OK && areq != AUTH_REQ_SASL_FIN) pq_flush(); CHECK_FOR_INTERRUPTS(); @@ -850,7 +851,10 @@ CheckSCRAMAuth(Port *port, char *shadow_pass, char **logdetail) void *scram_opaq; char *output = NULL; int outputlen = 0; + char *input; + int inputlen; int result; + bool initial; /* * SASL auth is not supported for protocol versions before 3, because it @@ -866,10 +870,13 @@ CheckSCRAMAuth(Port *port, char *shadow_pass, char **logdetail) errmsg("SASL authentication is not supported in protocol version 2"))); /* - * Send first the authentication request to user. + * Send the SASL authentication request to user. It includes the list of + * authentication mechanisms (which is trivial, because we only support + * SCRAM-SHA-256 at the moment). The extra "\0" is for an empty string to + * terminate the list. */ - sendAuthRequest(port, AUTH_REQ_SASL, SCRAM_SHA256_NAME, - strlen(SCRAM_SHA256_NAME) + 1); + sendAuthRequest(port, AUTH_REQ_SASL, SCRAM_SHA256_NAME "\0", + strlen(SCRAM_SHA256_NAME) + 2); /* * Initialize the status tracker for message exchanges. @@ -890,6 +897,7 @@ CheckSCRAMAuth(Port *port, char *shadow_pass, char **logdetail) * from the client. All messages from client to server are password * packets (type 'p'). */ + initial = true; do { pq_startmsgread(); @@ -920,11 +928,52 @@ CheckSCRAMAuth(Port *port, char *shadow_pass, char **logdetail) elog(DEBUG4, "Processing received SASL response of length %d", buf.len); + /* + * The first SASLInitialResponse message is different from the others. + * It indicates which SASL mechanism the client selected, and contains + * an optional Initial Client Response payload. The subsequent + * SASLResponse messages contain just the SASL payload. + */ + if (initial) + { + const char *selected_mech; + + /* + * We only support SCRAM-SHA-256 at the moment, so anything else + * is an error. + */ + selected_mech = pq_getmsgrawstring(&buf); + if (strcmp(selected_mech, SCRAM_SHA256_NAME) != 0) + ereport(COMMERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("client selected an invalid SASL authentication mechanism"))); + + inputlen = pq_getmsgint(&buf, 4); + if (inputlen == -1) + input = NULL; + else + input = (char *) pq_getmsgbytes(&buf, inputlen); + + initial = false; + } + else + { + inputlen = buf.len; + input = (char *) pq_getmsgbytes(&buf, buf.len); + } + pq_getmsgend(&buf); + + /* + * The StringInfo guarantees that there's a \0 byte after the + * response. + */ + Assert(input == NULL || input[inputlen] == '\0'); + /* * we pass 'logdetail' as NULL when doing a mock authentication, * because we should already have a better error message in that case */ - result = pg_be_scram_exchange(scram_opaq, buf.data, buf.len, + result = pg_be_scram_exchange(scram_opaq, input, inputlen, &output, &outputlen, logdetail); @@ -938,7 +987,10 @@ CheckSCRAMAuth(Port *port, char *shadow_pass, char **logdetail) */ elog(DEBUG4, "sending SASL challenge of length %u", outputlen); - sendAuthRequest(port, AUTH_REQ_SASL_CONT, output, outputlen); + if (result == SASL_EXCHANGE_SUCCESS) + sendAuthRequest(port, AUTH_REQ_SASL_FIN, output, outputlen); + else + sendAuthRequest(port, AUTH_REQ_SASL_CONT, output, outputlen); pfree(output); } diff --git a/src/include/libpq/pqcomm.h b/src/include/libpq/pqcomm.h index 5441aaa93a..b6de569c5c 100644 --- a/src/include/libpq/pqcomm.h +++ b/src/include/libpq/pqcomm.h @@ -172,8 +172,9 @@ extern bool Db_user_namespace; #define AUTH_REQ_GSS 7 /* GSSAPI without wrap() */ #define AUTH_REQ_GSS_CONT 8 /* Continue GSS exchanges */ #define AUTH_REQ_SSPI 9 /* SSPI negotiate without wrap() */ -#define AUTH_REQ_SASL 10 /* SASL */ -#define AUTH_REQ_SASL_CONT 11 /* continue SASL exchange */ +#define AUTH_REQ_SASL 10 /* Begin SASL authentication */ +#define AUTH_REQ_SASL_CONT 11 /* Continue SASL authentication */ +#define AUTH_REQ_SASL_FIN 12 /* Final SASL message */ typedef uint32 AuthRequest; diff --git a/src/interfaces/libpq/fe-auth.c b/src/interfaces/libpq/fe-auth.c index 14e00a69e2..d81ee4f944 100644 --- a/src/interfaces/libpq/fe-auth.c +++ b/src/interfaces/libpq/fe-auth.c @@ -475,88 +475,129 @@ pg_SSPI_startup(PGconn *conn, int use_negotiate, int payloadlen) static int pg_SASL_init(PGconn *conn, int payloadlen) { - char auth_mechanism[21]; - char *initialresponse; + char *initialresponse = NULL; int initialresponselen; bool done; bool success; - int res; + const char *selected_mechanism; + PQExpBufferData mechanism_buf; - /* - * Read the authentication mechanism the server told us to use. - */ - if (payloadlen > sizeof(auth_mechanism) - 1) - printfPQExpBuffer(&conn->errorMessage, - libpq_gettext("SASL authentication mechanism not supported\n")); - if (pqGetnchar(auth_mechanism, payloadlen, conn)) + initPQExpBuffer(&mechanism_buf); + + if (conn->sasl_state) { printfPQExpBuffer(&conn->errorMessage, - "fe_sendauth: invalid authentication request from server: invalid authentication mechanism\n"); - - return STATUS_ERROR; + libpq_gettext("duplicate SASL authentication request\n")); + goto error; } - auth_mechanism[payloadlen] = '\0'; /* - * Check the authentication mechanism (only SCRAM-SHA-256 is supported at - * the moment.) + * Parse the list of SASL authentication mechanisms in the + * AuthenticationSASL message, and select the best mechanism that we + * support. (Only SCRAM-SHA-256 is supported at the moment.) */ - if (strcmp(auth_mechanism, SCRAM_SHA256_NAME) == 0) + selected_mechanism = NULL; + for (;;) { - char *password; - - conn->password_needed = true; - password = conn->connhost[conn->whichhost].password; - if (password == NULL) - password = conn->pgpass; - if (password == NULL || password[0] == '\0') + if (pqGets(&mechanism_buf, conn)) { printfPQExpBuffer(&conn->errorMessage, - PQnoPasswordSupplied); - return STATUS_ERROR; + "fe_sendauth: invalid authentication request from server: invalid list of authentication mechanisms\n"); + goto error; } + if (PQExpBufferDataBroken(mechanism_buf)) + goto oom_error; + + /* An empty string indicates end of list */ + if (mechanism_buf.data[0] == '\0') + break; - conn->sasl_state = pg_fe_scram_init(conn->pguser, password); - if (!conn->sasl_state) + /* + * If we have already selected a mechanism, just skip through the rest + * of the list. + */ + if (selected_mechanism) + continue; + + /* + * Do we support this mechanism? + */ + if (strcmp(mechanism_buf.data, SCRAM_SHA256_NAME) == 0) { - printfPQExpBuffer(&conn->errorMessage, - libpq_gettext("out of memory\n")); - return STATUS_ERROR; + char *password; + + conn->password_needed = true; + password = conn->connhost[conn->whichhost].password; + if (password == NULL) + password = conn->pgpass; + if (password == NULL || password[0] == '\0') + { + printfPQExpBuffer(&conn->errorMessage, + PQnoPasswordSupplied); + goto error; + } + + conn->sasl_state = pg_fe_scram_init(conn->pguser, password); + if (!conn->sasl_state) + goto oom_error; + selected_mechanism = SCRAM_SHA256_NAME; } } - else + + if (!selected_mechanism) { printfPQExpBuffer(&conn->errorMessage, - libpq_gettext("SASL authentication mechanism %s not supported\n"), - auth_mechanism); - return STATUS_ERROR; + libpq_gettext("none of the server's SASL authentication mechanisms are supported\n")); + goto error; } - /* Send the initial client response */ + /* Get the mechanism-specific Initial Client Response, if any */ pg_fe_scram_exchange(conn->sasl_state, NULL, -1, &initialresponse, &initialresponselen, &done, &success, &conn->errorMessage); + if (done && !success) + goto error; + + /* + * Build a SASLInitialResponse message, and send it. + */ + if (pqPutMsgStart('p', true, conn)) + goto error; + if (pqPuts(selected_mechanism, conn)) + goto error; if (initialresponse) { - res = pqPacketSend(conn, 'p', initialresponse, initialresponselen); - free(initialresponse); - - if (res != STATUS_OK) - return STATUS_ERROR; + if (pqPutInt(initialresponselen, 4, conn)) + goto error; + if (pqPutnchar(initialresponse, initialresponselen, conn)) + goto error; } + if (pqPutMsgEnd(conn)) + goto error; + if (pqFlush(conn)) + goto error; - if (done && !success) - { - /* Use error message, if set already */ - if (conn->errorMessage.len == 0) - printfPQExpBuffer(&conn->errorMessage, - "fe_sendauth: error in SASL authentication\n"); - return STATUS_ERROR; - } + termPQExpBuffer(&mechanism_buf); + if (initialresponse) + free(initialresponse); return STATUS_OK; + +error: + termPQExpBuffer(&mechanism_buf); + if (initialresponse) + free(initialresponse); + return STATUS_ERROR; + +oom_error: + termPQExpBuffer(&mechanism_buf); + if (initialresponse) + free(initialresponse); + printfPQExpBuffer(&conn->errorMessage, + libpq_gettext("out of memory\n")); + return STATUS_ERROR; } /* @@ -565,7 +606,7 @@ pg_SASL_init(PGconn *conn, int payloadlen) * the protocol. */ static int -pg_SASL_continue(PGconn *conn, int payloadlen) +pg_SASL_continue(PGconn *conn, int payloadlen, bool final) { char *output; int outputlen; @@ -598,9 +639,20 @@ pg_SASL_continue(PGconn *conn, int payloadlen) &done, &success, &conn->errorMessage); free(challenge); /* don't need the input anymore */ - /* Send the SASL response to the server, if any. */ + if (final && !done) + { + if (outputlen != 0) + free(output); + + printfPQExpBuffer(&conn->errorMessage, + libpq_gettext("AuthenticationSASLFinal received from server, but SASL authentication was not completed\n")); + return STATUS_ERROR; + } if (outputlen != 0) { + /* + * Send the SASL response to the server. + */ res = pqPacketSend(conn, 'p', output, outputlen); free(output); @@ -918,13 +970,15 @@ pg_fe_sendauth(AuthRequest areq, int payloadlen, PGconn *conn) break; case AUTH_REQ_SASL_CONT: + case AUTH_REQ_SASL_FIN: if (conn->sasl_state == NULL) { printfPQExpBuffer(&conn->errorMessage, "fe_sendauth: invalid authentication request from server: AUTH_REQ_SASL_CONT without AUTH_REQ_SASL\n"); return STATUS_ERROR; } - if (pg_SASL_continue(conn, payloadlen) != STATUS_OK) + if (pg_SASL_continue(conn, payloadlen, + (areq == AUTH_REQ_SASL_FIN)) != STATUS_OK) { /* Use error message, if set already */ if (conn->errorMessage.len == 0) -- cgit v1.2.3 From 885fea5a34b036983379688d707b2a20735ebe4a Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Thu, 13 Apr 2017 13:12:58 -0400 Subject: doc: add missing sect1 close tag Fixes commit 4f3b87ab780b95c2cc8a591259baefaff4852037 --- doc/src/sgml/protocol.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index bc7809544e..e04064ac6a 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -1470,7 +1470,7 @@ the password is in. - + Streaming Replication Protocol -- cgit v1.2.3 From 887227a1cc861d87ca0f175cf8bd1447554090eb Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 14 Apr 2017 13:58:46 -0400 Subject: Add option to modify sync commit per subscription This also changes default behaviour of subscription workers to synchronous_commit = off. Author: Petr Jelinek --- doc/src/sgml/catalogs.sgml | 10 ++++++ doc/src/sgml/ref/alter_subscription.sgml | 2 ++ doc/src/sgml/ref/create_subscription.sgml | 31 +++++++++++++++++ src/backend/catalog/pg_subscription.c | 8 +++++ src/backend/commands/subscriptioncmds.c | 54 +++++++++++++++++++++++++----- src/backend/replication/logical/launcher.c | 6 +--- src/backend/replication/logical/worker.c | 8 +++++ src/bin/pg_dump/pg_dump.c | 11 +++++- src/bin/pg_dump/pg_dump.h | 1 + src/bin/psql/describe.c | 5 ++- src/include/catalog/pg_subscription.h | 8 +++-- src/test/regress/expected/subscription.out | 30 ++++++++++------- src/test/regress/sql/subscription.sql | 4 ++- 13 files changed, 145 insertions(+), 33 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 5883673448..5254bb3025 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -6530,6 +6530,16 @@ If true, the subscription is enabled and should be replicating. + + subsynccommit + text + + + Contains the value of the synchronous_commit + setting for the subscription workers. + + + subconninfo text diff --git a/doc/src/sgml/ref/alter_subscription.sgml b/doc/src/sgml/ref/alter_subscription.sgml index 640fac0a15..f71ee38b40 100644 --- a/doc/src/sgml/ref/alter_subscription.sgml +++ b/doc/src/sgml/ref/alter_subscription.sgml @@ -26,6 +26,7 @@ ALTER SUBSCRIPTION name WITH ( where suboption can be: SLOT NAME = slot_name + | SYNCHRONOUS_COMMIT = synchronous_commit ALTER SUBSCRIPTION name SET PUBLICATION publication_name [, ...] { REFRESH WITH ( puboption [, ... ] ) | NOREFRESH } ALTER SUBSCRIPTION name REFRESH PUBLICATION WITH ( puboption [, ... ] ) @@ -91,6 +92,7 @@ ALTER SUBSCRIPTION name DISABLE CONNECTION 'conninfo' SLOT NAME = slot_name + SYNCHRONOUS_COMMIT = synchronous_commit These clauses alter properties originally set by diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml index 3410d6fc8c..3c51012df8 100644 --- a/doc/src/sgml/ref/create_subscription.sgml +++ b/doc/src/sgml/ref/create_subscription.sgml @@ -32,6 +32,7 @@ CREATE SUBSCRIPTION subscription_nameslot_name | COPY DATA | NOCOPY DATA + | SYNCHRONOUS_COMMIT = synchronous_commit | NOCONNECT     @@ -147,6 +148,36 @@ CREATE SUBSCRIPTION subscription_name + + SYNCHRONOUS_COMMIT = synchronous_commit + + + The value of this parameter overrides the + setting. The default value is + off. + + + + It is safe to use off for logical replication: If the + subscriber loses transactions because of missing synchronization, the + data will be resent from the publisher. + + + + A different setting might be appropriate when doing synchronous logical + replication. The logical replication workers report the positions of + writes and flushes to the publisher, and when using synchronous + replication, the publisher will wait for the actual flush. This means + that setting SYNCHRONOUS_COMMIT for the subscriber + to off when the subscription is used for synchronous + replication might increase the latency for COMMIT on + the publisher. In this scenario, it can be advantageous to set + SYNCHRONOUS_COMMIT to local or + higher. + + + + NOCONNECT diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c index 7e38b1a31c..a18385055e 100644 --- a/src/backend/catalog/pg_subscription.c +++ b/src/backend/catalog/pg_subscription.c @@ -85,6 +85,14 @@ GetSubscription(Oid subid, bool missing_ok) Assert(!isnull); sub->slotname = pstrdup(NameStr(*DatumGetName(datum))); + /* Get synccommit */ + datum = SysCacheGetAttr(SUBSCRIPTIONOID, + tup, + Anum_pg_subscription_subsynccommit, + &isnull); + Assert(!isnull); + sub->synccommit = TextDatumGetCString(datum); + /* Get publications */ datum = SysCacheGetAttr(SUBSCRIPTIONOID, tup, diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c index 7b8b11cb81..519c6846e3 100644 --- a/src/backend/commands/subscriptioncmds.c +++ b/src/backend/commands/subscriptioncmds.c @@ -44,6 +44,7 @@ #include "storage/lmgr.h" #include "utils/builtins.h" +#include "utils/guc.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/syscache.h" @@ -60,7 +61,7 @@ static List *fetch_table_list(WalReceiverConn *wrconn, List *publications); static void parse_subscription_options(List *options, bool *connect, bool *enabled_given, bool *enabled, bool *create_slot, char **slot_name, - bool *copy_data) + bool *copy_data, char **synchronous_commit) { ListCell *lc; bool connect_given = false; @@ -80,6 +81,8 @@ parse_subscription_options(List *options, bool *connect, bool *enabled_given, *slot_name = NULL; if (copy_data) *copy_data = true; + if (synchronous_commit) + *synchronous_commit = NULL; /* Parse options */ foreach (lc, options) @@ -165,6 +168,21 @@ parse_subscription_options(List *options, bool *connect, bool *enabled_given, copy_data_given = true; *copy_data = !defGetBoolean(defel); } + else if (strcmp(defel->defname, "synchronous_commit") == 0 && + synchronous_commit) + { + if (*synchronous_commit) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"))); + + *synchronous_commit = defGetString(defel); + + /* Test if the given value is valid for synchronous_commit GUC. */ + (void) set_config_option("synchronous_commit", *synchronous_commit, + PGC_BACKEND, PGC_S_TEST, GUC_ACTION_SET, + false, 0, false); + } else elog(ERROR, "unrecognized option: %s", defel->defname); } @@ -269,6 +287,7 @@ CreateSubscription(CreateSubscriptionStmt *stmt, bool isTopLevel) bool enabled_given; bool enabled; bool copy_data; + char *synchronous_commit; char *conninfo; char *slotname; char originname[NAMEDATALEN]; @@ -280,7 +299,8 @@ CreateSubscription(CreateSubscriptionStmt *stmt, bool isTopLevel) * Connection and publication should not be specified here. */ parse_subscription_options(stmt->options, &connect, &enabled_given, - &enabled, &create_slot, &slotname, ©_data); + &enabled, &create_slot, &slotname, ©_data, + &synchronous_commit); /* * Since creating a replication slot is not transactional, rolling back @@ -311,6 +331,9 @@ CreateSubscription(CreateSubscriptionStmt *stmt, bool isTopLevel) if (slotname == NULL) slotname = stmt->subname; + /* The default for synchronous_commit of subscriptions is off. */ + if (synchronous_commit == NULL) + synchronous_commit = "off"; conninfo = stmt->conninfo; publications = stmt->publication; @@ -334,6 +357,8 @@ CreateSubscription(CreateSubscriptionStmt *stmt, bool isTopLevel) CStringGetTextDatum(conninfo); values[Anum_pg_subscription_subslotname - 1] = DirectFunctionCall1(namein, CStringGetDatum(slotname)); + values[Anum_pg_subscription_subsynccommit - 1] = + CStringGetTextDatum(synchronous_commit); values[Anum_pg_subscription_subpublications - 1] = publicationListToArray(publications); @@ -582,13 +607,24 @@ AlterSubscription(AlterSubscriptionStmt *stmt) case ALTER_SUBSCRIPTION_OPTIONS: { char *slot_name; + char *synchronous_commit; parse_subscription_options(stmt->options, NULL, NULL, NULL, - NULL, &slot_name, NULL); + NULL, &slot_name, NULL, + &synchronous_commit); - values[Anum_pg_subscription_subslotname - 1] = - DirectFunctionCall1(namein, CStringGetDatum(slot_name)); - replaces[Anum_pg_subscription_subslotname - 1] = true; + if (slot_name) + { + values[Anum_pg_subscription_subslotname - 1] = + DirectFunctionCall1(namein, CStringGetDatum(slot_name)); + replaces[Anum_pg_subscription_subslotname - 1] = true; + } + if (synchronous_commit) + { + values[Anum_pg_subscription_subsynccommit - 1] = + CStringGetTextDatum(synchronous_commit); + replaces[Anum_pg_subscription_subsynccommit - 1] = true; + } update_tuple = true; break; @@ -601,7 +637,7 @@ AlterSubscription(AlterSubscriptionStmt *stmt) parse_subscription_options(stmt->options, NULL, &enabled_given, &enabled, NULL, - NULL, NULL); + NULL, NULL, NULL); Assert(enabled_given); values[Anum_pg_subscription_subenabled - 1] = @@ -626,7 +662,7 @@ AlterSubscription(AlterSubscriptionStmt *stmt) Subscription *sub = GetSubscription(subid, false); parse_subscription_options(stmt->options, NULL, NULL, NULL, - NULL, NULL, ©_data); + NULL, NULL, ©_data, NULL); values[Anum_pg_subscription_subpublications - 1] = publicationListToArray(stmt->publication); @@ -652,7 +688,7 @@ AlterSubscription(AlterSubscriptionStmt *stmt) Subscription *sub = GetSubscription(subid, false); parse_subscription_options(stmt->options, NULL, NULL, NULL, - NULL, NULL, ©_data); + NULL, NULL, ©_data, NULL); AlterSubscription_refresh(sub, copy_data); diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c index 7ba239c02c..2d663f6308 100644 --- a/src/backend/replication/logical/launcher.c +++ b/src/backend/replication/logical/launcher.c @@ -129,17 +129,13 @@ get_subscription_list(void) */ oldcxt = MemoryContextSwitchTo(resultcxt); - sub = (Subscription *) palloc(sizeof(Subscription)); + sub = (Subscription *) palloc0(sizeof(Subscription)); sub->oid = HeapTupleGetOid(tup); sub->dbid = subform->subdbid; sub->owner = subform->subowner; sub->enabled = subform->subenabled; sub->name = pstrdup(NameStr(subform->subname)); - /* We don't fill fields we are not interested in. */ - sub->conninfo = NULL; - sub->slotname = NULL; - sub->publications = NIL; res = lappend(res, sub); MemoryContextSwitchTo(oldcxt); diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 3313448e7b..29b6c6a168 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -1416,6 +1416,10 @@ reread_subscription(void) MemoryContextSwitchTo(oldctx); + /* Change synchronous commit according to the user's wishes */ + SetConfigOption("synchronous_commit", MySubscription->synccommit, + PGC_BACKEND, PGC_S_OVERRIDE); + if (started_tx) CommitTransactionCommand(); @@ -1485,6 +1489,10 @@ ApplyWorkerMain(Datum main_arg) MySubscriptionValid = true; MemoryContextSwitchTo(oldctx); + /* Setup synchronous commit according to the user's wishes */ + SetConfigOption("synchronous_commit", MySubscription->synccommit, + PGC_BACKEND, PGC_S_OVERRIDE); + if (!MySubscription->enabled) { ereport(LOG, diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 1029354462..3eccfa626b 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -3683,6 +3683,7 @@ getSubscriptions(Archive *fout) int i_rolname; int i_subconninfo; int i_subslotname; + int i_subsynccommit; int i_subpublications; int i, ntups; @@ -3714,7 +3715,8 @@ getSubscriptions(Archive *fout) appendPQExpBuffer(query, "SELECT s.tableoid, s.oid, s.subname," "(%s s.subowner) AS rolname, " - " s.subconninfo, s.subslotname, s.subpublications " + " s.subconninfo, s.subslotname, s.subsynccommit, " + " s.subpublications " "FROM pg_catalog.pg_subscription s " "WHERE s.subdbid = (SELECT oid FROM pg_catalog.pg_database" " WHERE datname = current_database())", @@ -3729,6 +3731,7 @@ getSubscriptions(Archive *fout) i_rolname = PQfnumber(res, "rolname"); i_subconninfo = PQfnumber(res, "subconninfo"); i_subslotname = PQfnumber(res, "subslotname"); + i_subsynccommit = PQfnumber(res, "subsynccommit"); i_subpublications = PQfnumber(res, "subpublications"); subinfo = pg_malloc(ntups * sizeof(SubscriptionInfo)); @@ -3744,6 +3747,8 @@ getSubscriptions(Archive *fout) subinfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_rolname)); subinfo[i].subconninfo = pg_strdup(PQgetvalue(res, i, i_subconninfo)); subinfo[i].subslotname = pg_strdup(PQgetvalue(res, i, i_subslotname)); + subinfo[i].subsynccommit = + pg_strdup(PQgetvalue(res, i, i_subsynccommit)); subinfo[i].subpublications = pg_strdup(PQgetvalue(res, i, i_subpublications)); @@ -3810,6 +3815,10 @@ dumpSubscription(Archive *fout, SubscriptionInfo *subinfo) appendPQExpBuffer(query, " PUBLICATION %s WITH (NOCONNECT, SLOT NAME = ", publications->data); appendStringLiteralAH(query, subinfo->subslotname, fout); + + if (strcmp(subinfo->subsynccommit, "off") != 0) + appendPQExpBuffer(query, ", SYNCHRONOUS_COMMIT = %s", fmtId(subinfo->subsynccommit)); + appendPQExpBufferStr(query, ");\n"); appendPQExpBuffer(labelq, "SUBSCRIPTION %s", fmtId(subinfo->dobj.name)); diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h index ba85392f11..471cfce92a 100644 --- a/src/bin/pg_dump/pg_dump.h +++ b/src/bin/pg_dump/pg_dump.h @@ -616,6 +616,7 @@ typedef struct _SubscriptionInfo char *rolname; char *subconninfo; char *subslotname; + char *subsynccommit; char *subpublications; } SubscriptionInfo; diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index 2494d046b2..59121b8d1b 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -5199,7 +5199,8 @@ describeSubscriptions(const char *pattern, bool verbose) PQExpBufferData buf; PGresult *res; printQueryOpt myopt = pset.popt; - static const bool translate_columns[] = {false, false, false, false, false}; + static const bool translate_columns[] = {false, false, false, false, + false, false}; if (pset.sversion < 100000) { @@ -5225,7 +5226,9 @@ describeSubscriptions(const char *pattern, bool verbose) if (verbose) { appendPQExpBuffer(&buf, + ", subsynccommit AS \"%s\"\n" ", subconninfo AS \"%s\"\n", + gettext_noop("Synchronous commit"), gettext_noop("Conninfo")); } diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h index 0811880a8f..fae542b612 100644 --- a/src/include/catalog/pg_subscription.h +++ b/src/include/catalog/pg_subscription.h @@ -43,7 +43,7 @@ CATALOG(pg_subscription,6100) BKI_SHARED_RELATION BKI_ROWTYPE_OID(6101) BKI_SCHE #ifdef CATALOG_VARLEN /* variable-length fields start here */ text subconninfo; /* Connection string to the publisher */ NameData subslotname; /* Slot name on publisher */ - + text subsynccommit; /* Synchronous commit setting for worker */ text subpublications[1]; /* List of publications subscribed to */ #endif } FormData_pg_subscription; @@ -54,14 +54,15 @@ typedef FormData_pg_subscription *Form_pg_subscription; * compiler constants for pg_subscription * ---------------- */ -#define Natts_pg_subscription 7 +#define Natts_pg_subscription 8 #define Anum_pg_subscription_subdbid 1 #define Anum_pg_subscription_subname 2 #define Anum_pg_subscription_subowner 3 #define Anum_pg_subscription_subenabled 4 #define Anum_pg_subscription_subconninfo 5 #define Anum_pg_subscription_subslotname 6 -#define Anum_pg_subscription_subpublications 7 +#define Anum_pg_subscription_subsynccommit 7 +#define Anum_pg_subscription_subpublications 8 typedef struct Subscription @@ -73,6 +74,7 @@ typedef struct Subscription bool enabled; /* Indicates if the subscription is enabled */ char *conninfo; /* Connection string to the publisher */ char *slotname; /* Name of the replication slot */ + char *synccommit; /* Synchronous commit setting for worker */ List *publications; /* List of publication names to subscribe to */ } Subscription; diff --git a/src/test/regress/expected/subscription.out b/src/test/regress/expected/subscription.out index 8760d5970a..47531edd1b 100644 --- a/src/test/regress/expected/subscription.out +++ b/src/test/regress/expected/subscription.out @@ -46,10 +46,10 @@ CREATE SUBSCRIPTION testsub2 CONNECTION 'dbname=doesnotexist' PUBLICATION foo WI ERROR: must be superuser to create subscriptions SET SESSION AUTHORIZATION 'regress_subscription_user'; \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Conninfo ----------+---------------------------+---------+-------------+--------------------- - testsub | regress_subscription_user | f | {testpub} | dbname=doesnotexist + List of subscriptions + Name | Owner | Enabled | Publication | Synchronous commit | Conninfo +---------+---------------------------+---------+-------------+--------------------+--------------------- + testsub | regress_subscription_user | f | {testpub} | off | dbname=doesnotexist (1 row) ALTER SUBSCRIPTION testsub SET PUBLICATION testpub2, testpub3 NOREFRESH; @@ -59,10 +59,10 @@ ALTER SUBSCRIPTION testsub WITH (SLOT NAME = 'newname'); ALTER SUBSCRIPTION doesnotexist CONNECTION 'dbname=doesnotexist2'; ERROR: subscription "doesnotexist" does not exist \dRs+ - List of subscriptions - Name | Owner | Enabled | Publication | Conninfo ----------+---------------------------+---------+---------------------+---------------------- - testsub | regress_subscription_user | f | {testpub2,testpub3} | dbname=doesnotexist2 + List of subscriptions + Name | Owner | Enabled | Publication | Synchronous commit | Conninfo +---------+---------------------------+---------+---------------------+--------------------+---------------------- + testsub | regress_subscription_user | f | {testpub2,testpub3} | off | dbname=doesnotexist2 (1 row) BEGIN; @@ -89,11 +89,15 @@ ALTER SUBSCRIPTION testsub RENAME TO testsub_dummy; ERROR: must be owner of subscription testsub RESET ROLE; ALTER SUBSCRIPTION testsub RENAME TO testsub_foo; -\dRs - List of subscriptions - Name | Owner | Enabled | Publication --------------+---------------------------+---------+--------------------- - testsub_foo | regress_subscription_user | f | {testpub2,testpub3} +ALTER SUBSCRIPTION testsub_foo WITH (SYNCHRONOUS_COMMIT = local); +ALTER SUBSCRIPTION testsub_foo WITH (SYNCHRONOUS_COMMIT = foobar); +ERROR: invalid value for parameter "synchronous_commit": "foobar" +HINT: Available values: local, remote_write, remote_apply, on, off. +\dRs+ + List of subscriptions + Name | Owner | Enabled | Publication | Synchronous commit | Conninfo +-------------+---------------------------+---------+---------------------+--------------------+---------------------- + testsub_foo | regress_subscription_user | f | {testpub2,testpub3} | local | dbname=doesnotexist2 (1 row) -- rename back to keep the rest simple diff --git a/src/test/regress/sql/subscription.sql b/src/test/regress/sql/subscription.sql index 7bdc2b3503..1b30d150ce 100644 --- a/src/test/regress/sql/subscription.sql +++ b/src/test/regress/sql/subscription.sql @@ -66,8 +66,10 @@ ALTER SUBSCRIPTION testsub RENAME TO testsub_dummy; RESET ROLE; ALTER SUBSCRIPTION testsub RENAME TO testsub_foo; +ALTER SUBSCRIPTION testsub_foo WITH (SYNCHRONOUS_COMMIT = local); +ALTER SUBSCRIPTION testsub_foo WITH (SYNCHRONOUS_COMMIT = foobar); -\dRs +\dRs+ -- rename back to keep the rest simple ALTER SUBSCRIPTION testsub_foo RENAME TO testsub; -- cgit v1.2.3 From 6e5f9a6dc0a9e417544692db56d2c80a64dd83c7 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 14 Apr 2017 14:07:44 -0400 Subject: Fix typo in comment --- doc/src/sgml/ref/drop_foreign_table.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/drop_foreign_table.sgml b/doc/src/sgml/ref/drop_foreign_table.sgml index f9d1e459d2..5a2b235d4e 100644 --- a/doc/src/sgml/ref/drop_foreign_table.sgml +++ b/doc/src/sgml/ref/drop_foreign_table.sgml @@ -1,4 +1,4 @@ - + -- cgit v1.2.3 From 5a617ab3e691aec56725960e6d28c98c8af6ddaa Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 14 Apr 2017 19:36:34 -0400 Subject: doc: Fix typo --- doc/src/sgml/client-auth.sgml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/client-auth.sgml b/doc/src/sgml/client-auth.sgml index c2fc6d3261..d871c041ce 100644 --- a/doc/src/sgml/client-auth.sgml +++ b/doc/src/sgml/client-auth.sgml @@ -922,8 +922,8 @@ omicron bryanh guest1 - The password-based authentication methods are scram - md5 and password. These methods operate + The password-based authentication methods are scram, + md5, and password. These methods operate similarly except for the way that the password is sent across the connection. -- cgit v1.2.3 From 1fe33252a08c285de9e84615cfde0569b9a75e58 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Mon, 17 Apr 2017 09:14:22 -0400 Subject: Document that ONLY can be specified in publication commands Author: Amit Langote --- doc/src/sgml/ref/alter_publication.sgml | 12 ++++++++---- doc/src/sgml/ref/create_publication.sgml | 9 +++++++-- src/test/regress/expected/publication.out | 23 +++++++++++++++++++++++ src/test/regress/sql/publication.sql | 10 ++++++++++ 4 files changed, 48 insertions(+), 6 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/alter_publication.sgml b/doc/src/sgml/ref/alter_publication.sgml index 0a965b3bbf..858231fbcb 100644 --- a/doc/src/sgml/ref/alter_publication.sgml +++ b/doc/src/sgml/ref/alter_publication.sgml @@ -31,9 +31,9 @@ ALTER PUBLICATION name WITH ( name OWNER TO { new_owner | CURRENT_USER | SESSION_USER } ALTER PUBLICATION name RENAME TO new_name -ALTER PUBLICATION name ADD TABLE table_name [, ...] -ALTER PUBLICATION name SET TABLE table_name [, ...] -ALTER PUBLICATION name DROP TABLE table_name [, ...] +ALTER PUBLICATION name ADD TABLE [ ONLY ] table_name [ * ] [, ...] +ALTER PUBLICATION name SET TABLE [ ONLY ] table_name [ * ] [, ...] +ALTER PUBLICATION name DROP TABLE [ ONLY ] table_name [ * ] [, ...] @@ -116,7 +116,11 @@ ALTER PUBLICATION name DROP TABLE < table_name - Name of an existing table. + Name of an existing table. If ONLY is specified before the + table name, only that table is affected. If ONLY is not + specified, the table and all its descendant tables (if any) are + affected. Optionally, * can be specified after the table + name to explicitly indicate that descendant tables are included. diff --git a/doc/src/sgml/ref/create_publication.sgml b/doc/src/sgml/ref/create_publication.sgml index 3cdde801fa..0369b579c5 100644 --- a/doc/src/sgml/ref/create_publication.sgml +++ b/doc/src/sgml/ref/create_publication.sgml @@ -22,7 +22,7 @@ PostgreSQL documentation CREATE PUBLICATION name - [ FOR TABLE table_name [, ...] + [ FOR TABLE [ ONLY ] table_name [ * ] [, ...] | FOR ALL TABLES ] [ WITH ( option [, ... ] ) ] @@ -68,7 +68,12 @@ CREATE PUBLICATION name FOR TABLE - Specifies a list of tables to add to the publication. + Specifies a list of tables to add to the publication. If + ONLY is specified before the table name, only + that table is added to the publication. If ONLY is not + specified, the table and all its descendant tables (if any) are added. + Optionally, * can be specified after the table name to + explicitly indicate that descendant tables are included. diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out index 0964718a60..5b7fb674da 100644 --- a/src/test/regress/expected/publication.out +++ b/src/test/regress/expected/publication.out @@ -71,6 +71,29 @@ Publications: DROP TABLE testpub_tbl2; DROP PUBLICATION testpub_foralltables; +CREATE TABLE testpub_tbl3 (a int); +CREATE TABLE testpub_tbl3a (b text) INHERITS (testpub_tbl3); +CREATE PUBLICATION testpub3 FOR TABLE testpub_tbl3; +CREATE PUBLICATION testpub4 FOR TABLE ONLY testpub_tbl3; +\dRp+ testpub3 + Publication testpub3 + Inserts | Updates | Deletes +---------+---------+--------- + t | t | t +Tables: + "public.testpub_tbl3" + "public.testpub_tbl3a" + +\dRp+ testpub4 + Publication testpub4 + Inserts | Updates | Deletes +---------+---------+--------- + t | t | t +Tables: + "public.testpub_tbl3" + +DROP TABLE testpub_tbl3, testpub_tbl3a; +DROP PUBLICATION testpub3, testpub4; -- fail - view CREATE PUBLICATION testpub_fortbl FOR TABLE testpub_view; ERROR: "testpub_view" is not a table diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql index 85530bec0e..b118bc9906 100644 --- a/src/test/regress/sql/publication.sql +++ b/src/test/regress/sql/publication.sql @@ -44,6 +44,16 @@ SELECT pubname, puballtables FROM pg_publication WHERE pubname = 'testpub_forall DROP TABLE testpub_tbl2; DROP PUBLICATION testpub_foralltables; +CREATE TABLE testpub_tbl3 (a int); +CREATE TABLE testpub_tbl3a (b text) INHERITS (testpub_tbl3); +CREATE PUBLICATION testpub3 FOR TABLE testpub_tbl3; +CREATE PUBLICATION testpub4 FOR TABLE ONLY testpub_tbl3; +\dRp+ testpub3 +\dRp+ testpub4 + +DROP TABLE testpub_tbl3, testpub_tbl3a; +DROP PUBLICATION testpub3, testpub4; + -- fail - view CREATE PUBLICATION testpub_fortbl FOR TABLE testpub_view; CREATE PUBLICATION testpub_fortbl FOR TABLE testpub_tbl1, pub_test.testpub_nopk; -- cgit v1.2.3 From 8c5cdb7f4f6e1d6a6104cb58ce4f23453891651b Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Mon, 17 Apr 2017 17:55:17 -0300 Subject: Tighten up relation kind checks for extended statistics We were accepting creation of extended statistics only for regular tables, but they can usefully be created for foreign tables, partitioned tables, and materialized views, too. Allow those cases. While at it, make sure all the rejected cases throw a consistent error message, and add regression tests for the whole thing. Author: David Rowley, Álvaro Herrera Discussion: https://postgr.es/m/CAKJS1f-BmGo410bh5RSPZUvOO0LhmHL2NYmdrC_Jm8pk_FfyCA@mail.gmail.com --- doc/src/sgml/ref/create_statistics.sgml | 2 +- src/backend/commands/statscmds.c | 10 +++++---- src/bin/pg_dump/pg_dump.c | 9 ++++++-- src/test/regress/expected/stats_ext.out | 40 +++++++++++++++++++++++++++++++++ src/test/regress/sql/stats_ext.sql | 38 +++++++++++++++++++++++++++++++ 5 files changed, 92 insertions(+), 7 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/create_statistics.sgml b/doc/src/sgml/ref/create_statistics.sgml index dbe28d6685..edbcf5840b 100644 --- a/doc/src/sgml/ref/create_statistics.sgml +++ b/doc/src/sgml/ref/create_statistics.sgml @@ -34,7 +34,7 @@ CREATE STATISTICS [ IF NOT EXISTS ] statistics_na CREATE STATISTICS will create a new extended statistics - object on the specified table. + object on the specified table, foreign table or materialized view. The statistics will be created in the current database and will be owned by the user issuing the command. diff --git a/src/backend/commands/statscmds.c b/src/backend/commands/statscmds.c index 46abadcc81..2dd32d9318 100644 --- a/src/backend/commands/statscmds.c +++ b/src/backend/commands/statscmds.c @@ -102,14 +102,16 @@ CreateStatistics(CreateStatsStmt *stmt) * take only ShareUpdateExclusiveLock on relation, conflicting with * ANALYZE and other DDL that sets statistical information. */ - rel = heap_openrv(stmt->relation, ShareUpdateExclusiveLock); + rel = relation_openrv(stmt->relation, ShareUpdateExclusiveLock); relid = RelationGetRelid(rel); if (rel->rd_rel->relkind != RELKIND_RELATION && - rel->rd_rel->relkind != RELKIND_MATVIEW) + rel->rd_rel->relkind != RELKIND_MATVIEW && + rel->rd_rel->relkind != RELKIND_FOREIGN_TABLE && + rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("relation \"%s\" is not a table or materialized view", + errmsg("relation \"%s\" is not a table, foreign table, or materialized view", RelationGetRelationName(rel)))); /* @@ -248,7 +250,7 @@ CreateStatistics(CreateStatsStmt *stmt) CatalogTupleInsert(statrel, htup); statoid = HeapTupleGetOid(htup); heap_freetuple(htup); - heap_close(statrel, RowExclusiveLock); + relation_close(statrel, RowExclusiveLock); /* * Invalidate relcache so that others see the new statistics. diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 22b5f784dc..8824018786 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -6676,9 +6676,14 @@ getExtendedStatistics(Archive *fout, TableInfo tblinfo[], int numTables) { TableInfo *tbinfo = &tblinfo[i]; - /* Only plain tables and materialized views can have extended statistics. */ + /* + * Only plain tables, materialized views, foreign tables and + * partitioned tables can have extended statistics. + */ if (tbinfo->relkind != RELKIND_RELATION && - tbinfo->relkind != RELKIND_MATVIEW) + tbinfo->relkind != RELKIND_MATVIEW && + tbinfo->relkind != RELKIND_FOREIGN_TABLE && + tbinfo->relkind != RELKIND_PARTITIONED_TABLE) continue; /* diff --git a/src/test/regress/expected/stats_ext.out b/src/test/regress/expected/stats_ext.out index 6a3345548a..658d285769 100644 --- a/src/test/regress/expected/stats_ext.out +++ b/src/test/regress/expected/stats_ext.out @@ -47,6 +47,46 @@ ANALYZE ab1 (a); WARNING: extended statistics "public.ab1_a_b_stats" could not be collected for relation public.ab1 ANALYZE ab1; DROP TABLE ab1; +-- Verify supported object types for extended statistics +CREATE schema tststats; +CREATE TABLE tststats.t (a int, b int, c text); +CREATE INDEX ti ON tststats.t (a, b); +CREATE SEQUENCE tststats.s; +CREATE VIEW tststats.v AS SELECT * FROM tststats.t; +CREATE MATERIALIZED VIEW tststats.mv AS SELECT * FROM tststats.t; +CREATE TYPE tststats.ty AS (a int, b int, c text); +CREATE FOREIGN DATA WRAPPER extstats_dummy_fdw; +CREATE SERVER extstats_dummy_srv FOREIGN DATA WRAPPER extstats_dummy_fdw; +CREATE FOREIGN TABLE tststats.f (a int, b int, c text) SERVER extstats_dummy_srv; +CREATE TABLE tststats.pt (a int, b int, c text) PARTITION BY RANGE (a, b); +CREATE TABLE tststats.pt1 PARTITION OF tststats.pt FOR VALUES FROM (-10, -10) TO (10, 10); +CREATE STATISTICS tststats.s1 ON (a, b) FROM tststats.t; +CREATE STATISTICS tststats.s2 ON (a, b) FROM tststats.ti; +ERROR: relation "ti" is not a table, foreign table, or materialized view +CREATE STATISTICS tststats.s3 ON (a, b) FROM tststats.s; +ERROR: relation "s" is not a table, foreign table, or materialized view +CREATE STATISTICS tststats.s4 ON (a, b) FROM tststats.v; +ERROR: relation "v" is not a table, foreign table, or materialized view +CREATE STATISTICS tststats.s5 ON (a, b) FROM tststats.mv; +CREATE STATISTICS tststats.s6 ON (a, b) FROM tststats.ty; +ERROR: relation "ty" is not a table, foreign table, or materialized view +CREATE STATISTICS tststats.s7 ON (a, b) FROM tststats.f; +CREATE STATISTICS tststats.s8 ON (a, b) FROM tststats.pt; +CREATE STATISTICS tststats.s9 ON (a, b) FROM tststats.pt1; +DO $$ +DECLARE + relname text := reltoastrelid::regclass FROM pg_class WHERE oid = 'tststats.t'::regclass; +BEGIN + EXECUTE 'CREATE STATISTICS tststats.s10 ON (a, b) FROM ' || relname; +EXCEPTION WHEN wrong_object_type THEN + RAISE NOTICE 'stats on toast table not created'; +END; +$$; +NOTICE: stats on toast table not created +SET client_min_messages TO warning; +DROP SCHEMA tststats CASCADE; +DROP FOREIGN DATA WRAPPER extstats_dummy_fdw CASCADE; +RESET client_min_messages; -- n-distinct tests CREATE TABLE ndistinct ( filler1 TEXT, diff --git a/src/test/regress/sql/stats_ext.sql b/src/test/regress/sql/stats_ext.sql index ebb6a78383..3c7e0684d3 100644 --- a/src/test/regress/sql/stats_ext.sql +++ b/src/test/regress/sql/stats_ext.sql @@ -40,6 +40,44 @@ ANALYZE ab1 (a); ANALYZE ab1; DROP TABLE ab1; +-- Verify supported object types for extended statistics +CREATE schema tststats; + +CREATE TABLE tststats.t (a int, b int, c text); +CREATE INDEX ti ON tststats.t (a, b); +CREATE SEQUENCE tststats.s; +CREATE VIEW tststats.v AS SELECT * FROM tststats.t; +CREATE MATERIALIZED VIEW tststats.mv AS SELECT * FROM tststats.t; +CREATE TYPE tststats.ty AS (a int, b int, c text); +CREATE FOREIGN DATA WRAPPER extstats_dummy_fdw; +CREATE SERVER extstats_dummy_srv FOREIGN DATA WRAPPER extstats_dummy_fdw; +CREATE FOREIGN TABLE tststats.f (a int, b int, c text) SERVER extstats_dummy_srv; +CREATE TABLE tststats.pt (a int, b int, c text) PARTITION BY RANGE (a, b); +CREATE TABLE tststats.pt1 PARTITION OF tststats.pt FOR VALUES FROM (-10, -10) TO (10, 10); + +CREATE STATISTICS tststats.s1 ON (a, b) FROM tststats.t; +CREATE STATISTICS tststats.s2 ON (a, b) FROM tststats.ti; +CREATE STATISTICS tststats.s3 ON (a, b) FROM tststats.s; +CREATE STATISTICS tststats.s4 ON (a, b) FROM tststats.v; +CREATE STATISTICS tststats.s5 ON (a, b) FROM tststats.mv; +CREATE STATISTICS tststats.s6 ON (a, b) FROM tststats.ty; +CREATE STATISTICS tststats.s7 ON (a, b) FROM tststats.f; +CREATE STATISTICS tststats.s8 ON (a, b) FROM tststats.pt; +CREATE STATISTICS tststats.s9 ON (a, b) FROM tststats.pt1; +DO $$ +DECLARE + relname text := reltoastrelid::regclass FROM pg_class WHERE oid = 'tststats.t'::regclass; +BEGIN + EXECUTE 'CREATE STATISTICS tststats.s10 ON (a, b) FROM ' || relname; +EXCEPTION WHEN wrong_object_type THEN + RAISE NOTICE 'stats on toast table not created'; +END; +$$; + +SET client_min_messages TO warning; +DROP SCHEMA tststats CASCADE; +DROP FOREIGN DATA WRAPPER extstats_dummy_fdw CASCADE; +RESET client_min_messages; -- n-distinct tests CREATE TABLE ndistinct ( -- cgit v1.2.3 From ee6922112e9b3c02b995bd1d838c9a261f060133 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Mon, 17 Apr 2017 18:34:29 -0300 Subject: Rename columns in new pg_statistic_ext catalog The new catalog reused a column prefix "sta" from pg_statistic, but this is undesirable, so change the catalog to use prefix "stx" instead. Also, rename the column that lists enabled statistic kinds as "stxkind" rather than "enabled". Discussion: https://postgr.es/m/CAKJS1f_2t5jhSN7huYRFH3w3rrHfG2QU7hiUHsu-Vdjd1rYT3w@mail.gmail.com --- doc/src/sgml/catalogs.sgml | 22 ++++++++++--------- doc/src/sgml/planstats.sgml | 4 ++-- src/backend/catalog/aclchk.c | 2 +- src/backend/catalog/heap.c | 8 +++---- src/backend/catalog/objectaddress.c | 12 +++++----- src/backend/commands/statscmds.c | 32 +++++++++++++-------------- src/backend/optimizer/util/plancat.c | 4 ++-- src/backend/statistics/dependencies.c | 4 ++-- src/backend/statistics/extended_stats.c | 34 ++++++++++++++--------------- src/backend/statistics/mvdistinct.c | 2 +- src/backend/utils/adt/ruleutils.c | 18 +++++++-------- src/backend/utils/cache/relcache.c | 4 ++-- src/backend/utils/cache/syscache.c | 4 ++-- src/bin/pg_dump/pg_dump.c | 20 ++++++++--------- src/bin/psql/describe.c | 14 ++++++------ src/include/catalog/catversion.h | 2 +- src/include/catalog/indexing.h | 4 ++-- src/include/catalog/pg_statistic_ext.h | 34 ++++++++++++++--------------- src/test/regress/expected/alter_generic.out | 8 +++---- src/test/regress/expected/stats_ext.out | 30 ++++++++++++------------- src/test/regress/sql/alter_generic.sql | 6 ++--- src/test/regress/sql/stats_ext.sql | 14 ++++++------ 22 files changed, 142 insertions(+), 140 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 5254bb3025..ef36e87a72 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -4291,21 +4291,21 @@ - starelid + stxrelid oid pg_class.oid The table that the described columns belongs to - staname + stxname name Name of the statistic. - stanamespace + stxnamespace oid pg_namespace.oid @@ -4314,24 +4314,26 @@ - staowner + stxowner oid pg_authid.oid Owner of the statistic - staenabled + stxkind char[] - An array with the modes of the enabled statistic types, encoded as - d for ndistinct coefficients. + An array with the modes of the enabled statistic types. Valid values + are: + d for ndistinct coefficients, + f for functional dependencies. - stakeys + stxkeys int2vector pg_attribute.attnum @@ -4342,7 +4344,7 @@ - standistinct + stxndistinct pg_ndistinct @@ -4351,7 +4353,7 @@ - stadependencies + stxdependencies pg_dependencies diff --git a/doc/src/sgml/planstats.sgml b/doc/src/sgml/planstats.sgml index a4f91c737a..124e7e20ce 100644 --- a/doc/src/sgml/planstats.sgml +++ b/doc/src/sgml/planstats.sgml @@ -525,8 +525,8 @@ EXPLAIN ANALYZE SELECT * FROM t WHERE a = 1 AND b = 1; you may do this: -SELECT staname,stadependencies FROM pg_statistic_ext WHERE staname = 's1'; - staname | stadependencies +SELECT stxname,stxdependencies FROM pg_statistic_ext WHERE stxname = 's1'; + stxname | stxdependencies ---------+-------------------------------------------- s1 | [{1 => 2 : 1.000000}, {2 => 1 : 1.000000}] (1 row) diff --git a/src/backend/catalog/aclchk.c b/src/backend/catalog/aclchk.c index 70e3e6229c..32989df2b5 100644 --- a/src/backend/catalog/aclchk.c +++ b/src/backend/catalog/aclchk.c @@ -5148,7 +5148,7 @@ pg_statistics_ownercheck(Oid stat_oid, Oid roleid) (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("statistics with OID %u do not exist", stat_oid))); - ownerId = ((Form_pg_statistic_ext) GETSTRUCT(tuple))->staowner; + ownerId = ((Form_pg_statistic_ext) GETSTRUCT(tuple))->stxowner; ReleaseSysCache(tuple); diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index 4a5f545dc6..ece4df02cd 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -2805,7 +2805,7 @@ RemoveStatisticsExt(Oid relid, AttrNumber attnum) pgstatisticext = heap_open(StatisticExtRelationId, RowExclusiveLock); ScanKeyInit(&key, - Anum_pg_statistic_ext_starelid, + Anum_pg_statistic_ext_stxrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relid)); @@ -2825,13 +2825,13 @@ RemoveStatisticsExt(Oid relid, AttrNumber attnum) int i; /* - * Decode the stakeys array and delete any stats that involve the + * Decode the stxkeys array and delete any stats that involve the * specified column. */ staForm = (Form_pg_statistic_ext) GETSTRUCT(tuple); - for (i = 0; i < staForm->stakeys.dim1; i++) + for (i = 0; i < staForm->stxkeys.dim1; i++) { - if (staForm->stakeys.values[i] == attnum) + if (staForm->stxkeys.values[i] == attnum) { delete = true; break; diff --git a/src/backend/catalog/objectaddress.c b/src/backend/catalog/objectaddress.c index 1492722865..a9e529fba0 100644 --- a/src/backend/catalog/objectaddress.c +++ b/src/backend/catalog/objectaddress.c @@ -485,9 +485,9 @@ static const ObjectPropertyType ObjectProperty[] = StatisticExtOidIndexId, STATEXTOID, STATEXTNAMENSP, - Anum_pg_statistic_ext_staname, - Anum_pg_statistic_ext_stanamespace, - Anum_pg_statistic_ext_staowner, + Anum_pg_statistic_ext_stxname, + Anum_pg_statistic_ext_stxnamespace, + Anum_pg_statistic_ext_stxowner, InvalidAttrNumber, /* no ACL (same as relation) */ ACL_KIND_STATISTICS, true @@ -4936,13 +4936,13 @@ getObjectIdentityParts(const ObjectAddress *object, elog(ERROR, "cache lookup failed for statistics %u", object->objectId); formStatistic = (Form_pg_statistic_ext) GETSTRUCT(tup); - schema = get_namespace_name_or_temp(formStatistic->stanamespace); + schema = get_namespace_name_or_temp(formStatistic->stxnamespace); appendStringInfoString(&buffer, quote_qualified_identifier(schema, - NameStr(formStatistic->staname))); + NameStr(formStatistic->stxname))); if (objname) *objname = list_make2(schema, - pstrdup(NameStr(formStatistic->staname))); + pstrdup(NameStr(formStatistic->stxname))); ReleaseSysCache(tup); } break; diff --git a/src/backend/commands/statscmds.c b/src/backend/commands/statscmds.c index 2dd32d9318..f95cd153f5 100644 --- a/src/backend/commands/statscmds.c +++ b/src/backend/commands/statscmds.c @@ -50,13 +50,13 @@ CreateStatistics(CreateStatsStmt *stmt) int numcols = 0; ObjectAddress address = InvalidObjectAddress; char *namestr; - NameData staname; + NameData stxname; Oid statoid; Oid namespaceId; HeapTuple htup; Datum values[Natts_pg_statistic_ext]; bool nulls[Natts_pg_statistic_ext]; - int2vector *stakeys; + int2vector *stxkeys; Relation statrel; Relation rel; Oid relid; @@ -64,7 +64,7 @@ CreateStatistics(CreateStatsStmt *stmt) childobject; Datum types[2]; /* one for each possible type of statistics */ int ntypes; - ArrayType *staenabled; + ArrayType *stxkind; bool build_ndistinct; bool build_dependencies; bool requested_type = false; @@ -73,13 +73,13 @@ CreateStatistics(CreateStatsStmt *stmt) /* resolve the pieces of the name (namespace etc.) */ namespaceId = QualifiedNameGetCreationNamespace(stmt->defnames, &namestr); - namestrcpy(&staname, namestr); + namestrcpy(&stxname, namestr); /* * If if_not_exists was given and the statistics already exists, bail out. */ if (SearchSysCacheExists2(STATEXTNAMENSP, - PointerGetDatum(&staname), + PointerGetDatum(&stxname), ObjectIdGetDatum(namespaceId))) { if (stmt->if_not_exists) @@ -184,7 +184,7 @@ CreateStatistics(CreateStatsStmt *stmt) (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("duplicate column name in statistics definition"))); - stakeys = buildint2vector(attnums, numcols); + stxkeys = buildint2vector(attnums, numcols); /* * Parse the statistics options. Currently only statistics types are @@ -226,23 +226,23 @@ CreateStatistics(CreateStatsStmt *stmt) if (build_dependencies) types[ntypes++] = CharGetDatum(STATS_EXT_DEPENDENCIES); Assert(ntypes > 0); - staenabled = construct_array(types, ntypes, CHAROID, 1, true, 'c'); + stxkind = construct_array(types, ntypes, CHAROID, 1, true, 'c'); /* * Everything seems fine, so let's build the pg_statistic_ext tuple. */ memset(values, 0, sizeof(values)); memset(nulls, false, sizeof(nulls)); - values[Anum_pg_statistic_ext_starelid - 1] = ObjectIdGetDatum(relid); - values[Anum_pg_statistic_ext_staname - 1] = NameGetDatum(&staname); - values[Anum_pg_statistic_ext_stanamespace - 1] = ObjectIdGetDatum(namespaceId); - values[Anum_pg_statistic_ext_staowner - 1] = ObjectIdGetDatum(GetUserId()); - values[Anum_pg_statistic_ext_stakeys - 1] = PointerGetDatum(stakeys); - values[Anum_pg_statistic_ext_staenabled - 1] = PointerGetDatum(staenabled); + values[Anum_pg_statistic_ext_stxrelid - 1] = ObjectIdGetDatum(relid); + values[Anum_pg_statistic_ext_stxname - 1] = NameGetDatum(&stxname); + values[Anum_pg_statistic_ext_stxnamespace - 1] = ObjectIdGetDatum(namespaceId); + values[Anum_pg_statistic_ext_stxowner - 1] = ObjectIdGetDatum(GetUserId()); + values[Anum_pg_statistic_ext_stxkeys - 1] = PointerGetDatum(stxkeys); + values[Anum_pg_statistic_ext_stxkind - 1] = PointerGetDatum(stxkind); /* no statistics build yet */ - nulls[Anum_pg_statistic_ext_standistinct - 1] = true; - nulls[Anum_pg_statistic_ext_stadependencies - 1] = true; + nulls[Anum_pg_statistic_ext_stxndistinct - 1] = true; + nulls[Anum_pg_statistic_ext_stxdependencies - 1] = true; /* insert it into pg_statistic_ext */ statrel = heap_open(StatisticExtRelationId, RowExclusiveLock); @@ -303,7 +303,7 @@ RemoveStatisticsById(Oid statsOid) elog(ERROR, "cache lookup failed for statistics %u", statsOid); statext = (Form_pg_statistic_ext) GETSTRUCT(tup); - relid = statext->starelid; + relid = statext->stxrelid; rel = heap_open(relid, AccessExclusiveLock); diff --git a/src/backend/optimizer/util/plancat.c b/src/backend/optimizer/util/plancat.c index 28322cec0e..9207c8d809 100644 --- a/src/backend/optimizer/util/plancat.c +++ b/src/backend/optimizer/util/plancat.c @@ -1293,8 +1293,8 @@ get_relation_statistics(RelOptInfo *rel, Relation relation) * wasted if no stats are actually built, but it doesn't seem worth * troubling over that case. */ - for (i = 0; i < staForm->stakeys.dim1; i++) - keys = bms_add_member(keys, staForm->stakeys.values[i]); + for (i = 0; i < staForm->stxkeys.dim1; i++) + keys = bms_add_member(keys, staForm->stxkeys.values[i]); /* add one StatisticExtInfo for each kind built */ if (statext_is_kind_built(htup, STATS_EXT_NDISTINCT)) diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c index fee07c6b34..0890514bf7 100644 --- a/src/backend/statistics/dependencies.c +++ b/src/backend/statistics/dependencies.c @@ -411,7 +411,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows, Bitmapset *attrs, d = (MVDependency *) palloc0(offsetof(MVDependency, attributes) + k * sizeof(AttrNumber)); - /* copy the dependency (and keep the indexes into stakeys) */ + /* copy the dependency (and keep the indexes into stxkeys) */ d->degree = degree; d->nattributes = k; for (i = 0; i < k; i++) @@ -652,7 +652,7 @@ staext_dependencies_load(Oid mvoid) elog(ERROR, "cache lookup failed for extended statistics %u", mvoid); deps = SysCacheGetAttr(STATEXTOID, htup, - Anum_pg_statistic_ext_stadependencies, &isnull); + Anum_pg_statistic_ext_stxdependencies, &isnull); Assert(!isnull); diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c index 64d0cc3e69..b334140c48 100644 --- a/src/backend/statistics/extended_stats.c +++ b/src/backend/statistics/extended_stats.c @@ -145,11 +145,11 @@ statext_is_kind_built(HeapTuple htup, char type) switch (type) { case STATS_EXT_NDISTINCT: - attnum = Anum_pg_statistic_ext_standistinct; + attnum = Anum_pg_statistic_ext_stxndistinct; break; case STATS_EXT_DEPENDENCIES: - attnum = Anum_pg_statistic_ext_stadependencies; + attnum = Anum_pg_statistic_ext_stxdependencies; break; default: @@ -175,7 +175,7 @@ fetch_statentries_for_relation(Relation pg_statext, Oid relid) * rel. */ ScanKeyInit(&skey, - Anum_pg_statistic_ext_starelid, + Anum_pg_statistic_ext_stxrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relid)); @@ -195,23 +195,23 @@ fetch_statentries_for_relation(Relation pg_statext, Oid relid) entry = palloc0(sizeof(StatExtEntry)); entry->statOid = HeapTupleGetOid(htup); staForm = (Form_pg_statistic_ext) GETSTRUCT(htup); - entry->schema = get_namespace_name(staForm->stanamespace); - entry->name = pstrdup(NameStr(staForm->staname)); - for (i = 0; i < staForm->stakeys.dim1; i++) + entry->schema = get_namespace_name(staForm->stxnamespace); + entry->name = pstrdup(NameStr(staForm->stxname)); + for (i = 0; i < staForm->stxkeys.dim1; i++) { entry->columns = bms_add_member(entry->columns, - staForm->stakeys.values[i]); + staForm->stxkeys.values[i]); } - /* decode the staenabled char array into a list of chars */ + /* decode the stxkind char array into a list of chars */ datum = SysCacheGetAttr(STATEXTOID, htup, - Anum_pg_statistic_ext_staenabled, &isnull); + Anum_pg_statistic_ext_stxkind, &isnull); Assert(!isnull); arr = DatumGetArrayTypeP(datum); if (ARR_NDIM(arr) != 1 || ARR_HASNULL(arr) || ARR_ELEMTYPE(arr) != CHAROID) - elog(ERROR, "staenabled is not a 1-D char array"); + elog(ERROR, "stxkind is not a 1-D char array"); enabled = (char *) ARR_DATA_PTR(arr); for (i = 0; i < ARR_DIMS(arr)[0]; i++) { @@ -231,7 +231,7 @@ fetch_statentries_for_relation(Relation pg_statext, Oid relid) /* * Using 'vacatts' of size 'nvacatts' as input data, return a newly built * VacAttrStats array which includes only the items corresponding to - * attributes indicated by 'stakeys'. If we don't have all of the per column + * attributes indicated by 'stxkeys'. If we don't have all of the per column * stats available to compute the extended stats, then we return NULL to indicate * to the caller that the stats should not be built. */ @@ -310,21 +310,21 @@ statext_store(Relation pg_stext, Oid statOid, { bytea *data = statext_ndistinct_serialize(ndistinct); - nulls[Anum_pg_statistic_ext_standistinct - 1] = (data == NULL); - values[Anum_pg_statistic_ext_standistinct - 1] = PointerGetDatum(data); + nulls[Anum_pg_statistic_ext_stxndistinct - 1] = (data == NULL); + values[Anum_pg_statistic_ext_stxndistinct - 1] = PointerGetDatum(data); } if (dependencies != NULL) { bytea *data = statext_dependencies_serialize(dependencies); - nulls[Anum_pg_statistic_ext_stadependencies - 1] = (data == NULL); - values[Anum_pg_statistic_ext_stadependencies - 1] = PointerGetDatum(data); + nulls[Anum_pg_statistic_ext_stxdependencies - 1] = (data == NULL); + values[Anum_pg_statistic_ext_stxdependencies - 1] = PointerGetDatum(data); } /* always replace the value (either by bytea or NULL) */ - replaces[Anum_pg_statistic_ext_standistinct - 1] = true; - replaces[Anum_pg_statistic_ext_stadependencies - 1] = true; + replaces[Anum_pg_statistic_ext_stxndistinct - 1] = true; + replaces[Anum_pg_statistic_ext_stxdependencies - 1] = true; /* there should already be a pg_statistic_ext tuple */ oldtup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(statOid)); diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c index ece544b3a3..b77113fb39 100644 --- a/src/backend/statistics/mvdistinct.c +++ b/src/backend/statistics/mvdistinct.c @@ -134,7 +134,7 @@ statext_ndistinct_load(Oid mvoid) elog(ERROR, "cache lookup failed for statistics %u", mvoid); ndist = SysCacheGetAttr(STATEXTOID, htup, - Anum_pg_statistic_ext_standistinct, &isnull); + Anum_pg_statistic_ext_stxndistinct, &isnull); if (isnull) elog(ERROR, "requested statistic kind %c not yet built for statistics %u", diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c index 5f11af2ee6..184e5daa05 100644 --- a/src/backend/utils/adt/ruleutils.c +++ b/src/backend/utils/adt/ruleutils.c @@ -1473,23 +1473,23 @@ pg_get_statisticsext_worker(Oid statextid, bool missing_ok) initStringInfo(&buf); - nsp = get_namespace_name(statextrec->stanamespace); + nsp = get_namespace_name(statextrec->stxnamespace); appendStringInfo(&buf, "CREATE STATISTICS %s", quote_qualified_identifier(nsp, - NameStr(statextrec->staname))); + NameStr(statextrec->stxname))); /* - * Lookup the staenabled column so that we know how to handle the WITH + * Lookup the stxkind column so that we know how to handle the WITH * clause. */ datum = SysCacheGetAttr(STATEXTOID, statexttup, - Anum_pg_statistic_ext_staenabled, &isnull); + Anum_pg_statistic_ext_stxkind, &isnull); Assert(!isnull); arr = DatumGetArrayTypeP(datum); if (ARR_NDIM(arr) != 1 || ARR_HASNULL(arr) || ARR_ELEMTYPE(arr) != CHAROID) - elog(ERROR, "staenabled is not a 1-D char array"); + elog(ERROR, "stxkind is not a 1-D char array"); enabled = (char *) ARR_DATA_PTR(arr); ndistinct_enabled = false; @@ -1523,21 +1523,21 @@ pg_get_statisticsext_worker(Oid statextid, bool missing_ok) appendStringInfoString(&buf, " ON ("); - for (colno = 0; colno < statextrec->stakeys.dim1; colno++) + for (colno = 0; colno < statextrec->stxkeys.dim1; colno++) { - AttrNumber attnum = statextrec->stakeys.values[colno]; + AttrNumber attnum = statextrec->stxkeys.values[colno]; char *attname; if (colno > 0) appendStringInfoString(&buf, ", "); - attname = get_relid_attribute_name(statextrec->starelid, attnum); + attname = get_relid_attribute_name(statextrec->stxrelid, attnum); appendStringInfoString(&buf, quote_identifier(attname)); } appendStringInfo(&buf, ") FROM %s", - generate_relation_name(statextrec->starelid, NIL)); + generate_relation_name(statextrec->stxrelid, NIL)); ReleaseSysCache(statexttup); diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c index 5bf02d4c0e..85c6b61310 100644 --- a/src/backend/utils/cache/relcache.c +++ b/src/backend/utils/cache/relcache.c @@ -4497,9 +4497,9 @@ RelationGetStatExtList(Relation relation) */ result = NIL; - /* Prepare to scan pg_statistic_ext for entries having starelid = this rel. */ + /* Prepare to scan pg_statistic_ext for entries having stxrelid = this rel. */ ScanKeyInit(&skey, - Anum_pg_statistic_ext_starelid, + Anum_pg_statistic_ext_stxrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(RelationGetRelid(relation))); diff --git a/src/backend/utils/cache/syscache.c b/src/backend/utils/cache/syscache.c index d8c823f42b..edbc151f33 100644 --- a/src/backend/utils/cache/syscache.c +++ b/src/backend/utils/cache/syscache.c @@ -731,8 +731,8 @@ static const struct cachedesc cacheinfo[] = { StatisticExtNameIndexId, 2, { - Anum_pg_statistic_ext_staname, - Anum_pg_statistic_ext_stanamespace, + Anum_pg_statistic_ext_stxname, + Anum_pg_statistic_ext_stxnamespace, 0, 0 }, diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 8824018786..e9b5c8a448 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -6663,8 +6663,8 @@ getExtendedStatistics(Archive *fout, TableInfo tblinfo[], int numTables) int ntups; int i_tableoid; int i_oid; - int i_staname; - int i_stadef; + int i_stxname; + int i_stxdef; /* Extended statistics were new in v10 */ if (fout->remoteVersion < 100000) @@ -6707,11 +6707,11 @@ getExtendedStatistics(Archive *fout, TableInfo tblinfo[], int numTables) "SELECT " "tableoid, " "oid, " - "staname, " - "pg_catalog.pg_get_statisticsextdef(oid) AS stadef " + "stxname, " + "pg_catalog.pg_get_statisticsextdef(oid) AS stxdef " "FROM pg_statistic_ext " - "WHERE starelid = '%u' " - "ORDER BY staname", tbinfo->dobj.catId.oid); + "WHERE stxrelid = '%u' " + "ORDER BY stxname", tbinfo->dobj.catId.oid); res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); @@ -6719,8 +6719,8 @@ getExtendedStatistics(Archive *fout, TableInfo tblinfo[], int numTables) i_tableoid = PQfnumber(res, "tableoid"); i_oid = PQfnumber(res, "oid"); - i_staname = PQfnumber(res, "staname"); - i_stadef = PQfnumber(res, "stadef"); + i_stxname = PQfnumber(res, "stxname"); + i_stxdef = PQfnumber(res, "stxdef"); statsextinfo = (StatsExtInfo *) pg_malloc(ntups * sizeof(StatsExtInfo)); @@ -6730,10 +6730,10 @@ getExtendedStatistics(Archive *fout, TableInfo tblinfo[], int numTables) statsextinfo[j].dobj.catId.tableoid = atooid(PQgetvalue(res, j, i_tableoid)); statsextinfo[j].dobj.catId.oid = atooid(PQgetvalue(res, j, i_oid)); AssignDumpId(&statsextinfo[j].dobj); - statsextinfo[j].dobj.name = pg_strdup(PQgetvalue(res, j, i_staname)); + statsextinfo[j].dobj.name = pg_strdup(PQgetvalue(res, j, i_stxname)); statsextinfo[j].dobj.namespace = tbinfo->dobj.namespace; statsextinfo[j].statsexttable = tbinfo; - statsextinfo[j].statsextdef = pg_strdup(PQgetvalue(res, j, i_stadef)); + statsextinfo[j].statsextdef = pg_strdup(PQgetvalue(res, j, i_stxdef)); } PQclear(res); diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index 59121b8d1b..0f9f497c66 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -2344,16 +2344,16 @@ describeOneTableDetails(const char *schemaname, { printfPQExpBuffer(&buf, "SELECT oid, " - "stanamespace::pg_catalog.regnamespace AS nsp, " - "staname, stakeys,\n" + "stxnamespace::pg_catalog.regnamespace AS nsp, " + "stxname, stxkeys,\n" " (SELECT pg_catalog.string_agg(pg_catalog.quote_ident(attname),', ')\n" - " FROM pg_catalog.unnest(stakeys) s(attnum)\n" - " JOIN pg_catalog.pg_attribute a ON (starelid = a.attrelid AND\n" + " FROM pg_catalog.unnest(stxkeys) s(attnum)\n" + " JOIN pg_catalog.pg_attribute a ON (stxrelid = a.attrelid AND\n" " a.attnum = s.attnum AND NOT attisdropped)) AS columns,\n" - " (staenabled @> '{d}') AS ndist_enabled,\n" - " (staenabled @> '{f}') AS deps_enabled\n" + " (stxkind @> '{d}') AS ndist_enabled,\n" + " (stxkind @> '{f}') AS deps_enabled\n" "FROM pg_catalog.pg_statistic_ext stat " - "WHERE starelid = '%s'\n" + "WHERE stxrelid = '%s'\n" "ORDER BY 1;", oid); diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index e978b3a8a5..ab92fd88ed 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201704141 +#define CATALOG_VERSION_NO 201704171 #endif diff --git a/src/include/catalog/indexing.h b/src/include/catalog/indexing.h index a7266860ce..07300f8a2b 100644 --- a/src/include/catalog/indexing.h +++ b/src/include/catalog/indexing.h @@ -184,9 +184,9 @@ DECLARE_UNIQUE_INDEX(pg_largeobject_metadata_oid_index, 2996, on pg_largeobject_ DECLARE_UNIQUE_INDEX(pg_statistic_ext_oid_index, 3380, on pg_statistic_ext using btree(oid oid_ops)); #define StatisticExtOidIndexId 3380 -DECLARE_UNIQUE_INDEX(pg_statistic_ext_name_index, 3997, on pg_statistic_ext using btree(staname name_ops, stanamespace oid_ops)); +DECLARE_UNIQUE_INDEX(pg_statistic_ext_name_index, 3997, on pg_statistic_ext using btree(stxname name_ops, stxnamespace oid_ops)); #define StatisticExtNameIndexId 3997 -DECLARE_INDEX(pg_statistic_ext_relid_index, 3379, on pg_statistic_ext using btree(starelid oid_ops)); +DECLARE_INDEX(pg_statistic_ext_relid_index, 3379, on pg_statistic_ext using btree(stxrelid oid_ops)); #define StatisticExtRelidIndexId 3379 DECLARE_UNIQUE_INDEX(pg_namespace_nspname_index, 2684, on pg_namespace using btree(nspname name_ops)); diff --git a/src/include/catalog/pg_statistic_ext.h b/src/include/catalog/pg_statistic_ext.h index 0a1cc0446e..b0fda076fe 100644 --- a/src/include/catalog/pg_statistic_ext.h +++ b/src/include/catalog/pg_statistic_ext.h @@ -31,22 +31,22 @@ CATALOG(pg_statistic_ext,3381) { /* These fields form the unique key for the entry: */ - Oid starelid; /* relation containing attributes */ - NameData staname; /* statistics name */ - Oid stanamespace; /* OID of namespace containing this statistics */ - Oid staowner; /* statistics owner */ + Oid stxrelid; /* relation containing attributes */ + NameData stxname; /* statistics name */ + Oid stxnamespace; /* OID of namespace containing this statistics */ + Oid stxowner; /* statistics owner */ /* * variable-length fields start here, but we allow direct access to - * stakeys + * stxkeys */ - int2vector stakeys; /* array of column keys */ + int2vector stxkeys; /* array of column keys */ #ifdef CATALOG_VARLEN - char staenabled[1] BKI_FORCE_NOT_NULL; /* statistic types + char stxkind[1] BKI_FORCE_NOT_NULL; /* statistic types * requested to build */ - pg_ndistinct standistinct; /* ndistinct coefficients (serialized) */ - pg_dependencies stadependencies; /* dependencies (serialized) */ + pg_ndistinct stxndistinct; /* ndistinct coefficients (serialized) */ + pg_dependencies stxdependencies; /* dependencies (serialized) */ #endif } FormData_pg_statistic_ext; @@ -63,14 +63,14 @@ typedef FormData_pg_statistic_ext *Form_pg_statistic_ext; * ---------------- */ #define Natts_pg_statistic_ext 8 -#define Anum_pg_statistic_ext_starelid 1 -#define Anum_pg_statistic_ext_staname 2 -#define Anum_pg_statistic_ext_stanamespace 3 -#define Anum_pg_statistic_ext_staowner 4 -#define Anum_pg_statistic_ext_stakeys 5 -#define Anum_pg_statistic_ext_staenabled 6 -#define Anum_pg_statistic_ext_standistinct 7 -#define Anum_pg_statistic_ext_stadependencies 8 +#define Anum_pg_statistic_ext_stxrelid 1 +#define Anum_pg_statistic_ext_stxname 2 +#define Anum_pg_statistic_ext_stxnamespace 3 +#define Anum_pg_statistic_ext_stxowner 4 +#define Anum_pg_statistic_ext_stxkeys 5 +#define Anum_pg_statistic_ext_stxkind 6 +#define Anum_pg_statistic_ext_stxndistinct 7 +#define Anum_pg_statistic_ext_stxdependencies 8 #define STATS_EXT_NDISTINCT 'd' #define STATS_EXT_DEPENDENCIES 'f' diff --git a/src/test/regress/expected/alter_generic.out b/src/test/regress/expected/alter_generic.out index ce581bb93d..a81a4edfb2 100644 --- a/src/test/regress/expected/alter_generic.out +++ b/src/test/regress/expected/alter_generic.out @@ -525,12 +525,12 @@ ERROR: must be owner of statistics alt_stat3 ALTER STATISTICS alt_stat2 SET SCHEMA alt_nsp2; -- failed (name conflict) ERROR: statistics "alt_stat2" already exists in schema "alt_nsp2" RESET SESSION AUTHORIZATION; -SELECT nspname, staname, rolname +SELECT nspname, stxname, rolname FROM pg_statistic_ext s, pg_namespace n, pg_authid a - WHERE s.stanamespace = n.oid AND s.staowner = a.oid + WHERE s.stxnamespace = n.oid AND s.stxowner = a.oid AND n.nspname in ('alt_nsp1', 'alt_nsp2') - ORDER BY nspname, staname; - nspname | staname | rolname + ORDER BY nspname, stxname; + nspname | stxname | rolname ----------+-----------+--------------------- alt_nsp1 | alt_stat2 | regress_alter_user2 alt_nsp1 | alt_stat3 | regress_alter_user1 diff --git a/src/test/regress/expected/stats_ext.out b/src/test/regress/expected/stats_ext.out index 658d285769..0d6f65e604 100644 --- a/src/test/regress/expected/stats_ext.out +++ b/src/test/regress/expected/stats_ext.out @@ -12,7 +12,7 @@ DROP STATISTICS ab1_a_b_stats; CREATE SCHEMA regress_schema_2; CREATE STATISTICS regress_schema_2.ab1_a_b_stats ON (a, b) FROM ab1; -- Let's also verify the pg_get_statisticsextdef output looks sane. -SELECT pg_get_statisticsextdef(oid) FROM pg_statistic_ext WHERE staname = 'ab1_a_b_stats'; +SELECT pg_get_statisticsextdef(oid) FROM pg_statistic_ext WHERE stxname = 'ab1_a_b_stats'; pg_get_statisticsextdef --------------------------------------------------------------------- CREATE STATISTICS regress_schema_2.ab1_a_b_stats ON (a, b) FROM ab1 @@ -173,11 +173,11 @@ ERROR: duplicate column name in statistics definition -- correct command CREATE STATISTICS s10 ON (a, b, c) FROM ndistinct; ANALYZE ndistinct; -SELECT staenabled, standistinct - FROM pg_statistic_ext WHERE starelid = 'ndistinct'::regclass; - staenabled | standistinct -------------+------------------------------------------------------------------------------------------------ - {d,f} | [{(b 3 4), 301.000000}, {(b 3 6), 301.000000}, {(b 4 6), 301.000000}, {(b 3 4 6), 301.000000}] +SELECT stxkind, stxndistinct + FROM pg_statistic_ext WHERE stxrelid = 'ndistinct'::regclass; + stxkind | stxndistinct +---------+------------------------------------------------------------------------------------------------ + {d,f} | [{(b 3 4), 301.000000}, {(b 3 6), 301.000000}, {(b 4 6), 301.000000}, {(b 3 4 6), 301.000000}] (1 row) -- Hash Aggregate, thanks to estimates improved by the statistic @@ -239,11 +239,11 @@ INSERT INTO ndistinct (a, b, c, filler1) cash_words(mod(i,33)::int::money) FROM generate_series(1,10000) s(i); ANALYZE ndistinct; -SELECT staenabled, standistinct - FROM pg_statistic_ext WHERE starelid = 'ndistinct'::regclass; - staenabled | standistinct -------------+---------------------------------------------------------------------------------------------------- - {d,f} | [{(b 3 4), 2550.000000}, {(b 3 6), 800.000000}, {(b 4 6), 1632.000000}, {(b 3 4 6), 10000.000000}] +SELECT stxkind, stxndistinct + FROM pg_statistic_ext WHERE stxrelid = 'ndistinct'::regclass; + stxkind | stxndistinct +---------+---------------------------------------------------------------------------------------------------- + {d,f} | [{(b 3 4), 2550.000000}, {(b 3 6), 800.000000}, {(b 4 6), 1632.000000}, {(b 3 4 6), 10000.000000}] (1 row) -- plans using Group Aggregate, thanks to using correct esimates @@ -299,10 +299,10 @@ EXPLAIN (COSTS off) (3 rows) DROP STATISTICS s10; -SELECT staenabled, standistinct - FROM pg_statistic_ext WHERE starelid = 'ndistinct'::regclass; - staenabled | standistinct -------------+-------------- +SELECT stxkind, stxndistinct + FROM pg_statistic_ext WHERE stxrelid = 'ndistinct'::regclass; + stxkind | stxndistinct +---------+-------------- (0 rows) -- dropping the statistics switches the plans to Hash Aggregate, diff --git a/src/test/regress/sql/alter_generic.sql b/src/test/regress/sql/alter_generic.sql index f6fa8d8bfd..88e8d7eb86 100644 --- a/src/test/regress/sql/alter_generic.sql +++ b/src/test/regress/sql/alter_generic.sql @@ -459,11 +459,11 @@ ALTER STATISTICS alt_stat3 SET SCHEMA alt_nsp2; -- failed (not owner) ALTER STATISTICS alt_stat2 SET SCHEMA alt_nsp2; -- failed (name conflict) RESET SESSION AUTHORIZATION; -SELECT nspname, staname, rolname +SELECT nspname, stxname, rolname FROM pg_statistic_ext s, pg_namespace n, pg_authid a - WHERE s.stanamespace = n.oid AND s.staowner = a.oid + WHERE s.stxnamespace = n.oid AND s.stxowner = a.oid AND n.nspname in ('alt_nsp1', 'alt_nsp2') - ORDER BY nspname, staname; + ORDER BY nspname, stxname; -- -- Text Search Dictionary diff --git a/src/test/regress/sql/stats_ext.sql b/src/test/regress/sql/stats_ext.sql index 3c7e0684d3..72c7659c4b 100644 --- a/src/test/regress/sql/stats_ext.sql +++ b/src/test/regress/sql/stats_ext.sql @@ -16,7 +16,7 @@ CREATE SCHEMA regress_schema_2; CREATE STATISTICS regress_schema_2.ab1_a_b_stats ON (a, b) FROM ab1; -- Let's also verify the pg_get_statisticsextdef output looks sane. -SELECT pg_get_statisticsextdef(oid) FROM pg_statistic_ext WHERE staname = 'ab1_a_b_stats'; +SELECT pg_get_statisticsextdef(oid) FROM pg_statistic_ext WHERE stxname = 'ab1_a_b_stats'; DROP STATISTICS regress_schema_2.ab1_a_b_stats; @@ -130,8 +130,8 @@ CREATE STATISTICS s10 ON (a, b, c) FROM ndistinct; ANALYZE ndistinct; -SELECT staenabled, standistinct - FROM pg_statistic_ext WHERE starelid = 'ndistinct'::regclass; +SELECT stxkind, stxndistinct + FROM pg_statistic_ext WHERE stxrelid = 'ndistinct'::regclass; -- Hash Aggregate, thanks to estimates improved by the statistic EXPLAIN (COSTS off) @@ -161,8 +161,8 @@ INSERT INTO ndistinct (a, b, c, filler1) ANALYZE ndistinct; -SELECT staenabled, standistinct - FROM pg_statistic_ext WHERE starelid = 'ndistinct'::regclass; +SELECT stxkind, stxndistinct + FROM pg_statistic_ext WHERE stxrelid = 'ndistinct'::regclass; -- plans using Group Aggregate, thanks to using correct esimates EXPLAIN (COSTS off) @@ -182,8 +182,8 @@ EXPLAIN (COSTS off) DROP STATISTICS s10; -SELECT staenabled, standistinct - FROM pg_statistic_ext WHERE starelid = 'ndistinct'::regclass; +SELECT stxkind, stxndistinct + FROM pg_statistic_ext WHERE stxrelid = 'ndistinct'::regclass; -- dropping the statistics switches the plans to Hash Aggregate, -- due to under-estimates -- cgit v1.2.3 From 8efd1e08f063892be13de085e5e2c0e802fadf55 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Mon, 17 Apr 2017 23:32:54 -0400 Subject: doc: Clarify logical replication details Document more explicitly that the target table can have more columns than the source table. Reported-by: Euler Taveira --- doc/src/sgml/logical-replication.sgml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml index 8c70ce3b6e..7bfff24a68 100644 --- a/doc/src/sgml/logical-replication.sgml +++ b/doc/src/sgml/logical-replication.sgml @@ -200,7 +200,9 @@ Columns of a table are also matched by name. A different order of columns - in the target table is allowed, but the column types have to match. + in the target table is allowed, but the column types have to match. The + target table can have additional columns not provided by the published + table. Those will be filled with their default values. -- cgit v1.2.3 From b2188575c59462c0fd23b63c92fb802206162e49 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Tue, 18 Apr 2017 11:51:06 +0300 Subject: Fix example on creating a trigger with a transition table. Yugo Nagata Discussion: https://www.postgresql.org/message-id/20170417180921.3047f3b0.nagata@sraoss.co.jp --- doc/src/sgml/ref/create_trigger.sgml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/create_trigger.sgml b/doc/src/sgml/ref/create_trigger.sgml index 24195b3849..c5f7c75838 100644 --- a/doc/src/sgml/ref/create_trigger.sgml +++ b/doc/src/sgml/ref/create_trigger.sgml @@ -531,8 +531,8 @@ CREATE TRIGGER view_insert CREATE TRIGGER transfer_insert AFTER INSERT ON transfer - FOR EACH STATEMENT REFERENCING NEW TABLE AS inserted + FOR EACH STATEMENT EXECUTE PROCEDURE check_transfer_balances_to_zero(); @@ -543,8 +543,8 @@ CREATE TRIGGER transfer_insert CREATE TRIGGER paired_items_update AFTER UPDATE ON paired_items - FOR EACH ROW REFERENCING NEW TABLE AS newtab OLD TABLE AS oldtab + FOR EACH ROW EXECUTE PROCEDURE check_matching_pairs(); -- cgit v1.2.3 From c727f120ff50f624a1ee3abe700d995c18314a0b Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Tue, 18 Apr 2017 14:50:50 +0300 Subject: Rename "scram" to "scram-sha-256" in pg_hba.conf and password_encryption. Per discussion, plain "scram" is confusing because we actually implement SCRAM-SHA-256 rather than the original SCRAM that uses SHA-1 as the hash algorithm. If we add support for SCRAM-SHA-512 or some other mechanism in the SCRAM family in the future, that would become even more confusing. Most of the internal files and functions still use just "scram" as a shorthand for SCRMA-SHA-256, but I did change PASSWORD_TYPE_SCRAM to PASSWORD_TYPE_SCRAM_SHA_256, as that could potentially be used by 3rd party extensions that hook into the password-check hook. Michael Paquier did this in an earlier version of the SCRAM patch set already, but I didn't include that in the version that was committed. Discussion: https://www.postgresql.org/message-id/fde71ff1-5858-90c8-99a9-1c2427e7bafb@iki.fi --- doc/src/sgml/client-auth.sgml | 15 ++++++++------- doc/src/sgml/config.sgml | 4 ++-- src/backend/commands/user.c | 8 ++++---- src/backend/libpq/auth-scram.c | 2 +- src/backend/libpq/auth.c | 16 ++++++++-------- src/backend/libpq/crypt.c | 10 +++++----- src/backend/libpq/hba.c | 4 ++-- src/backend/libpq/pg_hba.conf.sample | 8 ++++---- src/backend/utils/misc/guc.c | 2 +- src/bin/initdb/initdb.c | 16 ++++++++-------- src/include/libpq/crypt.h | 2 +- src/test/authentication/t/001_password.pl | 14 +++++++------- src/test/authentication/t/002_saslprep.pl | 4 ++-- src/test/regress/expected/password.out | 8 ++++---- src/test/regress/sql/password.sql | 6 +++--- 15 files changed, 60 insertions(+), 59 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/client-auth.sgml b/doc/src/sgml/client-auth.sgml index d871c041ce..819db811b2 100644 --- a/doc/src/sgml/client-auth.sgml +++ b/doc/src/sgml/client-auth.sgml @@ -412,7 +412,7 @@ hostnossl database user - scram + scram-sha-256 Perform SCRAM-SHA-256 authentication to verify the user's @@ -683,7 +683,7 @@ host postgres all 192.168.93.0/24 ident # "postgres" if the user's password is correctly supplied. # # TYPE DATABASE USER ADDRESS METHOD -host postgres all 192.168.12.10/32 scram +host postgres all 192.168.12.10/32 scram-sha-256 # Allow any user from hosts in the example.com domain to connect to # any database if the user's password is correctly supplied. @@ -694,7 +694,7 @@ host postgres all 192.168.12.10/32 scram # # TYPE DATABASE USER ADDRESS METHOD host all mike .example.com md5 -host all all .example.com scram +host all all .example.com scram-sha-256 # In the absence of preceding "host" lines, these two lines will # reject all connections from 192.168.54.1 (since that entry will be @@ -922,7 +922,7 @@ omicron bryanh guest1 - The password-based authentication methods are scram, + The password-based authentication methods are scram-sha-256, md5, and password. These methods operate similarly except for the way that the password is sent across the connection. @@ -939,8 +939,9 @@ omicron bryanh guest1 - scram performs SCRAM-SHA-256 authentication, as described - in RFC5802. It + scram-sha-256 performs SCRAM-SHA-256 authentication, as + described in + RFC5802. It is a challenge-response scheme, that prevents password sniffing on untrusted connections. It is more secure than the md5 method, but might not be supported by older clients. @@ -953,7 +954,7 @@ omicron bryanh guest1 protection if an attacker manages to steal the password hash from the server, and it cannot be used with the feature. For all other users, - md5 works the same as scram. + md5 works the same as scram-sha-256. diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 744c5e8f37..e02b0c80df 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -1194,8 +1194,8 @@ include_dir 'conf.d' stores the password as an MD5 hash. Setting this to plain stores it in plaintext. on and off are also accepted, as aliases for md5 and plain, respectively. Setting - this parameter to scram will encrypt the password with - SCRAM-SHA-256. + this parameter to scram-sha-256 will encrypt the password + with SCRAM-SHA-256. diff --git a/src/backend/commands/user.c b/src/backend/commands/user.c index de264974ae..c719682274 100644 --- a/src/backend/commands/user.c +++ b/src/backend/commands/user.c @@ -140,8 +140,8 @@ CreateRole(ParseState *pstate, CreateRoleStmt *stmt) dpassword = defel; if (strcmp(defel->defname, "encryptedPassword") == 0) { - if (Password_encryption == PASSWORD_TYPE_SCRAM) - password_type = PASSWORD_TYPE_SCRAM; + if (Password_encryption == PASSWORD_TYPE_SCRAM_SHA_256) + password_type = PASSWORD_TYPE_SCRAM_SHA_256; else password_type = PASSWORD_TYPE_MD5; } @@ -548,8 +548,8 @@ AlterRole(AlterRoleStmt *stmt) dpassword = defel; if (strcmp(defel->defname, "encryptedPassword") == 0) { - if (Password_encryption == PASSWORD_TYPE_SCRAM) - password_type = PASSWORD_TYPE_SCRAM; + if (Password_encryption == PASSWORD_TYPE_SCRAM_SHA_256) + password_type = PASSWORD_TYPE_SCRAM_SHA_256; else password_type = PASSWORD_TYPE_MD5; } diff --git a/src/backend/libpq/auth-scram.c b/src/backend/libpq/auth-scram.c index 338afede9d..76c502d415 100644 --- a/src/backend/libpq/auth-scram.c +++ b/src/backend/libpq/auth-scram.c @@ -183,7 +183,7 @@ pg_be_scram_init(const char *username, const char *shadow_pass) { int password_type = get_password_type(shadow_pass); - if (password_type == PASSWORD_TYPE_SCRAM) + if (password_type == PASSWORD_TYPE_SCRAM_SHA_256) { if (parse_scram_verifier(shadow_pass, &state->salt, &state->iterations, state->StoredKey, state->ServerKey)) diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c index 848561e188..ab4be21943 100644 --- a/src/backend/libpq/auth.c +++ b/src/backend/libpq/auth.c @@ -50,7 +50,7 @@ static char *recv_password_packet(Port *port); /*---------------------------------------------------------------- - * Password-based authentication methods (password, md5, and scram) + * Password-based authentication methods (password, md5, and scram-sha-256) *---------------------------------------------------------------- */ static int CheckPasswordAuth(Port *port, char **logdetail); @@ -757,10 +757,10 @@ CheckPWChallengeAuth(Port *port, char **logdetail) * If the user does not exist, or has no password, we still go through the * motions of authentication, to avoid revealing to the client that the * user didn't exist. If 'md5' is allowed, we choose whether to use 'md5' - * or 'scram' authentication based on current password_encryption setting. - * The idea is that most genuine users probably have a password of that - * type, if we pretend that this user had a password of that type, too, it - * "blends in" best. + * or 'scram-sha-256' authentication based on current password_encryption + * setting. The idea is that most genuine users probably have a password + * of that type, if we pretend that this user had a password of that type, + * too, it "blends in" best. * * If the user had a password, but it was expired, we'll use the details * of the expired password for the authentication, but report it as @@ -773,9 +773,9 @@ CheckPWChallengeAuth(Port *port, char **logdetail) /* * If 'md5' authentication is allowed, decide whether to perform 'md5' or - * 'scram' authentication based on the type of password the user has. If - * it's an MD5 hash, we must do MD5 authentication, and if it's a SCRAM - * verifier, we must do SCRAM authentication. If it's stored in + * 'scram-sha-256' authentication based on the type of password the user + * has. If it's an MD5 hash, we must do MD5 authentication, and if it's + * a SCRAM verifier, we must do SCRAM authentication. If it's stored in * plaintext, we could do either one, so we opt for the more secure * mechanism, SCRAM. * diff --git a/src/backend/libpq/crypt.c b/src/backend/libpq/crypt.c index 34beab5334..03ef3cc652 100644 --- a/src/backend/libpq/crypt.c +++ b/src/backend/libpq/crypt.c @@ -101,7 +101,7 @@ get_password_type(const char *shadow_pass) if (strncmp(shadow_pass, "md5", 3) == 0 && strlen(shadow_pass) == MD5_PASSWD_LEN) return PASSWORD_TYPE_MD5; if (strncmp(shadow_pass, "scram-sha-256:", strlen("scram-sha-256:")) == 0) - return PASSWORD_TYPE_SCRAM; + return PASSWORD_TYPE_SCRAM_SHA_256; return PASSWORD_TYPE_PLAINTEXT; } @@ -141,7 +141,7 @@ encrypt_password(PasswordType target_type, const char *role, elog(ERROR, "password encryption failed"); return encrypted_password; - case PASSWORD_TYPE_SCRAM: + case PASSWORD_TYPE_SCRAM_SHA_256: /* * cannot convert a SCRAM verifier to an MD5 hash, so fall @@ -152,7 +152,7 @@ encrypt_password(PasswordType target_type, const char *role, } break; - case PASSWORD_TYPE_SCRAM: + case PASSWORD_TYPE_SCRAM_SHA_256: switch (guessed_type) { case PASSWORD_TYPE_PLAINTEXT: @@ -164,7 +164,7 @@ encrypt_password(PasswordType target_type, const char *role, * cannot convert an MD5 hash to a SCRAM verifier, so fall * through to save the MD5 hash instead. */ - case PASSWORD_TYPE_SCRAM: + case PASSWORD_TYPE_SCRAM_SHA_256: return pstrdup(password); } break; @@ -280,7 +280,7 @@ plain_crypt_verify(const char *role, const char *shadow_pass, */ switch (get_password_type(shadow_pass)) { - case PASSWORD_TYPE_SCRAM: + case PASSWORD_TYPE_SCRAM_SHA_256: if (scram_verify_plain_password(role, client_pass, shadow_pass)) diff --git a/src/backend/libpq/hba.c b/src/backend/libpq/hba.c index af89fe898a..5561c399da 100644 --- a/src/backend/libpq/hba.c +++ b/src/backend/libpq/hba.c @@ -126,7 +126,7 @@ static const char *const UserAuthName[] = "ident", "password", "md5", - "scram", + "scram-sha256", "gss", "sspi", "pam", @@ -1327,7 +1327,7 @@ parse_hba_line(TokenizedLine *tok_line, int elevel) } parsedline->auth_method = uaMD5; } - else if (strcmp(token->string, "scram") == 0) + else if (strcmp(token->string, "scram-sha-256") == 0) parsedline->auth_method = uaSCRAM; else if (strcmp(token->string, "pam") == 0) #ifdef USE_PAM diff --git a/src/backend/libpq/pg_hba.conf.sample b/src/backend/libpq/pg_hba.conf.sample index 6b1778a721..c853e36232 100644 --- a/src/backend/libpq/pg_hba.conf.sample +++ b/src/backend/libpq/pg_hba.conf.sample @@ -42,10 +42,10 @@ # or "samenet" to match any address in any subnet that the server is # directly connected to. # -# METHOD can be "trust", "reject", "md5", "password", "scram", "gss", -# "sspi", "ident", "peer", "pam", "ldap", "radius" or "cert". Note that -# "password" sends passwords in clear text; "md5" or "scram" are preferred -# since they send encrypted passwords. +# METHOD can be "trust", "reject", "md5", "password", "scram-sha-256", +# "gss", "sspi", "ident", "peer", "pam", "ldap", "radius" or "cert". +# Note that "password" sends passwords in clear text; "md5" or +# "scram-sha-256" are preferred since they send encrypted passwords. # # OPTIONS are a set of options for the authentication in the format # NAME=VALUE. The available options depend on the different diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 9ad8361a9b..a414fb2c76 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -410,7 +410,7 @@ static const struct config_enum_entry force_parallel_mode_options[] = { static const struct config_enum_entry password_encryption_options[] = { {"plain", PASSWORD_TYPE_PLAINTEXT, false}, {"md5", PASSWORD_TYPE_MD5, false}, - {"scram", PASSWORD_TYPE_SCRAM, false}, + {"scram-sha-256", PASSWORD_TYPE_SCRAM_SHA_256, false}, {"off", PASSWORD_TYPE_PLAINTEXT, false}, {"on", PASSWORD_TYPE_MD5, false}, {"true", PASSWORD_TYPE_MD5, true}, diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index d40ed412fc..95da8b7722 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -77,7 +77,7 @@ extern const char *select_default_timezone(const char *share_path); static const char *const auth_methods_host[] = { - "trust", "reject", "md5", "password", "scram", "ident", "radius", + "trust", "reject", "scram-sha-256", "md5", "password", "ident", "radius", #ifdef ENABLE_GSS "gss", #endif @@ -99,7 +99,7 @@ static const char *const auth_methods_host[] = { NULL }; static const char *const auth_methods_local[] = { - "trust", "reject", "md5", "scram", "password", "peer", "radius", + "trust", "reject", "scram-sha-256", "md5", "password", "peer", "radius", #ifdef USE_PAM "pam", "pam ", #endif @@ -1130,12 +1130,12 @@ setup_config(void) "#update_process_title = off"); #endif - if (strcmp(authmethodlocal, "scram") == 0 || - strcmp(authmethodhost, "scram") == 0) + if (strcmp(authmethodlocal, "scram-sha-256") == 0 || + strcmp(authmethodhost, "scram-sha-256") == 0) { conflines = replace_token(conflines, "#password_encryption = md5", - "password_encryption = scram"); + "password_encryption = scram-sha-256"); } snprintf(path, sizeof(path), "%s/postgresql.conf", pg_data); @@ -2329,16 +2329,16 @@ check_need_password(const char *authmethodlocal, const char *authmethodhost) { if ((strcmp(authmethodlocal, "md5") == 0 || strcmp(authmethodlocal, "password") == 0 || - strcmp(authmethodlocal, "scram") == 0) && + strcmp(authmethodlocal, "scram-sha-256") == 0) && (strcmp(authmethodhost, "md5") == 0 || strcmp(authmethodhost, "password") == 0 || - strcmp(authmethodhost, "scram") == 0) && + strcmp(authmethodhost, "scram-sha-256") == 0) && !(pwprompt || pwfilename)) { fprintf(stderr, _("%s: must specify a password for the superuser to enable %s authentication\n"), progname, (strcmp(authmethodlocal, "md5") == 0 || strcmp(authmethodlocal, "password") == 0 || - strcmp(authmethodlocal, "scram") == 0) + strcmp(authmethodlocal, "scram-sha-256") == 0) ? authmethodlocal : authmethodhost); exit(1); diff --git a/src/include/libpq/crypt.h b/src/include/libpq/crypt.h index 3b5da69b08..63724f39ee 100644 --- a/src/include/libpq/crypt.h +++ b/src/include/libpq/crypt.h @@ -25,7 +25,7 @@ typedef enum PasswordType { PASSWORD_TYPE_PLAINTEXT = 0, PASSWORD_TYPE_MD5, - PASSWORD_TYPE_SCRAM + PASSWORD_TYPE_SCRAM_SHA_256 } PasswordType; extern PasswordType get_password_type(const char *shadow_pass); diff --git a/src/test/authentication/t/001_password.pl b/src/test/authentication/t/001_password.pl index d7bc13bd58..216bdc031c 100644 --- a/src/test/authentication/t/001_password.pl +++ b/src/test/authentication/t/001_password.pl @@ -51,7 +51,7 @@ SKIP: # Create 3 roles with different password methods for each one. The same # password is used for all of them. - $node->safe_psql('postgres', "SET password_encryption='scram'; CREATE ROLE scram_role LOGIN PASSWORD 'pass';"); + $node->safe_psql('postgres', "SET password_encryption='scram-sha-256'; CREATE ROLE scram_role LOGIN PASSWORD 'pass';"); $node->safe_psql('postgres', "SET password_encryption='md5'; CREATE ROLE md5_role LOGIN PASSWORD 'pass';"); $node->safe_psql('postgres', "SET password_encryption='plain'; CREATE ROLE plain_role LOGIN PASSWORD 'pass';"); $ENV{"PGPASSWORD"} = 'pass'; @@ -68,12 +68,12 @@ SKIP: test_role($node, 'md5_role', 'password', 0); test_role($node, 'plain_role', 'password', 0); - # For "scram" method, user "plain_role" and "scram_role" should be able to - # connect. - reset_pg_hba($node, 'scram'); - test_role($node, 'scram_role', 'scram', 0); - test_role($node, 'md5_role', 'scram', 2); - test_role($node, 'plain_role', 'scram', 0); + # For "scram-sha-256" method, user "plain_role" and "scram_role" should + # be able to connect. + reset_pg_hba($node, 'scram-sha-256'); + test_role($node, 'scram_role', 'scram-sha-256', 0); + test_role($node, 'md5_role', 'scram-sha-256', 2); + test_role($node, 'plain_role', 'scram-sha-256', 0); # For "md5" method, all users should be able to connect (SCRAM # authentication will be performed for the user with a scram verifier.) diff --git a/src/test/authentication/t/002_saslprep.pl b/src/test/authentication/t/002_saslprep.pl index 7e373ed7bf..67ba92cdd9 100644 --- a/src/test/authentication/t/002_saslprep.pl +++ b/src/test/authentication/t/002_saslprep.pl @@ -63,7 +63,7 @@ SKIP: # Create test roles. $node->safe_psql('postgres', -"SET password_encryption='scram'; +"SET password_encryption='scram-sha-256'; SET client_encoding='utf8'; CREATE ROLE saslpreptest1_role LOGIN PASSWORD 'IX'; CREATE ROLE saslpreptest4a_role LOGIN PASSWORD 'a'; @@ -73,7 +73,7 @@ SKIP: "); # Require password from now on. - reset_pg_hba($node, 'scram'); + reset_pg_hba($node, 'scram-sha-256'); # Check that #1 and #5 are treated the same as just 'IX' test_login($node, 'saslpreptest1_role', "I\xc2\xadX", 0); diff --git a/src/test/regress/expected/password.out b/src/test/regress/expected/password.out index c503e43abe..676b3e6ff3 100644 --- a/src/test/regress/expected/password.out +++ b/src/test/regress/expected/password.out @@ -4,11 +4,11 @@ -- Tests for GUC password_encryption SET password_encryption = 'novalue'; -- error ERROR: invalid value for parameter "password_encryption": "novalue" -HINT: Available values: plain, md5, scram, off, on. +HINT: Available values: plain, md5, scram-sha-256, off, on. SET password_encryption = true; -- ok SET password_encryption = 'md5'; -- ok SET password_encryption = 'plain'; -- ok -SET password_encryption = 'scram'; -- ok +SET password_encryption = 'scram-sha-256'; -- ok -- consistency of password entries SET password_encryption = 'plain'; CREATE ROLE regress_passwd1 PASSWORD 'role_pwd1'; @@ -16,7 +16,7 @@ SET password_encryption = 'md5'; CREATE ROLE regress_passwd2 PASSWORD 'role_pwd2'; SET password_encryption = 'on'; CREATE ROLE regress_passwd3 PASSWORD 'role_pwd3'; -SET password_encryption = 'scram'; +SET password_encryption = 'scram-sha-256'; CREATE ROLE regress_passwd4 PASSWORD 'role_pwd4'; SET password_encryption = 'plain'; CREATE ROLE regress_passwd5 PASSWORD NULL; @@ -60,7 +60,7 @@ ALTER ROLE regress_passwd2 UNENCRYPTED PASSWORD 'md5dfa155cadd5f4ad57860162f3fab SET password_encryption = 'md5'; ALTER ROLE regress_passwd3 ENCRYPTED PASSWORD 'foo'; -- encrypted with MD5 ALTER ROLE regress_passwd4 ENCRYPTED PASSWORD 'scram-sha-256:VLK4RMaQLCvNtQ==:4096:3ded2376f7aafa93b1bdbd71bcc18b7d6ee50ed018029cc583d152ef3fc7d430:a6dd36dfc94c181956a6ae95f05e01b1864f0a22a2657d1de4ba84d2a24dc438'; -- client-supplied SCRAM verifier, use as it is -SET password_encryption = 'scram'; +SET password_encryption = 'scram-sha-256'; ALTER ROLE regress_passwd5 ENCRYPTED PASSWORD 'foo'; -- create SCRAM verifier CREATE ROLE regress_passwd6 ENCRYPTED PASSWORD 'md53725413363ab045e20521bf36b8d8d7f'; -- encrypted with MD5, use as it is SELECT rolname, regexp_replace(rolpassword, '(scram-sha-256):([a-zA-Z0-9+/]+==):(\d+):(\w+):(\w+)', '\1::\3::') as rolpassword_masked diff --git a/src/test/regress/sql/password.sql b/src/test/regress/sql/password.sql index f4b3a9ac3a..95557e4566 100644 --- a/src/test/regress/sql/password.sql +++ b/src/test/regress/sql/password.sql @@ -7,7 +7,7 @@ SET password_encryption = 'novalue'; -- error SET password_encryption = true; -- ok SET password_encryption = 'md5'; -- ok SET password_encryption = 'plain'; -- ok -SET password_encryption = 'scram'; -- ok +SET password_encryption = 'scram-sha-256'; -- ok -- consistency of password entries SET password_encryption = 'plain'; @@ -16,7 +16,7 @@ SET password_encryption = 'md5'; CREATE ROLE regress_passwd2 PASSWORD 'role_pwd2'; SET password_encryption = 'on'; CREATE ROLE regress_passwd3 PASSWORD 'role_pwd3'; -SET password_encryption = 'scram'; +SET password_encryption = 'scram-sha-256'; CREATE ROLE regress_passwd4 PASSWORD 'role_pwd4'; SET password_encryption = 'plain'; CREATE ROLE regress_passwd5 PASSWORD NULL; @@ -50,7 +50,7 @@ ALTER ROLE regress_passwd3 ENCRYPTED PASSWORD 'foo'; -- encrypted with MD5 ALTER ROLE regress_passwd4 ENCRYPTED PASSWORD 'scram-sha-256:VLK4RMaQLCvNtQ==:4096:3ded2376f7aafa93b1bdbd71bcc18b7d6ee50ed018029cc583d152ef3fc7d430:a6dd36dfc94c181956a6ae95f05e01b1864f0a22a2657d1de4ba84d2a24dc438'; -- client-supplied SCRAM verifier, use as it is -SET password_encryption = 'scram'; +SET password_encryption = 'scram-sha-256'; ALTER ROLE regress_passwd5 ENCRYPTED PASSWORD 'foo'; -- create SCRAM verifier CREATE ROLE regress_passwd6 ENCRYPTED PASSWORD 'md53725413363ab045e20521bf36b8d8d7f'; -- encrypted with MD5, use as it is -- cgit v1.2.3 From 2b67c9d207d428c2476af0dd39043d83469d70c2 Mon Sep 17 00:00:00 2001 From: Andrew Dunstan Date: Tue, 18 Apr 2017 08:50:15 -0400 Subject: Simplify docs on creating a self-signed SSL certificate Discussion: --- doc/src/sgml/runtime.sgml | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/runtime.sgml b/doc/src/sgml/runtime.sgml index 01153f9a37..6865b73011 100644 --- a/doc/src/sgml/runtime.sgml +++ b/doc/src/sgml/runtime.sgml @@ -2389,28 +2389,14 @@ pg_dumpall -p 5432 | psql -d postgres -p 5433 Creating a Self-signed Certificate - To create a quick self-signed certificate for the server, use the - following OpenSSL command: + To create a quick self-signed certificate for the server, valid for 365 + days, use the following OpenSSL command, using + the local host name in the subject argument: -openssl req -new -text -out server.req +openssl req -new -x509 -days 365 -nodes -text -out server.crt \ + -keyout server.key -subj "/CN=yourdomain.com" - Fill out the information that openssl asks for. Make sure - you enter the local host name as Common Name; the challenge - password can be left blank. The program will generate a key that is - passphrase protected; it will not accept a passphrase that is less - than four characters long. To remove the passphrase again (as you must - if you want automatic start-up of the server), next run the commands: - -openssl rsa -in privkey.pem -out server.key -rm privkey.pem - - Enter the old passphrase to unlock the existing key. Now do: - -openssl req -x509 -in server.req -text -key server.key -out server.crt - - to turn the certificate into a self-signed certificate and to copy - the key and certificate to where the server will look for them. - Finally do: + Then do: chmod og-rwx server.key -- cgit v1.2.3 From a790ed9f69ef584c12aec68d0d80e6b6b543bacb Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Wed, 19 Apr 2017 02:58:28 +0900 Subject: Improve documentation and comment for quorum-based sync replication. Author: Masahiko Sawada, heavily modified by me Discussion: http://postgr.es/m/CAHGQGwEKOw=SmPLxJzkBsH6wwDBgOnVz46QjHbtsiZ-d-2RGUg@mail.gmail.com --- doc/src/sgml/high-availability.sgml | 21 ++++++++++++++++----- src/backend/replication/syncrep.c | 5 +++++ 2 files changed, 21 insertions(+), 5 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml index 51359d6236..9e2be5f67c 100644 --- a/doc/src/sgml/high-availability.sgml +++ b/doc/src/sgml/high-availability.sgml @@ -1084,8 +1084,8 @@ primary_slot_name = 'node_a_slot' In the case that synchronous_commit is set to remote_apply, the standby sends reply messages when the commit record is replayed, making the transaction visible. - If the standby is chosen as a synchronous standby, from a priority - list of synchronous_standby_names on the primary, the reply + If the standby is chosen as a synchronous standby, according to the setting + of synchronous_standby_names on the primary, the reply messages from that standby will be considered along with those from other synchronous standbys to decide when to release transactions waiting for confirmation that the commit record has been received. These parameters @@ -1246,9 +1246,20 @@ synchronous_standby_names = 'FIRST 2 (s1, s2, s3)' The best solution for high availability is to ensure you keep as many synchronous standbys as requested. This can be achieved by naming multiple potential synchronous standbys using synchronous_standby_names. - The standbys whose names appear earlier in the list will be used as - synchronous standbys. Standbys listed after these will take over - the role of synchronous standby if one of current ones should fail. + + + + In a priority-based synchronous replication, the standbys whose names + appear earlier in the list will be used as synchronous standbys. + Standbys listed after these will take over the role of synchronous standby + if one of current ones should fail. + + + + In a quorum-based synchronous replication, all the standbys appearing + in the list will be used as candidates for synchronous standbys. + Even if one of them should fail, the other standbys will keep performing + the role of candidates of synchronous standby. diff --git a/src/backend/replication/syncrep.c b/src/backend/replication/syncrep.c index 20a1441f0a..25c67aaac7 100644 --- a/src/backend/replication/syncrep.c +++ b/src/backend/replication/syncrep.c @@ -53,6 +53,10 @@ * in the list. All the standbys appearing in the list are considered as * candidates for quorum synchronous standbys. * + * If neither FIRST nor ANY is specified, FIRST is used as the method. + * This is for backward compatibility with 9.6 or before where only a + * priority-based sync replication was supported. + * * Before the standbys chosen from synchronous_standby_names can * become the synchronous standbys they must have caught up with * the primary; that may take some time. Once caught up, @@ -629,6 +633,7 @@ SyncRepGetNthLatestSyncRecPtr(XLogRecPtr *writePtr, XLogRecPtr *flushPtr, i++; } + /* Sort each array in descending order */ qsort(write_array, len, sizeof(XLogRecPtr), cmp_lsn); qsort(flush_array, len, sizeof(XLogRecPtr), cmp_lsn); qsort(apply_array, len, sizeof(XLogRecPtr), cmp_lsn); -- cgit v1.2.3 From 539f67012ec978e931054e413a4ab3c4c68bb737 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 18 Apr 2017 14:21:57 -0400 Subject: Doc: improve markup in self-signed certificate example. --- doc/src/sgml/runtime.sgml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/runtime.sgml b/doc/src/sgml/runtime.sgml index 6865b73011..6d57525515 100644 --- a/doc/src/sgml/runtime.sgml +++ b/doc/src/sgml/runtime.sgml @@ -2390,11 +2390,11 @@ pg_dumpall -p 5432 | psql -d postgres -p 5433 To create a quick self-signed certificate for the server, valid for 365 - days, use the following OpenSSL command, using - the local host name in the subject argument: + days, use the following OpenSSL command, + replacing yourdomain.com with the server's host name: openssl req -new -x509 -days 365 -nodes -text -out server.crt \ - -keyout server.key -subj "/CN=yourdomain.com" + -keyout server.key -subj "/CN=yourdomain.com" Then do: -- cgit v1.2.3 From e68432a0e1ad07b6848fc9bbc11e382eb43b8618 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 19 Apr 2017 21:42:27 +0300 Subject: Fix typo in docs on SASL authentication. Word "sends" was missing. Jaime Casanova --- doc/src/sgml/protocol.sgml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index e04064ac6a..cb72c5d9dd 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -1353,9 +1353,9 @@ general, while the next subsection gives more details on SCRAM-SHA-256. - To begin a SASL authentication exchange, the server an AuthenticationSASL - message. It includes a list of SASL authentication mechanisms that the - server can accept, in the server's preferred order. + To begin a SASL authentication exchange, the server sends an + AuthenticationSASL message. It includes a list of SASL authentication + mechanisms that the server can accept, in the server's preferred order. -- cgit v1.2.3 From 594b526bcf32e307b963ea77f10c9720d9ca423f Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 20 Apr 2017 14:18:33 -0400 Subject: Modify message when partitioned table is added to publication Give a more specific error message than "xyz is not a table". Also document in CREATE PUBLICATION which kinds of relations are not supported. based on patch by Amit Langote --- doc/src/sgml/ref/create_publication.sgml | 8 ++++++++ src/backend/catalog/pg_publication.c | 9 +++++++++ src/test/regress/expected/publication.out | 7 +++++++ src/test/regress/sql/publication.sql | 4 ++++ 4 files changed, 28 insertions(+) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/create_publication.sgml b/doc/src/sgml/ref/create_publication.sgml index 0369b579c5..521376ef4b 100644 --- a/doc/src/sgml/ref/create_publication.sgml +++ b/doc/src/sgml/ref/create_publication.sgml @@ -75,6 +75,14 @@ CREATE PUBLICATION name Optionally, * can be specified after the table name to explicitly indicate that descendant tables are included. + + + Only persistent base tables can be part of a publication. Temporary + tables, unlogged tables, foreign tables, materialized views, regular + views, and partitioned tables cannot be part of a publication. To + replicate a partitioned table, add the individual partitions to the + publication. + diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c index 9330e2380a..15f68a915b 100644 --- a/src/backend/catalog/pg_publication.c +++ b/src/backend/catalog/pg_publication.c @@ -50,6 +50,15 @@ static void check_publication_add_relation(Relation targetrel) { + /* Give more specific error for partitioned tables */ + if (RelationGetForm(targetrel)->relkind == RELKIND_PARTITIONED_TABLE) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("\"%s\" is a partitioned table", + RelationGetRelationName(targetrel)), + errdetail("Adding partitioned tables to publications is not supported."), + errhint("You can add the table partitions individually."))); + /* Must be table */ if (RelationGetForm(targetrel)->relkind != RELKIND_RELATION) ereport(ERROR, diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out index 5b7fb674da..f3a348d368 100644 --- a/src/test/regress/expected/publication.out +++ b/src/test/regress/expected/publication.out @@ -37,6 +37,7 @@ CREATE SCHEMA pub_test; CREATE TABLE testpub_tbl1 (id serial primary key, data text); CREATE TABLE pub_test.testpub_nopk (foo int, bar int); CREATE VIEW testpub_view AS SELECT 1; +CREATE TABLE testpub_parted (a int) PARTITION BY LIST (a); CREATE PUBLICATION testpub_foralltables FOR ALL TABLES WITH (nopublish delete, nopublish update); ALTER PUBLICATION testpub_foralltables WITH (publish update); CREATE TABLE testpub_tbl2 (id serial primary key, data text); @@ -118,6 +119,11 @@ Tables: ALTER PUBLICATION testpub_default ADD TABLE testpub_view; ERROR: "testpub_view" is not a table DETAIL: Only tables can be added to publications. +-- fail - partitioned table +ALTER PUBLICATION testpub_fortbl ADD TABLE testpub_parted; +ERROR: "testpub_parted" is a partitioned table +DETAIL: Adding partitioned tables to publications is not supported. +HINT: You can add the table partitions individually. ALTER PUBLICATION testpub_default ADD TABLE testpub_tbl1; ALTER PUBLICATION testpub_default SET TABLE testpub_tbl1; ALTER PUBLICATION testpub_default ADD TABLE pub_test.testpub_nopk; @@ -188,6 +194,7 @@ ALTER PUBLICATION testpub2 ADD TABLE testpub_tbl1; -- ok DROP PUBLICATION testpub2; SET ROLE regress_publication_user; REVOKE CREATE ON DATABASE regression FROM regress_publication_user2; +DROP TABLE testpub_parted; DROP VIEW testpub_view; DROP TABLE testpub_tbl1; \dRp+ testpub_default diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql index b118bc9906..7d1cba5db3 100644 --- a/src/test/regress/sql/publication.sql +++ b/src/test/regress/sql/publication.sql @@ -26,6 +26,7 @@ CREATE SCHEMA pub_test; CREATE TABLE testpub_tbl1 (id serial primary key, data text); CREATE TABLE pub_test.testpub_nopk (foo int, bar int); CREATE VIEW testpub_view AS SELECT 1; +CREATE TABLE testpub_parted (a int) PARTITION BY LIST (a); CREATE PUBLICATION testpub_foralltables FOR ALL TABLES WITH (nopublish delete, nopublish update); ALTER PUBLICATION testpub_foralltables WITH (publish update); @@ -66,6 +67,8 @@ CREATE PUBLICATION testpub_fortbl FOR TABLE testpub_tbl1; -- fail - view ALTER PUBLICATION testpub_default ADD TABLE testpub_view; +-- fail - partitioned table +ALTER PUBLICATION testpub_fortbl ADD TABLE testpub_parted; ALTER PUBLICATION testpub_default ADD TABLE testpub_tbl1; ALTER PUBLICATION testpub_default SET TABLE testpub_tbl1; @@ -104,6 +107,7 @@ DROP PUBLICATION testpub2; SET ROLE regress_publication_user; REVOKE CREATE ON DATABASE regression FROM regress_publication_user2; +DROP TABLE testpub_parted; DROP VIEW testpub_view; DROP TABLE testpub_tbl1; -- cgit v1.2.3 From 8bcb31ad5a7b3f4a0a6e4fbd8cbc1a7a8dde423d Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 20 Apr 2017 14:41:48 -0400 Subject: Sync pg_ctl documentation and usage message with reality. Commit 05cd12ed5 ("pg_ctl: Change default to wait for all actions") was a tad sloppy about updating the documentation to match. The documentation was also sorely in need of a copy-editing pass, having been adjusted at different times by different people who took little care to maintain consistency of style. --- doc/src/sgml/ref/pg_ctl-ref.sgml | 205 +++++++++++++++++++++------------------ src/bin/pg_ctl/pg_ctl.c | 29 +++--- 2 files changed, 123 insertions(+), 111 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/pg_ctl-ref.sgml b/doc/src/sgml/ref/pg_ctl-ref.sgml index 176dfaf98a..71e52c4c35 100644 --- a/doc/src/sgml/ref/pg_ctl-ref.sgml +++ b/doc/src/sgml/ref/pg_ctl-ref.sgml @@ -23,19 +23,19 @@ PostgreSQL documentation pg_ctl - datadir + initdb-options pg_ctl + datadir + filename seconds - datadir - filename options path @@ -44,9 +44,6 @@ PostgreSQL documentation pg_ctl - - seconds - datadir @@ -55,16 +52,15 @@ PostgreSQL documentation + + seconds + pg_ctl - - seconds - datadir - @@ -72,14 +68,18 @@ PostgreSQL documentation + + seconds + options + pg_ctl - datadir + @@ -91,10 +91,10 @@ PostgreSQL documentation pg_ctl + datadir seconds - datadir @@ -104,19 +104,22 @@ PostgreSQL documentation process_id + On Microsoft Windows, also: + pg_ctl + datadir servicename username password - datadir + source seconds @@ -147,14 +150,14 @@ PostgreSQL documentation The or mode creates a new - PostgreSQL database cluster. A database - cluster is a collection of databases that are managed by a single + PostgreSQL database cluster, that is, + a collection of databases that will be managed by a single server instance. This mode invokes the initdb command. See for details. - In mode, a new server is launched. The + mode launches a new server. The server is started in the background, and its standard input is attached to /dev/null (or nul on Windows). On Unix-like systems, by default, the server's standard output and @@ -172,8 +175,8 @@ PostgreSQL documentation - In mode, the server that is running in - the specified data directory is shut down. Three different + mode shuts down the server that is running in + the specified data directory. Three different shutdown methods can be selected with the option. Smart mode waits for all active clients to disconnect and any online backup to finish. @@ -183,60 +186,63 @@ PostgreSQL documentation will terminate an online backup in progress. All active transactions are rolled back and clients are forcibly disconnected, then the server is shut down. Immediate mode will abort - all server processes immediately, without a clean shutdown. - This will lead to a crash-recovery run on the next restart. + all server processes immediately, without a clean shutdown. This choice + will lead to a crash-recovery cycle during the next server start. mode effectively executes a stop followed by a start. This allows changing the postgres - command-line options. might fail if - relative paths specified were specified on the command-line during - server start. + command-line options, or changing configuration-file options that + cannot be changed without restarting the server. + If relative paths were used on the command line during server + start, might fail unless + pg_ctl is executed in the same current + directory as it was during server start. mode simply sends the - postgres process a SIGHUP + postgres server process a SIGHUP signal, causing it to reread its configuration files (postgresql.conf, - pg_hba.conf, etc.). This allows changing of - configuration-file options that do not require a complete restart + pg_hba.conf, etc.). This allows changing + configuration-file options that do not require a full server restart to take effect. mode checks whether a server is running in - the specified data directory. If it is, the PID - and the command line options that were used to invoke it are - displayed. If the server is not running, the process returns an - exit status of 3. If an accessible data directory is not specified, - the process returns an exit status of 4. + the specified data directory. If it is, the server's PID + and the command line options that were used to invoke it are displayed. + If the server is not running, pg_ctl returns + an exit status of 3. If an accessible data directory is not + specified, pg_ctl returns an exit status of 4. - In mode, the standby server that is - running in the specified data directory is commanded to exit - recovery and begin read-write operations. + mode commands the standby server that is + running in the specified data directory to end standby mode + and begin read-write operations. - mode allows you to send a signal to a specified - process. This is particularly valuable for Microsoft Windows - which does not have a kill command. Use - --help to see a list of supported signal names. + mode sends a signal to a specified process. + This is primarily valuable on Microsoft Windows + which does not have a built-in kill command. Use + --help to see a list of supported signal names. - mode allows you to register a system service - on Microsoft Windows. The option - allows selection of service start type, either auto (start - service automatically on system startup) or demand (start - service on demand). + mode registers the PostgreSQL + server as a system service on Microsoft Windows. + The option allows selection of service start type, + either auto (start service automatically on system startup) + or demand (start service on demand). - mode allows you to unregister a system service + mode unregisters a system service on Microsoft Windows. This undoes the effects of the command. @@ -249,7 +255,7 @@ PostgreSQL documentation - + Attempt to allow server crashes to produce core files, on platforms @@ -267,7 +273,7 @@ PostgreSQL documentation Specifies the file system location of the database configuration files. If - this is omitted, the environment variable + this option is omitted, the environment variable PGDATA is used. @@ -294,7 +300,8 @@ PostgreSQL documentation Specifies the shutdown mode. mode can be smart, fast, or immediate, or the first letter of one of - these three. If this is omitted, fast is used. + these three. If this option is omitted, fast is + the default. @@ -305,12 +312,13 @@ PostgreSQL documentation Specifies options to be passed directly to the - postgres command; multiple - option invocations are appended. + postgres command. + - The options should usually be surrounded by single or double - quotes to ensure that they are passed through as a group. + The options should usually be surrounded by single or + double quotes to ensure that they are passed through as a group. @@ -322,10 +330,12 @@ PostgreSQL documentation Specifies options to be passed directly to the initdb command. + - The options should usually be surrounded by single or double - quotes to ensure that they are passed through as a group. + The options should usually be surrounded by single or + double quotes to ensure that they are passed through as a group. @@ -361,14 +371,14 @@ PostgreSQL documentation - - + + - The maximum number of seconds to wait when waiting for an operation - to complete (see option ). Defaults to the value of the - PGCTLTIMEOUT environment variable or, if not set, to 60 - seconds. + Specifies the maximum number of seconds to wait when waiting for an + operation to complete (see option ). Defaults to + the value of the PGCTLTIMEOUT environment variable or, if + not set, to 60 seconds. @@ -388,7 +398,7 @@ PostgreSQL documentation - Wait for an operation to complete. This is supported for the + Wait for the operation to complete. This is supported for the modes start, stop, restart, promote, and register, and is the default for those modes. @@ -399,7 +409,6 @@ PostgreSQL documentation attempts to connect to the server. When waiting for shutdown, pg_ctl waits for the server to remove its PID file. - This option allows the entry of an SSL passphrase on startup. pg_ctl returns an exit code based on the success of the startup or shutdown. @@ -411,8 +420,8 @@ PostgreSQL documentation - Do not wait for an operation to complete. This is the opposite of the - option . + Do not wait for the operation to complete. This is the opposite of + the option . @@ -441,6 +450,11 @@ PostgreSQL documentation + + If an option is specified that is valid, but not relevant to the selected + operating mode, pg_ctl ignores it. + + Options for Windows @@ -452,11 +466,12 @@ PostgreSQL documentation Name of the event source for pg_ctl to use for logging to the event log when running as a Windows service. The default is PostgreSQL. Note that this only controls - the logging from pg_ctl itself; once + messages sent from pg_ctl itself; once started, the server will use the event source specified - by . Should the server fail during - early startup, it might also log using the default event - source PostgreSQL. + by its parameter. Should the server + fail very early in startup, before that parameter has been set, + it might also log using the default event + source name PostgreSQL. @@ -465,8 +480,9 @@ PostgreSQL documentation - Name of the system service to register. The name will be used + Name of the system service to register. This name will be used as both the service name and the display name. + The default is PostgreSQL. @@ -475,7 +491,7 @@ PostgreSQL documentation - Password for the user to start the service. + Password for the user to run the service as. @@ -484,10 +500,10 @@ PostgreSQL documentation - Start type of the system service to register. start-type can + Start type of the system service. start-type can be auto, or demand, or - the first letter of one of these two. If this is omitted, - auto is used. + the first letter of one of these two. If this option is omitted, + auto is the default. @@ -496,7 +512,7 @@ PostgreSQL documentation - User name for the user to start the service. For domain users, use the + User name for the user to run the service as. For domain users, use the format DOMAIN\username. @@ -534,12 +550,22 @@ PostgreSQL documentation + + Most pg_ctl modes require knowing the data directory + location; therefore, the + pg_ctl, like most other PostgreSQL utilities, also uses the environment variables supported by libpq (see ). - For additional server variables, see . + + + + For additional variables that affect the server, + see . @@ -553,9 +579,8 @@ PostgreSQL documentation - The existence of this file in the data directory is used to help - pg_ctl determine if the server is - currently running. + pg_ctl examines this file in the data + directory to determine whether the server is currently running. @@ -584,13 +609,6 @@ PostgreSQL documentation Starting the Server - - To start the server: - -$ pg_ctl start - - - To start the server, waiting until the server is accepting connections: @@ -626,24 +644,17 @@ PostgreSQL documentation Restarting the server is almost equivalent to stopping the - server and starting it again, - except that pg_ctl saves and reuses the command line options that - were passed to the previously running instance. To restart - the server in the simplest form, use: - -$ pg_ctl restart - - - - - To restart the server, - waiting for it to shut down and restart: + server and starting it again, except that by default, + pg_ctl saves and reuses the command line options that + were passed to the previously-running instance. To restart + the server using the same options as before, use: $ pg_ctl restart + But if diff --git a/src/bin/pg_ctl/pg_ctl.c b/src/bin/pg_ctl/pg_ctl.c index c63819b88b..f34dd28c6e 100644 --- a/src/bin/pg_ctl/pg_ctl.c +++ b/src/bin/pg_ctl/pg_ctl.c @@ -1932,18 +1932,19 @@ do_help(void) { printf(_("%s is a utility to initialize, start, stop, or control a PostgreSQL server.\n\n"), progname); printf(_("Usage:\n")); - printf(_(" %s init[db] [-D DATADIR] [-s] [-o \"OPTIONS\"]\n"), progname); - printf(_(" %s start [-w] [-t SECS] [-D DATADIR] [-s] [-l FILENAME] [-o \"OPTIONS\"]\n"), progname); - printf(_(" %s stop [-W] [-t SECS] [-D DATADIR] [-s] [-m SHUTDOWN-MODE]\n"), progname); - printf(_(" %s restart [-w] [-t SECS] [-D DATADIR] [-s] [-m SHUTDOWN-MODE]\n" - " [-o \"OPTIONS\"]\n"), progname); - printf(_(" %s reload [-D DATADIR] [-s]\n"), progname); - printf(_(" %s status [-D DATADIR]\n"), progname); - printf(_(" %s promote [-w] [-t SECS] [-D DATADIR] [-s]\n"), progname); - printf(_(" %s kill SIGNALNAME PID\n"), progname); + printf(_(" %s init[db] [-D DATADIR] [-s] [-o OPTIONS]\n"), progname); + printf(_(" %s start [-D DATADIR] [-l FILENAME] [-W] [-t SECS] [-s]\n" + " [-o OPTIONS] [-p PATH] [-c]\n"), progname); + printf(_(" %s stop [-D DATADIR] [-m SHUTDOWN-MODE] [-W] [-t SECS] [-s]\n"), progname); + printf(_(" %s restart [-D DATADIR] [-m SHUTDOWN-MODE] [-W] [-t SECS] [-s]\n" + " [-o OPTIONS] [-c]\n"), progname); + printf(_(" %s reload [-D DATADIR] [-s]\n"), progname); + printf(_(" %s status [-D DATADIR]\n"), progname); + printf(_(" %s promote [-D DATADIR] [-W] [-t SECS] [-s]\n"), progname); + printf(_(" %s kill SIGNALNAME PID\n"), progname); #ifdef WIN32 - printf(_(" %s register [-N SERVICENAME] [-U USERNAME] [-P PASSWORD] [-D DATADIR]\n" - " [-S START-TYPE] [-w] [-t SECS] [-o \"OPTIONS\"]\n"), progname); + printf(_(" %s register [-D DATADIR] [-N SERVICENAME] [-U USERNAME] [-P PASSWORD]\n" + " [-S START-TYPE] [-e SOURCE] [-W] [-t SECS] [-s] [-o OPTIONS]\n"), progname); printf(_(" %s unregister [-N SERVICENAME]\n"), progname); #endif @@ -1958,7 +1959,6 @@ do_help(void) printf(_(" -w, --wait wait until operation completes (default)\n")); printf(_(" -W, --no-wait do not wait until operation completes\n")); printf(_(" -?, --help show this help, then exit\n")); - printf(_("(The default is to wait for shutdown, but not for start or restart.)\n\n")); printf(_("If the -D option is omitted, the environment variable PGDATA is used.\n")); printf(_("\nOptions for start or restart:\n")); @@ -1976,7 +1976,7 @@ do_help(void) printf(_("\nShutdown modes are:\n")); printf(_(" smart quit after all clients have disconnected\n")); - printf(_(" fast quit directly, with proper shutdown\n")); + printf(_(" fast quit directly, with proper shutdown (default)\n")); printf(_(" immediate quit without complete shutdown; will lead to recovery on restart\n")); printf(_("\nAllowed signal names for kill:\n")); @@ -2242,7 +2242,8 @@ main(int argc, char **argv) /* process command-line options */ while (optind < argc) { - while ((c = getopt_long(argc, argv, "cD:e:l:m:N:o:p:P:sS:t:U:wW", long_options, &option_index)) != -1) + while ((c = getopt_long(argc, argv, "cD:e:l:m:N:o:p:P:sS:t:U:wW", + long_options, &option_index)) != -1) { switch (c) { -- cgit v1.2.3 From 919f6d746e45c8fe84d02799053ef47c3bb11050 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Thu, 20 Apr 2017 15:43:33 -0300 Subject: Improve multivariate statistics documentation Extended statistics commit 7b504eb282c did not include appropriate documentation next to where we document regular planner statistics (I ripped what was submitted before commit and then forgot to put it back), and while later commit 2686ee1b7ccf added some material, it structurally depended on what I had ripped out, so the end result wasn't proper. Fix those problems by shuffling what was added by 2686ee1b7ccf and including some additional material, so that now chapter 14 "Performance Tips" now describes the types of multivariate statistics we currently have, and chapter 68 "How the Planner Uses Statistics" shows some examples. The new text should be more in line with previous material, in (hopefully) the appropriate depth. While at it, fix a small bug in pg_statistic_ext docs: one column was listed in the wrong spot. --- doc/src/sgml/catalogs.sgml | 22 ++--- doc/src/sgml/perform.sgml | 195 +++++++++++++++++++++++++++++++++++++ doc/src/sgml/planstats.sgml | 228 ++++++++++++++++++++------------------------ 3 files changed, 311 insertions(+), 134 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index ef36e87a72..ed74704b2a 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -4320,6 +4320,17 @@ Owner of the statistic + + stxkeys + int2vector + pg_attribute.attnum + + This is an array of values that indicate which table columns this + statistic covers. For example a value of 1 3 would + mean that the first and the third table columns make up the statistic key. + + + stxkind char[] @@ -4332,17 +4343,6 @@ - - stxkeys - int2vector - pg_attribute.attnum - - This is an array of values that indicate which table columns this - statistic covers. For example a value of 1 3 would - mean that the first and the third table columns make up the statistic key. - - - stxndistinct pg_ndistinct diff --git a/doc/src/sgml/perform.sgml b/doc/src/sgml/perform.sgml index 8d30fd1384..a8bebb1436 100644 --- a/doc/src/sgml/perform.sgml +++ b/doc/src/sgml/perform.sgml @@ -906,6 +906,8 @@ EXPLAIN ANALYZE SELECT * FROM tenk1 WHERE unique1 < 100 AND unique2 > 9000 of the planner + + Single-Column Statistics As we saw in the previous section, the query planner needs to estimate the number of rows retrieved by a query in order to make good choices @@ -1043,7 +1045,200 @@ WHERE tablename = 'road'; Further details about the planner's use of statistics can be found in . + + + + Extended Statistics + + + statistics + of the planner + + + correlation + in the query planner + + + + pg_statistic_ext + + + + It is common to see slow queries running bad execution plans because + multiple columns used in the query clauses are correlated. + The planner normally assumes that multiple conditions + are independent of each other, + an assumption that does not hold when column values are correlated. + Regular statistics, because of their per-individual-column nature, + do not capture the knowledge of cross-column correlation; + multivariate statistics can be used to instruct + the server to obtain statistics across such a set of columns, + which are later used by the query optimizer + to determine cardinality and selectivity + of clauses involving those columns. + Multivariate statistics are currently the only use of + extended statistics. + + + + Extended statistics are created using + , which see for more details. + Data collection is deferred until the next ANALYZE + on the table, after which the stored values can be examined in the + pg_statistic_ext + catalog. + + + + The following subsections describe the types of extended statistics + that are currently supported. + + + + Functional Dependencies + + + The simplest type of extended statistics are functional dependencies, + a concept used in definitions of database normal forms. + Put simply, it is said that column b is functionally + dependent on column a if knowledge of the value of + a is sufficient to determine the value of b. + In normalized databases, functional dependencies are allowed only on + primary keys and superkeys. However, many data sets are in practice not + fully normalized for various reasons; intentional denormalization for + performance reasons is a common example. + + + + The existance of functional dependencies directly affects the accuracy + of estimates in certain queries. + The reason is that conditions on the dependent columns do not + restrict the result set, but the query planner (lacking functional + dependency knowledge) considers them independent, resulting in + underestimates. + To inform the planner about the functional dependencies, we collect + measurements of dependency during ANALYZE. Assessing + the degree of dependency between all sets of columns would be + prohibitively expensive, so the search is limited to potential + dependencies defined using the dependencies option of + extended statistics. It is advisable to create + dependencies statistics if and only if functional + dependencies actually exist, to avoid unnecessary overhead on both + ANALYZE and query planning. + + + + To inspect functional dependencies on a statistics + stts, you may do this: + +CREATE STATISTICS stts WITH (dependencies) + ON (zip, city) FROM zipcodes; +ANALYZE zipcodes; +SELECT stxname, stxkeys, stxdependencies + FROM pg_statistic_ext + WHERE stxname = 'stts'; + stxname | stxkeys | stxdependencies +---------+---------+-------------------------------------------- + stts | 1 5 | [{1 => 5 : 1.000000}, {5 => 1 : 0.423130}] +(1 row) + + where it can be seen that column 1 (a zip code) fully determines column + 5 (city) so the coefficient is 1.0, while city only determines zip code + about 42% of the time, meaning that there are many cities (58%) that are + represented by more than a single ZIP code. + + + + When computing the selectivity, the planner inspects all conditions and + attempts to identify which conditions are already implied by other + conditions. The selectivity estimates from any redundant conditions are + ignored from a selectivity point of view. In the example query above, + the selectivity estimates for either of the conditions may be eliminated, + thus improving the overall estimate. + + + + Limitations of Functional Dependencies + + + Functional dependencies are a very simple type of statistics, and + as such have several limitations. The first limitation is that they + only work with simple equality conditions, comparing columns and constant + values. It's not possible to use them to eliminate equality conditions + comparing two columns or a column to an expression, range clauses, + LIKE or any other type of conditions. + + + + When eliminating the implied conditions, the planner assumes that the + conditions are compatible. Consider the following example, where + this assumption does not hold: + + +EXPLAIN (ANALYZE, TIMING OFF) SELECT * FROM t WHERE a = 1 AND b = 10; + QUERY PLAN +----------------------------------------------------------------------------- + Seq Scan on t (cost=0.00..195.00 rows=100 width=8) (actual rows=0 loops=1) + Filter: ((a = 1) AND (b = 10)) + Rows Removed by Filter: 10000 + + + While there are no rows with such combination of values, the planner + is unable to verify whether the values match — it only knows that + the columns are functionally dependent. + + + + This assumption is related to queries executed on the database; in many + cases, it's actually satisfied (e.g. when the GUI only allows selecting + compatible values). But if that's not the case, functional dependencies + may not be a viable option. + + + + + + Multivariate N-Distinct Coefficients + + + Single-column statistics store the number of distinct values in each + column. Estimates of the number of distinct values on more than one + column (for example, for GROUP BY a, b) are + frequently wrong when the planner only has single-column statistical + data, however, causing it to select bad plans. + In order to improve n-distinct estimation when multiple columns are + grouped together, the ndistinct option of extended statistics + can be used, which instructs ANALYZE to collect n-distinct + estimates for all possible combinations of two or more columns of the set + of columns in the statistics object (the per-column estimates are already + available in pg_statistic). + + + + Continuing the above example, the n-distinct coefficients in a ZIP + code table may look like the following: + +CREATE STATISTICS stts2 WITH (ndistinct) + ON (zip, state, city) FROM zipcodes; +ANALYZE zipcodes; +SELECT stxkeys AS k, stxndistinct AS nd + FROM pg_statistic_ext + WHERE stxname = 'stts2'; +-[ RECORD 1 ]--------------------------------------------- +k | 1 2 5 +nd | [{(b 1 2), 33178.000000}, {(b 1 5), 33178.000000}, + {(b 2 5), 27435.000000}, {(b 1 2 5), 33178.000000}] +(1 row) + + which indicates that there are three combinations of columns that + have 33178 distinct values: ZIP code and state; ZIP code and city; + and ZIP code, city and state (the fact that they are all equal is + expected given the nature of ZIP-code data). On the other hand, + the combination of city and state only has 27435 distinct values. + + + diff --git a/doc/src/sgml/planstats.sgml b/doc/src/sgml/planstats.sgml index 124e7e20ce..f4430eb23c 100644 --- a/doc/src/sgml/planstats.sgml +++ b/doc/src/sgml/planstats.sgml @@ -445,159 +445,141 @@ rows = (outer_cardinality * inner_cardinality) * selectivity operator-specific selectivity functions are mostly found in src/backend/utils/adt/selfuncs.c. + - - Functional Dependencies + + Multivariate Statistics Examples - - The simplest type of extended statistics are functional dependencies, - used in definitions of database normal forms. When simplified, saying that - b is functionally dependent on a means that - knowledge of value of a is sufficient to determine value of - b. - + + row estimation + multivariate + + + Functional dependencies - In normalized databases, only functional dependencies on primary keys - and superkeys are allowed. However, in practice, many data sets are not - fully normalized, for example, due to intentional denormalization for - performance reasons. - + Multivariate correlation can be seen with a very simple data set — a + table with two columns, both containing the same values: - - Functional dependencies directly affect accuracy of the estimates, as - conditions on the dependent column(s) do not restrict the result set, - resulting in underestimates. + +CREATE TABLE t (a INT, b INT); +INSERT INTO t SELECT i % 100, i % 100 FROM generate_series(1, 10000) s(i); +ANALYZE t; + + + As explained in , the planner can determine + cardinality of t using the number of pages and + rows obtained from pg_class: + + +SELECT relpages, reltuples FROM pg_class WHERE relname = 't'; + + relpages | reltuples +----------+----------- + 45 | 10000 + + + The data distribution is very simple; there are only 100 distinct values + in each column, uniformly distributed. - To inform the planner about the functional dependencies, we collect - measurements of dependency during ANALYZE. Assessing - dependency between all sets of columns would be prohibitively - expensive, so we limit our search to potential dependencies defined - using the CREATE STATISTICS command. + The following example shows the result of estimating a WHERE + condition on the a column: -CREATE TABLE t (a INT, b INT); -INSERT INTO t SELECT i/100, i/100 FROM generate_series(1,10000) s(i); -CREATE STATISTICS s1 WITH (dependencies) ON (a, b) FROM t; -ANALYZE t; -EXPLAIN ANALYZE SELECT * FROM t WHERE a = 1 AND b = 1; - QUERY PLAN -------------------------------------------------------------------------------------------------- - Seq Scan on t (cost=0.00..195.00 rows=100 width=8) (actual time=0.095..3.118 rows=100 loops=1) - Filter: ((a = 1) AND (b = 1)) +EXPLAIN (ANALYZE, TIMING OFF) SELECT * FROM t WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------- + Seq Scan on t (cost=0.00..170.00 rows=100 width=8) (actual rows=100 loops=1) + Filter: (a = 1) Rows Removed by Filter: 9900 - Planning time: 0.367 ms - Execution time: 3.380 ms -(5 rows) - The planner is now aware of the functional dependencies and considers - them when computing the selectivity of the second condition. Running - the query without the statistics would lead to quite different estimates. + The planner examines the condition and determines the selectivity + of this clause to be 1%. By comparing this estimate and the actual + number of rows, we see that the estimate is very accurate + (in fact exact, as the table is very small). Changing the + WHERE to use the b column, an identical + plan is generated. Observe what happens if we apply the same + condition on both columns combining them with AND: -DROP STATISTICS s1; -EXPLAIN ANALYZE SELECT * FROM t WHERE a = 1 AND b = 1; - QUERY PLAN ------------------------------------------------------------------------------------------------ - Seq Scan on t (cost=0.00..195.00 rows=1 width=8) (actual time=0.000..6.379 rows=100 loops=1) +EXPLAIN (ANALYZE, TIMING OFF) SELECT * FROM t WHERE a = 1 AND b = 1; + QUERY PLAN +----------------------------------------------------------------------------- + Seq Scan on t (cost=0.00..195.00 rows=1 width=8) (actual rows=100 loops=1) Filter: ((a = 1) AND (b = 1)) Rows Removed by Filter: 9900 - Planning time: 0.000 ms - Execution time: 6.379 ms -(5 rows) - - - If no dependency exists, the collected statistics do not influence the - query plan. The only effect is to slow down ANALYZE. Should - partial dependencies exist these will also be stored and applied - during planning. + The planner estimates the selectivity for each condition individually, + arriving to the 1% estimates as above, and then multiplies them, getting + the final 0.01% estimate. The actual figures, however, + show that this results in a significant underestimate, as the actual + number of rows matching the conditions (100) is two orders of magnitude + higher than the estimated value. - Similarly to per-column statistics, extended statistics are stored in - a system catalog called pg_statistic_ext. - To inspect the statistics s1 defined above, - you may do this: + This problem can be fixed by applying functional-dependency + multivariate statistics on the two columns: -SELECT stxname,stxdependencies FROM pg_statistic_ext WHERE stxname = 's1'; - stxname | stxdependencies ----------+-------------------------------------------- - s1 | [{1 => 2 : 1.000000}, {2 => 1 : 1.000000}] -(1 row) +CREATE STATISTICS stts WITH (dependencies) ON (a, b) FROM t; +ANALYZE t; +EXPLAIN (ANALYZE, TIMING OFF) SELECT * FROM t WHERE a = 1 AND b = 1; + QUERY PLAN +------------------------------------------------------------------------------- + Seq Scan on t (cost=0.00..195.00 rows=100 width=8) (actual rows=100 loops=1) + Filter: ((a = 1) AND (b = 1)) + Rows Removed by Filter: 9900 - - This shows that the statistics are defined on table t, - attnums lists attribute numbers of columns - (references pg_attribute). It also shows - the length in bytes of the functional dependencies, as found by - ANALYZE when serialized into a bytea column. + + + Multivariate N-Distinct coefficients - When computing the selectivity, the planner inspects all conditions and - attempts to identify which conditions are already implied by other - conditions. The selectivity estimates from any redundant conditions are - ignored from a selectivity point of view. In the example query above, - the selectivity estimates for either of the conditions may be eliminated, - thus improving the overall estimate. + A similar problem occurs with estimation of the cardinality of distinct + elements, used to determine the number of groups that would be generated + by a GROUP BY clause. When GROUP BY + lists a single column, the n-distinct estimate (which can be seen as the + number of rows returned by the aggregate execution node) is very accurate: + +EXPLAIN (ANALYZE, TIMING OFF) SELECT COUNT(*) FROM t GROUP BY a; + QUERY PLAN +----------------------------------------------------------------------------------------- + HashAggregate (cost=195.00..196.00 rows=100 width=12) (actual rows=100 loops=1) + Group Key: a + -> Seq Scan on t (cost=0.00..145.00 rows=10000 width=4) (actual rows=10000 loops=1) + + But without multivariate statistics, the estimate for the number of + groups in a query with two columns in GROUP BY, as + in the following example, is off by an order of magnitude: + +EXPLAIN (ANALYZE, TIMING OFF) SELECT COUNT(*) FROM t GROUP BY a, b; + QUERY PLAN +-------------------------------------------------------------------------------------------- + HashAggregate (cost=220.00..230.00 rows=1000 width=16) (actual rows=100 loops=1) + Group Key: a, b + -> Seq Scan on t (cost=0.00..145.00 rows=10000 width=8) (actual rows=10000 loops=1) + + By dropping the existing statistics and re-creating it to include n-distinct + calculation, the estimate is much improved: + +DROP STATISTICS stts; +CREATE STATISTICS stts WITH (dependencies, ndistinct) ON (a, b) FROM t; +ANALYZE t; +EXPLAIN (ANALYZE, TIMING OFF) SELECT COUNT(*) FROM t GROUP BY a, b; + QUERY PLAN +-------------------------------------------------------------------------------------------- + HashAggregate (cost=220.00..221.00 rows=100 width=16) (actual rows=100 loops=1) + Group Key: a, b + -> Seq Scan on t (cost=0.00..145.00 rows=10000 width=8) (actual rows=10000 loops=1) + - - Limitations of functional dependencies - - - Functional dependencies are a very simple type of statistics, and - as such have several limitations. The first limitation is that they - only work with simple equality conditions, comparing columns and constant - values. It's not possible to use them to eliminate equality conditions - comparing two columns or a column to an expression, range clauses, - LIKE or any other type of conditions. - - - - When eliminating the implied conditions, the planner assumes that the - conditions are compatible. Consider the following example, violating - this assumption: - - -EXPLAIN ANALYZE SELECT * FROM t WHERE a = 1 AND b = 10; - QUERY PLAN ------------------------------------------------------------------------------------------------ - Seq Scan on t (cost=0.00..195.00 rows=100 width=8) (actual time=2.992..2.992 rows=0 loops=1) - Filter: ((a = 1) AND (b = 10)) - Rows Removed by Filter: 10000 - Planning time: 0.232 ms - Execution time: 3.033 ms -(5 rows) - - - While there are no rows with such combination of values, the planner - is unable to verify whether the values match - it only knows that - the columns are functionally dependent. - - - - This assumption is more about queries executed on the database - in many - cases, it's actually satisfied (e.g. when the GUI only allows selecting - compatible values). But if that's not the case, functional dependencies - may not be a viable option. - - - - For additional information about functional dependencies, see - src/backend/statistics/README.dependencies. - - - - - - -- cgit v1.2.3 From c29a752c683d9b08ee1376709b825532e94c2709 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 21 Apr 2017 15:33:25 -0400 Subject: doc: Fix typo --- doc/src/sgml/logical-replication.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/logical-replication.sgml b/doc/src/sgml/logical-replication.sgml index 7bfff24a68..e7526905f8 100644 --- a/doc/src/sgml/logical-replication.sgml +++ b/doc/src/sgml/logical-replication.sgml @@ -24,7 +24,7 @@ - Logical replication of a table typically starts with a taking a snapshot + Logical replication of a table typically starts with taking a snapshot of the data on the publisher database and copying that to the subscriber. Once that is done, the changes on the publisher are sent to the subscriber as they occur in real-time. The subscriber applies the data in the same -- cgit v1.2.3 From 68e61ee72eb6914f493f08be98363c2f980ee242 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Fri, 21 Apr 2017 22:51:57 +0300 Subject: Change the on-disk format of SCRAM verifiers to conform to RFC 5803. It doesn't make any immediate difference to PostgreSQL, but might as well follow the standard, since one exists. (I looked at RFC 5803 earlier, but didn't fully understand it back then.) The new format uses Base64 instead of hex to encode StoredKey and ServerKey, which makes the verifiers slightly smaller. Using the same encoding for the salt and the keys also means that you only need one encoder/decoder instead of two. Although we have code in the backend to do both, we are talking about teaching libpq how to create SCRAM verifiers for PQencodePassword(), and libpq doesn't currently have any code for hex encoding. Bump catversion, because this renders any existing SCRAM verifiers in pg_authid invalid. Discussion: https://www.postgresql.org/message-id/351ba574-85ea-d9b8-9689-8c928dd0955d@iki.fi --- doc/src/sgml/catalogs.sgml | 24 ++++-- src/backend/libpq/auth-scram.c | 142 +++++++++++++++++++++------------ src/backend/libpq/crypt.c | 2 +- src/include/catalog/catversion.h | 2 +- src/test/regress/expected/password.out | 14 ++-- src/test/regress/sql/password.sql | 8 +- 6 files changed, 119 insertions(+), 73 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index ed74704b2a..787fcbd51a 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -1376,14 +1376,22 @@ 32-character hexadecimal MD5 hash. The MD5 hash will be of the user's password concatenated to their user name. For example, if user joe has password xyzzy, PostgreSQL - will store the md5 hash of xyzzyjoe. If the password is - encrypted with SCRAM-SHA-256, it consists of 5 fields separated by colons. - The first field is the constant scram-sha-256, to - identify the password as a SCRAM-SHA-256 verifier. The second field is a - salt, Base64-encoded, and the third field is the number of iterations used - to generate the password. The fourth field and fifth field are the stored - key and server key, respectively, in hexadecimal format. A password that - does not follow either of those formats is assumed to be unencrypted. + will store the md5 hash of xyzzyjoe. + + + + If the password is encrypted with SCRAM-SHA-256, it has the format: + +SCRAM-SHA-256$<iteration count>:<salt>$<StoredKey>:<ServerKey> + + where salt, StoredKey and + ServerKey are in Base64 encoded format. This format is + the same as that specified by RFC 5803. + + + + A password that does not follow either of those formats is assumed to be + unencrypted. diff --git a/src/backend/libpq/auth-scram.c b/src/backend/libpq/auth-scram.c index 76c502d415..16bea446e3 100644 --- a/src/backend/libpq/auth-scram.c +++ b/src/backend/libpq/auth-scram.c @@ -5,6 +5,7 @@ * * See the following RFCs for more details: * - RFC 5802: https://tools.ietf.org/html/rfc5802 + * - RFC 5803: https://tools.ietf.org/html/rfc5803 * - RFC 7677: https://tools.ietf.org/html/rfc7677 * * Here are some differences: @@ -19,7 +20,7 @@ * - Channel binding is not supported yet. * * - * The password stored in pg_authid consists of the salt, iteration count, + * The password stored in pg_authid consists of the iteration count, salt, * StoredKey and ServerKey. * * SASLprep usage @@ -111,8 +112,8 @@ typedef struct const char *username; /* username from startup packet */ - char *salt; /* base64-encoded */ int iterations; + char *salt; /* base64-encoded */ uint8 StoredKey[SCRAM_KEY_LEN]; uint8 ServerKey[SCRAM_KEY_LEN]; @@ -146,10 +147,10 @@ static char *build_server_first_message(scram_state *state); static char *build_server_final_message(scram_state *state); static bool verify_client_proof(scram_state *state); static bool verify_final_nonce(scram_state *state); -static bool parse_scram_verifier(const char *verifier, char **salt, - int *iterations, uint8 *stored_key, uint8 *server_key); -static void mock_scram_verifier(const char *username, char **salt, int *iterations, - uint8 *stored_key, uint8 *server_key); +static bool parse_scram_verifier(const char *verifier, int *iterations, + char **salt, uint8 *stored_key, uint8 *server_key); +static void mock_scram_verifier(const char *username, int *iterations, + char **salt, uint8 *stored_key, uint8 *server_key); static bool is_scram_printable(char *p); static char *sanitize_char(char c); static char *scram_MockSalt(const char *username); @@ -185,7 +186,7 @@ pg_be_scram_init(const char *username, const char *shadow_pass) if (password_type == PASSWORD_TYPE_SCRAM_SHA_256) { - if (parse_scram_verifier(shadow_pass, &state->salt, &state->iterations, + if (parse_scram_verifier(shadow_pass, &state->iterations, &state->salt, state->StoredKey, state->ServerKey)) got_verifier = true; else @@ -208,7 +209,7 @@ pg_be_scram_init(const char *username, const char *shadow_pass) verifier = scram_build_verifier(username, shadow_pass, 0); - (void) parse_scram_verifier(verifier, &state->salt, &state->iterations, + (void) parse_scram_verifier(verifier, &state->iterations, &state->salt, state->StoredKey, state->ServerKey); pfree(verifier); @@ -243,7 +244,7 @@ pg_be_scram_init(const char *username, const char *shadow_pass) */ if (!got_verifier) { - mock_scram_verifier(username, &state->salt, &state->iterations, + mock_scram_verifier(username, &state->iterations, &state->salt, state->StoredKey, state->ServerKey); state->doomed = true; } @@ -393,14 +394,15 @@ char * scram_build_verifier(const char *username, const char *password, int iterations) { - uint8 keybuf[SCRAM_KEY_LEN + 1]; - char storedkey_hex[SCRAM_KEY_LEN * 2 + 1]; - char serverkey_hex[SCRAM_KEY_LEN * 2 + 1]; - char salt[SCRAM_SALT_LEN]; - char *encoded_salt; - int encoded_len; char *prep_password = NULL; pg_saslprep_rc rc; + char saltbuf[SCRAM_SALT_LEN]; + uint8 keybuf[SCRAM_KEY_LEN]; + char *encoded_salt; + char *encoded_storedkey; + char *encoded_serverkey; + int encoded_len; + char *result; /* * Normalize the password with SASLprep. If that doesn't work, because @@ -414,7 +416,8 @@ scram_build_verifier(const char *username, const char *password, if (iterations <= 0) iterations = SCRAM_ITERATIONS_DEFAULT; - if (!pg_backend_random(salt, SCRAM_SALT_LEN)) + /* Generate salt, and encode it in base64 */ + if (!pg_backend_random(saltbuf, SCRAM_SALT_LEN)) { ereport(LOG, (errcode(ERRCODE_INTERNAL_ERROR), @@ -423,26 +426,38 @@ scram_build_verifier(const char *username, const char *password, } encoded_salt = palloc(pg_b64_enc_len(SCRAM_SALT_LEN) + 1); - encoded_len = pg_b64_encode(salt, SCRAM_SALT_LEN, encoded_salt); + encoded_len = pg_b64_encode(saltbuf, SCRAM_SALT_LEN, encoded_salt); encoded_salt[encoded_len] = '\0'; - /* Calculate StoredKey, and encode it in hex */ - scram_ClientOrServerKey(password, salt, SCRAM_SALT_LEN, + /* Calculate StoredKey, and encode it in base64 */ + scram_ClientOrServerKey(password, saltbuf, SCRAM_SALT_LEN, iterations, SCRAM_CLIENT_KEY_NAME, keybuf); scram_H(keybuf, SCRAM_KEY_LEN, keybuf); /* StoredKey */ - (void) hex_encode((const char *) keybuf, SCRAM_KEY_LEN, storedkey_hex); - storedkey_hex[SCRAM_KEY_LEN * 2] = '\0'; + + encoded_storedkey = palloc(pg_b64_enc_len(SCRAM_KEY_LEN) + 1); + encoded_len = pg_b64_encode((const char *) keybuf, SCRAM_KEY_LEN, + encoded_storedkey); + encoded_storedkey[encoded_len] = '\0'; /* And same for ServerKey */ - scram_ClientOrServerKey(password, salt, SCRAM_SALT_LEN, iterations, + scram_ClientOrServerKey(password, saltbuf, SCRAM_SALT_LEN, iterations, SCRAM_SERVER_KEY_NAME, keybuf); - (void) hex_encode((const char *) keybuf, SCRAM_KEY_LEN, serverkey_hex); - serverkey_hex[SCRAM_KEY_LEN * 2] = '\0'; + + encoded_serverkey = palloc(pg_b64_enc_len(SCRAM_KEY_LEN) + 1); + encoded_len = pg_b64_encode((const char *) keybuf, SCRAM_KEY_LEN, + encoded_serverkey); + encoded_serverkey[encoded_len] = '\0'; + + result = psprintf("SCRAM-SHA-256$%d:%s$%s:%s", iterations, encoded_salt, + encoded_storedkey, encoded_serverkey); if (prep_password) pfree(prep_password); + pfree(encoded_salt); + pfree(encoded_storedkey); + pfree(encoded_serverkey); - return psprintf("scram-sha-256:%s:%d:%s:%s", encoded_salt, iterations, storedkey_hex, serverkey_hex); + return result; } /* @@ -464,7 +479,7 @@ scram_verify_plain_password(const char *username, const char *password, char *prep_password = NULL; pg_saslprep_rc rc; - if (!parse_scram_verifier(verifier, &encoded_salt, &iterations, + if (!parse_scram_verifier(verifier, &iterations, &encoded_salt, stored_key, server_key)) { /* @@ -509,13 +524,14 @@ scram_verify_plain_password(const char *username, const char *password, bool is_scram_verifier(const char *verifier) { - char *salt = NULL; int iterations; + char *salt = NULL; uint8 stored_key[SCRAM_KEY_LEN]; uint8 server_key[SCRAM_KEY_LEN]; bool result; - result = parse_scram_verifier(verifier, &salt, &iterations, stored_key, server_key); + result = parse_scram_verifier(verifier, &iterations, &salt, + stored_key, server_key); if (salt) pfree(salt); @@ -529,60 +545,82 @@ is_scram_verifier(const char *verifier) * Returns true if the SCRAM verifier has been parsed, and false otherwise. */ static bool -parse_scram_verifier(const char *verifier, char **salt, int *iterations, +parse_scram_verifier(const char *verifier, int *iterations, char **salt, uint8 *stored_key, uint8 *server_key) { char *v; char *p; + char *scheme_str; + char *salt_str; + char *iterations_str; + char *storedkey_str; + char *serverkey_str; + int decoded_len; + char *decoded_salt_buf; /* * The verifier is of form: * - * scram-sha-256:::: + * SCRAM-SHA-256$:$: */ - if (strncmp(verifier, "scram-sha-256:", strlen("scram-sha-256:")) != 0) - return false; - - v = pstrdup(verifier + strlen("scram-sha-256:")); - - /* salt */ - if ((p = strtok(v, ":")) == NULL) + v = pstrdup(verifier); + if ((scheme_str = strtok(v, "$")) == NULL) + goto invalid_verifier; + if ((iterations_str = strtok(NULL, ":")) == NULL) + goto invalid_verifier; + if ((salt_str = strtok(NULL, "$")) == NULL) + goto invalid_verifier; + if ((storedkey_str = strtok(NULL, ":")) == NULL) + goto invalid_verifier; + if ((serverkey_str = strtok(NULL, "")) == NULL) goto invalid_verifier; - *salt = pstrdup(p); - /* iterations */ - if ((p = strtok(NULL, ":")) == NULL) + /* Parse the fields */ + if (strcmp(scheme_str, "SCRAM-SHA-256") != 0) goto invalid_verifier; + errno = 0; - *iterations = strtol(p, &p, 10); + *iterations = strtol(iterations_str, &p, 10); if (*p || errno != 0) goto invalid_verifier; - /* storedkey */ - if ((p = strtok(NULL, ":")) == NULL) - goto invalid_verifier; - if (strlen(p) != SCRAM_KEY_LEN * 2) + /* + * Verify that the salt is in Base64-encoded format, by decoding it, + * although we return the encoded version to the caller. + */ + decoded_salt_buf = palloc(pg_b64_dec_len(strlen(salt_str))); + decoded_len = pg_b64_decode(salt_str, strlen(salt_str), decoded_salt_buf); + if (decoded_len < 0) goto invalid_verifier; + *salt = pstrdup(salt_str); - hex_decode(p, SCRAM_KEY_LEN * 2, (char *) stored_key); + /* + * Decode StoredKey and ServerKey. + */ + if (pg_b64_dec_len(strlen(storedkey_str) != SCRAM_KEY_LEN)) + goto invalid_verifier; + decoded_len = pg_b64_decode(storedkey_str, strlen(storedkey_str), + (char *) stored_key); + if (decoded_len != SCRAM_KEY_LEN) + goto invalid_verifier; - /* serverkey */ - if ((p = strtok(NULL, ":")) == NULL) + if (pg_b64_dec_len(strlen(serverkey_str) != SCRAM_KEY_LEN)) goto invalid_verifier; - if (strlen(p) != SCRAM_KEY_LEN * 2) + decoded_len = pg_b64_decode(serverkey_str, strlen(serverkey_str), + (char *) server_key); + if (decoded_len != SCRAM_KEY_LEN) goto invalid_verifier; - hex_decode(p, SCRAM_KEY_LEN * 2, (char *) server_key); - pfree(v); return true; invalid_verifier: pfree(v); + *salt = NULL; return false; } static void -mock_scram_verifier(const char *username, char **salt, int *iterations, +mock_scram_verifier(const char *username, int *iterations, char **salt, uint8 *stored_key, uint8 *server_key) { char *raw_salt; diff --git a/src/backend/libpq/crypt.c b/src/backend/libpq/crypt.c index 03ef3cc652..d0030f2b6d 100644 --- a/src/backend/libpq/crypt.c +++ b/src/backend/libpq/crypt.c @@ -100,7 +100,7 @@ get_password_type(const char *shadow_pass) { if (strncmp(shadow_pass, "md5", 3) == 0 && strlen(shadow_pass) == MD5_PASSWD_LEN) return PASSWORD_TYPE_MD5; - if (strncmp(shadow_pass, "scram-sha-256:", strlen("scram-sha-256:")) == 0) + if (strncmp(shadow_pass, "SCRAM-SHA-256$", strlen("SCRAM-SHA-256$")) == 0) return PASSWORD_TYPE_SCRAM_SHA_256; return PASSWORD_TYPE_PLAINTEXT; } diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index ab92fd88ed..6cc9e30ec2 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201704171 +#define CATALOG_VERSION_NO 201704211 #endif diff --git a/src/test/regress/expected/password.out b/src/test/regress/expected/password.out index 676b3e6ff3..9ec5a52bba 100644 --- a/src/test/regress/expected/password.out +++ b/src/test/regress/expected/password.out @@ -23,11 +23,11 @@ CREATE ROLE regress_passwd5 PASSWORD NULL; -- check list of created entries -- -- The scram verifier will look something like: --- scram-sha-256:E4HxLGtnRzsYwg==:4096:5ebc825510cb7862efd87dfa638d8337179e6913a724441dc9e888a856fbc10c:e966b1c72fad89d69aaebb156eae04edc9581286f92207c044711e79cd461bee +-- SCRAM-SHA-256$4096:E4HxLGtnRzsYwg==$6YtlR4t69SguDiwFvbVgVZtuz6gpJQQqUMZ7IQJK5yI=:ps75jrHeYU4lXCcXI4O8oIdJ3eO8o2jirjruw9phBTo= -- -- Since the salt is random, the exact value stored will be different on every test -- run. Use a regular expression to mask the changing parts. -SELECT rolname, regexp_replace(rolpassword, '(scram-sha-256):([a-zA-Z0-9+/]+==):(\d+):(\w+):(\w+)', '\1::\3::') as rolpassword_masked +SELECT rolname, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/]+==)\$([a-zA-Z0-9+/]+=):([a-zA-Z0-9+/]+=)', '\1$\2:$:') as rolpassword_masked FROM pg_authid WHERE rolname LIKE 'regress_passwd%' ORDER BY rolname, rolpassword; @@ -36,7 +36,7 @@ SELECT rolname, regexp_replace(rolpassword, '(scram-sha-256):([a-zA-Z0-9+/]+==): regress_passwd1 | role_pwd1 regress_passwd2 | md54044304ba511dd062133eb5b4b84a2a3 regress_passwd3 | md50e5699b6911d87f17a08b8d76a21e8b8 - regress_passwd4 | scram-sha-256::4096:: + regress_passwd4 | SCRAM-SHA-256$4096:$: regress_passwd5 | (5 rows) @@ -59,11 +59,11 @@ ALTER ROLE regress_passwd1 UNENCRYPTED PASSWORD 'foo'; -- unencrypted ALTER ROLE regress_passwd2 UNENCRYPTED PASSWORD 'md5dfa155cadd5f4ad57860162f3fab9cdb'; -- encrypted with MD5 SET password_encryption = 'md5'; ALTER ROLE regress_passwd3 ENCRYPTED PASSWORD 'foo'; -- encrypted with MD5 -ALTER ROLE regress_passwd4 ENCRYPTED PASSWORD 'scram-sha-256:VLK4RMaQLCvNtQ==:4096:3ded2376f7aafa93b1bdbd71bcc18b7d6ee50ed018029cc583d152ef3fc7d430:a6dd36dfc94c181956a6ae95f05e01b1864f0a22a2657d1de4ba84d2a24dc438'; -- client-supplied SCRAM verifier, use as it is +ALTER ROLE regress_passwd4 ENCRYPTED PASSWORD 'SCRAM-SHA-256$4096:VLK4RMaQLCvNtQ==$6YtlR4t69SguDiwFvbVgVZtuz6gpJQQqUMZ7IQJK5yI=:ps75jrHeYU4lXCcXI4O8oIdJ3eO8o2jirjruw9phBTo='; -- client-supplied SCRAM verifier, use as it is SET password_encryption = 'scram-sha-256'; ALTER ROLE regress_passwd5 ENCRYPTED PASSWORD 'foo'; -- create SCRAM verifier CREATE ROLE regress_passwd6 ENCRYPTED PASSWORD 'md53725413363ab045e20521bf36b8d8d7f'; -- encrypted with MD5, use as it is -SELECT rolname, regexp_replace(rolpassword, '(scram-sha-256):([a-zA-Z0-9+/]+==):(\d+):(\w+):(\w+)', '\1::\3::') as rolpassword_masked +SELECT rolname, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/]+==)\$([a-zA-Z0-9+/]+=):([a-zA-Z0-9+/]+=)', '\1$\2:$:') as rolpassword_masked FROM pg_authid WHERE rolname LIKE 'regress_passwd%' ORDER BY rolname, rolpassword; @@ -72,8 +72,8 @@ SELECT rolname, regexp_replace(rolpassword, '(scram-sha-256):([a-zA-Z0-9+/]+==): regress_passwd1 | foo regress_passwd2 | md5dfa155cadd5f4ad57860162f3fab9cdb regress_passwd3 | md5530de4c298af94b3b9f7d20305d2a1bf - regress_passwd4 | scram-sha-256::4096:: - regress_passwd5 | scram-sha-256::4096:: + regress_passwd4 | SCRAM-SHA-256$4096:$: + regress_passwd5 | SCRAM-SHA-256$4096:$: regress_passwd6 | md53725413363ab045e20521bf36b8d8d7f (6 rows) diff --git a/src/test/regress/sql/password.sql b/src/test/regress/sql/password.sql index 95557e4566..1e022dbf2d 100644 --- a/src/test/regress/sql/password.sql +++ b/src/test/regress/sql/password.sql @@ -24,11 +24,11 @@ CREATE ROLE regress_passwd5 PASSWORD NULL; -- check list of created entries -- -- The scram verifier will look something like: --- scram-sha-256:E4HxLGtnRzsYwg==:4096:5ebc825510cb7862efd87dfa638d8337179e6913a724441dc9e888a856fbc10c:e966b1c72fad89d69aaebb156eae04edc9581286f92207c044711e79cd461bee +-- SCRAM-SHA-256$4096:E4HxLGtnRzsYwg==$6YtlR4t69SguDiwFvbVgVZtuz6gpJQQqUMZ7IQJK5yI=:ps75jrHeYU4lXCcXI4O8oIdJ3eO8o2jirjruw9phBTo= -- -- Since the salt is random, the exact value stored will be different on every test -- run. Use a regular expression to mask the changing parts. -SELECT rolname, regexp_replace(rolpassword, '(scram-sha-256):([a-zA-Z0-9+/]+==):(\d+):(\w+):(\w+)', '\1::\3::') as rolpassword_masked +SELECT rolname, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/]+==)\$([a-zA-Z0-9+/]+=):([a-zA-Z0-9+/]+=)', '\1$\2:$:') as rolpassword_masked FROM pg_authid WHERE rolname LIKE 'regress_passwd%' ORDER BY rolname, rolpassword; @@ -48,13 +48,13 @@ ALTER ROLE regress_passwd2 UNENCRYPTED PASSWORD 'md5dfa155cadd5f4ad57860162f3fab SET password_encryption = 'md5'; ALTER ROLE regress_passwd3 ENCRYPTED PASSWORD 'foo'; -- encrypted with MD5 -ALTER ROLE regress_passwd4 ENCRYPTED PASSWORD 'scram-sha-256:VLK4RMaQLCvNtQ==:4096:3ded2376f7aafa93b1bdbd71bcc18b7d6ee50ed018029cc583d152ef3fc7d430:a6dd36dfc94c181956a6ae95f05e01b1864f0a22a2657d1de4ba84d2a24dc438'; -- client-supplied SCRAM verifier, use as it is +ALTER ROLE regress_passwd4 ENCRYPTED PASSWORD 'SCRAM-SHA-256$4096:VLK4RMaQLCvNtQ==$6YtlR4t69SguDiwFvbVgVZtuz6gpJQQqUMZ7IQJK5yI=:ps75jrHeYU4lXCcXI4O8oIdJ3eO8o2jirjruw9phBTo='; -- client-supplied SCRAM verifier, use as it is SET password_encryption = 'scram-sha-256'; ALTER ROLE regress_passwd5 ENCRYPTED PASSWORD 'foo'; -- create SCRAM verifier CREATE ROLE regress_passwd6 ENCRYPTED PASSWORD 'md53725413363ab045e20521bf36b8d8d7f'; -- encrypted with MD5, use as it is -SELECT rolname, regexp_replace(rolpassword, '(scram-sha-256):([a-zA-Z0-9+/]+==):(\d+):(\w+):(\w+)', '\1::\3::') as rolpassword_masked +SELECT rolname, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/]+==)\$([a-zA-Z0-9+/]+=):([a-zA-Z0-9+/]+=)', '\1$\2:$:') as rolpassword_masked FROM pg_authid WHERE rolname LIKE 'regress_passwd%' ORDER BY rolname, rolpassword; -- cgit v1.2.3 From f58b664393dcfd02c2f57b3ff20fc0aee6dfebf1 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 21 Apr 2017 19:42:01 -0400 Subject: doc: Update link The reference "That is the topic of the next section." has been incorrect since the materialized views documentation got inserted between the section "rules-views" and "rules-update". Author: Zertrin --- doc/src/sgml/rules.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/rules.sgml b/doc/src/sgml/rules.sgml index ca1b767d69..bcbc170335 100644 --- a/doc/src/sgml/rules.sgml +++ b/doc/src/sgml/rules.sgml @@ -862,7 +862,7 @@ SELECT t1.a, t2.b, t1.ctid FROM t1, t2 WHERE t1.a = t2.a; UPDATE, and DELETE commands on a view. These rules will rewrite the command, typically into a command that updates one or more tables, rather than views. That is the topic - of the next section. + of . -- cgit v1.2.3 From 66fade8a0447093e6bd5a39ee6fec38790479664 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Mon, 24 Apr 2017 19:04:28 -0400 Subject: doc: update release doc markup instructions --- doc/src/sgml/release.sgml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/release.sgml b/doc/src/sgml/release.sgml index 472c1f6f12..9919332278 100644 --- a/doc/src/sgml/release.sgml +++ b/doc/src/sgml/release.sgml @@ -9,12 +9,13 @@ postgresql.conf, pg_hba.conf, recovery.conf [A-Z][A-Z_ ]+[A-Z_] , , , [A-Za-z_][A-Za-z0-9_]+() --[-A-Za-z_]+ - - This avoids some limits on SysV semaphores usage. -- cgit v1.2.3 From d103e6718bfe4a1f50eb25b94db2cf9f84c69766 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Mon, 24 Apr 2017 22:53:16 -0400 Subject: doc: fix PG 10 release note doc markup --- doc/src/sgml/release-10.sgml | 6 ------ 1 file changed, 6 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/release-10.sgml b/doc/src/sgml/release-10.sgml index 45f74d0af8..bfcb3c2a40 100644 --- a/doc/src/sgml/release-10.sgml +++ b/doc/src/sgml/release-10.sgml @@ -411,12 +411,6 @@ Allow merge joins to be performed in parallel (Dilip Kumar) - -This allows a single index scan to dispatch parallel workers to process -different areas of the heap. - - - Increase parallel query usage in procedural language functions (Robert -Haas) +Haas, Rafia Sabih) @@ -2530,7 +2530,7 @@ class="osname">Linux and FreeBSD -This avoids some limits on SysV semaphores usage. +This avoids some limits on SysV semaphore usage. -- cgit v1.2.3 From cef5dbbf2b8d8a5e7b536bcbf6d9066588c6173b Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Mon, 24 Apr 2017 23:29:14 -0400 Subject: doc: move hash performance item into index section The requirement to rebuild pg_upgrade-ed hash indexes was kept in the incompatibilities section. Reported-by: Amit Kapila --- doc/src/sgml/release-10.sgml | 42 +++++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 15 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/release-10.sgml b/doc/src/sgml/release-10.sgml index 93bcb079be..18483238f1 100644 --- a/doc/src/sgml/release-10.sgml +++ b/doc/src/sgml/release-10.sgml @@ -49,26 +49,14 @@ -Improve hash bucket split performance by reducing locking requirements -(Amit Kapila, Mithun Cy) +pg_upgrade-ed hash indexes from previous major Postgres versions must be rebuilt. -Also cache hash index meta-information for faster lookups. Additional -hash performance improvements have also been made. pg_upgrade'd hash -indexes from previous major Postgres versions must be rebuilt. +Major hash index performance improvements necessitated this requirement. @@ -475,6 +463,30 @@ warning message about their use. + + + +Improve hash bucket split performance by reducing locking requirements +(Amit Kapila, Mithun Cy) + + + +Also cache hash index meta-information for faster lookups. Additional +hash performance improvements have also been made. + + + pg_upgrade-ed hash indexes from previous major Postgres versions must be rebuilt. -Major hash index performance improvements necessitated this requirement. +Major hash storage improvements necessitated this requirement. @@ -450,130 +452,148 @@ processes for non-parallel purposes. -Add write-ahead logging support to hash indexes (Amit Kapila) +Add full text search support for JSON and JSONB (Dmitry Dolgov) -This makes hash indexes crash-safe and replicated, and removes the -warning message about their use. +This is accessed via ts_headline() and to_tsvector. RIGHT SECTION? -Improve hash bucket split performance by reducing locking requirements -(Amit Kapila, Mithun Cy) +Add SP-GiST index support for INET and CIDR data types (Emre Hasegeli) -Also cache hash index meta-information for faster lookups. Additional -hash performance improvements have also been made. +These data types already had GiST support. -Allow single-page hash pruning (Ashutosh Sharma) +Reduce page locking during vacuuming of GIN indexes (Andrey Borodin) -Add full text search support for JSON and JSONB (Dmitry Dolgov) +Cause BRIN index summarization to happen more aggressively (Álvaro +Herrera) -This is accessed via ts_headline() and to_tsvector. RIGHT SECTION? +Specifically, summarize the previous page range when a new page range is +created. -Add SP-GiST index support for INET and CIDR data types (Emre Hasegeli) +Add function brin_desummarize_range() to remove BRIN summarization of a +specified range (Álvaro Herrera) -These data types already had GiST support. +This allows future BRIN index summarization to be more compact. CLARIFY + + + + <link linkend="indexes-types">Hash Indexes</link> + + + -Reduce page locking during vacuuming of GIN indexes (Andrey Borodin) +Add write-ahead logging support to hash indexes (Amit Kapila) + + + +This makes hash indexes crash-safe and replicated, and removes the +warning message about their use. -Cause BRIN index summarization to happen more aggressively (Álvaro -Herrera) +Improve hash bucket split performance by reducing locking requirements +(Amit Kapila, Mithun Cy) -Specifically, summarize the previous page range when a new page range is -created. +Also cache hash index meta-information for faster lookups. -Add function brin_desummarize_range() to remove BRIN summarization of a -specified range (Álvaro Herrera) +Improve efficiency of hash index growth (Amit Kapila, Mithun Cy) + + + -This allows future BRIN index summarization to be more compact. CLARIFY +Allow single-page hash pruning (Ashutosh Sharma) - - - + - - Locking + - + + + + + Locking + + - -Improve optimization of FULL JOIN queries containing subqueries in the -FROM clause (Etsuro Fujita) - - - Allow explicit control over EXPLAIN's display of planning and execution -time (Stephen Frost) +time (Ashutosh Bapat) @@ -2694,6 +2684,17 @@ optimization. + + + +Allow push down of FULL JOIN queries containing subqueries in the +FROM clause to foreign servers (Etsuro Fujita) + + + Add GUC to add details to WAL that can be -sanity-checked on the standby (Kuntal Ghosh, Michael Paquier, Robert +sanity-checked on the standby (Kuntal Ghosh, Robert Haas) @@ -1389,8 +1389,8 @@ Add specification of a Log Sequence Number (LSN) stopping point in -Previously only specification of the stop name, time, and xid were -supported. +Previously only specification of the stop name, time, timeline, xid, +and immediate were supported. @@ -1440,7 +1440,8 @@ Author: Simon Riggs 2017-04-04 [728bd991c] Speedup 2PC recovery by skipping two phase state files i --> -Speed up two-phase commit recovery performance (Simon Riggs) +Speed up two-phase commit recovery performance (Stas Kelvich, Nikhil +Sontakke, Michael Paquier) @@ -2467,35 +2468,35 @@ It is called - - - - Locking + + + <link linkend="indexes-types">Hash Indexes</link> - - - -Only check for REFERENCES permission on referenced tables (Tom Lane) - - - -Previously REFERENCES permission on the referencing table was also -required. - - - - - - -Reduce locking required for adding values to enum types (Andrew Dunstan, -Tom Lane) - - - -Previously it was impossible to run ALTER TYPE ... ADD VALUE in a -transaction block unless the enum type was created in the same block. -Now, only references to uncommitted enum values from other transactions -is prohibited. - - - - - - -Allow tuning of predicate lock promotion thresholds (Dagfinn Ilmari -Mannsåker) - - - -The new settings are and -max_pred_locks_per_page. - - + + + + Add write-ahead logging support to hash indexes (Amit Kapila) + + + + This makes hash indexes crash-safe and replicated, and removes + the warning message about their use. + + + + + + + Improve hash bucket split performance by reducing locking + requirements (Amit Kapila, Mithun Cy) + + + + Also cache hash index meta-information for faster lookups. + + + + + + + Improve efficiency of hash index growth (Amit Kapila, Mithun Cy) + + + + + + + Allow single-page hash pruning (Ashutosh Sharma) + + - + + + + + + + + + Locking + + + + + + + Only check for REFERENCES permission on referenced + tables (Tom Lane) + + + + Previously REFERENCES permission on the referencing + table was also required. + + + + + + + Reduce locking required for adding values to enum types (Andrew + Dunstan, Tom Lane) + + + + Previously it was impossible to run ALTER TYPE ... ADD + VALUE in a transaction block unless the enum type was created + in the same block. Now, only references to uncommitted enum + values from other transactions is prohibited. + + + + + + + Allow tuning of predicate lock promotion thresholds (Dagfinn + Ilmari Mannsåker) + + + + The new settings are and + max_pred_locks_per_page. + + + + @@ -653,129 +678,134 @@ The new settings are and - - - -Add the ability to compute a correlation ratio and the number of -distinct values on several columns (Tomas Vondra, David Rowley) - - - -New commands are CREATE, -ALTER, and -DROP STATISTICS. This is helpful in -estimating query memory usage and when combining the statistics from -individual columns. - - - - - - -Improve planner matching of boolean indexes (Tom Lane) - - - - - - -Improve performance of queries referencing row-level security -restrictions (Tom Lane) - - - -The optimizer now has more flexibility in reordering executor behavior. - - + + + + Add the ability to compute a correlation ratio and the number of + distinct values on several columns (Tomas Vondra, David Rowley) + + + + New commands are CREATE, + ALTER, and + DROP STATISTICS. + This is helpful in + estimating query memory usage and when combining the statistics + from individual columns. + + + + + + + Improve planner matching of boolean indexes (Tom Lane) + + + + + + + Improve performance of queries referencing row-level security + restrictions (Tom Lane) + + + + The optimizer now has more flexibility in reordering executor + behavior. + + - + General Performance - - - -Speed up SUM() calculations (Heikki Linnakangas) - - - -This uses an optimized numeric accumulator. - - - - - - -Improve the performance of character encoding conversions by using radix -trees (Kyotaro Horiguchi, Heikki Linnakangas) - - - - - - -Reduce the function call overhead during query execution (Andres Freund) - - - -This is particularly helpful for queries that process many rows. - - - - - - -Improve the performance of grouping sets (Andrew Gierth) - - - - - - -Use uniqueness guarantees to optimize certain join types (David Rowley) - - - - - - -Improve sort performance of the macaddr data type (Brandur Leach) - - + + + + Speed up SUM() calculations (Heikki Linnakangas) + + + + This uses an optimized numeric accumulator. + + + + + + + Improve the performance of character encoding conversions by + using radix trees (Kyotaro Horiguchi, Heikki Linnakangas) + + + + + + + Reduce the function call overhead during query execution (Andres + Freund) + + + + This is particularly helpful for queries that process many rows. + + + + + + + Improve the performance of grouping sets (Andrew Gierth) + + + + + + + Use uniqueness guarantees to optimize certain join types (David + Rowley) + + + + + + + Improve sort performance of the macaddr data type (Brandur Leach) + + @@ -786,266 +816,288 @@ Improve sort performance of the macaddr data type (Brandur Leach) - - - -Add pg_sequences view to show all sequences (Peter Eisentraut) - - - - - - -Create a pg_sequence system catalog to store sequence metadata -(Peter Eisentraut) - - - -Sequence metadata includes start, increment, etc, which is now -transactional. Sequence counters are still stored in separate heap -relations. - - - - - - -Allow explicit control over EXPLAIN's display of planning and execution -time (Ashutosh Bapat) - - - -By default planning and execution is display by EXPLAIN ANALYZE and not -display in other cases. The new EXPLAIN option SUMMARY allows explicit -control of this. - - - - - - -Properly update the statistics collector during REFRESH MATERIALIZED VIEW -(Jim Mlodgenski) - - - - - - -Add default monitoring roles (Dave Page) - - - -New roles pg_monitor, pg_read_all_settings, pg_read_all_stats, and -pg_stat_scan_tables allow simplified permission configuration. - - + + + + Add pg_sequences view + to show all sequences (Peter Eisentraut) + + + + + + + Create a pg_sequence + system catalog to store sequence metadata (Peter Eisentraut) + + + + Sequence metadata includes start, increment, etc, which is now + transactional. Sequence counters are still stored in separate + heap relations. + + + + + + + Allow explicit control over EXPLAIN's display of planning and + execution time (Ashutosh Bapat) + + + + By default planning and execution is display by EXPLAIN + ANALYZE and not display in other cases. The new + EXPLAIN option SUMMARY allows explicit + control of this. + + + + + + + Properly update the statistics collector during REFRESH MATERIALIZED + VIEW (Jim Mlodgenski) + + + + + + + Add default monitoring roles (Dave Page) + + + + New roles pg_monitor, pg_read_all_settings, + pg_read_all_stats, and pg_stat_scan_tables + allow simplified permission configuration. + + - - - + + + Logging - - - -Change default to include current timestamp with milliseconds -and the process id (Christoph Berg) - - - -The previous default was not to output a prefix. - - - - - - -Add functions to return the log and WAL directory names (Dave Page) - - - -The new functions are pg_ls_logdir() and -pg_ls_waldir() and can be -executed by non-super users with the proper permissions. - - - - - - -Add function pg_current_logfile() to read syslog's current stderr and -csvlog output file names (Gilles Darold) - - - - - - -Report the address and port number of successful startup socket binding -in the server logs (Tom Lane) - - - -Also, report bind socket failure details in the server logs. - - - - - - -Reduce log chatter about the starting and stopping of launcher -subprocesses (Tom Lane) - - - -These are now DEBUG1-level messages. - - - - - - -Reduce message verbosity of lower-numbered debug levels controlled by - (Robert Haas) - - - -This also changes the verbosity of debug levels. - - + + + + Change default to include + current timestamp with milliseconds and the process id (Christoph + Berg) + + + + The previous default was not to output a prefix. + + + + + + + Add functions to return the log and WAL directory + names (Dave Page) + + + + The new functions are pg_ls_logdir() + and + pg_ls_waldir() + and can be + executed by non-super users with the proper permissions. + + + + + + + Add function pg_current_logfile() + to read syslog's current stderr and csvlog output file names + (Gilles Darold) + + + + + + + Report the address and port number of successful startup socket + binding in the server logs (Tom Lane) + + + + Also, report bind socket failure details in the server logs. + + + + + + + Reduce log chatter about the starting and stopping of launcher + subprocesses (Tom Lane) + + + + These are now DEBUG1-level messages. + + + + + + + Reduce message verbosity of lower-numbered debug levels + controlled by + (Robert Haas) + + + + This also changes the verbosity of debug levels. + + - - - + + + + <link linkend="pg-stat-activity-view"><structname>pg_stat_activity</></link> - - - -Add pg_stat_activity reporting of latch wait states (Michael Paquier, -Robert Haas) - - - -This includes the remaining wait events, like client reads, client -writes, and synchronous replication. - - - - - - -Add pg_stat_activity reporting of waits on reads, writes, and fsyncs -(Rushabh Lathia) - - - - - - -Show auxiliary processes and background workers in pg_stat_activity -(Kuntal Ghosh) - - - -New column backend_type identifies the process type. - - - - - - -Display walsender processes in pg_stat_activity (Michael Paquier) - - - -This simplifies monitoring. - - - - - - -Allow pg_stat_activity to show the source query being executed by parallel workers -(Rafia Sabih) - - - - - - -Rename pg_stat_activity.wait_event_type values LWLockTranche and -LWLockNamed to LWLock (Robert Haas) - - - -This makes the output more consistent. - - + + + + Add pg_stat_activity reporting of latch wait states + (Michael Paquier, Robert Haas) + + + + This includes the remaining wait events, like client reads, + client writes, and synchronous replication. + + + + + + + Add pg_stat_activity reporting of waits on reads, + writes, and fsyncs (Rushabh Lathia) + + + + + + + Show auxiliary processes and background workers in + pg_stat_activity (Kuntal Ghosh) + + + + New column backend_type identifies the process + type. + + + + + + + Display walsender processes in pg_stat_activity + (Michael Paquier) + + + + This simplifies monitoring. + + + + + + + Allow pg_stat_activity to show the source query + being executed by parallel workers (Rafia Sabih) + + + + + + + Rename + pg_stat_activity.wait_event_type + values LWLockTranche and + LWLockNamed to LWLock (Robert Haas) + + + + This makes the output more consistent. + + - - + + @@ -1053,71 +1105,75 @@ This makes the output more consistent. - - - -Add SCRAM-SHA-256 support for password negotiation and storage (Michael -Paquier, Heikki Linnakangas) - - - -This proves better security than the existing md5 negotiation and -storage method. - - - - - - -Change GUC from boolean to enum (Michael Paquier) - - - -This was necessary to support additional password hashing options. - - - - - - -Add view pg_hba_file_rules to display the contents of pg_hba.conf -(Haribabu Kommi) - - - -This shows the file contents, not the currently active settings. - - - - - - -Support multiple RADIUS servers (Magnus Hagander) - - - -All the RADIUS related parameters are now plural and support a -comma-separated list of servers. - - + + + + Add SCRAM-SHA-256 + support for password negotiation and storage (Michael Paquier, + Heikki Linnakangas) + + + + This proves better security than the existing md5 + negotiation and storage method. + + + + + + + Change GUC + from boolean to enum (Michael Paquier) + + + + This was necessary to support additional password hashing options. + + + + + + + Add view pg_hba_file_rules + to display the contents of pg_hba.conf (Haribabu + Kommi) + + + + This shows the file contents, not the currently active settings. + + + + + + + Support multiple RADIUS servers (Magnus Hagander) + + + + All the RADIUS related parameters are now plural and + support a comma-separated list of servers. + + @@ -1128,67 +1184,70 @@ comma-separated list of servers. - - - -Allow SSL configuration to be updated at SIGHUP (Andreas Karlsson, Tom Lane) - - - -This allows SSL to be reconfigured without a server restart by using -pg_ctl reload, SELECT pg_reload_conf(), or sending a -SIGHUP signal. -Reload SSL configuration updates do not work if the SSL key requires a -passphrase. - - - - - - -Remove documented restriction about using large shared buffers on -Windows (Takayuki Tsunakawa) - - - - - - -Reduce locking required to change table params (Simon Riggs, Fabrízio -Mello) - - - -For example, changing a table's setting can now -be done with a more lightweight lock. - - - - - - -Make the maximum value of effectively unlimited -(Jim Nasby) - - + + + + Allow SSL configuration to be updated at + SIGHUP (Andreas Karlsson, Tom Lane) + + + + This allows SSL to be reconfigured without a server + restart by using pg_ctl reload, SELECT + pg_reload_conf(), or sending a SIGHUP signal. + Reload SSL configuration updates do not work if the + SSL key requires a passphrase. + + + + + + + Remove documented restriction about using large shared buffers on + Windows (Takayuki Tsunakawa) + + + + + + + Reduce locking required to change table params (Simon Riggs, + Fabrízio Mello) + + + + For example, changing a table's setting can now be done + with a more lightweight lock. + + + + + + + Make the maximum value of effectively unlimited + (Jim Nasby) + + @@ -1199,86 +1258,88 @@ Make the maximum value of effectively - - - -Perform an fsync on the directory after creating or unlinking files -(Michael Paquier) - - - -This reduces the risk of data loss after a power failure. - - - - - - -Remove orphaned temporary tables more aggressively (Robert Haas, Tom Lane) - - - -Previously such tables were removed only when necessary. SECTION? - - + + + + Perform an fsync on the directory after creating or unlinking files + (Michael Paquier) + + + + This reduces the risk of data loss after a power failure. + + + + + + + Remove orphaned temporary tables more aggressively (Robert Haas, + Tom Lane) + + + + Previously such tables were removed only when necessary. SECTION? + + - + - + <link linkend="wal">Write-Ahead Log</> (<acronym>WAL</>) - - - -Prevent checkpoints and WAL archiving on otherwise-idle systems (Michael -Paquier) - - - - - - -Add GUC to add details to WAL that can be -sanity-checked on the standby (Kuntal Ghosh, Robert -Haas) - - - -Any sanity-check failure generates a fatal error on the standby. - - - - - - -Increase the maximum configurable WAL size to 1 gigabyte (Beena Emerson) - - + + + + Prevent checkpoints and WAL archiving on + otherwise-idle systems (Michael Paquier) + + + + + + + Add GUC + to add details to WAL that can be sanity-checked on + the standby (Kuntal Ghosh, Robert Haas) + + + + Any sanity-check failure generates a fatal error on the standby. + + + + + + + Increase the maximum configurable WAL size to 1 + gigabyte (Beena Emerson) + + - - + + @@ -1287,383 +1348,405 @@ Increase the maximum configurable WAL size to 1 gigabyte (Beena Emer Replication and Recovery - + - - - -Add the ability to logically replicate tables to standby servers (Petr -Jelinek) - - - -This allows more fine-grained replication options, including replication -between different major versions of Postgres and selective-table -replication. - - - - - - -Allow waiting for commit acknowledgement from standby servers -irrespective of the order they appear in -(Masahiko Sawada) - - - -Previously the server always waited for the active standbys that -appeared first in synchronous_standby_names. The new -synchronous_standby_names keyword ANY allows waiting for any number of -standbys irrespective of their ordering. This is known as quorum commit. - - - - - - - -Reduce configuration necessary to perform streaming backup and -replication (Magnus Hagander) - - - -Specifically, defaults were changed for , -, and -. - - - - - - -Enable replication from localhost connections by default in pg_hba.conf -(Michael Paquier) - - - -Previously pg_hba.conf's replication connection lines were commented -out. This is particularly useful for . - - - - - - -Add columns to pg_stat_replication to report replication delay times -(Thomas Munro) - - - -The new columns are write_lag, flush_lag, and replay_lag. - - - - - - -Add specification of a Log Sequence Number (LSN) stopping point in -recovery.conf (Michael Paquier) - - - -Previously only specification of the stop name, time, timeline, xid, -and immediate were supported. - - - - - - -Allow users to disable pg_stop_backup()'s waiting for all WAL to be -archived (David Steele) - - - -An optional second argument to pg_stop_backup() controls that behavior. - - - - - - -Allow creation of temporary replication slots (Petr Jelinek) - - - -Temporary slots are automatically removed on session exit or error. - - - - - - -Improve performance of hot standby replay with better tracking of -Access Exclusive locks (Simon Riggs, David Rowley) - - - - - - -Speed up two-phase commit recovery performance (Stas Kelvich, Nikhil -Sontakke, Michael Paquier) - - + + + + Add the ability to logically + replicate tables to standby servers (Petr Jelinek) + + + + This allows more fine-grained replication options, including + replication between different major versions of Postgres and + selective-table replication. + + + + + + + Allow waiting for commit acknowledgement from standby + servers irrespective of the order they appear in (Masahiko Sawada) + + + + Previously the server always waited for the active standbys that + appeared first in synchronous_standby_names. The new + synchronous_standby_names keyword ANY allows + waiting for any number of standbys irrespective of their ordering. + This is known as quorum commit. + + + + + + + Reduce configuration necessary to perform streaming backup and + replication (Magnus Hagander) + + + + Specifically, defaults were changed for , , + and . + + + + + + + Enable replication from localhost connections by default in + pg_hba.conf + (Michael Paquier) + + + + Previously pg_hba.conf's replication connection + lines were commented out. This is particularly useful for + . + + + + + + + Add columns to pg_stat_replication + to report replication delay times (Thomas Munro) + + + + The new columns are write_lag, + flush_lag, and replay_lag. + + + + + + + Add specification of a Log Sequence Number (LSN) + stopping point in + recovery.conf + (Michael Paquier) + + + + Previously only specification of the stop name, time, timeline, + xid, and immediate were supported. + + + + + + + Allow users to disable pg_stop_backup()'s + waiting for all WAL to be archived (David Steele) + + + + An optional second argument to pg_stop_backup() + controls that behavior. + + + + + + + Allow creation of temporary replication slots + (Petr Jelinek) + + + + Temporary slots are automatically removed on session exit or error. + + + + + + + Improve performance of hot standby replay with better tracking of + Access Exclusive locks (Simon Riggs, David Rowley) + + + + + + + Speed up two-phase commit recovery performance (Stas Kelvich, + Nikhil Sontakke, Michael Paquier) + + - + Queries - + - - - -Allow ROW to supply values to UPDATE ... SET (column_list) (Tom Lane) - - - -Also allow row values to be supplied by table.*. - - - - - - -Fix regular expression locale class handling for bytes greater than U+7FF -(Tom Lane) - - - -Previously such classes were not recognized. - - + + + + Allow ROW to supply values to UPDATE ... SET + (column_list) (Tom Lane) + + + + Also allow row values to be supplied by table.*. + + + + + + + Fix regular expression locale class handling for bytes greater + than U+7FF (Tom Lane) + + + + Previously such classes were not recognized. + + - + Utility Commands - + - - - -Add table partitioning syntax that automatically creates partition -constraints and INSERT routing (Amit Langote) - - - -The syntax supports range and list partitioning. - - - - - - -Add AFTER trigger transition table to record changed rows (Kevin Grittner) - - - -Transition table contents are accessible from server-side languages. - - - - - - -Allow restrictive row-level security policies (Stephen Frost) - - - -Previously all security policies were permissive, meaning that any -matching policy allowed access. Optional restrictive policies must -match for access to be granted. These policy types can be combined. - - - - - - -Allow default permissions on schemas (Matheus Oliveira) - - - -This is done using the ALTER DEFAULT PRIVILEGES command. - - - - - - -Add CREATE SEQUENCE AS command to create a sequence matching -an integer data type (Peter Eisentraut) - - - -This simplifies the creation of sequences matching the range of base -columns. - - - - - - -Allow COPY view FROM on views with INSTEAD INSERT triggers (Haribabu -Kommi) - - - -The triggers are fed the rows from COPY. - - - - - - -Allow the specification of a function name without arguments in DDL -commands, when unique (Peter Eisentraut) - - - -For example, allow DROP FUNCTION on a function name without arguments if -there is only one function with that name. This is required by the SQL -standard. - - - - - - -Allow multiple functions, operators, and aggregates to be dropped with a -single DROP command (Peter Eisentraut) - - - - - - -Add IF NOT EXISTS for CREATE SERVER and -CREATE USER MAPPING (Anastasia -Lubennikova) - - - - - - -Add IF NOT EXISTS clause to CREATE COLLATION (Peter Eisentraut) - - - - - - -Have VACUUM VERBOSE report the number of skipped frozen pages (Masahiko -Sawada) - - - -This information is also included in output. - - - - - - -Fix check_srf_call_placement() to handle VALUES cases correctly (Tom -Lane) - - - -NEED TEXT. - - + + + + Add table partitioning + syntax that automatically creates partition constraints and + INSERT routing (Amit Langote) + + + + The syntax supports range and list partitioning. + + + + + + + Add AFTER trigger + transition table to record changed rows (Kevin Grittner) + + + + Transition table contents are accessible from server-side languages. + + + + + + + Allow restrictive row-level + security policies (Stephen Frost) + + + + Previously all security policies were permissive, meaning that any + matching policy allowed access. Optional restrictive policies must + match for access to be granted. These policy types can be combined. + + + + + + + Allow default + permissions on schemas (Matheus Oliveira) + + + + This is done using the ALTER DEFAULT PRIVILEGES command. + + + + + + + Add CREATE SEQUENCE + AS command to create a sequence matching an integer data type + (Peter Eisentraut) + + + + This simplifies the creation of sequences matching the range of + base columns. + + + + + + + Allow COPY view FROM on views with INSTEAD + INSERT triggers (Haribabu Kommi) + + + + The triggers are fed the rows from COPY. + + + + + + + Allow the specification of a function name without arguments in + DDL commands, when unique (Peter Eisentraut) + + + + For example, allow DROP + FUNCTION on a function name without arguments if there + is only one function with that name. This is required by the + SQL standard. + + + + + + + Allow multiple functions, operators, and aggregates to be dropped + with a single DROP command (Peter Eisentraut) + + + + + + + Add IF NOT EXISTS for CREATE SERVER and + CREATE USER + MAPPING (Anastasia + Lubennikova) + + + + + + + Add IF NOT EXISTS clause to CREATE COLLATION + (Peter Eisentraut) + + + + + + + Have VACUUM VERBOSE + report the number of skipped frozen pages (Masahiko Sawada) + + + + This information is also included in output. + + + + + + + Fix check_srf_call_placement() to handle + VALUES cases correctly (Tom Lane) + + + + NEED TEXT. + + @@ -1674,85 +1757,95 @@ NEED TEXT. - - - -Add support for EUI-64 MAC addresses as MACADDR8 (Haribabu Kommi) - - - -This complements support for EUI-48 MAC addresses as macaddr. - - - - - - -Add identity columns for assigning a numeric value to columns on insert -(Peter Eisentraut) - - - -These are similar to SERIAL columns, but are SQL standard compliant. - - - - - - -Allow ENUM values to be renamed (Dagfinn Ilmari Mannsåker) - - - -This uses the syntax ALTER TYPE ... RENAME VALUE. - - - - - - -Properly treat array pseudotypes (anyarray) as arrays in to_json() and to_jsonb() (Andrew -Dunstan) - - - -Previously "anyarray" values were converted to JSON strings. - - - - - - -Add MONEY operators for multiplication and division with INT8 values -(Peter Eisentraut) - - - - - - -More strictly check the MONEY type for overflow operations (Peter -Eisentraut) - - + + + + Add support for EUI-64 MAC addresses as + MACADDR8 (Haribabu + Kommi) + + + + This complements support for EUI-48 MAC + addresses as macaddr. + + + + + + + Add identity columns for + assigning a numeric value to columns on insert (Peter Eisentraut) + + + + These are similar to SERIAL columns, but are + SQL standard compliant. + + + + + + + Allow ENUM values to be + renamed (Dagfinn Ilmari Mannsåker) + + + + This uses the syntax ALTER + TYPE ... RENAME VALUE. + + + + + + + Properly treat array pseudotypes + (anyarray) as arrays in to_json() + and to_jsonb() (Andrew Dunstan) + + + + Previously "anyarray" values were converted to JSON + strings. + + + + + + + Add MONEY operators + for multiplication and division with INT8 values (Peter + Eisentraut) + + + + + + + More strictly check the MONEY type for overflow operations + (Peter Eisentraut) + + @@ -1761,130 +1854,142 @@ Eisentraut) Functions - + - - - -Add simplified regexp_match() function (Emre Hasegeli) - - - -Similar to regexp_matches(), but only returns results from the first -match so it is easier use for simple cases. - - - - - - -Add support for converting XML-formatted data into a row set (Pavel -Stehule, Álvaro Herrera) - - - -This is done by referencing the new XMLTABLE function. - - - - - - -Add version of jsonb's delete operator that takes an array of keys to delete -(Magnus Hagander) - - - -The JSONB delete operator also now supports arrays. - - - - - - -Improve json_populate_record and friends operate recursively (Nikita -Glukhov) - - - -CLARIFY - - - - - - -Add function txid_current_ifassigned() to return NULL if no transaction -id has been assigned (Craig Ringer) - - - -This is different from txid_current(), which always returns a -transaction id by assigning one if necessary. This can be also run on -standby servers. - - - - - - -Add function txid_status() to check if a transaction was committed -(Craig Ringer) - - - -This is useful for checking after an abrupt disconnection if your -previous transaction committed and you just didn't receive the -acknowledgement. - - - - - - -Allow make_date() to interpret negative years as BC years (Álvaro -Herrera) - - - - - - -Have to_timestamp() and to_date() check input values for validity -(Artur Zakirov) - - - -Previously to_date('2009-06-40','YYYY-MM-DD') was accepted and returned -'2009-07-10'. It will now generate an error. - - + + + + Add simplified regexp_match() + function (Emre Hasegeli) + + + + Similar to regexp_matches(), but only returns results + from the first match so it is easier use for simple cases. + + + + + + + Add support for converting XML-formatted data into a row + set (Pavel Stehule, Álvaro Herrera) + + + + This is done by referencing the new XMLTABLE + function. + + + + + + + Add version of jsonb's delete operator that takes + an array of keys to delete (Magnus Hagander) + + + + The JSONB delete operator also now supports arrays. + + + + + + + Improve json_populate_record + and friends operate recursively (Nikita Glukhov) + + + + CLARIFY + + + + + + + Add function txid_current_ifassigned() + to return NULL if no transaction id has been assigned + (Craig Ringer) + + + + This is different from txid_current(), + which always returns a transaction id by assigning one if necessary. + This can be also run on standby servers. + + + + + + + Add function txid_status() + to check if a transaction was committed (Craig Ringer) + + + + This is useful for checking after an abrupt disconnection if + your previous transaction committed and you just didn't receive + the acknowledgement. + + + + + + + Allow make_date() + to interpret negative years as BC years (Álvaro + Herrera) + + + + + + + Have to_timestamp() and to_date() check + input values for validity (Artur Zakirov) + + + + Previously to_date('2009-06-40','YYYY-MM-DD') was + accepted and returned '2009-07-10'. It will now generate an error. + + - + @@ -1893,99 +1998,102 @@ Previously to_date('2009-06-40','YYYY-MM-DD') was accepted and retu - - - -Allow the PL/Python plan object to call cursor and execute methods -(Peter Eisentraut) - - - -This is a more object oriented style. - - - - - - -Allow PL/pgSQL's GET DIAGNOSTICS to retrieve values into array elements -(Tom Lane) - - - - - - -Remove SPI functions SPI_push(), SPI_pop(), SPI_restore_connection() as -unnecessary (Tom Lane) - - - -Their functionality now happens automatically. Also, SPI_palloc() now -requires an active connection. - - + + + + Allow the PL/Python plan object to call cursor and execute methods + (Peter Eisentraut) + + + + This is a more object oriented style. + + + + + + + Allow PL/pgSQL's GET DIAGNOSTICS to retrieve values + into array elements (Tom Lane) + + + + + + + Remove SPI functions SPI_push(), + SPI_pop(), SPI_restore_connection() + as unnecessary (Tom Lane) + + + + Their functionality now happens automatically. Also, + SPI_palloc() now requires an active connection. + + - <link linkend="pltcl">PL/Tcl</> + <link linkend="pltcl">PL/Tcl</> - + - - - -Allow PL/Tcl functions to return composite types and sets (Jim Nasby) - - - - - - -Add a subtransaction command to PL/Tcl (Victor Wagner) - - - -This allows PL/Tcl queries to fail without aborting the entire function. - - - - - - -Add GUCs to allow initialization routines to be called on PL/Tcl startup -(Tom Lane) - - - -The GUCs are pltcl.start_proc and pltclu.start_proc. - - + + + + Allow PL/Tcl functions to return composite types and sets + (Jim Nasby) + + + + + + + Add a subtransaction command to PL/Tcl (Victor Wagner) + + + + This allows PL/Tcl queries to fail without aborting the entire + function. + + + + + + + Add GUCs to allow initialization routines to be called + on PL/Tcl startup (Tom Lane) + + + + The GUCs are pltcl.start_proc and + pltclu.start_proc. + + - - + + - + @@ -1993,64 +2101,69 @@ The GUCs are pltcl.start_proc and pltclu.start_ - - - -Allow libpq to connect to multiple specified host names (Robert Haas) - - - -libpq will connect with the first responsive host name. - - - - - - -Allow the libpq connection string to request a read/write host (Victor -Wagner, Mithun Cy) - - - -This is useful when multiple libpq host names are specified, and is -controlled by libpq connection parameter - - - - - -Allow password file name to be specified as a libpq connection parameter -(Julian Markwort) - - - -Previously this could only be specified via an environment variable. - - - - - - -ecpg preprocessor version changed from 4.12 to 10 (Tom Lane) - - - -The ecpg version now matches the Postgres distribution version number. - - + + + + Allow libpq to connect to multiple specified host names + (Robert Haas) + + + + libpq will connect with the first responsive host name. + + + + + + + Allow the libpq connection string to request a read/write host + (Victor Wagner, Mithun Cy) + + + + This is useful when multiple libpq host names are + specified, and is controlled by libpq connection parameter + + + + + + + Allow password file name + to be specified as a libpq connection parameter (Julian Markwort) + + + + Previously this could only be specified via an environment variable. + + + + + + + ecpg preprocessor version changed from 4.12 to 10 (Tom Lane) + + + + The ecpg version now matches the Postgres distribution version + number. + + @@ -2059,138 +2172,143 @@ The ecpg version now matches the Postgres distribution version number. Client Applications - - <xref linkend="APP-PSQL"> + + <xref linkend="APP-PSQL"> - + - - - -Add conditional branch support to psql (Corey Huinker) - - - -The new syntax uses \if, \elif, \else, and \endif. This is particularly -helpful for scripting. - - - - - - -Add psql \gx command to perform \g(execute) in expanded mode (\x) -(Christoph Berg) - - - - - - -Improve psql's \d (relation) and \dD (domain) commands to specify -collation, nullable, and default in separate columns (Peter Eisentraut) - - - -Previous they were in a single "Modifiers" column. - - - - - - -Expand psql colon variables when used in backtick-executed contexts (Tom Lane) - - - -This is particularly useful for the new psql conditional branch support -commands. - - - - - - -Prevent psql special variables from being set to invalid values (Daniel -Vérité, Tom Lane) - - - -Previously setting psql special variables to invalid values produced the -default behavior. \set and \unset of special variables now sets them to -"on" and its default value, rather than a zero-length string and -undefined. Also have \set always display values for FETCH_COUNT, -HISTSIZE, and IGNOREEOF. - - - - - - -Fix psql \p to always print what would be executed by \g or \w (Daniel -Vérité) - - - -Previously \p didn't properly print the reverted-to command after a -buffer contents reset. CLARIFY? - - - - - - -Improve psql's tab completion (Jeff Janes, Ian Barwick, Andreas Karlsson, -Sehrope Sarkuni, Thomas Munro, Kevin Grittner, Dagfinn Ilmari Mannsåker) - - + + + + Add conditional branch support to psql (Corey + Huinker) + + + + The new syntax uses \if, \elif, \else, and \endif. This is + particularly helpful for scripting. + + + + + + + Add psql \gx command to perform \g(execute) + in expanded mode (\x) (Christoph Berg) + + + + + + + Improve psql's \d (relation) and \dD (domain) + commands to specify collation, nullable, and default in separate + columns (Peter Eisentraut) + + + + Previous they were in a single "Modifiers" column. + + + + + + + Expand psql colon variables when used in + backtick-executed contexts (Tom Lane) + + + + This is particularly useful for the new psql + conditional branch support commands. + + + + + + + Prevent psql special variables from being set to + invalid values (Daniel Vérité, Tom Lane) + + + + Previously setting psql special variables + to invalid values produced the default behavior. \set and + \unset of special variables now sets them to "on" and its + default value, rather than a zero-length string and undefined. + Also have \set always display values for FETCH_COUNT, + HISTSIZE, and IGNOREEOF. + + + + + + + Fix psql \p to always print what would be executed + by \g or \w (Daniel Vérité) + + + + Previously \p didn't properly print the reverted-to command after + a buffer contents reset. CLARIFY? + + + + + + + Improve psql's tab completion (Jeff Janes, + Ian Barwick, Andreas Karlsson, Sehrope Sarkuni, Thomas Munro, + Kevin Grittner, Dagfinn Ilmari Mannsåker) + + - + @@ -2199,27 +2317,27 @@ Sehrope Sarkuni, Thomas Munro, Kevin Grittner, Dagfinn Ilmari Mannsåker) - - - -Add pgbench option - - - - - -Allow pgbench's meta commands to span multiple lines via a -line-terminating backslash (Fabien Coelho) - - + + + + Add pgbench option + + + + + + Allow pgbench's meta commands to span multiple lines via a + line-terminating backslash (Fabien Coelho) + + @@ -2232,421 +2350,445 @@ line-terminating backslash (Fabien Coelho) - - - -Add pg_receivewal option - - - - - -Add pg_recvlogical option ( - - -This complements the existing - - - - - -Rename initdb options - - -The old spellings are still supported. - - + + + + Add pg_receivewal + option + + + + + + Add pg_recvlogical option + ( + + + This complements the existing + + + + + + Rename initdb + options + + + The old spellings are still supported. + + - <link linkend="APP-PGDUMP"><application>pg_dump</></>, <link linkend="APP-PG-DUMPALL"><application>pg_dumpall</></>. - <link linkend="APP-PGRESTORE"><application>pg_restore</></> + <link linkend="APP-PGDUMP"><application>pg_dump</></>, <link linkend="APP-PG-DUMPALL"><application>pg_dumpall</></>. + <link linkend="APP-PGRESTORE"><application>pg_restore</></> - + - - - -Allow pg_restore to exclude schemas (Michael Banck) - - - -This added a new - - - - - -Add - - -This suppresses the dumping of large objects. - - - - - - -Add pg_dumpall option - - -This allows easier dumping for less-privileged users. - - - - - - -Issue fsync on the output files generated by pg_dump and -pg_dumpall (Michael Paquier) - - - -This can be disabled with the - + + + + Allow pg_restore to exclude schemas (Michael Banck) + + + + This added a new + + + + + + Add + + + This suppresses the dumping of large objects. + + + + + + + Add pg_dumpall option + + + + This allows easier dumping for less-privileged users. + + + + + + + Issue fsync on the output files generated by + pg_dump and + pg_dumpall (Michael Paquier) + + + + This can be disabled with the + - + - - - + + + <xref linkend="app-pgbasebackup"> - + - - - -Allow pg_basebackup to stream transaction log in tar mode (Magnus -Hagander) - - - -The WAL will be stored in a separate tar file from the base backup. - - - - - - -Make pg_basebackup use temporary replication slots (Magnus Hagander) - - - -Temporary replication slots will be used by default when pg_basebackup -uses wal streaming with default options. - - - - - - -Improve fsync handling of pg_basebackup and pg_receivewal (Michael Paquier) - - - -Also add - - - - - -Improve pg_basebackup's handling of which directories to skip (David -Steele) - - - -Also improve the documentation of skipped directories. - - + + + + Allow pg_basebackup to stream transaction log in + tar mode (Magnus Hagander) + + + + The WAL will be stored in a separate tar file from + the base backup. + + + + + + + Make pg_basebackup use temporary replication slots + (Magnus Hagander) + + + + Temporary replication slots will be used by default when + pg_basebackup uses wal streaming with default + options. + + + + + + + Improve fsync handling of pg_basebackup and + pg_receivewal (Michael Paquier) + + + + Also add + + + + + + Improve pg_basebackup's handling of which + directories to skip (David Steele) + + + + Also improve the documentation of skipped directories. + + - + - - - - <application><xref linkend="app-pg-ctl"></> + + + + <application><xref linkend="app-pg-ctl"></> - + - - - -Add wait option for 's promote operation (Peter Eisentraut) - - - - - - -Add log options for pg_ctl wait ( - - - - - -Add long options flag for pg_ctl options (Peter Eisentraut) - - - -It is called - + + + + Add wait option for 's + promote operation (Peter Eisentraut) + + + + + + + Add log options for pg_ctl wait ( + + + + + + Add long options flag for pg_ctl + options (Peter Eisentraut) + + + + It is called + - - - + + + Source Code - + - - - -New major version numbering (Peter Eisentraut, Tom Lane) - - - -Major versions will now increase just the first number, and minor -releases will increase just the second number. A third number will no -longer be used in Postgres version numbers. - - - - - - -Allow the ICU library to optionally be used for collation support (Peter -Eisentraut) - - - -The ICU library has versioning that allows detection of collation -changes between versions. It is enabled via configure option - - - - - - -Automatically mark all PG_FUNCTION_INFO_V1 functions as DLLEXPORT-ed on -Windows (Laurenz Albe) - - - -If third-party code is using extern function declarations, they should -also add DLLEXPORT markers to those declarations. - - - - - - -Allow shared memory to be dynamically allocated (Thomas Munro, Robert Haas) - - - - - - -Add slab-like memory allocator for efficient fixed-size allocations -(Tomas Vondra) - - - - - - -Use POSIX semaphores rather than SysV semaphores on Linux and FreeBSD -(Tom Lane) - - - -This avoids some limits on SysV semaphore usage. - - - - - - -Improve support for 64-bit atomics (Andres Freund) - - - - - - -Enable 64-bit atomic operations on ARM64 (Roman Shaposhnik) - - - - - - -Switch to using clock_gettime(), if available, for duration measurements -(Tom Lane) - - - -gettimeofday() is used if clock_gettime() is not available. - - - - - - -Add more robust random number generators to be used for -cryptographic secure uses (Magnus Hagander, Michael Paquier, Heikki -Linnakangas) - - - -If no strong random number generator can be found, configure will fail -unless the configure - - - - - -Overhaul documentation build process (Alexander Lakhin, Alexander Law) - - - - - - -Use XSLT to build the Postgres documentation (Peter Eisentraut) - - - -Previously Jade, DSSSL, and JadeTex were used. - - - - - - -Build HTML documentation using XSLT stylesheets by default (Peter -Eisentraut) - - + + + + New major version numbering (Peter Eisentraut, Tom Lane) + + + + Major versions will now increase just the first number, and minor + releases will increase just the second number. A third number + will no longer be used in Postgres version numbers. + + + + + + + Allow the ICU library to + optionally be used for collation support (Peter Eisentraut) + + + + The ICU library has versioning that allows detection + of collation changes between versions. It is enabled via configure + option + + + + + + Automatically mark all PG_FUNCTION_INFO_V1 functions + as DLLEXPORT-ed on + Windows (Laurenz Albe) + + + + If third-party code is using extern function + declarations, they should also add DLLEXPORT markers + to those declarations. + + + + + + + Allow shared memory to be dynamically allocated (Thomas Munro, + Robert Haas) + + + + + + + Add slab-like memory allocator for efficient fixed-size allocations + (Tomas Vondra) + + + + + + + Use POSIX semaphores rather than SysV semaphores + on Linux and FreeBSD (Tom Lane) + + + + This avoids some limits on SysV semaphore usage. + + + + + + + Improve support for 64-bit atomics (Andres Freund) + + + + + + + Enable 64-bit atomic operations on ARM64 (Roman + Shaposhnik) + + + + + + + Switch to using clock_gettime(), if available, for + duration measurements (Tom Lane) + + + + gettimeofday() is used if clock_gettime() + is not available. + + + + + + + Add more robust random number generators to be used for + cryptographic secure uses (Magnus Hagander, Michael Paquier, + Heikki Linnakangas) + + + + If no strong random number generator can be found, configure + will fail unless the configure + + + + + + + Overhaul documentation build + process (Alexander Lakhin, Alexander Law) + + + + + + + Use XSLT to build the Postgres documentation (Peter + Eisentraut) + + + + Previously Jade, DSSSL, and + JadeTex were used. + + + + + + + Build HTML documentation using XSLT + stylesheets by default (Peter Eisentraut) + + @@ -2657,210 +2799,230 @@ Eisentraut) - - - -Allow file_fdw to read from program output as well as files (Corey Huinker, Adam Gomaa) - - - - - - -Push aggregates to foreign data wrapper servers, where possible (Jeevan -Chalke, Ashutosh Bapat) - - - -This reduces the amount of data that must be passed from the foreign -data wrapper server, and offloads aggregate computation from the -requesting server. The postgres_fdw is able to perform this -optimization. - - - - - - -Allow push down of FULL JOIN queries containing subqueries in the -FROM clause to foreign servers (Etsuro Fujita) - - - - - - -Properly support OID columns in postgres_fdw tables (Etsuro Fujita) - - - -Previously OID columns always returned zeros. - - - - - - -Allow btree_gist and btree_gin to index enum types -(Andrew Dunstan) - - - -This allows enums to be used in exclusion constraints. - - - - - - -Add indexing support to btree_gist for the UUID data type (Paul -Jungwirth) - - - - - - -Add amcheck which can check the validity of btree indexes (Peter -Geoghegan) - - - - - - -Show ignored constants as $N rather than ? in -pg_stat_statements (Lukas Fittl) - - - - - - -Improve cube's handling of zero-dimensional cubes (Tom Lane) - - - -This also improves handling of infinite and NaN values. - - - - - - -Allow pg_buffercache to run with fewer locks (Ivan Kartyshov) - - - -This allows it be less disruptive when run on production systems. - - - - - - -Add pgstathashindex() function to pgstattuple to view hash index -statistics (Ashutosh Sharma) - - - - - - -Allow pgstattuple to use GRANT permissions (Stephen Frost) - - - -This allows non-superusers to run these functions if permissions allow. - - - - - - -Reduce locking when pgstattuple examines hash indexes (Amit Kapila) - - - - - - -Add page_checksum() function to pageinspect (Tomas Vondra) - - - - - - -Add hash index support to pageinspect (Jesper Pedersen, Ashutosh -Sharma) - - - - - - -Add pageinspect function bt_page_items() to print page items -from a page image (Tomas Vondra) - - - -Previously only block numbers were supported. - - + + + + Allow file_fdw to read + from program output as well as files (Corey Huinker, Adam Gomaa) + + + + + + + Push aggregates to foreign data wrapper servers, where possible + (Jeevan Chalke, Ashutosh Bapat) + + + + This reduces the amount of data that must be passed + from the foreign data wrapper server, and offloads + aggregate computation from the requesting server. The postgres_fdw is able to + perform this optimization. + + + + + + + Allow push down of FULL JOIN queries containing + subqueries in the + FROM clause to foreign servers (Etsuro Fujita) + + + + + + + Properly support OID columns in + postgres_fdw tables (Etsuro Fujita) + + + + Previously OID columns always returned zeros. + + + + + + + Allow btree_gist + and btree_gin to + index enum types (Andrew Dunstan) + + + + This allows enums to be used in exclusion constraints. + + + + + + + Add indexing support to btree_gist for the + UUID data type (Paul Jungwirth) + + + + + + + Add amcheck which can + check the validity of btree indexes (Peter Geoghegan) + + + + + + + Show ignored constants as $N rather than ? + in + pg_stat_statements + (Lukas Fittl) + + + + + + + Improve cube's handling + of zero-dimensional cubes (Tom Lane) + + + + This also improves handling of infinite and + NaN values. + + + + + + + Allow pg_buffercache to run + with fewer locks (Ivan Kartyshov) + + + + This allows it be less disruptive when run on production systems. + + + + + + + Add pgstathashindex() function to pgstattuple to view hash + index statistics (Ashutosh Sharma) + + + + + + + Allow pgstattuple + to use GRANT permissions (Stephen Frost) + + + + This allows non-superusers to run these functions if permissions + allow. + + + + + + + Reduce locking when pgstattuple examines hash + indexes (Amit Kapila) + + + + + + + Add page_checksum() function to pageinspect (Tomas Vondra) + + + + + + + Add hash index support to pageinspect (Jesper + Pedersen, Ashutosh Sharma) + + + + + + + Add pageinspect + function bt_page_items() to print page items from a + page image (Tomas Vondra) + + + + Previously only block numbers were supported. + + - + - + -- cgit v1.2.3 From 3640cf5e1c8a0ae6ac55bf6384de0521dc77facf Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Tue, 25 Apr 2017 13:29:26 -0400 Subject: doc PG10: add commit 090010f2e and adjust EXPLAIN SUMMARY item Reported-by: Tels, Andres Freund --- doc/src/sgml/release-10.sgml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/release-10.sgml b/doc/src/sgml/release-10.sgml index 3ea33ca58b..8736770d57 100644 --- a/doc/src/sgml/release-10.sgml +++ b/doc/src/sgml/release-10.sgml @@ -807,6 +807,17 @@ + + + + Improve table creation speed in sessions that reference many + relations (Aleksander Alekseev) + + + @@ -858,10 +869,10 @@ - By default planning and execution is display by EXPLAIN - ANALYZE and not display in other cases. The new - EXPLAIN option SUMMARY allows explicit - control of this. + By default planning and execution time is display by + EXPLAIN ANALYZE and not display in other cases. + The new EXPLAIN option SUMMARY allows + explicit control of this. -- cgit v1.2.3 From 3d7741194a4688adea8404ab0a7bb57bac20232b Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Tue, 25 Apr 2017 13:45:47 -0400 Subject: doc: add PG 10 doc item about VACUUM truncation, 7e26e02ee Reported-by: Andres Freund --- doc/src/sgml/release-10.sgml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'doc/src') diff --git a/doc/src/sgml/release-10.sgml b/doc/src/sgml/release-10.sgml index 8736770d57..dfe003bddb 100644 --- a/doc/src/sgml/release-10.sgml +++ b/doc/src/sgml/release-10.sgml @@ -1744,6 +1744,22 @@ + + + + Improve speed of VACUUM's removal of trailing empty + heap pages (Alvaro Herrera) + + + + This information is also included in output. + + + - Improve table creation speed in sessions that reference many - relations (Aleksander Alekseev) + Reduce statistics tracking overhead in sessions that reference + many thousands of relations (Aleksander Alekseev) -- cgit v1.2.3 From 5f2b48d1dd17156c2021f9fa7c85d5c550bc2c6a Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Tue, 25 Apr 2017 15:30:45 -0400 Subject: doc PG10: update EXPLAIN SUMMARY item Reported-by: Tels --- doc/src/sgml/release-10.sgml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/release-10.sgml b/doc/src/sgml/release-10.sgml index 1d63e69122..175b127906 100644 --- a/doc/src/sgml/release-10.sgml +++ b/doc/src/sgml/release-10.sgml @@ -869,8 +869,8 @@ - By default planning and execution time is display by - EXPLAIN ANALYZE and not display in other cases. + By default planning and execution time are display by + EXPLAIN ANALYZE and are not display in other cases. The new EXPLAIN option SUMMARY allows explicit control of this. -- cgit v1.2.3 From 9139aa19423b736470f669e566f8ef6a7f19b801 Mon Sep 17 00:00:00 2001 From: Stephen Frost Date: Tue, 25 Apr 2017 16:57:43 -0400 Subject: Allow ALTER TABLE ONLY on partitioned tables There is no need to forbid ALTER TABLE ONLY on partitioned tables, when no partitions exist yet. This can be handy for users who are building up their partitioned table independently and will create actual partitions later. In addition, this is how pg_dump likes to operate in certain instances. Author: Amit Langote, with some error message word-smithing by me --- doc/src/sgml/ddl.sgml | 18 ++++++--- src/backend/commands/tablecmds.c | 66 +++++++++++++++++++------------ src/test/regress/expected/alter_table.out | 36 +++++++++++------ src/test/regress/expected/truncate.out | 8 ++++ src/test/regress/sql/alter_table.sql | 24 ++++++++--- src/test/regress/sql/truncate.sql | 4 ++ 6 files changed, 108 insertions(+), 48 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ddl.sgml b/doc/src/sgml/ddl.sgml index 340c961b3f..84c4f20990 100644 --- a/doc/src/sgml/ddl.sgml +++ b/doc/src/sgml/ddl.sgml @@ -2944,17 +2944,23 @@ VALUES ('Albany', NULL, NULL, 'NY'); Both CHECK and NOT NULL constraints of a partitioned table are always inherited by all its partitions. CHECK constraints that are marked - NO INHERIT are not allowed. + NO INHERIT are not allowed to be created on + partitioned tables. - The ONLY notation used to exclude child tables - will cause an error for partitioned tables in the case of - schema-modifying commands such as most ALTER TABLE - commands. For example, dropping a column from only the parent does - not make sense for partitioned tables. + Using ONLY to add or drop a constraint on only the + partitioned table is supported when there are no partitions. Once + partitions exist, using ONLY will result in an error + as adding or dropping constraints on only the partitioned table, when + partitions exist, is not supported. Instead, constraints can be added + or dropped, when they are not present in the parent table, directly on + the partitions. As a partitioned table does not have any data + directly, attempts to use TRUNCATE + ONLY on a partitioned table will always return an + error. diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index a02904c85c..a35713096d 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -1259,7 +1259,8 @@ ExecuteTruncate(TruncateStmt *stmt) else if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("must truncate child tables too"))); + errmsg("cannot truncate only a partitioned table"), + errhint("Do not specify the ONLY keyword, or use truncate only on the partitions directly."))); } /* @@ -5578,14 +5579,20 @@ static void ATPrepDropNotNull(Relation rel, bool recurse, bool recursing) { /* - * If the parent is a partitioned table, like check constraints, NOT NULL - * constraints must be dropped from child tables. + * If the parent is a partitioned table, like check constraints, we do + * not support removing the NOT NULL while partitions exist. */ - if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE && - !recurse && !recursing) - ereport(ERROR, - (errcode(ERRCODE_INVALID_TABLE_DEFINITION), - errmsg("constraint must be dropped from child tables too"))); + if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + { + PartitionDesc partdesc = RelationGetPartitionDesc(rel); + + Assert(partdesc != NULL); + if (partdesc->nparts > 0 && !recurse && !recursing) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("cannot remove constraint from only the partitioned table when partitions exist"), + errhint("Do not specify the ONLY keyword."))); + } } static ObjectAddress ATExecDropNotNull(Relation rel, const char *colName, LOCKMODE lockmode) @@ -5746,13 +5753,19 @@ ATPrepSetNotNull(Relation rel, bool recurse, bool recursing) { /* * If the parent is a partitioned table, like check constraints, NOT NULL - * constraints must be added to the child tables. + * constraints must be added to the child tables. Complain if requested + * otherwise and partitions exist. */ - if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE && - !recurse && !recursing) - ereport(ERROR, - (errcode(ERRCODE_INVALID_TABLE_DEFINITION), - errmsg("constraint must be added to child tables too"))); + if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + { + PartitionDesc partdesc = RelationGetPartitionDesc(rel); + + if (partdesc && partdesc->nparts > 0 && !recurse && !recursing) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("cannot add constraint to only the partitioned table when partitions exist"), + errhint("Do not specify the ONLY keyword."))); + } } static ObjectAddress @@ -6547,7 +6560,8 @@ ATExecDropColumn(List **wqueue, Relation rel, const char *colName, if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE && !recurse) ereport(ERROR, (errcode(ERRCODE_INVALID_TABLE_DEFINITION), - errmsg("column must be dropped from child tables too"))); + errmsg("cannot drop column from only the partitioned table when partitions exist"), + errhint("Do not specify the ONLY keyword."))); attr_rel = heap_open(AttributeRelationId, RowExclusiveLock); foreach(child, children) @@ -8561,16 +8575,6 @@ ATExecDropConstraint(Relation rel, const char *constrName, } } - /* - * In case of a partitioned table, the constraint must be dropped from the - * partitions too. There is no such thing as NO INHERIT constraints in - * case of partitioned tables. - */ - if (!recurse && rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) - ereport(ERROR, - (errcode(ERRCODE_INVALID_TABLE_DEFINITION), - errmsg("constraint must be dropped from child tables too"))); - /* * Propagate to children as appropriate. Unlike most other ALTER * routines, we have to do this one level of recursion at a time; we can't @@ -8581,6 +8585,18 @@ ATExecDropConstraint(Relation rel, const char *constrName, else children = NIL; + /* + * For a partitioned table, if partitions exist and we are told not to + * recurse, it's a user error. It doesn't make sense to have a constraint + * be defined only on the parent, especially if it's a partitioned table. + */ + if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE && + children != NIL && !recurse) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("cannot remove constraint from only the partitioned table when partitions exist"), + errhint("Do not specify the ONLY keyword."))); + foreach(child, children) { Oid childrelid = lfirst_oid(child); diff --git a/src/test/regress/expected/alter_table.out b/src/test/regress/expected/alter_table.out index 883a5c9864..375a0f618a 100644 --- a/src/test/regress/expected/alter_table.out +++ b/src/test/regress/expected/alter_table.out @@ -3295,7 +3295,8 @@ DROP TABLE part_3_4; ALTER TABLE ONLY list_parted2 ADD COLUMN c int; ERROR: column must be added to child tables too ALTER TABLE ONLY list_parted2 DROP COLUMN b; -ERROR: column must be dropped from child tables too +ERROR: cannot drop column from only the partitioned table when partitions exist +HINT: Do not specify the ONLY keyword. -- cannot add a column to partition or drop an inherited one ALTER TABLE part_2 ADD COLUMN c text; ERROR: cannot add column to a partition @@ -3306,24 +3307,37 @@ ALTER TABLE part_2 RENAME COLUMN b to c; ERROR: cannot rename inherited column "b" ALTER TABLE part_2 ALTER COLUMN b TYPE text; ERROR: cannot alter inherited column "b" --- cannot add NOT NULL or check constraints to *only* the parent (ie, non-inherited) +-- cannot add/drop NOT NULL or check constraints to *only* the parent, when +-- partitions exist ALTER TABLE ONLY list_parted2 ALTER b SET NOT NULL; +ERROR: cannot add constraint to only the partitioned table when partitions exist +HINT: Do not specify the ONLY keyword. +ALTER TABLE ONLY list_parted2 ADD CONSTRAINT check_b CHECK (b <> 'zz'); ERROR: constraint must be added to child tables too -ALTER TABLE ONLY list_parted2 add constraint check_b check (b <> 'zz'); -ERROR: constraint must be added to child tables too -ALTER TABLE list_parted2 add constraint check_b check (b <> 'zz') NO INHERIT; -ERROR: cannot add NO INHERIT constraint to partitioned table "list_parted2" +ALTER TABLE list_parted2 ALTER b SET NOT NULL; +ALTER TABLE ONLY list_parted2 ALTER b DROP NOT NULL; +ERROR: cannot remove constraint from only the partitioned table when partitions exist +HINT: Do not specify the ONLY keyword. +ALTER TABLE list_parted2 ADD CONSTRAINT check_b CHECK (b <> 'zz'); +ALTER TABLE ONLY list_parted2 DROP CONSTRAINT check_b; +ERROR: cannot remove constraint from only the partitioned table when partitions exist +HINT: Do not specify the ONLY keyword. +-- It's alright though, if no partitions are yet created +CREATE TABLE parted_no_parts (a int) PARTITION BY LIST (a); +ALTER TABLE ONLY parted_no_parts ALTER a SET NOT NULL; +ALTER TABLE ONLY parted_no_parts ADD CONSTRAINT check_a CHECK (a > 0); +ALTER TABLE ONLY parted_no_parts ALTER a DROP NOT NULL; +ALTER TABLE ONLY parted_no_parts DROP CONSTRAINT check_a; +DROP TABLE parted_no_parts; -- cannot drop inherited NOT NULL or check constraints from partition ALTER TABLE list_parted2 ALTER b SET NOT NULL, ADD CONSTRAINT check_a2 CHECK (a > 0); ALTER TABLE part_2 ALTER b DROP NOT NULL; ERROR: column "b" is marked NOT NULL in parent table ALTER TABLE part_2 DROP CONSTRAINT check_a2; ERROR: cannot drop inherited constraint "check_a2" of relation "part_2" --- cannot drop NOT NULL or check constraints from *only* the parent -ALTER TABLE ONLY list_parted2 ALTER a DROP NOT NULL; -ERROR: constraint must be dropped from child tables too -ALTER TABLE ONLY list_parted2 DROP CONSTRAINT check_a2; -ERROR: constraint must be dropped from child tables too +-- Doesn't make sense to add NO INHERIT constraints on partitioned tables +ALTER TABLE list_parted2 add constraint check_b2 check (b <> 'zz') NO INHERIT; +ERROR: cannot add NO INHERIT constraint to partitioned table "list_parted2" -- check that a partition cannot participate in regular inheritance CREATE TABLE inh_test () INHERITS (part_2); ERROR: cannot inherit from partition "part_2" diff --git a/src/test/regress/expected/truncate.out b/src/test/regress/expected/truncate.out index b652562f5b..d967e8dd21 100644 --- a/src/test/regress/expected/truncate.out +++ b/src/test/regress/expected/truncate.out @@ -452,7 +452,15 @@ LINE 1: SELECT nextval('truncate_a_id1'); ^ -- partitioned table CREATE TABLE truncparted (a int, b char) PARTITION BY LIST (a); +-- error, can't truncate a partitioned table +TRUNCATE ONLY truncparted; +ERROR: cannot truncate only a partitioned table +HINT: Do not specify the ONLY keyword, or use truncate only on the partitions directly. CREATE TABLE truncparted1 PARTITION OF truncparted FOR VALUES IN (1); INSERT INTO truncparted VALUES (1, 'a'); +-- error, must truncate partitions +TRUNCATE ONLY truncparted; +ERROR: cannot truncate only a partitioned table +HINT: Do not specify the ONLY keyword, or use truncate only on the partitions directly. TRUNCATE truncparted; DROP TABLE truncparted; diff --git a/src/test/regress/sql/alter_table.sql b/src/test/regress/sql/alter_table.sql index eb1b4b536f..85c848f620 100644 --- a/src/test/regress/sql/alter_table.sql +++ b/src/test/regress/sql/alter_table.sql @@ -2173,19 +2173,31 @@ ALTER TABLE part_2 DROP COLUMN b; ALTER TABLE part_2 RENAME COLUMN b to c; ALTER TABLE part_2 ALTER COLUMN b TYPE text; --- cannot add NOT NULL or check constraints to *only* the parent (ie, non-inherited) +-- cannot add/drop NOT NULL or check constraints to *only* the parent, when +-- partitions exist ALTER TABLE ONLY list_parted2 ALTER b SET NOT NULL; -ALTER TABLE ONLY list_parted2 add constraint check_b check (b <> 'zz'); -ALTER TABLE list_parted2 add constraint check_b check (b <> 'zz') NO INHERIT; +ALTER TABLE ONLY list_parted2 ADD CONSTRAINT check_b CHECK (b <> 'zz'); + +ALTER TABLE list_parted2 ALTER b SET NOT NULL; +ALTER TABLE ONLY list_parted2 ALTER b DROP NOT NULL; +ALTER TABLE list_parted2 ADD CONSTRAINT check_b CHECK (b <> 'zz'); +ALTER TABLE ONLY list_parted2 DROP CONSTRAINT check_b; + +-- It's alright though, if no partitions are yet created +CREATE TABLE parted_no_parts (a int) PARTITION BY LIST (a); +ALTER TABLE ONLY parted_no_parts ALTER a SET NOT NULL; +ALTER TABLE ONLY parted_no_parts ADD CONSTRAINT check_a CHECK (a > 0); +ALTER TABLE ONLY parted_no_parts ALTER a DROP NOT NULL; +ALTER TABLE ONLY parted_no_parts DROP CONSTRAINT check_a; +DROP TABLE parted_no_parts; -- cannot drop inherited NOT NULL or check constraints from partition ALTER TABLE list_parted2 ALTER b SET NOT NULL, ADD CONSTRAINT check_a2 CHECK (a > 0); ALTER TABLE part_2 ALTER b DROP NOT NULL; ALTER TABLE part_2 DROP CONSTRAINT check_a2; --- cannot drop NOT NULL or check constraints from *only* the parent -ALTER TABLE ONLY list_parted2 ALTER a DROP NOT NULL; -ALTER TABLE ONLY list_parted2 DROP CONSTRAINT check_a2; +-- Doesn't make sense to add NO INHERIT constraints on partitioned tables +ALTER TABLE list_parted2 add constraint check_b2 check (b <> 'zz') NO INHERIT; -- check that a partition cannot participate in regular inheritance CREATE TABLE inh_test () INHERITS (part_2); diff --git a/src/test/regress/sql/truncate.sql b/src/test/regress/sql/truncate.sql index 9d3d8de54a..fbd1d1a8a5 100644 --- a/src/test/regress/sql/truncate.sql +++ b/src/test/regress/sql/truncate.sql @@ -236,7 +236,11 @@ SELECT nextval('truncate_a_id1'); -- fail, seq should have been dropped -- partitioned table CREATE TABLE truncparted (a int, b char) PARTITION BY LIST (a); +-- error, can't truncate a partitioned table +TRUNCATE ONLY truncparted; CREATE TABLE truncparted1 PARTITION OF truncparted FOR VALUES IN (1); INSERT INTO truncparted VALUES (1, 'a'); +-- error, must truncate partitions +TRUNCATE ONLY truncparted; TRUNCATE truncparted; DROP TABLE truncparted; -- cgit v1.2.3 From 309191f66a947c5b63dd348a13aafa52b5847f94 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Wed, 26 Apr 2017 06:33:25 -0400 Subject: doc PG10: add Rafia Sabih to parallel index scan item Reported-by: Amit Kapila --- doc/src/sgml/release-10.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/release-10.sgml b/doc/src/sgml/release-10.sgml index 175b127906..ef3686e6f2 100644 --- a/doc/src/sgml/release-10.sgml +++ b/doc/src/sgml/release-10.sgml @@ -380,7 +380,7 @@ --> Support parallel btree index scans (Rahila Syed, Amit Kapila, - Robert Haas) + Robert Haas, Rafia Sabih) -- cgit v1.2.3 From e315346d839ef27f9d5f2584f44de09f08573df2 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 26 Apr 2017 12:05:11 -0400 Subject: doc: ALTER SUBSCRIPTION documentation fixes WITH is optional for REFRESH PUBLICATION. Also, remove a spurious bracket and fix a punctuation. Author: Euler Taveira --- doc/src/sgml/ref/alter_subscription.sgml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/alter_subscription.sgml b/doc/src/sgml/ref/alter_subscription.sgml index f71ee38b40..35aeb6af41 100644 --- a/doc/src/sgml/ref/alter_subscription.sgml +++ b/doc/src/sgml/ref/alter_subscription.sgml @@ -21,7 +21,7 @@ PostgreSQL documentation -ALTER SUBSCRIPTION name WITH ( suboption [, ... ] ) ] +ALTER SUBSCRIPTION name WITH ( suboption [, ... ] ) where suboption can be: @@ -29,7 +29,7 @@ ALTER SUBSCRIPTION name WITH ( synchronous_commit ALTER SUBSCRIPTION name SET PUBLICATION publication_name [, ...] { REFRESH WITH ( puboption [, ... ] ) | NOREFRESH } -ALTER SUBSCRIPTION name REFRESH PUBLICATION WITH ( puboption [, ... ] ) +ALTER SUBSCRIPTION name REFRESH PUBLICATION [ WITH ( puboption [, ... ] ) ] where puboption can be: @@ -54,7 +54,7 @@ ALTER SUBSCRIPTION name DISABLE To alter the owner, you must also be a direct or indirect member of the - new owning role. The new owner has to be a superuser + new owning role. The new owner has to be a superuser. -- cgit v1.2.3 From fda4fec578505a624e4c0f0244816ae5bc10a3d1 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Wed, 26 Apr 2017 15:50:51 -0400 Subject: doc PG10: add commit for transition table item --- doc/src/sgml/release-10.sgml | 2 ++ 1 file changed, 2 insertions(+) (limited to 'doc/src') diff --git a/doc/src/sgml/release-10.sgml b/doc/src/sgml/release-10.sgml index ef3686e6f2..de2129758c 100644 --- a/doc/src/sgml/release-10.sgml +++ b/doc/src/sgml/release-10.sgml @@ -1597,6 +1597,8 @@ 2016-11-04 [8c48375e5] Implement syntax for transition tables in AFTER triggers Author: Kevin Grittner 2017-04-04 [5ebeb579b] Follow-on cleanup for the transition table patch. + Author: Kevin Grittner + 2017-03-31 [597027163] Add transition table support to plpgsql. --> Add AFTER trigger -- cgit v1.2.3 From f8ab08ad0e411139060bf943fa9931f7fbe97cde Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Thu, 27 Apr 2017 10:17:08 -0400 Subject: doc PG10rel: adjust hash index commits and add parallel subquery Reported-by: Amit Kapila --- doc/src/sgml/release-10.sgml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/release-10.sgml b/doc/src/sgml/release-10.sgml index de2129758c..8b1c950634 100644 --- a/doc/src/sgml/release-10.sgml +++ b/doc/src/sgml/release-10.sgml @@ -416,6 +416,16 @@ + + + + Allow non-correlated subqueries to be run in parallel (Amit Kapila) + + + Add write-ahead logging support to hash indexes (Amit Kapila) @@ -585,10 +599,6 @@ -- cgit v1.2.3 From a311d2a04f849f76d33b59995097999a311a4c62 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Thu, 27 Apr 2017 10:21:44 -0400 Subject: doc: PG10 release note typo fix Reported-by: daniel.westermann --- doc/src/sgml/release-10.sgml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/release-10.sgml b/doc/src/sgml/release-10.sgml index 8b1c950634..241064e58e 100644 --- a/doc/src/sgml/release-10.sgml +++ b/doc/src/sgml/release-10.sgml @@ -879,8 +879,8 @@ - By default planning and execution time are display by - EXPLAIN ANALYZE and are not display in other cases. + By default planning and execution time are displayed by + EXPLAIN ANALYZE and are not displayed in other cases. The new EXPLAIN option SUMMARY allows explicit control of this. -- cgit v1.2.3 From b9a3ef55b253d885081c2d0e9dc45802cab71c7b Mon Sep 17 00:00:00 2001 From: Stephen Frost Date: Thu, 27 Apr 2017 20:14:39 -0400 Subject: Remove unnecessairly duplicated gram.y productions Declarative partitioning duplicated the TypedTableElement productions, evidently to remove the need to specify WITH OPTIONS when creating partitions. Instead, simply make WITH OPTIONS optional in the TypedTableElement production and remove all of the duplicate PartitionElement-related productions. This change simplifies the syntax and makes WITH OPTIONS optional when adding defaults, constraints or storage parameters to columns when creating either typed tables or partitions. Also update pg_dump to no longer include WITH OPTIONS, since it's not necessary, and update the documentation to reflect that WITH OPTIONS is now optional. --- doc/src/sgml/ref/create_foreign_table.sgml | 2 +- doc/src/sgml/ref/create_table.sgml | 4 +- src/backend/parser/gram.y | 68 ++++++++++-------------------- src/bin/pg_dump/pg_dump.c | 17 +++++--- src/test/regress/expected/create_table.out | 2 +- src/test/regress/expected/sanity_check.out | 3 ++ src/test/regress/expected/typed_table.out | 37 ++++++++++++++-- src/test/regress/sql/create_table.sql | 2 +- src/test/regress/sql/typed_table.sql | 21 ++++++++- 9 files changed, 94 insertions(+), 62 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/create_foreign_table.sgml b/doc/src/sgml/ref/create_foreign_table.sgml index 5d0dcf567b..065c982082 100644 --- a/doc/src/sgml/ref/create_foreign_table.sgml +++ b/doc/src/sgml/ref/create_foreign_table.sgml @@ -29,7 +29,7 @@ CREATE FOREIGN TABLE [ IF NOT EXISTS ] table_name CREATE FOREIGN TABLE [ IF NOT EXISTS ] table_name PARTITION OF parent_table [ ( - { column_name WITH OPTIONS [ column_constraint [ ... ] ] + { column_name [ WITH OPTIONS ] [ column_constraint [ ... ] ] | table_constraint } [, ... ] ) ] partition_bound_spec diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml index a3dc744efa..484f81898b 100644 --- a/doc/src/sgml/ref/create_table.sgml +++ b/doc/src/sgml/ref/create_table.sgml @@ -35,7 +35,7 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXISTS ] table_name OF type_name [ ( - { column_name WITH OPTIONS [ column_constraint [ ... ] ] + { column_name [ WITH OPTIONS ] [ column_constraint [ ... ] ] | table_constraint } [, ... ] ) ] @@ -46,7 +46,7 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXISTS ] table_name PARTITION OF parent_table [ ( - { column_name [ column_constraint [ ... ] ] + { column_name [ WITH OPTIONS ] [ column_constraint [ ... ] ] | table_constraint } [, ... ] ) ] FOR VALUES partition_bound_spec diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 89d2836c49..21cdc7c7da 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -576,8 +576,6 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); %type part_strategy %type part_elem %type part_params -%type OptPartitionElementList PartitionElementList -%type PartitionElement %type ForValues %type partbound_datum %type partbound_datum_list @@ -3131,7 +3129,7 @@ CreateStmt: CREATE OptTemp TABLE qualified_name '(' OptTableElementList ')' $$ = (Node *)n; } | CREATE OptTemp TABLE qualified_name PARTITION OF qualified_name - OptPartitionElementList ForValues OptPartitionSpec OptWith + OptTypedTableElementList ForValues OptPartitionSpec OptWith OnCommitOption OptTableSpace { CreateStmt *n = makeNode(CreateStmt); @@ -3150,7 +3148,7 @@ CreateStmt: CREATE OptTemp TABLE qualified_name '(' OptTableElementList ')' $$ = (Node *)n; } | CREATE OptTemp TABLE IF_P NOT EXISTS qualified_name PARTITION OF - qualified_name OptPartitionElementList ForValues OptPartitionSpec + qualified_name OptTypedTableElementList ForValues OptPartitionSpec OptWith OnCommitOption OptTableSpace { CreateStmt *n = makeNode(CreateStmt); @@ -3213,11 +3211,6 @@ OptTypedTableElementList: | /*EMPTY*/ { $$ = NIL; } ; -OptPartitionElementList: - '(' PartitionElementList ')' { $$ = $2; } - | /*EMPTY*/ { $$ = NIL; } - ; - TableElementList: TableElement { @@ -3240,17 +3233,6 @@ TypedTableElementList: } ; -PartitionElementList: - PartitionElement - { - $$ = list_make1($1); - } - | PartitionElementList ',' PartitionElement - { - $$ = lappend($1, $3); - } - ; - TableElement: columnDef { $$ = $1; } | TableLikeClause { $$ = $1; } @@ -3262,28 +3244,6 @@ TypedTableElement: | TableConstraint { $$ = $1; } ; -PartitionElement: - TableConstraint { $$ = $1; } - | ColId ColQualList - { - ColumnDef *n = makeNode(ColumnDef); - n->colname = $1; - n->typeName = NULL; - n->inhcount = 0; - n->is_local = true; - n->is_not_null = false; - n->is_from_type = false; - n->storage = 0; - n->raw_default = NULL; - n->cooked_default = NULL; - n->collOid = InvalidOid; - SplitColQualList($2, &n->constraints, &n->collClause, - yyscanner); - n->location = @1; - $$ = (Node *) n; - } - ; - columnDef: ColId Typename create_generic_options ColQualList { ColumnDef *n = makeNode(ColumnDef); @@ -3305,7 +3265,25 @@ columnDef: ColId Typename create_generic_options ColQualList } ; -columnOptions: ColId WITH OPTIONS ColQualList +columnOptions: ColId ColQualList + { + ColumnDef *n = makeNode(ColumnDef); + n->colname = $1; + n->typeName = NULL; + n->inhcount = 0; + n->is_local = true; + n->is_not_null = false; + n->is_from_type = false; + n->storage = 0; + n->raw_default = NULL; + n->cooked_default = NULL; + n->collOid = InvalidOid; + SplitColQualList($2, &n->constraints, &n->collClause, + yyscanner); + n->location = @1; + $$ = (Node *)n; + } + | ColId WITH OPTIONS ColQualList { ColumnDef *n = makeNode(ColumnDef); n->colname = $1; @@ -4872,7 +4850,7 @@ CreateForeignTableStmt: $$ = (Node *) n; } | CREATE FOREIGN TABLE qualified_name - PARTITION OF qualified_name OptPartitionElementList ForValues + PARTITION OF qualified_name OptTypedTableElementList ForValues SERVER name create_generic_options { CreateForeignTableStmt *n = makeNode(CreateForeignTableStmt); @@ -4893,7 +4871,7 @@ CreateForeignTableStmt: $$ = (Node *) n; } | CREATE FOREIGN TABLE IF_P NOT EXISTS qualified_name - PARTITION OF qualified_name OptPartitionElementList ForValues + PARTITION OF qualified_name OptTypedTableElementList ForValues SERVER name create_generic_options { CreateForeignTableStmt *n = makeNode(CreateForeignTableStmt); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index e9b5c8a448..2fda350faa 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -15267,13 +15267,16 @@ dumpTableSchema(Archive *fout, TableInfo *tbinfo) continue; } - /* Attribute type */ - if ((tbinfo->reloftype || tbinfo->partitionOf) && - !dopt->binary_upgrade) - { - appendPQExpBufferStr(q, " WITH OPTIONS"); - } - else + /* + * Attribute type + * + * In binary-upgrade mode, we always include the type. + * If we aren't in binary-upgrade mode, then we skip the + * type when creating a typed table ('OF type_name') or a + * partition ('PARTITION OF'), since the type comes from + * the parent/partitioned table. + */ + if (dopt->binary_upgrade || (!tbinfo->reloftype && !tbinfo->partitionOf)) { appendPQExpBuffer(q, " %s", tbinfo->atttypnames[j]); diff --git a/src/test/regress/expected/create_table.out b/src/test/regress/expected/create_table.out index b6c75d2e81..3f94250df2 100644 --- a/src/test/regress/expected/create_table.out +++ b/src/test/regress/expected/create_table.out @@ -624,7 +624,7 @@ SELECT conislocal, coninhcount FROM pg_constraint WHERE conrelid = 'part_b'::reg -- specify PARTITION BY for a partition CREATE TABLE fail_part_col_not_found PARTITION OF parted FOR VALUES IN ('c') PARTITION BY RANGE (c); ERROR: column "c" named in partition key does not exist -CREATE TABLE part_c PARTITION OF parted FOR VALUES IN ('c') PARTITION BY RANGE ((b)); +CREATE TABLE part_c PARTITION OF parted (b WITH OPTIONS NOT NULL DEFAULT 0) FOR VALUES IN ('c') PARTITION BY RANGE ((b)); -- create a level-2 partition CREATE TABLE part_c_1_10 PARTITION OF part_c FOR VALUES FROM (1) TO (10); -- Partition bound in describe output diff --git a/src/test/regress/expected/sanity_check.out b/src/test/regress/expected/sanity_check.out index 753ad81e43..6750152e0f 100644 --- a/src/test/regress/expected/sanity_check.out +++ b/src/test/regress/expected/sanity_check.out @@ -93,6 +93,9 @@ onek|t onek2|t path_tbl|f person|f +persons|f +persons2|t +persons3|t pg_aggregate|t pg_am|t pg_amop|t diff --git a/src/test/regress/expected/typed_table.out b/src/test/regress/expected/typed_table.out index 141d3bcf87..5cdd019244 100644 --- a/src/test/regress/expected/typed_table.out +++ b/src/test/regress/expected/typed_table.out @@ -1,3 +1,7 @@ +-- Clean up in case a prior regression run failed +SET client_min_messages TO 'warning'; +DROP TYPE IF EXISTS person_type CASCADE; +RESET client_min_messages; CREATE TABLE ttable1 OF nothing; ERROR: type "nothing" does not exist CREATE TYPE person_type AS (id int, name text); @@ -102,7 +106,32 @@ SELECT id, namelen(persons) FROM persons; 1 | 4 (1 row) -DROP TYPE person_type CASCADE; -NOTICE: drop cascades to 2 other objects -DETAIL: drop cascades to table persons -drop cascades to function namelen(person_type) +CREATE TABLE persons2 OF person_type ( + id WITH OPTIONS PRIMARY KEY, + UNIQUE (name) +); +\d persons2 + Table "public.persons2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + id | integer | | not null | + name | text | | | +Indexes: + "persons2_pkey" PRIMARY KEY, btree (id) + "persons2_name_key" UNIQUE CONSTRAINT, btree (name) +Typed table of type: person_type + +CREATE TABLE persons3 OF person_type ( + PRIMARY KEY (id), + name NOT NULL DEFAULT '' +); +\d persons3 + Table "public.persons3" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+---------- + id | integer | | not null | + name | text | | not null | ''::text +Indexes: + "persons3_pkey" PRIMARY KEY, btree (id) +Typed table of type: person_type + diff --git a/src/test/regress/sql/create_table.sql b/src/test/regress/sql/create_table.sql index b00d9e87b8..f08942f7d2 100644 --- a/src/test/regress/sql/create_table.sql +++ b/src/test/regress/sql/create_table.sql @@ -578,7 +578,7 @@ SELECT conislocal, coninhcount FROM pg_constraint WHERE conrelid = 'part_b'::reg -- specify PARTITION BY for a partition CREATE TABLE fail_part_col_not_found PARTITION OF parted FOR VALUES IN ('c') PARTITION BY RANGE (c); -CREATE TABLE part_c PARTITION OF parted FOR VALUES IN ('c') PARTITION BY RANGE ((b)); +CREATE TABLE part_c PARTITION OF parted (b WITH OPTIONS NOT NULL DEFAULT 0) FOR VALUES IN ('c') PARTITION BY RANGE ((b)); -- create a level-2 partition CREATE TABLE part_c_1_10 PARTITION OF part_c FOR VALUES FROM (1) TO (10); diff --git a/src/test/regress/sql/typed_table.sql b/src/test/regress/sql/typed_table.sql index 25aaccb8bc..1cdceef363 100644 --- a/src/test/regress/sql/typed_table.sql +++ b/src/test/regress/sql/typed_table.sql @@ -1,3 +1,10 @@ +-- Clean up in case a prior regression run failed +SET client_min_messages TO 'warning'; + +DROP TYPE IF EXISTS person_type CASCADE; + +RESET client_min_messages; + CREATE TABLE ttable1 OF nothing; CREATE TYPE person_type AS (id int, name text); @@ -60,4 +67,16 @@ INSERT INTO persons VALUES (1, 'test'); CREATE FUNCTION namelen(person_type) RETURNS int LANGUAGE SQL AS $$ SELECT length($1.name) $$; SELECT id, namelen(persons) FROM persons; -DROP TYPE person_type CASCADE; +CREATE TABLE persons2 OF person_type ( + id WITH OPTIONS PRIMARY KEY, + UNIQUE (name) +); + +\d persons2 + +CREATE TABLE persons3 OF person_type ( + PRIMARY KEY (id), + name NOT NULL DEFAULT '' +); + +\d persons3 -- cgit v1.2.3 From bc920bee296ec4c1e8cd1598c71f21d80a59d351 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 28 Apr 2017 15:30:54 -0400 Subject: doc: Fix typo in 9.6 release notes Author: Huong Dangminh --- doc/src/sgml/release-9.6.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/release-9.6.sgml b/doc/src/sgml/release-9.6.sgml index 7a37f3f44b..dc7b479387 100644 --- a/doc/src/sgml/release-9.6.sgml +++ b/doc/src/sgml/release-9.6.sgml @@ -5004,7 +5004,7 @@ This commit is also listed under libpq and PL/pgSQL --> Allow the number of client connections ( -- cgit v1.2.3 From e180c8aa8caf5c55a273d4a8e6092e77ff3cff10 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Mon, 1 May 2017 08:23:01 -0400 Subject: Fire per-statement triggers on partitioned tables. Even though no actual tuples are ever inserted into a partitioned table (the actual tuples are in the partitions, not the partitioned table itself), we still need to have a ResultRelInfo for the partitioned table, or per-statement triggers won't get fired. Amit Langote, per a report from Rajkumar Raghuwanshi. Reviewed by me. Discussion: http://postgr.es/m/CAKcux6%3DwYospCRY2J4XEFuVy0L41S%3Dfic7rmkbsU-GXhhSbmBg%40mail.gmail.com --- doc/src/sgml/trigger.sgml | 19 ++++---- src/backend/executor/execMain.c | 55 ++++++++++++++++++++-- src/backend/executor/nodeModifyTable.c | 42 +++++++++++++---- src/backend/nodes/copyfuncs.c | 2 + src/backend/nodes/outfuncs.c | 3 ++ src/backend/nodes/readfuncs.c | 2 + src/backend/optimizer/plan/createplan.c | 1 + src/backend/optimizer/plan/planner.c | 3 ++ src/backend/optimizer/plan/setrefs.c | 19 ++++++-- src/include/nodes/execnodes.h | 12 +++++ src/include/nodes/plannodes.h | 13 +++++- src/include/nodes/relation.h | 1 + src/test/regress/expected/triggers.out | 81 +++++++++++++++++++++++++++++++++ src/test/regress/sql/triggers.sql | 70 ++++++++++++++++++++++++++++ 14 files changed, 296 insertions(+), 27 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/trigger.sgml b/doc/src/sgml/trigger.sgml index 2a718d7f47..6f8416dda7 100644 --- a/doc/src/sgml/trigger.sgml +++ b/doc/src/sgml/trigger.sgml @@ -33,7 +33,8 @@ A trigger is a specification that the database should automatically execute a particular function whenever a certain type of operation is - performed. Triggers can be attached to tables, views, and foreign tables. + performed. Triggers can be attached to tables (partitioned or not), + views, and foreign tables. @@ -111,14 +112,14 @@ Statement-level BEFORE triggers naturally fire before the statement starts to do anything, while statement-level AFTER triggers fire at the very end of the statement. These types of - triggers may be defined on tables or views. Row-level BEFORE - triggers fire immediately before a particular row is operated on, - while row-level AFTER triggers fire at the end of the - statement (but before any statement-level AFTER triggers). - These types of triggers may only be defined on tables and foreign tables. - Row-level INSTEAD OF triggers may only be defined on views, - and fire immediately as each row in the view is identified as needing to - be operated on. + triggers may be defined on tables, views, or foreign tables. Row-level + BEFORE triggers fire immediately before a particular row is + operated on, while row-level AFTER triggers fire at the end of + the statement (but before any statement-level AFTER triggers). + These types of triggers may only be defined on non-partitioned tables and + foreign tables. Row-level INSTEAD OF triggers may only be + defined on views, and fire immediately as each row in the view is + identified as needing to be operated on. diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c index 5c12fb457d..cdb1a6a5f5 100644 --- a/src/backend/executor/execMain.c +++ b/src/backend/executor/execMain.c @@ -861,17 +861,52 @@ InitPlan(QueryDesc *queryDesc, int eflags) /* * In the partitioned result relation case, lock the non-leaf result - * relations too. We don't however need ResultRelInfos for them. + * relations too. A subset of these are the roots of respective + * partitioned tables, for which we also allocate ResulRelInfos. */ + estate->es_root_result_relations = NULL; + estate->es_num_root_result_relations = 0; if (plannedstmt->nonleafResultRelations) { + int num_roots = list_length(plannedstmt->rootResultRelations); + + /* + * Firstly, build ResultRelInfos for all the partitioned table + * roots, because we will need them to fire the statement-level + * triggers, if any. + */ + resultRelInfos = (ResultRelInfo *) + palloc(num_roots * sizeof(ResultRelInfo)); + resultRelInfo = resultRelInfos; + foreach(l, plannedstmt->rootResultRelations) + { + Index resultRelIndex = lfirst_int(l); + Oid resultRelOid; + Relation resultRelDesc; + + resultRelOid = getrelid(resultRelIndex, rangeTable); + resultRelDesc = heap_open(resultRelOid, RowExclusiveLock); + InitResultRelInfo(resultRelInfo, + resultRelDesc, + lfirst_int(l), + NULL, + estate->es_instrument); + resultRelInfo++; + } + + estate->es_root_result_relations = resultRelInfos; + estate->es_num_root_result_relations = num_roots; + + /* Simply lock the rest of them. */ foreach(l, plannedstmt->nonleafResultRelations) { - Index resultRelationIndex = lfirst_int(l); - Oid resultRelationOid; + Index resultRelIndex = lfirst_int(l); - resultRelationOid = getrelid(resultRelationIndex, rangeTable); - LockRelationOid(resultRelationOid, RowExclusiveLock); + /* We locked the roots above. */ + if (!list_member_int(plannedstmt->rootResultRelations, + resultRelIndex)) + LockRelationOid(getrelid(resultRelIndex, rangeTable), + RowExclusiveLock); } } } @@ -883,6 +918,8 @@ InitPlan(QueryDesc *queryDesc, int eflags) estate->es_result_relations = NULL; estate->es_num_result_relations = 0; estate->es_result_relation_info = NULL; + estate->es_root_result_relations = NULL; + estate->es_num_root_result_relations = 0; } /* @@ -1565,6 +1602,14 @@ ExecEndPlan(PlanState *planstate, EState *estate) resultRelInfo++; } + /* Close the root target relation(s). */ + resultRelInfo = estate->es_root_result_relations; + for (i = estate->es_num_root_result_relations; i > 0; i--) + { + heap_close(resultRelInfo->ri_RelationDesc, NoLock); + resultRelInfo++; + } + /* * likewise close any trigger target relations */ diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c index 71e3b8ec2d..652cd97599 100644 --- a/src/backend/executor/nodeModifyTable.c +++ b/src/backend/executor/nodeModifyTable.c @@ -1328,19 +1328,29 @@ ExecOnConflictUpdate(ModifyTableState *mtstate, static void fireBSTriggers(ModifyTableState *node) { + ResultRelInfo *resultRelInfo = node->resultRelInfo; + + /* + * If the node modifies a partitioned table, we must fire its triggers. + * Note that in that case, node->resultRelInfo points to the first leaf + * partition, not the root table. + */ + if (node->rootResultRelInfo != NULL) + resultRelInfo = node->rootResultRelInfo; + switch (node->operation) { case CMD_INSERT: - ExecBSInsertTriggers(node->ps.state, node->resultRelInfo); + ExecBSInsertTriggers(node->ps.state, resultRelInfo); if (node->mt_onconflict == ONCONFLICT_UPDATE) ExecBSUpdateTriggers(node->ps.state, - node->resultRelInfo); + resultRelInfo); break; case CMD_UPDATE: - ExecBSUpdateTriggers(node->ps.state, node->resultRelInfo); + ExecBSUpdateTriggers(node->ps.state, resultRelInfo); break; case CMD_DELETE: - ExecBSDeleteTriggers(node->ps.state, node->resultRelInfo); + ExecBSDeleteTriggers(node->ps.state, resultRelInfo); break; default: elog(ERROR, "unknown operation"); @@ -1354,19 +1364,29 @@ fireBSTriggers(ModifyTableState *node) static void fireASTriggers(ModifyTableState *node) { + ResultRelInfo *resultRelInfo = node->resultRelInfo; + + /* + * If the node modifies a partitioned table, we must fire its triggers. + * Note that in that case, node->resultRelInfo points to the first leaf + * partition, not the root table. + */ + if (node->rootResultRelInfo != NULL) + resultRelInfo = node->rootResultRelInfo; + switch (node->operation) { case CMD_INSERT: if (node->mt_onconflict == ONCONFLICT_UPDATE) ExecASUpdateTriggers(node->ps.state, - node->resultRelInfo); - ExecASInsertTriggers(node->ps.state, node->resultRelInfo); + resultRelInfo); + ExecASInsertTriggers(node->ps.state, resultRelInfo); break; case CMD_UPDATE: - ExecASUpdateTriggers(node->ps.state, node->resultRelInfo); + ExecASUpdateTriggers(node->ps.state, resultRelInfo); break; case CMD_DELETE: - ExecASDeleteTriggers(node->ps.state, node->resultRelInfo); + ExecASDeleteTriggers(node->ps.state, resultRelInfo); break; default: elog(ERROR, "unknown operation"); @@ -1652,6 +1672,12 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags) mtstate->mt_plans = (PlanState **) palloc0(sizeof(PlanState *) * nplans); mtstate->resultRelInfo = estate->es_result_relations + node->resultRelIndex; + + /* If modifying a partitioned table, initialize the root table info */ + if (node->rootResultRelIndex >= 0) + mtstate->rootResultRelInfo = estate->es_root_result_relations + + node->rootResultRelIndex; + mtstate->mt_arowmarks = (List **) palloc0(sizeof(List *) * nplans); mtstate->mt_nplans = nplans; mtstate->mt_onconflict = node->onConflictAction; diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index 8fb872d288..35a237a000 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -91,6 +91,7 @@ _copyPlannedStmt(const PlannedStmt *from) COPY_NODE_FIELD(rtable); COPY_NODE_FIELD(resultRelations); COPY_NODE_FIELD(nonleafResultRelations); + COPY_NODE_FIELD(rootResultRelations); COPY_NODE_FIELD(subplans); COPY_BITMAPSET_FIELD(rewindPlanIDs); COPY_NODE_FIELD(rowMarks); @@ -205,6 +206,7 @@ _copyModifyTable(const ModifyTable *from) COPY_NODE_FIELD(partitioned_rels); COPY_NODE_FIELD(resultRelations); COPY_SCALAR_FIELD(resultRelIndex); + COPY_SCALAR_FIELD(rootResultRelIndex); COPY_NODE_FIELD(plans); COPY_NODE_FIELD(withCheckOptionLists); COPY_NODE_FIELD(returningLists); diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 05a78b32b7..98f67681a7 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -253,6 +253,7 @@ _outPlannedStmt(StringInfo str, const PlannedStmt *node) WRITE_NODE_FIELD(rtable); WRITE_NODE_FIELD(resultRelations); WRITE_NODE_FIELD(nonleafResultRelations); + WRITE_NODE_FIELD(rootResultRelations); WRITE_NODE_FIELD(subplans); WRITE_BITMAPSET_FIELD(rewindPlanIDs); WRITE_NODE_FIELD(rowMarks); @@ -350,6 +351,7 @@ _outModifyTable(StringInfo str, const ModifyTable *node) WRITE_NODE_FIELD(partitioned_rels); WRITE_NODE_FIELD(resultRelations); WRITE_INT_FIELD(resultRelIndex); + WRITE_INT_FIELD(rootResultRelIndex); WRITE_NODE_FIELD(plans); WRITE_NODE_FIELD(withCheckOptionLists); WRITE_NODE_FIELD(returningLists); @@ -2145,6 +2147,7 @@ _outPlannerGlobal(StringInfo str, const PlannerGlobal *node) WRITE_NODE_FIELD(finalrowmarks); WRITE_NODE_FIELD(resultRelations); WRITE_NODE_FIELD(nonleafResultRelations); + WRITE_NODE_FIELD(rootResultRelations); WRITE_NODE_FIELD(relationOids); WRITE_NODE_FIELD(invalItems); WRITE_INT_FIELD(nParamExec); diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index a883220a49..f9a227e237 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -1453,6 +1453,7 @@ _readPlannedStmt(void) READ_NODE_FIELD(rtable); READ_NODE_FIELD(resultRelations); READ_NODE_FIELD(nonleafResultRelations); + READ_NODE_FIELD(rootResultRelations); READ_NODE_FIELD(subplans); READ_BITMAPSET_FIELD(rewindPlanIDs); READ_NODE_FIELD(rowMarks); @@ -1548,6 +1549,7 @@ _readModifyTable(void) READ_NODE_FIELD(partitioned_rels); READ_NODE_FIELD(resultRelations); READ_INT_FIELD(resultRelIndex); + READ_INT_FIELD(rootResultRelIndex); READ_NODE_FIELD(plans); READ_NODE_FIELD(withCheckOptionLists); READ_NODE_FIELD(returningLists); diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index 95e6eb7d28..52daf43c81 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -6437,6 +6437,7 @@ make_modifytable(PlannerInfo *root, node->partitioned_rels = partitioned_rels; node->resultRelations = resultRelations; node->resultRelIndex = -1; /* will be set correctly in setrefs.c */ + node->rootResultRelIndex = -1; /* will be set correctly in setrefs.c */ node->plans = subplans; if (!onconflict) { diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index 649a233e11..c4a5651abd 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -240,6 +240,7 @@ standard_planner(Query *parse, int cursorOptions, ParamListInfo boundParams) glob->finalrowmarks = NIL; glob->resultRelations = NIL; glob->nonleafResultRelations = NIL; + glob->rootResultRelations = NIL; glob->relationOids = NIL; glob->invalItems = NIL; glob->nParamExec = 0; @@ -408,6 +409,7 @@ standard_planner(Query *parse, int cursorOptions, ParamListInfo boundParams) Assert(glob->finalrowmarks == NIL); Assert(glob->resultRelations == NIL); Assert(glob->nonleafResultRelations == NIL); + Assert(glob->rootResultRelations == NIL); top_plan = set_plan_references(root, top_plan); /* ... and the subplans (both regular subplans and initplans) */ Assert(list_length(glob->subplans) == list_length(glob->subroots)); @@ -434,6 +436,7 @@ standard_planner(Query *parse, int cursorOptions, ParamListInfo boundParams) result->rtable = glob->finalrtable; result->resultRelations = glob->resultRelations; result->nonleafResultRelations = glob->nonleafResultRelations; + result->rootResultRelations = glob->rootResultRelations; result->subplans = glob->subplans; result->rewindPlanIDs = glob->rewindPlanIDs; result->rowMarks = glob->finalrowmarks; diff --git a/src/backend/optimizer/plan/setrefs.c b/src/backend/optimizer/plan/setrefs.c index 1278371b65..c192dc4f70 100644 --- a/src/backend/optimizer/plan/setrefs.c +++ b/src/backend/optimizer/plan/setrefs.c @@ -882,11 +882,22 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) /* * If the main target relation is a partitioned table, the * following list contains the RT indexes of partitioned child - * relations, which are not included in the above list. + * relations including the root, which are not included in the + * above list. We also keep RT indexes of the roots separately + * to be identitied as such during the executor initialization. */ - root->glob->nonleafResultRelations = - list_concat(root->glob->nonleafResultRelations, - list_copy(splan->partitioned_rels)); + if (splan->partitioned_rels != NIL) + { + root->glob->nonleafResultRelations = + list_concat(root->glob->nonleafResultRelations, + list_copy(splan->partitioned_rels)); + /* Remember where this root will be in the global list. */ + splan->rootResultRelIndex = + list_length(root->glob->rootResultRelations); + root->glob->rootResultRelations = + lappend_int(root->glob->rootResultRelations, + linitial_int(splan->partitioned_rels)); + } } break; case T_Append: diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 4330a851c3..f289f3c3c2 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -422,6 +422,16 @@ typedef struct EState int es_num_result_relations; /* length of array */ ResultRelInfo *es_result_relation_info; /* currently active array elt */ + /* + * Info about the target partitioned target table root(s) for + * update/delete queries. They required only to fire any per-statement + * triggers defined on the table. It exists separately from + * es_result_relations, because partitioned tables don't appear in the + * plan tree for the update/delete cases. + */ + ResultRelInfo *es_root_result_relations; /* array of ResultRelInfos */ + int es_num_root_result_relations; /* length of the array */ + /* Stuff used for firing triggers: */ List *es_trig_target_relations; /* trigger-only ResultRelInfos */ TupleTableSlot *es_trig_tuple_slot; /* for trigger output tuples */ @@ -914,6 +924,8 @@ typedef struct ModifyTableState int mt_nplans; /* number of plans in the array */ int mt_whichplan; /* which one is being executed (0..n-1) */ ResultRelInfo *resultRelInfo; /* per-subplan target relations */ + ResultRelInfo *rootResultRelInfo; /* root target relation (partitioned + * table root) */ List **mt_arowmarks; /* per-subplan ExecAuxRowMark lists */ EPQState mt_epqstate; /* for evaluating EvalPlanQual rechecks */ bool fireBSTriggers; /* do we need to fire stmt triggers? */ diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h index cba915572e..164105a3a9 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h @@ -65,9 +65,19 @@ typedef struct PlannedStmt /* rtable indexes of target relations for INSERT/UPDATE/DELETE */ List *resultRelations; /* integer list of RT indexes, or NIL */ - /* rtable indexes of non-leaf target relations for INSERT/UPDATE/DELETE */ + /* + * rtable indexes of non-leaf target relations for UPDATE/DELETE on + * all the partitioned table mentioned in the query. + */ List *nonleafResultRelations; + /* + * rtable indexes of root target relations for UPDATE/DELETE; this list + * maintains a subset of the RT indexes in nonleafResultRelations, + * indicating the roots of the respective partition hierarchies. + */ + List *rootResultRelations; + List *subplans; /* Plan trees for SubPlan expressions; note * that some could be NULL */ @@ -211,6 +221,7 @@ typedef struct ModifyTable List *partitioned_rels; List *resultRelations; /* integer list of RT indexes */ int resultRelIndex; /* index of first resultRel in plan's list */ + int rootResultRelIndex; /* index of the partitioned table root */ List *plans; /* plan(s) producing source data */ List *withCheckOptionLists; /* per-target-table WCO lists */ List *returningLists; /* per-target-table RETURNING tlists */ diff --git a/src/include/nodes/relation.h b/src/include/nodes/relation.h index 7a8e2fd2b8..adbd3dd556 100644 --- a/src/include/nodes/relation.h +++ b/src/include/nodes/relation.h @@ -108,6 +108,7 @@ typedef struct PlannerGlobal List *resultRelations; /* "flat" list of integer RT indexes */ List *nonleafResultRelations; /* "flat" list of integer RT indexes */ + List *rootResultRelations; /* "flat" list of integer RT indexes */ List *relationOids; /* OIDs of relations the plan depends on */ diff --git a/src/test/regress/expected/triggers.out b/src/test/regress/expected/triggers.out index 4b0b3b7c42..10a301310b 100644 --- a/src/test/regress/expected/triggers.out +++ b/src/test/regress/expected/triggers.out @@ -1787,3 +1787,84 @@ create trigger my_trigger after update on my_table_42 referencing old table as o drop trigger my_trigger on my_table_42; drop table my_table_42; drop table my_table; +-- +-- Verify that per-statement triggers are fired for partitioned tables +-- +create table parted_stmt_trig (a int) partition by list (a); +create table parted_stmt_trig1 partition of parted_stmt_trig for values in (1); +create table parted_stmt_trig2 partition of parted_stmt_trig for values in (2); +create table parted2_stmt_trig (a int) partition by list (a); +create table parted2_stmt_trig1 partition of parted2_stmt_trig for values in (1); +create table parted2_stmt_trig2 partition of parted2_stmt_trig for values in (2); +create or replace function trigger_notice() returns trigger as $$ + begin + raise notice 'trigger on % % % for %', TG_TABLE_NAME, TG_WHEN, TG_OP, TG_LEVEL; + if TG_LEVEL = 'ROW' then + return NEW; + end if; + return null; + end; + $$ language plpgsql; +-- insert/update/delete statment-level triggers on the parent +create trigger trig_ins_before before insert on parted_stmt_trig + for each statement execute procedure trigger_notice(); +create trigger trig_ins_after after insert on parted_stmt_trig + for each statement execute procedure trigger_notice(); +create trigger trig_upd_before before update on parted_stmt_trig + for each statement execute procedure trigger_notice(); +create trigger trig_upd_after after update on parted_stmt_trig + for each statement execute procedure trigger_notice(); +create trigger trig_del_before before delete on parted_stmt_trig + for each statement execute procedure trigger_notice(); +create trigger trig_del_after after delete on parted_stmt_trig + for each statement execute procedure trigger_notice(); +-- insert/update/delete row-level triggers on the first partition +create trigger trig_ins_before before insert on parted_stmt_trig1 + for each row execute procedure trigger_notice(); +create trigger trig_ins_after after insert on parted_stmt_trig1 + for each row execute procedure trigger_notice(); +create trigger trig_upd_before before update on parted_stmt_trig1 + for each row execute procedure trigger_notice(); +create trigger trig_upd_after after update on parted_stmt_trig1 + for each row execute procedure trigger_notice(); +-- insert/update/delete statement-level triggers on the parent +create trigger trig_ins_before before insert on parted2_stmt_trig + for each statement execute procedure trigger_notice(); +create trigger trig_ins_after after insert on parted2_stmt_trig + for each statement execute procedure trigger_notice(); +create trigger trig_upd_before before update on parted2_stmt_trig + for each statement execute procedure trigger_notice(); +create trigger trig_upd_after after update on parted2_stmt_trig + for each statement execute procedure trigger_notice(); +create trigger trig_del_before before delete on parted2_stmt_trig + for each statement execute procedure trigger_notice(); +create trigger trig_del_after after delete on parted2_stmt_trig + for each statement execute procedure trigger_notice(); +with ins (a) as ( + insert into parted2_stmt_trig values (1), (2) returning a +) insert into parted_stmt_trig select a from ins returning tableoid::regclass, a; +NOTICE: trigger on parted_stmt_trig BEFORE INSERT for STATEMENT +NOTICE: trigger on parted2_stmt_trig BEFORE INSERT for STATEMENT +NOTICE: trigger on parted_stmt_trig1 BEFORE INSERT for ROW +NOTICE: trigger on parted_stmt_trig1 AFTER INSERT for ROW +NOTICE: trigger on parted2_stmt_trig AFTER INSERT for STATEMENT +NOTICE: trigger on parted_stmt_trig AFTER INSERT for STATEMENT + tableoid | a +-------------------+--- + parted_stmt_trig1 | 1 + parted_stmt_trig2 | 2 +(2 rows) + +with upd as ( + update parted2_stmt_trig set a = a +) update parted_stmt_trig set a = a; +NOTICE: trigger on parted_stmt_trig BEFORE UPDATE for STATEMENT +NOTICE: trigger on parted_stmt_trig1 BEFORE UPDATE for ROW +NOTICE: trigger on parted2_stmt_trig BEFORE UPDATE for STATEMENT +NOTICE: trigger on parted_stmt_trig1 AFTER UPDATE for ROW +NOTICE: trigger on parted_stmt_trig AFTER UPDATE for STATEMENT +NOTICE: trigger on parted2_stmt_trig AFTER UPDATE for STATEMENT +delete from parted_stmt_trig; +NOTICE: trigger on parted_stmt_trig BEFORE DELETE for STATEMENT +NOTICE: trigger on parted_stmt_trig AFTER DELETE for STATEMENT +drop table parted_stmt_trig, parted2_stmt_trig; diff --git a/src/test/regress/sql/triggers.sql b/src/test/regress/sql/triggers.sql index 4473ce0518..84b5ada554 100644 --- a/src/test/regress/sql/triggers.sql +++ b/src/test/regress/sql/triggers.sql @@ -1263,3 +1263,73 @@ create trigger my_trigger after update on my_table_42 referencing old table as o drop trigger my_trigger on my_table_42; drop table my_table_42; drop table my_table; + +-- +-- Verify that per-statement triggers are fired for partitioned tables +-- +create table parted_stmt_trig (a int) partition by list (a); +create table parted_stmt_trig1 partition of parted_stmt_trig for values in (1); +create table parted_stmt_trig2 partition of parted_stmt_trig for values in (2); + +create table parted2_stmt_trig (a int) partition by list (a); +create table parted2_stmt_trig1 partition of parted2_stmt_trig for values in (1); +create table parted2_stmt_trig2 partition of parted2_stmt_trig for values in (2); + +create or replace function trigger_notice() returns trigger as $$ + begin + raise notice 'trigger on % % % for %', TG_TABLE_NAME, TG_WHEN, TG_OP, TG_LEVEL; + if TG_LEVEL = 'ROW' then + return NEW; + end if; + return null; + end; + $$ language plpgsql; + +-- insert/update/delete statment-level triggers on the parent +create trigger trig_ins_before before insert on parted_stmt_trig + for each statement execute procedure trigger_notice(); +create trigger trig_ins_after after insert on parted_stmt_trig + for each statement execute procedure trigger_notice(); +create trigger trig_upd_before before update on parted_stmt_trig + for each statement execute procedure trigger_notice(); +create trigger trig_upd_after after update on parted_stmt_trig + for each statement execute procedure trigger_notice(); +create trigger trig_del_before before delete on parted_stmt_trig + for each statement execute procedure trigger_notice(); +create trigger trig_del_after after delete on parted_stmt_trig + for each statement execute procedure trigger_notice(); + +-- insert/update/delete row-level triggers on the first partition +create trigger trig_ins_before before insert on parted_stmt_trig1 + for each row execute procedure trigger_notice(); +create trigger trig_ins_after after insert on parted_stmt_trig1 + for each row execute procedure trigger_notice(); +create trigger trig_upd_before before update on parted_stmt_trig1 + for each row execute procedure trigger_notice(); +create trigger trig_upd_after after update on parted_stmt_trig1 + for each row execute procedure trigger_notice(); + +-- insert/update/delete statement-level triggers on the parent +create trigger trig_ins_before before insert on parted2_stmt_trig + for each statement execute procedure trigger_notice(); +create trigger trig_ins_after after insert on parted2_stmt_trig + for each statement execute procedure trigger_notice(); +create trigger trig_upd_before before update on parted2_stmt_trig + for each statement execute procedure trigger_notice(); +create trigger trig_upd_after after update on parted2_stmt_trig + for each statement execute procedure trigger_notice(); +create trigger trig_del_before before delete on parted2_stmt_trig + for each statement execute procedure trigger_notice(); +create trigger trig_del_after after delete on parted2_stmt_trig + for each statement execute procedure trigger_notice(); + +with ins (a) as ( + insert into parted2_stmt_trig values (1), (2) returning a +) insert into parted_stmt_trig select a from ins returning tableoid::regclass, a; + +with upd as ( + update parted2_stmt_trig set a = a +) update parted_stmt_trig set a = a; + +delete from parted_stmt_trig; +drop table parted_stmt_trig, parted2_stmt_trig; -- cgit v1.2.3 From 34fc61673829f0cb63811fb63cc3c510fc2aadfc Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Tue, 2 May 2017 11:12:30 +0200 Subject: Change hot_standby default value to 'on' This goes together with the changes made to enable replication on the sending side by default (wal_level, max_wal_senders etc) by making the receiving stadby node also enable it by default. Huong Dangminh --- doc/src/sgml/config.sgml | 2 +- doc/src/sgml/high-availability.sgml | 4 ++-- src/backend/utils/misc/guc.c | 2 +- src/backend/utils/misc/postgresql.conf.sample | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index e02b0c80df..0b9e3002fb 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3227,7 +3227,7 @@ ANY num_sync ( . - The default value is off. + The default value is on. This parameter can only be set at server start. It only has effect during archive recovery or in standby mode. diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml index 9e2be5f67c..22ac10f428 100644 --- a/doc/src/sgml/high-availability.sgml +++ b/doc/src/sgml/high-availability.sgml @@ -2057,8 +2057,8 @@ if (!triggered) Administrator's Overview - If hot_standby is turned on in - postgresql.conf and there is a recovery.conf + If hot_standby is on in postgresql.conf + (the default value) and there is a recovery.conf file present, the server will run in Hot Standby mode. However, it may take some time for Hot Standby connections to be allowed, because the server will not accept connections until it has completed diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index a414fb2c76..587fbce147 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -1571,7 +1571,7 @@ static struct config_bool ConfigureNamesBool[] = NULL }, &EnableHotStandby, - false, + true, NULL, NULL, NULL }, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index c02f7f3645..fceef14c78 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -254,7 +254,7 @@ # These settings are ignored on a master server. -#hot_standby = off # "on" allows queries during recovery +#hot_standby = on # "off" disallows queries during recovery # (change requires restart) #max_standby_archive_delay = 30s # max delay before canceling queries # when reading WAL from archive; -- cgit v1.2.3 From a35ac7c4e3ccf93876b4652d94a418fc82e0eda3 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 2 May 2017 10:34:49 -0400 Subject: doc: Update ALTER SEQUENCE claims about changes being nontransactional Clarify that all changes except RESTART are transactional (since 1753b1b027035029c2a2a1649065762fafbf63f3). Reported-by: Michael Paquier --- doc/src/sgml/ref/alter_sequence.sgml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/alter_sequence.sgml b/doc/src/sgml/ref/alter_sequence.sgml index 5c912ab892..3fb3400d6b 100644 --- a/doc/src/sgml/ref/alter_sequence.sgml +++ b/doc/src/sgml/ref/alter_sequence.sgml @@ -180,6 +180,14 @@ ALTER SEQUENCE [ IF EXISTS ] name S the start value that was recorded by CREATE SEQUENCE or last set by ALTER SEQUENCE START WITH. + + + Like a setval call, a RESTART + operation on a sequence is never rolled back, to avoid blocking of + concurrent transactions that obtain numbers from the same sequence. + (The other clauses cause ordinary catalog updates that can be rolled + back.) + @@ -281,15 +289,6 @@ ALTER SEQUENCE [ IF EXISTS ] name S Notes - - To avoid blocking of concurrent transactions that obtain numbers from the - same sequence, ALTER SEQUENCE's effects on the sequence - generation parameters are never rolled back; those changes take effect - immediately and are not reversible. However, the OWNED BY, - OWNER TO, RENAME TO, and SET SCHEMA - clauses cause ordinary catalog updates that can be rolled back. - - ALTER SEQUENCE will not immediately affect nextval results in backends, -- cgit v1.2.3 From 460c89f46c1fdf11baa8e76e6d04e1ff87d7e008 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 2 May 2017 14:33:19 -0400 Subject: doc: Add missing markup --- doc/src/sgml/datatype.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/datatype.sgml b/doc/src/sgml/datatype.sgml index 7a546a0399..c96374a201 100644 --- a/doc/src/sgml/datatype.sgml +++ b/doc/src/sgml/datatype.sgml @@ -943,7 +943,7 @@ ALTER SEQUENCE tablename_ - money + money 8 bytes currency amount -92233720368547758.08 to +92233720368547758.07 -- cgit v1.2.3 From e9500240661c03750923e6f539bfa2d75cfaa32a Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 2 May 2017 15:29:30 -0400 Subject: doc: Improve order in ALTER PUBLICATION/SUBSCRIPTION ref pages Move the OWNER and RENAME clauses to the end, so the interesting functionality is listed first. This is more typical on nearby reference pages, whereas the previous order was the order in which the clauses were added. --- doc/src/sgml/ref/alter_publication.sgml | 40 ++++++++++++++++---------------- doc/src/sgml/ref/alter_subscription.sgml | 39 +++++++++++++++---------------- 2 files changed, 39 insertions(+), 40 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/alter_publication.sgml b/doc/src/sgml/ref/alter_publication.sgml index 858231fbcb..05bd57d9ca 100644 --- a/doc/src/sgml/ref/alter_publication.sgml +++ b/doc/src/sgml/ref/alter_publication.sgml @@ -29,11 +29,11 @@ ALTER PUBLICATION name WITH ( name     OWNER TO { new_owner | CURRENT_USER | SESSION_USER } -ALTER PUBLICATION name RENAME TO new_name ALTER PUBLICATION name ADD TABLE [ ONLY ] table_name [ * ] [, ...] ALTER PUBLICATION name SET TABLE [ ONLY ] table_name [ * ] [, ...] ALTER PUBLICATION name DROP TABLE [ ONLY ] table_name [ * ] [, ...] +ALTER PUBLICATION name OWNER TO { new_owner | CURRENT_USER | SESSION_USER } +ALTER PUBLICATION name RENAME TO new_name @@ -79,24 +79,6 @@ ALTER PUBLICATION name DROP TABLE [ - - new_owner - - - The user name of the new owner of the publication. - - - - - - new_name - - - The new name for the publication. - - - - PUBLISH INSERT NOPUBLISH INSERT @@ -124,6 +106,24 @@ ALTER PUBLICATION name DROP TABLE [ + + + new_owner + + + The user name of the new owner of the publication. + + + + + + new_name + + + The new name for the publication. + + + diff --git a/doc/src/sgml/ref/alter_subscription.sgml b/doc/src/sgml/ref/alter_subscription.sgml index 35aeb6af41..5dae4aebd6 100644 --- a/doc/src/sgml/ref/alter_subscription.sgml +++ b/doc/src/sgml/ref/alter_subscription.sgml @@ -35,11 +35,11 @@ ALTER SUBSCRIPTION name REFRESH PUB COPY DATA | NOCOPY DATA -ALTER SUBSCRIPTION name OWNER TO { new_owner | CURRENT_USER | SESSION_USER } -ALTER SUBSCRIPTION name RENAME TO new_name ALTER SUBSCRIPTION name CONNECTION 'conninfo' ALTER SUBSCRIPTION name ENABLE ALTER SUBSCRIPTION name DISABLE +ALTER SUBSCRIPTION name OWNER TO { new_owner | CURRENT_USER | SESSION_USER } +ALTER SUBSCRIPTION name RENAME TO new_name @@ -71,24 +71,6 @@ ALTER SUBSCRIPTION name DISABLE - - new_owner - - - The user name of the new owner of the subscription. - - - - - - new_name - - - The new name for the subscription. - - - - CONNECTION 'conninfo' SLOT NAME = slot_name @@ -156,6 +138,23 @@ ALTER SUBSCRIPTION name DISABLE + + new_owner + + + The user name of the new owner of the subscription. + + + + + + new_name + + + The new name for the subscription. + + + -- cgit v1.2.3 From 93bbeec6a21b76612d77176a8054b41277135684 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Tue, 2 May 2017 18:49:32 -0300 Subject: extstats: change output functions to emit valid JSON Manipulating extended statistics is more convenient as JSON than the current ad-hoc format, so let's change before it's too late. Discussion: https://postgr.es/m/20170420193828.k3fliiock5hdnehn@alvherre.pgsql --- doc/src/sgml/perform.sgml | 11 +++++------ src/backend/statistics/dependencies.c | 23 +++++++---------------- src/backend/statistics/mvdistinct.c | 15 ++++++++++----- src/test/regress/expected/stats_ext.out | 12 ++++++------ 4 files changed, 28 insertions(+), 33 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/perform.sgml b/doc/src/sgml/perform.sgml index a8bebb1436..b10b734b90 100644 --- a/doc/src/sgml/perform.sgml +++ b/doc/src/sgml/perform.sgml @@ -1138,9 +1138,9 @@ ANALYZE zipcodes; SELECT stxname, stxkeys, stxdependencies FROM pg_statistic_ext WHERE stxname = 'stts'; - stxname | stxkeys | stxdependencies ----------+---------+-------------------------------------------- - stts | 1 5 | [{1 => 5 : 1.000000}, {5 => 1 : 0.423130}] + stxname | stxkeys | stxdependencies +---------+---------+------------------------------------------ + stts | 1 5 | {"1 => 5": 1.000000, "5 => 1": 0.423130} (1 row) where it can be seen that column 1 (a zip code) fully determines column @@ -1225,10 +1225,9 @@ ANALYZE zipcodes; SELECT stxkeys AS k, stxndistinct AS nd FROM pg_statistic_ext WHERE stxname = 'stts2'; --[ RECORD 1 ]--------------------------------------------- +-[ RECORD 1 ]-------------------------------------------------------- k | 1 2 5 -nd | [{(b 1 2), 33178.000000}, {(b 1 5), 33178.000000}, - {(b 2 5), 27435.000000}, {(b 1 2 5), 33178.000000}] +nd | {"1, 2": 33178, "1, 5": 33178, "2, 5": 27435, "1, 2, 5": 33178} (1 row) which indicates that there are three combinations of columns that diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c index 0890514bf7..fe9a9ef5de 100644 --- a/src/backend/statistics/dependencies.c +++ b/src/backend/statistics/dependencies.c @@ -624,7 +624,7 @@ dependency_is_fully_matched(MVDependency * dependency, Bitmapset *attnums) * check that the attnum matches is implied by the functional dependency */ static bool -dependency_implies_attribute(MVDependency * dependency, AttrNumber attnum) +dependency_implies_attribute(MVDependency *dependency, AttrNumber attnum) { if (attnum == dependency->attributes[dependency->nattributes - 1]) return true; @@ -641,11 +641,6 @@ staext_dependencies_load(Oid mvoid) { bool isnull; Datum deps; - - /* - * Prepare to scan pg_statistic_ext for entries having indrelid = this - * rel. - */ HeapTuple htup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(mvoid)); if (!HeapTupleIsValid(htup)) @@ -653,7 +648,6 @@ staext_dependencies_load(Oid mvoid) deps = SysCacheGetAttr(STATEXTOID, htup, Anum_pg_statistic_ext_stxdependencies, &isnull); - Assert(!isnull); ReleaseSysCache(htup); @@ -687,16 +681,14 @@ pg_dependencies_in(PG_FUNCTION_ARGS) Datum pg_dependencies_out(PG_FUNCTION_ARGS) { + bytea *data = PG_GETARG_BYTEA_PP(0); + MVDependencies *dependencies = statext_dependencies_deserialize(data); int i, j; StringInfoData str; - bytea *data = PG_GETARG_BYTEA_PP(0); - - MVDependencies *dependencies = statext_dependencies_deserialize(data); - initStringInfo(&str); - appendStringInfoChar(&str, '['); + appendStringInfoChar(&str, '{'); for (i = 0; i < dependencies->ndeps; i++) { @@ -705,7 +697,7 @@ pg_dependencies_out(PG_FUNCTION_ARGS) if (i > 0) appendStringInfoString(&str, ", "); - appendStringInfoChar(&str, '{'); + appendStringInfoChar(&str, '"'); for (j = 0; j < dependency->nattributes; j++) { if (j == dependency->nattributes - 1) @@ -715,11 +707,10 @@ pg_dependencies_out(PG_FUNCTION_ARGS) appendStringInfo(&str, "%d", dependency->attributes[j]); } - appendStringInfo(&str, " : %f", dependency->degree); - appendStringInfoChar(&str, '}'); + appendStringInfo(&str, "\": %f", dependency->degree); } - appendStringInfoChar(&str, ']'); + appendStringInfoChar(&str, '}'); PG_RETURN_CSTRING(str.data); } diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c index b77113fb39..f67f576236 100644 --- a/src/backend/statistics/mvdistinct.c +++ b/src/backend/statistics/mvdistinct.c @@ -354,21 +354,26 @@ pg_ndistinct_out(PG_FUNCTION_ARGS) StringInfoData str; initStringInfo(&str); - appendStringInfoChar(&str, '['); + appendStringInfoChar(&str, '{'); for (i = 0; i < ndist->nitems; i++) { MVNDistinctItem item = ndist->items[i]; + int x = -1; + bool first = true; if (i > 0) appendStringInfoString(&str, ", "); - appendStringInfoChar(&str, '{'); - outBitmapset(&str, item.attrs); - appendStringInfo(&str, ", %f}", item.ndistinct); + while ((x = bms_next_member(item.attrs, x)) >= 0) + { + appendStringInfo(&str, "%s%d", first ? "\"" : ", ", x); + first = false; + } + appendStringInfo(&str, "\": %d", (int) item.ndistinct); } - appendStringInfoChar(&str, ']'); + appendStringInfoChar(&str, '}'); PG_RETURN_CSTRING(str.data); } diff --git a/src/test/regress/expected/stats_ext.out b/src/test/regress/expected/stats_ext.out index 72b1014195..92ac84ac67 100644 --- a/src/test/regress/expected/stats_ext.out +++ b/src/test/regress/expected/stats_ext.out @@ -175,9 +175,9 @@ CREATE STATISTICS s10 ON (a, b, c) FROM ndistinct; ANALYZE ndistinct; SELECT stxkind, stxndistinct FROM pg_statistic_ext WHERE stxrelid = 'ndistinct'::regclass; - stxkind | stxndistinct ----------+------------------------------------------------------------------------------------------------ - {d,f} | [{(b 3 4), 301.000000}, {(b 3 6), 301.000000}, {(b 4 6), 301.000000}, {(b 3 4 6), 301.000000}] + stxkind | stxndistinct +---------+--------------------------------------------------------- + {d,f} | {"3, 4": 301, "3, 6": 301, "4, 6": 301, "3, 4, 6": 301} (1 row) -- Hash Aggregate, thanks to estimates improved by the statistic @@ -241,9 +241,9 @@ INSERT INTO ndistinct (a, b, c, filler1) ANALYZE ndistinct; SELECT stxkind, stxndistinct FROM pg_statistic_ext WHERE stxrelid = 'ndistinct'::regclass; - stxkind | stxndistinct ----------+---------------------------------------------------------------------------------------------------- - {d,f} | [{(b 3 4), 2550.000000}, {(b 3 6), 800.000000}, {(b 4 6), 1632.000000}, {(b 3 4 6), 10000.000000}] + stxkind | stxndistinct +---------+------------------------------------------------------------- + {d,f} | {"3, 4": 2550, "3, 6": 800, "4, 6": 1632, "3, 4, 6": 10000} (1 row) -- plans using Group Aggregate, thanks to using correct esimates -- cgit v1.2.3 From 8f8b9be51fd788bb11276df89606bc653163524e Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 3 May 2017 11:19:07 +0300 Subject: Add PQencryptPasswordConn function to libpq, use it in psql and createuser. The new function supports creating SCRAM verifiers, in addition to md5 hashes. The algorithm is chosen based on password_encryption, by default. This fixes the issue reported by Jeff Janes, that there was previously no way to create a SCRAM verifier with "\password". Michael Paquier and me Discussion: https://www.postgresql.org/message-id/CAMkU%3D1wfBgFPbfAMYZQE78p%3DVhZX7nN86aWkp0QcCp%3D%2BKxZ%3Dbg%40mail.gmail.com --- doc/src/sgml/libpq.sgml | 67 ++++++++++++++++--- src/backend/libpq/auth-scram.c | 51 ++------------ src/backend/libpq/crypt.c | 2 +- src/bin/psql/command.c | 4 +- src/bin/scripts/createuser.c | 9 ++- src/common/scram-common.c | 64 ++++++++++++++++++ src/include/common/scram-common.h | 3 + src/include/libpq/scram.h | 4 +- src/interfaces/libpq/exports.txt | 1 + src/interfaces/libpq/fe-auth-scram.c | 35 ++++++++++ src/interfaces/libpq/fe-auth.c | 125 +++++++++++++++++++++++++++++++---- src/interfaces/libpq/fe-auth.h | 1 + src/interfaces/libpq/libpq-fe.h | 1 + 13 files changed, 291 insertions(+), 76 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/libpq.sgml b/doc/src/sgml/libpq.sgml index 4bc5bf3192..4f60b203fb 100644 --- a/doc/src/sgml/libpq.sgml +++ b/doc/src/sgml/libpq.sgml @@ -5875,11 +5875,11 @@ void PQconninfoFree(PQconninfoOption *connOptions); - + - PQencryptPassword + PQencryptPasswordConn - PQencryptPassword + PQencryptPasswordConn @@ -5887,20 +5887,65 @@ void PQconninfoFree(PQconninfoOption *connOptions); Prepares the encrypted form of a PostgreSQL password. -char * PQencryptPassword(const char *passwd, const char *user); +char *PQencryptPasswordConn(PGconn *conn, const char *passwd, const char *user, const char *algorithm); This function is intended to be used by client applications that wish to send commands like ALTER USER joe PASSWORD 'pwd'. It is good practice not to send the original cleartext password in such a command, because it might be exposed in command logs, activity displays, and so on. Instead, use this function to - convert the password to encrypted form before it is sent. The - arguments are the cleartext password, and the SQL name of the user - it is for. The return value is a string allocated by - malloc, or NULL if out of - memory. The caller can assume the string doesn't contain any - special characters that would require escaping. Use - PQfreemem to free the result when done with it. + convert the password to encrypted form before it is sent. + + + + The passwd and user arguments + are the cleartext password, and the SQL name of the user it is for. + algorithm specifies the encryption algorithm + to use to encrypt the password. Currently supported algorithms are + md5, scram-sha-256 and plain. + scram-sha-256 was introduced in PostgreSQL + version 10, and will not work correctly with older server versions. If + algorithm is NULL, this function will query + the server for the current value of the + setting. That can block, and + will fail if the current transaction is aborted, or if the connection + is busy executing another query. If you wish to use the default + algorithm for the server but want to avoid blocking, query + password_encryption yourself before calling + PQencryptPasswordConn, and pass that value as the + algorithm. + + + + The return value is a string allocated by malloc. + The caller can assume the string doesn't contain any special characters + that would require escaping. Use PQfreemem to free the + result when done with it. On error, returns NULL, and + a suitable message is stored in the connection object. + + + + + + + + PQencryptPassword + + PQencryptPassword + + + + + + Prepares the md5-encrypted form of a PostgreSQL password. + +char *PQencryptPassword(const char *passwd, const char *user); + + PQencryptPassword is an older, deprecated version of + PQencryptPasswodConn. The difference is that + PQencryptPassword does not + require a connection object, and md5 is always used as the + encryption algorithm. diff --git a/src/backend/libpq/auth-scram.c b/src/backend/libpq/auth-scram.c index 5c85af943c..6e7a140582 100644 --- a/src/backend/libpq/auth-scram.c +++ b/src/backend/libpq/auth-scram.c @@ -207,7 +207,7 @@ pg_be_scram_init(const char *username, const char *shadow_pass) */ char *verifier; - verifier = scram_build_verifier(username, shadow_pass, 0); + verifier = pg_be_scram_build_verifier(shadow_pass); (void) parse_scram_verifier(verifier, &state->iterations, &state->salt, state->StoredKey, state->ServerKey); @@ -387,22 +387,14 @@ pg_be_scram_exchange(void *opaq, char *input, int inputlen, /* * Construct a verifier string for SCRAM, stored in pg_authid.rolpassword. * - * If iterations is 0, default number of iterations is used. The result is - * palloc'd, so caller is responsible for freeing it. + * The result is palloc'd, so caller is responsible for freeing it. */ char * -scram_build_verifier(const char *username, const char *password, - int iterations) +pg_be_scram_build_verifier(const char *password) { char *prep_password = NULL; pg_saslprep_rc rc; char saltbuf[SCRAM_DEFAULT_SALT_LEN]; - uint8 salted_password[SCRAM_KEY_LEN]; - uint8 keybuf[SCRAM_KEY_LEN]; - char *encoded_salt; - char *encoded_storedkey; - char *encoded_serverkey; - int encoded_len; char *result; /* @@ -414,10 +406,7 @@ scram_build_verifier(const char *username, const char *password, if (rc == SASLPREP_SUCCESS) password = (const char *) prep_password; - if (iterations <= 0) - iterations = SCRAM_DEFAULT_ITERATIONS; - - /* Generate salt, and encode it in base64 */ + /* Generate random salt */ if (!pg_backend_random(saltbuf, SCRAM_DEFAULT_SALT_LEN)) { ereport(LOG, @@ -426,37 +415,11 @@ scram_build_verifier(const char *username, const char *password, return NULL; } - encoded_salt = palloc(pg_b64_enc_len(SCRAM_DEFAULT_SALT_LEN) + 1); - encoded_len = pg_b64_encode(saltbuf, SCRAM_DEFAULT_SALT_LEN, encoded_salt); - encoded_salt[encoded_len] = '\0'; - - /* Calculate StoredKey, and encode it in base64 */ - scram_SaltedPassword(password, saltbuf, SCRAM_DEFAULT_SALT_LEN, - iterations, salted_password); - scram_ClientKey(salted_password, keybuf); - scram_H(keybuf, SCRAM_KEY_LEN, keybuf); /* StoredKey */ - - encoded_storedkey = palloc(pg_b64_enc_len(SCRAM_KEY_LEN) + 1); - encoded_len = pg_b64_encode((const char *) keybuf, SCRAM_KEY_LEN, - encoded_storedkey); - encoded_storedkey[encoded_len] = '\0'; - - /* And same for ServerKey */ - scram_ServerKey(salted_password, keybuf); - - encoded_serverkey = palloc(pg_b64_enc_len(SCRAM_KEY_LEN) + 1); - encoded_len = pg_b64_encode((const char *) keybuf, SCRAM_KEY_LEN, - encoded_serverkey); - encoded_serverkey[encoded_len] = '\0'; - - result = psprintf("SCRAM-SHA-256$%d:%s$%s:%s", iterations, encoded_salt, - encoded_storedkey, encoded_serverkey); + result = scram_build_verifier(saltbuf, SCRAM_DEFAULT_SALT_LEN, + SCRAM_DEFAULT_ITERATIONS, password); if (prep_password) pfree(prep_password); - pfree(encoded_salt); - pfree(encoded_storedkey); - pfree(encoded_serverkey); return result; } @@ -1194,7 +1157,7 @@ scram_MockSalt(const char *username) * Generate salt using a SHA256 hash of the username and the cluster's * mock authentication nonce. (This works as long as the salt length is * not larger the SHA256 digest length. If the salt is smaller, the caller - * will just ignore the extra data)) + * will just ignore the extra data.) */ StaticAssertStmt(PG_SHA256_DIGEST_LENGTH >= SCRAM_DEFAULT_SALT_LEN, "salt length greater than SHA256 digest length"); diff --git a/src/backend/libpq/crypt.c b/src/backend/libpq/crypt.c index d0030f2b6d..9fe79b4894 100644 --- a/src/backend/libpq/crypt.c +++ b/src/backend/libpq/crypt.c @@ -156,7 +156,7 @@ encrypt_password(PasswordType target_type, const char *role, switch (guessed_type) { case PASSWORD_TYPE_PLAINTEXT: - return scram_build_verifier(role, password, 0); + return pg_be_scram_build_verifier(password); case PASSWORD_TYPE_MD5: diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c index 859ded71f6..b3263a9570 100644 --- a/src/bin/psql/command.c +++ b/src/bin/psql/command.c @@ -1878,11 +1878,11 @@ exec_command_password(PsqlScanState scan_state, bool active_branch) else user = PQuser(pset.db); - encrypted_password = PQencryptPassword(pw1, user); + encrypted_password = PQencryptPasswordConn(pset.db, pw1, user, NULL); if (!encrypted_password) { - psql_error("Password encryption failed.\n"); + psql_error("%s", PQerrorMessage(pset.db)); success = false; } else diff --git a/src/bin/scripts/createuser.c b/src/bin/scripts/createuser.c index 3d74797a8f..35a53bf206 100644 --- a/src/bin/scripts/createuser.c +++ b/src/bin/scripts/createuser.c @@ -274,11 +274,14 @@ main(int argc, char *argv[]) { char *encrypted_password; - encrypted_password = PQencryptPassword(newpassword, - newuser); + encrypted_password = PQencryptPasswordConn(conn, + newpassword, + newuser, + NULL); if (!encrypted_password) { - fprintf(stderr, _("Password encryption failed.\n")); + fprintf(stderr, _("%s: password encryption failed: %s"), + progname, PQerrorMessage(conn)); exit(1); } appendStringLiteralConn(&sql, encrypted_password, conn); diff --git a/src/common/scram-common.c b/src/common/scram-common.c index a8ea44944c..77b54c8a5e 100644 --- a/src/common/scram-common.c +++ b/src/common/scram-common.c @@ -23,6 +23,7 @@ #include #include +#include "common/base64.h" #include "common/scram-common.h" #define HMAC_IPAD 0x36 @@ -180,3 +181,66 @@ scram_ServerKey(const uint8 *salted_password, uint8 *result) scram_HMAC_update(&ctx, "Server Key", strlen("Server Key")); scram_HMAC_final(result, &ctx); } + + +/* + * Construct a verifier string for SCRAM, stored in pg_authid.rolpassword. + * + * The password should already have been processed with SASLprep, if necessary! + * + * If iterations is 0, default number of iterations is used. The result is + * palloc'd or malloc'd, so caller is responsible for freeing it. + */ +char * +scram_build_verifier(const char *salt, int saltlen, int iterations, + const char *password) +{ + uint8 salted_password[SCRAM_KEY_LEN]; + uint8 stored_key[SCRAM_KEY_LEN]; + uint8 server_key[SCRAM_KEY_LEN]; + char *result; + char *p; + int maxlen; + + if (iterations <= 0) + iterations = SCRAM_DEFAULT_ITERATIONS; + + /* Calculate StoredKey and ServerKey */ + scram_SaltedPassword(password, salt, saltlen, iterations, + salted_password); + scram_ClientKey(salted_password, stored_key); + scram_H(stored_key, SCRAM_KEY_LEN, stored_key); + + scram_ServerKey(salted_password, server_key); + + /* + * The format is: + * SCRAM-SHA-256$:$: + */ + maxlen = strlen("SCRAM-SHA-256") + 1 + + 10 + 1 /* iteration count */ + + pg_b64_enc_len(saltlen) + 1 /* Base64-encoded salt */ + + pg_b64_enc_len(SCRAM_KEY_LEN) + 1 /* Base64-encoded StoredKey */ + + pg_b64_enc_len(SCRAM_KEY_LEN) + 1; /* Base64-encoded ServerKey */ + +#ifdef FRONTEND + result = malloc(maxlen); + if (!result) + return NULL; +#else + result = palloc(maxlen); +#endif + + p = result + sprintf(result, "SCRAM-SHA-256$%d:", iterations); + + p += pg_b64_encode(salt, saltlen, p); + *(p++) = '$'; + p += pg_b64_encode((char *) stored_key, SCRAM_KEY_LEN, p); + *(p++) = ':'; + p += pg_b64_encode((char *) server_key, SCRAM_KEY_LEN, p); + *(p++) = '\0'; + + Assert(p - result <= maxlen); + + return result; +} diff --git a/src/include/common/scram-common.h b/src/include/common/scram-common.h index 656d9e1e6b..307f92b54a 100644 --- a/src/include/common/scram-common.h +++ b/src/include/common/scram-common.h @@ -53,4 +53,7 @@ extern void scram_H(const uint8 *str, int len, uint8 *result); extern void scram_ClientKey(const uint8 *salted_password, uint8 *result); extern void scram_ServerKey(const uint8 *salted_password, uint8 *result); +extern char *scram_build_verifier(const char *salt, int saltlen, int iterations, + const char *password); + #endif /* SCRAM_COMMON_H */ diff --git a/src/include/libpq/scram.h b/src/include/libpq/scram.h index e373f0c07e..060b8af69e 100644 --- a/src/include/libpq/scram.h +++ b/src/include/libpq/scram.h @@ -27,9 +27,7 @@ extern int pg_be_scram_exchange(void *opaq, char *input, int inputlen, char **output, int *outputlen, char **logdetail); /* Routines to handle and check SCRAM-SHA-256 verifier */ -extern char *scram_build_verifier(const char *username, - const char *password, - int iterations); +extern char *pg_be_scram_build_verifier(const char *password); extern bool is_scram_verifier(const char *verifier); extern bool scram_verify_plain_password(const char *username, const char *password, const char *verifier); diff --git a/src/interfaces/libpq/exports.txt b/src/interfaces/libpq/exports.txt index 21dd772ca9..d6a38d0df8 100644 --- a/src/interfaces/libpq/exports.txt +++ b/src/interfaces/libpq/exports.txt @@ -171,3 +171,4 @@ PQsslAttributeNames 168 PQsslAttribute 169 PQsetErrorContextVisibility 170 PQresultVerboseErrorMessage 171 +PQencryptPasswordConn 172 diff --git a/src/interfaces/libpq/fe-auth-scram.c b/src/interfaces/libpq/fe-auth-scram.c index be271ce8ac..52dae49abf 100644 --- a/src/interfaces/libpq/fe-auth-scram.c +++ b/src/interfaces/libpq/fe-auth-scram.c @@ -614,6 +614,41 @@ verify_server_signature(fe_scram_state *state) return true; } +/* + * Build a new SCRAM verifier. + */ +char * +pg_fe_scram_build_verifier(const char *password) +{ + char *prep_password = NULL; + pg_saslprep_rc rc; + char saltbuf[SCRAM_DEFAULT_SALT_LEN]; + char *result; + + /* + * Normalize the password with SASLprep. If that doesn't work, because + * the password isn't valid UTF-8 or contains prohibited characters, just + * proceed with the original password. (See comments at top of file.) + */ + rc = pg_saslprep(password, &prep_password); + if (rc == SASLPREP_OOM) + return NULL; + if (rc == SASLPREP_SUCCESS) + password = (const char *) prep_password; + + /* Generate a random salt */ + if (!pg_frontend_random(saltbuf, SCRAM_DEFAULT_SALT_LEN)) + return NULL; + + result = scram_build_verifier(saltbuf, SCRAM_DEFAULT_SALT_LEN, + SCRAM_DEFAULT_ITERATIONS, password); + + if (prep_password) + free(prep_password); + + return result; +} + /* * Random number generator. */ diff --git a/src/interfaces/libpq/fe-auth.c b/src/interfaces/libpq/fe-auth.c index d81ee4f944..daa7cc9585 100644 --- a/src/interfaces/libpq/fe-auth.c +++ b/src/interfaces/libpq/fe-auth.c @@ -1077,7 +1077,33 @@ pg_fe_getauthname(PQExpBuffer errorMessage) /* - * PQencryptPassword -- exported routine to encrypt a password + * PQencryptPassword -- exported routine to encrypt a password with MD5 + * + * This function is equivalent to calling PQencryptPasswordConn with + * "md5" as the encryption method, except that this doesn't require + * a connection object. This function is deprecated, use + * PQencryptPasswordConn instead. + */ +char * +PQencryptPassword(const char *passwd, const char *user) +{ + char *crypt_pwd; + + crypt_pwd = malloc(MD5_PASSWD_LEN + 1); + if (!crypt_pwd) + return NULL; + + if (!pg_md5_encrypt(passwd, user, strlen(user), crypt_pwd)) + { + free(crypt_pwd); + return NULL; + } + + return crypt_pwd; +} + +/* + * PQencryptPasswordConn -- exported routine to encrypt a password * * This is intended to be used by client applications that wish to send * commands like ALTER USER joe PASSWORD 'pwd'. The password need not @@ -1087,27 +1113,102 @@ pg_fe_getauthname(PQExpBuffer errorMessage) * be dependent on low-level details like whether the encryption is MD5 * or something else. * - * Arguments are the cleartext password, and the SQL name of the user it - * is for. + * Arguments are a connection object, the cleartext password, the SQL + * name of the user it is for, and a string indicating the algorithm to + * use for encrypting the password. If algorithm is NULL, this queries + * the server for the current 'password_encryption' value. If you wish + * to avoid that, e.g. to avoid blocking, you can execute + * 'show password_encryption' yourself before calling this function, and + * pass it as the algorithm. * - * Return value is a malloc'd string, or NULL if out-of-memory. The client - * may assume the string doesn't contain any special characters that would - * require escaping. + * Return value is a malloc'd string. The client may assume the string + * doesn't contain any special characters that would require escaping. + * On error, an error message is stored in the connection object, and + * returns NULL. */ char * -PQencryptPassword(const char *passwd, const char *user) +PQencryptPasswordConn(PGconn *conn, const char *passwd, const char *user, + const char *algorithm) { - char *crypt_pwd; +#define MAX_ALGORITHM_NAME_LEN 50 + char algobuf[MAX_ALGORITHM_NAME_LEN + 1]; + char *crypt_pwd = NULL; - crypt_pwd = malloc(MD5_PASSWD_LEN + 1); - if (!crypt_pwd) + if (!conn) return NULL; - if (!pg_md5_encrypt(passwd, user, strlen(user), crypt_pwd)) + /* If no algorithm was given, ask the server. */ + if (algorithm == NULL) { - free(crypt_pwd); + PGresult *res; + char *val; + + res = PQexec(conn, "show password_encryption"); + if (res == NULL) + { + /* PQexec() should've set conn->errorMessage already */ + return NULL; + } + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + /* PQexec() should've set conn->errorMessage already */ + PQclear(res); + return NULL; + } + if (PQntuples(res) != 1 || PQnfields(res) != 1) + { + PQclear(res); + printfPQExpBuffer(&conn->errorMessage, + libpq_gettext("unexpected shape of result set returned for SHOW\n")); + return NULL; + } + val = PQgetvalue(res, 0, 0); + + if (strlen(val) > MAX_ALGORITHM_NAME_LEN) + { + PQclear(res); + printfPQExpBuffer(&conn->errorMessage, + libpq_gettext("password_encryption value too long\n")); + return NULL; + } + strcpy(algobuf, val); + PQclear(res); + + algorithm = algobuf; + } + + /* Ok, now we know what algorithm to use */ + + if (strcmp(algorithm, "scram-sha-256") == 0) + { + crypt_pwd = pg_fe_scram_build_verifier(passwd); + } + else if (strcmp(algorithm, "md5") == 0) + { + crypt_pwd = malloc(MD5_PASSWD_LEN + 1); + if (crypt_pwd) + { + if (!pg_md5_encrypt(passwd, user, strlen(user), crypt_pwd)) + { + free(crypt_pwd); + crypt_pwd = NULL; + } + } + } + else if (strcmp(algorithm, "plain") == 0) + { + crypt_pwd = strdup(passwd); + } + else + { + printfPQExpBuffer(&conn->errorMessage, + libpq_gettext("unknown password encryption algorithm\n")); return NULL; } + if (!crypt_pwd) + printfPQExpBuffer(&conn->errorMessage, + libpq_gettext("out of memory\n")); + return crypt_pwd; } diff --git a/src/interfaces/libpq/fe-auth.h b/src/interfaces/libpq/fe-auth.h index a5c739f01a..9f4c2a50d8 100644 --- a/src/interfaces/libpq/fe-auth.h +++ b/src/interfaces/libpq/fe-auth.h @@ -28,5 +28,6 @@ extern void pg_fe_scram_free(void *opaq); extern void pg_fe_scram_exchange(void *opaq, char *input, int inputlen, char **output, int *outputlen, bool *done, bool *success, PQExpBuffer errorMessage); +extern char *pg_fe_scram_build_verifier(const char *password); #endif /* FE_AUTH_H */ diff --git a/src/interfaces/libpq/libpq-fe.h b/src/interfaces/libpq/libpq-fe.h index 635af5b50e..093c4986d8 100644 --- a/src/interfaces/libpq/libpq-fe.h +++ b/src/interfaces/libpq/libpq-fe.h @@ -597,6 +597,7 @@ extern int PQenv2encoding(void); /* === in fe-auth.c === */ extern char *PQencryptPassword(const char *passwd, const char *user); +extern char *PQencryptPasswordConn(PGconn *conn, const char *passwd, const char *user, const char *algorithm); /* === in encnames.c === */ -- cgit v1.2.3 From 20bf7b2b0afcb53608ec37005ee7f831132925d2 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Thu, 4 May 2017 12:28:25 +0300 Subject: Fix PQencryptPasswordConn to work with older server versions. password_encryption was a boolean before version 10, so cope with "on" and "off". Also, change the behavior with "plain", to treat it the same as "md5". We're discussing removing the password_encryption='plain' option from the server altogether, which will make this the only reasonable choice, but even if we kept it, it seems best to never send the password in cleartext. --- doc/src/sgml/libpq.sgml | 4 +++- src/interfaces/libpq/fe-auth.c | 21 ++++++++++++++------- 2 files changed, 17 insertions(+), 8 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/libpq.sgml b/doc/src/sgml/libpq.sgml index 4f60b203fb..c2b7abc603 100644 --- a/doc/src/sgml/libpq.sgml +++ b/doc/src/sgml/libpq.sgml @@ -5902,7 +5902,9 @@ char *PQencryptPasswordConn(PGconn *conn, const char *passwd, const char *user, are the cleartext password, and the SQL name of the user it is for. algorithm specifies the encryption algorithm to use to encrypt the password. Currently supported algorithms are - md5, scram-sha-256 and plain. + md5 and scram-sha-256 (on and + off are also accepted as aliases for md5, for + compatibility with older server versions). Note that support for scram-sha-256 was introduced in PostgreSQL version 10, and will not work correctly with older server versions. If algorithm is NULL, this function will query diff --git a/src/interfaces/libpq/fe-auth.c b/src/interfaces/libpq/fe-auth.c index daa7cc9585..54acd0f6bf 100644 --- a/src/interfaces/libpq/fe-auth.c +++ b/src/interfaces/libpq/fe-auth.c @@ -1168,7 +1168,7 @@ PQencryptPasswordConn(PGconn *conn, const char *passwd, const char *user, { PQclear(res); printfPQExpBuffer(&conn->errorMessage, - libpq_gettext("password_encryption value too long\n")); + libpq_gettext("password_encryption value too long\n")); return NULL; } strcpy(algobuf, val); @@ -1177,8 +1177,19 @@ PQencryptPasswordConn(PGconn *conn, const char *passwd, const char *user, algorithm = algobuf; } - /* Ok, now we know what algorithm to use */ + /* + * Also accept "on" and "off" as aliases for "md5", because + * password_encryption was a boolean before PostgreSQL 10. We refuse to + * send the password in plaintext even if it was "off". + */ + if (strcmp(algorithm, "on") == 0 || + strcmp(algorithm, "off") == 0 || + strcmp(algorithm, "plain") == 0) + algorithm = "md5"; + /* + * Ok, now we know what algorithm to use + */ if (strcmp(algorithm, "scram-sha-256") == 0) { crypt_pwd = pg_fe_scram_build_verifier(passwd); @@ -1195,14 +1206,10 @@ PQencryptPasswordConn(PGconn *conn, const char *passwd, const char *user, } } } - else if (strcmp(algorithm, "plain") == 0) - { - crypt_pwd = strdup(passwd); - } else { printfPQExpBuffer(&conn->errorMessage, - libpq_gettext("unknown password encryption algorithm\n")); + libpq_gettext("unknown password encryption algorithm\n")); return NULL; } -- cgit v1.2.3 From c22b59edd26fb69661188b040ea3e8776a533685 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Thu, 4 May 2017 17:50:54 -0300 Subject: Credit Claudio as main author of feature --- doc/src/sgml/release-10.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/release-10.sgml b/doc/src/sgml/release-10.sgml index 241064e58e..3fa207f3ca 100644 --- a/doc/src/sgml/release-10.sgml +++ b/doc/src/sgml/release-10.sgml @@ -1763,7 +1763,7 @@ --> Improve speed of VACUUM's removal of trailing empty - heap pages (Alvaro Herrera) + heap pages (Claudio Freire, Álvaro Herrera) -- cgit v1.2.3 From 4f45beba7aea007cbae68eaea5d04d4cab7e4f62 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Thu, 4 May 2017 19:33:41 -0400 Subject: doc: PG 10 release note updates for psql, GiST, and markup Reported-by: Andrew Borodin, Fabien COELHO, Dagfinn Ilmari Mannsaker --- doc/src/sgml/release-10.sgml | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/release-10.sgml b/doc/src/sgml/release-10.sgml index 3fa207f3ca..5c26f2d0ea 100644 --- a/doc/src/sgml/release-10.sgml +++ b/doc/src/sgml/release-10.sgml @@ -552,6 +552,17 @@ + + + + Allow faster GiST inserts and updates by reusing + index space more efficiently (Andrey Borodin) + + + @@ -675,7 +686,7 @@ The new settings are and - max_pred_locks_per_page. + . @@ -1840,7 +1851,7 @@ - This uses the syntax ALTER + This uses the syntax ALTER TYPE ... RENAME VALUE. @@ -2220,6 +2231,10 @@ Add conditional branch support to psql (Corey @@ -2299,24 +2314,6 @@ - - - - Fix psql \p to always print what would be executed - by \g or \w (Daniel Vérité) - - - - Previously \p didn't properly print the reverted-to command after - a buffer contents reset. CLARIFY? - - - + + Improve accuracy in determining if a BRIN index scan + is beneficial (David Rowley, Emre Hasegeli) + + + - Add the ability to compute a correlation ratio and the number of - distinct values on several columns (Tomas Vondra, David Rowley) + Add multi-column optimizer statistics to compute the correlation + ratio and number of distinct values (Tomas Vondra, David Rowley, + Álvaro Herrera) @@ -1755,10 +1767,12 @@ - Have VACUUM VERBOSE - report the number of skipped frozen pages (Masahiko Sawada) + Have VACUUM VERBOSE report + the number of skipped frozen pages and oldest xmin (Masahiko Sawada) -- cgit v1.2.3 From 086221cf6b1727c2baed4703c582f657b7c5350e Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Mon, 1 May 2017 15:09:06 -0400 Subject: Prevent panic during shutdown checkpoint When the checkpointer writes the shutdown checkpoint, it checks afterwards whether any WAL has been written since it started and throws a PANIC if so. At that point, only walsenders are still active, so one might think this could not happen, but walsenders can also generate WAL, for instance in BASE_BACKUP and certain variants of CREATE_REPLICATION_SLOT. So they can trigger this panic if such a command is run while the shutdown checkpoint is being written. To fix this, divide the walsender shutdown into two phases. First, the postmaster sends a SIGUSR2 signal to all walsenders. The walsenders then put themselves into the "stopping" state. In this state, they reject any new commands. (For simplicity, we reject all new commands, so that in the future we do not have to track meticulously which commands might generate WAL.) The checkpointer waits for all walsenders to reach this state before proceeding with the shutdown checkpoint. After the shutdown checkpoint is done, the postmaster sends SIGINT (previously unused) to the walsenders. This triggers the existing shutdown behavior of sending out the shutdown checkpoint record and then terminating. Author: Michael Paquier Reported-by: Fujii Masao --- doc/src/sgml/monitoring.sgml | 5 + src/backend/access/transam/xlog.c | 6 ++ src/backend/postmaster/postmaster.c | 7 +- src/backend/replication/walsender.c | 143 ++++++++++++++++++++++++---- src/include/replication/walsender.h | 1 + src/include/replication/walsender_private.h | 3 +- 6 files changed, 141 insertions(+), 24 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 2a83671b53..80d12b26d7 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -1690,6 +1690,11 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i backup: This WAL sender is sending a backup. + + + stopping: This WAL sender is stopping. + + diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index a89d99838a..5d6f8b75b8 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -8325,6 +8325,12 @@ ShutdownXLOG(int code, Datum arg) ereport(IsPostmasterEnvironment ? LOG : NOTICE, (errmsg("shutting down"))); + /* + * Wait for WAL senders to be in stopping state. This prevents commands + * from writing new WAL. + */ + WalSndWaitStopping(); + if (RecoveryInProgress()) CreateRestartPoint(CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_IMMEDIATE); else diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 4a25ed8f5b..01f1c2805f 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -2918,7 +2918,7 @@ reaper(SIGNAL_ARGS) * Waken walsenders for the last time. No regular backends * should be around anymore. */ - SignalChildren(SIGUSR2); + SignalChildren(SIGINT); pmState = PM_SHUTDOWN_2; @@ -3656,7 +3656,9 @@ PostmasterStateMachine(void) /* * If we get here, we are proceeding with normal shutdown. All * the regular children are gone, and it's time to tell the - * checkpointer to do a shutdown checkpoint. + * checkpointer to do a shutdown checkpoint. All WAL senders + * are told to switch to a stopping state so that the shutdown + * checkpoint can go ahead. */ Assert(Shutdown > NoShutdown); /* Start the checkpointer if not running */ @@ -3665,6 +3667,7 @@ PostmasterStateMachine(void) /* And tell it to shut down */ if (CheckpointerPID != 0) { + SignalSomeChildren(SIGUSR2, BACKEND_TYPE_WALSND); signal_child(CheckpointerPID, SIGUSR2); pmState = PM_SHUTDOWN; } diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 2a6c8bb62d..45d027803a 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -24,11 +24,14 @@ * are treated as not a crash but approximately normal termination; * the walsender will exit quickly without sending any more XLOG records. * - * If the server is shut down, postmaster sends us SIGUSR2 after all - * regular backends have exited and the shutdown checkpoint has been written. - * This instructs walsender to send any outstanding WAL, including the - * shutdown checkpoint record, wait for it to be replicated to the standby, - * and then exit. + * If the server is shut down, postmaster sends us SIGUSR2 after all regular + * backends have exited. This causes the walsender to switch to the "stopping" + * state. In this state, the walsender will reject any replication command + * that may generate WAL activity. The checkpointer begins the shutdown + * checkpoint once all walsenders are confirmed as stopping. When the shutdown + * checkpoint finishes, the postmaster sends us SIGINT. This instructs + * walsender to send any outstanding WAL, including the shutdown checkpoint + * record, wait for it to be replicated to the standby, and then exit. * * * Portions Copyright (c) 2010-2017, PostgreSQL Global Development Group @@ -177,13 +180,14 @@ static bool WalSndCaughtUp = false; /* Flags set by signal handlers for later service in main loop */ static volatile sig_atomic_t got_SIGHUP = false; -static volatile sig_atomic_t walsender_ready_to_stop = false; +static volatile sig_atomic_t got_SIGINT = false; +static volatile sig_atomic_t got_SIGUSR2 = false; /* - * This is set while we are streaming. When not set, SIGUSR2 signal will be + * This is set while we are streaming. When not set, SIGINT signal will be * handled like SIGTERM. When set, the main loop is responsible for checking - * walsender_ready_to_stop and terminating when it's set (after streaming any - * remaining WAL). + * got_SIGINT and terminating when it's set (after streaming any remaining + * WAL). */ static volatile sig_atomic_t replication_active = false; @@ -213,6 +217,7 @@ static struct /* Signal handlers */ static void WalSndSigHupHandler(SIGNAL_ARGS); static void WalSndXLogSendHandler(SIGNAL_ARGS); +static void WalSndSwitchStopping(SIGNAL_ARGS); static void WalSndLastCycleHandler(SIGNAL_ARGS); /* Prototypes for private functions */ @@ -299,11 +304,14 @@ WalSndErrorCleanup(void) ReplicationSlotCleanup(); replication_active = false; - if (walsender_ready_to_stop) + if (got_SIGINT) proc_exit(0); /* Revert back to startup state */ WalSndSetState(WALSNDSTATE_STARTUP); + + if (got_SIGUSR2) + WalSndSetState(WALSNDSTATE_STOPPING); } /* @@ -676,7 +684,7 @@ StartReplication(StartReplicationCmd *cmd) WalSndLoop(XLogSendPhysical); replication_active = false; - if (walsender_ready_to_stop) + if (got_SIGINT) proc_exit(0); WalSndSetState(WALSNDSTATE_STARTUP); @@ -1053,7 +1061,7 @@ StartLogicalReplication(StartReplicationCmd *cmd) { ereport(LOG, (errmsg("terminating walsender process after promotion"))); - walsender_ready_to_stop = true; + got_SIGINT = true; } WalSndSetState(WALSNDSTATE_CATCHUP); @@ -1103,7 +1111,7 @@ StartLogicalReplication(StartReplicationCmd *cmd) ReplicationSlotRelease(); replication_active = false; - if (walsender_ready_to_stop) + if (got_SIGINT) proc_exit(0); WalSndSetState(WALSNDSTATE_STARTUP); @@ -1290,6 +1298,14 @@ WalSndWaitForWal(XLogRecPtr loc) else RecentFlushPtr = GetXLogReplayRecPtr(NULL); + /* + * If postmaster asked us to switch to the stopping state, do so. + * Shutdown is in progress and this will allow the checkpointer to + * move on with the shutdown checkpoint. + */ + if (got_SIGUSR2) + WalSndSetState(WALSNDSTATE_STOPPING); + /* * If postmaster asked us to stop, don't wait here anymore. This will * cause the xlogreader to return without reading a full record, which @@ -1299,7 +1315,7 @@ WalSndWaitForWal(XLogRecPtr loc) * RecentFlushPtr, so we can send all remaining data before shutting * down. */ - if (walsender_ready_to_stop) + if (got_SIGINT) break; /* @@ -1373,6 +1389,22 @@ exec_replication_command(const char *cmd_string) MemoryContext cmd_context; MemoryContext old_context; + /* + * If WAL sender has been told that shutdown is getting close, switch its + * status accordingly to handle the next replication commands correctly. + */ + if (got_SIGUSR2) + WalSndSetState(WALSNDSTATE_STOPPING); + + /* + * Throw error if in stopping mode. We need prevent commands that could + * generate WAL while the shutdown checkpoint is being written. To be + * safe, we just prohibit all new commands. + */ + if (MyWalSnd->state == WALSNDSTATE_STOPPING) + ereport(ERROR, + (errmsg("cannot execute new commands while WAL sender is in stopping mode"))); + /* * CREATE_REPLICATION_SLOT ... LOGICAL exports a snapshot until the next * command arrives. Clean up the old stuff if there's anything. @@ -2095,13 +2127,20 @@ WalSndLoop(WalSndSendDataCallback send_data) } /* - * When SIGUSR2 arrives, we send any outstanding logs up to the + * At the reception of SIGUSR2, switch the WAL sender to the stopping + * state. + */ + if (got_SIGUSR2) + WalSndSetState(WALSNDSTATE_STOPPING); + + /* + * When SIGINT arrives, we send any outstanding logs up to the * shutdown checkpoint record (i.e., the latest record), wait for * them to be replicated to the standby, and exit. This may be a * normal termination at shutdown, or a promotion, the walsender * is not sure which. */ - if (walsender_ready_to_stop) + if (got_SIGINT) WalSndDone(send_data); } @@ -2841,7 +2880,23 @@ WalSndXLogSendHandler(SIGNAL_ARGS) errno = save_errno; } -/* SIGUSR2: set flag to do a last cycle and shut down afterwards */ +/* SIGUSR2: set flag to switch to stopping state */ +static void +WalSndSwitchStopping(SIGNAL_ARGS) +{ + int save_errno = errno; + + got_SIGUSR2 = true; + SetLatch(MyLatch); + + errno = save_errno; +} + +/* + * SIGINT: set flag to do a last cycle and shut down afterwards. The WAL + * sender should already have been switched to WALSNDSTATE_STOPPING at + * this point. + */ static void WalSndLastCycleHandler(SIGNAL_ARGS) { @@ -2856,7 +2911,7 @@ WalSndLastCycleHandler(SIGNAL_ARGS) if (!replication_active) kill(MyProcPid, SIGTERM); - walsender_ready_to_stop = true; + got_SIGINT = true; SetLatch(MyLatch); errno = save_errno; @@ -2869,14 +2924,14 @@ WalSndSignals(void) /* Set up signal handlers */ pqsignal(SIGHUP, WalSndSigHupHandler); /* set flag to read config * file */ - pqsignal(SIGINT, SIG_IGN); /* not used */ + pqsignal(SIGINT, WalSndLastCycleHandler); /* request a last cycle and + * shutdown */ pqsignal(SIGTERM, die); /* request shutdown */ pqsignal(SIGQUIT, quickdie); /* hard crash time */ InitializeTimeouts(); /* establishes SIGALRM handler */ pqsignal(SIGPIPE, SIG_IGN); pqsignal(SIGUSR1, WalSndXLogSendHandler); /* request WAL sending */ - pqsignal(SIGUSR2, WalSndLastCycleHandler); /* request a last cycle and - * shutdown */ + pqsignal(SIGUSR2, WalSndSwitchStopping); /* switch to stopping state */ /* Reset some signals that are accepted by postmaster but not here */ pqsignal(SIGCHLD, SIG_DFL); @@ -2954,6 +3009,50 @@ WalSndWakeup(void) } } +/* + * Wait that all the WAL senders have reached the stopping state. This is + * used by the checkpointer to control when shutdown checkpoints can + * safely begin. + */ +void +WalSndWaitStopping(void) +{ + for (;;) + { + int i; + bool all_stopped = true; + + for (i = 0; i < max_wal_senders; i++) + { + WalSndState state; + WalSnd *walsnd = &WalSndCtl->walsnds[i]; + + SpinLockAcquire(&walsnd->mutex); + + if (walsnd->pid == 0) + { + SpinLockRelease(&walsnd->mutex); + continue; + } + + state = walsnd->state; + SpinLockRelease(&walsnd->mutex); + + if (state != WALSNDSTATE_STOPPING) + { + all_stopped = false; + break; + } + } + + /* safe to leave if confirmation is done for all WAL senders */ + if (all_stopped) + return; + + pg_usleep(10000L); /* wait for 10 msec */ + } +} + /* Set state for current walsender (only called in walsender) */ void WalSndSetState(WalSndState state) @@ -2987,6 +3086,8 @@ WalSndGetStateString(WalSndState state) return "catchup"; case WALSNDSTATE_STREAMING: return "streaming"; + case WALSNDSTATE_STOPPING: + return "stopping"; } return "UNKNOWN"; } diff --git a/src/include/replication/walsender.h b/src/include/replication/walsender.h index 2ca903872e..99f12377e0 100644 --- a/src/include/replication/walsender.h +++ b/src/include/replication/walsender.h @@ -44,6 +44,7 @@ extern void WalSndSignals(void); extern Size WalSndShmemSize(void); extern void WalSndShmemInit(void); extern void WalSndWakeup(void); +extern void WalSndWaitStopping(void); extern void WalSndRqstFileReload(void); /* diff --git a/src/include/replication/walsender_private.h b/src/include/replication/walsender_private.h index 2c59056cef..36311e124c 100644 --- a/src/include/replication/walsender_private.h +++ b/src/include/replication/walsender_private.h @@ -24,7 +24,8 @@ typedef enum WalSndState WALSNDSTATE_STARTUP = 0, WALSNDSTATE_BACKUP, WALSNDSTATE_CATCHUP, - WALSNDSTATE_STREAMING + WALSNDSTATE_STREAMING, + WALSNDSTATE_STOPPING } WalSndState; /* -- cgit v1.2.3 From 54dbd4dc78b045ffcc046b9a43681770c3992dd4 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 5 May 2017 19:33:34 -0400 Subject: First-draft release notes for 9.6.3. As usual, the release notes for other branches will be made by cutting these down, but put them up for community review first. Note there are some entries that really only apply to pre-9.6 branches. --- doc/src/sgml/release-9.6.sgml | 902 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 902 insertions(+) (limited to 'doc/src') diff --git a/doc/src/sgml/release-9.6.sgml b/doc/src/sgml/release-9.6.sgml index dc7b479387..cdff0849ea 100644 --- a/doc/src/sgml/release-9.6.sgml +++ b/doc/src/sgml/release-9.6.sgml @@ -1,6 +1,908 @@ + + Release 9.6.3 + + + Release Date + 2017-05-11 + + + + This release contains a variety of fixes from 9.6.2. + For information about new features in the 9.6 major release, see + . + + + + Migration to Version 9.6.3 + + + A dump/restore is not required for those running 9.6.X. + + + + However, if you are upgrading from a version earlier than 9.6.2, + see . + + + + + Changes + + + + + + + Fix possible corruption of init forks of unlogged indexes + (Robert Haas, Michael Paquier) + + + + This could result in an unlogged index being set to an invalid state + after a crash and restart. Such a problem would persist until the + index was dropped and rebuilt. + + + + + + + Fix incorrect reconstruction of pg_subtrans entries + when a standby server replays a prepared but uncommitted two-phase + transaction (Tom Lane) + + + + In most cases this turned out to have no visible ill effects, but in + corner cases it could result in circular references + in pg_subtrans, potentially causing infinite loops + in queries that examine rows modified by the two-phase transaction. + + + + + + + Fix possibly-corrupt initial snapshot during logical decoding + (Petr Jelinek, Andres Freund) + + + + If a logical decoding replication slot was created while another slot + already exists, it was initialized with a potentially-corrupted + snapshot, allowing wrong data to be returned during decoding. + The time window during which this snapshot continued to be used + depended on how busy the server was; under low load it would be hard + to see any problem. + + + + + + + Avoid possible crash in walsender due to failure + to initialize a string buffer (Stas Kelvich, Fujii Masao) + + + + + + + Fix possible crash when rescanning a nearest-neighbor index-only scan + on a GiST index (Tom Lane) + + + + + + + Prevent delays in postmaster's launching of parallel worker processes + (Tom Lane) + + + + There could be a significant delay (up to tens of seconds) before + satisfying a query's request for more than one worker process. On most + platforms this required unlucky timing, but on some it was the typical + case. + + + + + + + Fix postmaster's handling of fork() failure for a + background worker process (Tom Lane) + + + + Previously, the postmaster partially updated its state as though + the process had been successfully launched, resulting in subsequent + confusion. + + + + + + + Fix crash or wrong answers when a GROUPING SETS column's + data type is hashable but not sortable (Pavan Deolasee) + + + + + + + Fix possible no relation entry for relid 0 error when + planning nested set operations (Tom Lane) + + + + + + + Fix assorted minor issues in planning of parallel queries (Robert Haas) + + + + + + + Avoid applying physical targetlist optimization to custom + scans (Dmitry Ivanov, Tom Lane) + + + + This optimization supposed that retrieving all columns of a tuple + is inexpensive, which is true for ordinary Postgres tuples; but it + might not be the case at all for a custom scan provider. + + + + + + + Ensure parsing of queries in extension scripts sees the results of + immediately-preceding DDL (Julien Rouhaud, Tom Lane) + + + + Due to lack of a cache flush step between commands in an extension + script file, non-utility queries might not see the effects of an + immediately preceding catalog change, such as ALTER TABLE + ... RENAME. + + + + + + + Skip tablespace privilege checks when ALTER TABLE ... ALTER + COLUMN TYPE rebuilds an existing index (Noah Misch) + + + + The command failed if the calling user did not currently have + privileges to use the tablespace an index had been created in. + That behavior seems unhelpful, so skip the check, allowing the + index to be rebuilt where it is. + + + + + + + Fix ALTER TABLE ... VALIDATE CONSTRAINT to not recurse + to child tables if the constraint is marked NO INHERIT + (Amit Langote) + + + + This fix prevents unwanted constraint does not exist failures + when no matching constraint is present in the child tables. + + + + + + + Avoid dangling pointer in COPY ... TO when row-level + security is active for the source table (Tom Lane) + + + + Usually this had no ill effects, but sometimes it would cause + unexpected errors or crashes. + + + + + + + Avoid accessing an already-closed relcache entry in CLUSTER + and VACUUM FULL (Tom Lane) + + + + With some bad luck, this could lead to indexes on the target + relation getting rebuilt with the wrong persistence setting. + + + + + + + Fix VACUUM to account properly for pages that could not + be scanned due to conflicting page pins (Andrew Gierth) + + + + This tended to lead to underestimation of the number of tuples in + the table. In the worst case of a small heavily-contended + table, VACUUM could incorrectly report that the table + contained no tuples, leading to very bad planning choices. + + + + + + + Ensure that bulk-tuple-transfer loops within a hash join are + interruptible by query cancel requests (Tom Lane, Thomas Munro) + + + + + + + Fix incorrect support for certain box operators in SP-GiST + (Nikita Glukhov) + + + + SP-GiST index scans using the operators &< + &> &<| and |&> + would yield incorrect answers. + + + + + + + Fix integer-overflow problems in interval comparison (Kyotaro + Horiguchi and Tom Lane) + + + + The comparison operators for type interval could yield wrong + answers for intervals larger than about 296000 years. Indexes on + columns containing such large values should be reindexed, since they + may be corrupt. + + + + + + + Fix cursor_to_xml() to produce valid output + with tableforest = false + (Thomas Munro, Peter Eisentraut) + + + + Previously it failed to produce a wrapping <table> + element. + + + + + + + Fix roundoff problems in float8_timestamptz() + and make_interval() (Tom Lane) + + + + These functions truncated, rather than rounded, when converting a + floating-point value to integer microseconds; that could cause + unexpectedly off-by-one results. + + + + + + + Fix pg_get_object_address() to handle members of operator + families correctly (Álvaro Herrera) + + + + + + + Fix cancelling of pg_stop_backup() when attempting to stop + a non-exclusive backup (Michael Paquier, David Steele) + + + + If pg_stop_backup() was cancelled while waiting for a + non-exclusive backup to end, related state was left inconsistent; + a new exclusive backup could not be started, and there were other minor + problems. + + + + + + + Improve performance of pg_timezone_names view (Tom Lane) + + + + + + + Reduce memory management overhead for contexts containing many large + blocks (Tom Lane) + + + + + + + Fix sloppy handling of corner-case errors from lseek() + and close() (Tom Lane) + + + + Neither of these system calls are likely to fail in typical situations, + but if they did, fd.c could get quite confused. + + + + + + + Fix incorrect check for whether postmaster is running as a Windows + service (Michael Paquier) + + + + This could result in attempting to write to the event log when that + isn't accessible, so that no logging happens at all. + + + + + + + Fix ecpg to support COMMIT PREPARED + and ROLLBACK PREPARED (Masahiko Sawada) + + + + + + + Fix a double-free error when processing dollar-quoted string literals + in ecpg (Michael Meskes) + + + + + + + Fix pgbench to handle the combination + of + + + + + + Fix pgbench to honor the long-form option + spelling + + + + + + Fix pg_dump/pg_restore to correctly + handle privileges for the public schema when + using + + + Other schemas start out with no privileges granted, + but public does not; this requires special-case treatment + when it is dropped and restored due to the + + + + + + In pg_dump, fix incorrect schema and owner marking for + comments and security labels of some types of database objects + (Giuseppe Broccolo, Tom Lane) + + + + In simple cases this caused no ill effects; but for example, a + schema-selective restore might omit comments it should include, because + they were not marked as belonging to the schema of their associated + object. + + + + + + + Fix typo in pg_dump's query for initial privileges + of a procedural language (Peter Eisentraut) + + + + This resulted in pg_dump always thinking that the + language had no initial privileges; since that's true for most + procedural languages, bad effects from this bug are probably rare. + + + + + + + Avoid emitting an invalid list file in pg_restore -l + when SQL object names contain newlines (Tom Lane) + + + + Replace newlines by spaces, which is sufficient to make the output + valid for pg_restore -L's purposes. + + + + + + + Fix pg_upgrade to transfer comments and security labels + attached to large objects (blobs) (Stephen Frost) + + + + Previously, blobs were correctly transferred to the new database, but + any comments or security labels attached to them were lost. + + + + + + + Improve error handling + in contrib/adminpack's pg_file_write() + function (Noah Misch) + + + + Notably, it failed to detect errors reported + by fclose(). + + + + + + + In contrib/dblink, avoid connection leak when establishing + a new unnamed connection (Joe Conway) + + + + + + + Fix contrib/pg_trgm's extraction of trigrams from regular + expressions (Tom Lane) + + + + In some cases it would produce a broken data structure that could never + match anything, leading to GIN or GiST indexscans that use a trigram + index not finding any matches to the regular expression. + + + + + + + In contrib/postgres_fdw, allow join conditions that + contain shippable extension-provided functions to be pushed to the + remote server (David Rowley, Ashutosh Bapat) + + + + + + + Support OpenSSL 1.1.0 (Heikki Linnakangas, Andreas Karlsson, Tom Lane) + + + + This is a back-patch of work previously done in newer branches; + it's needed since many platforms are adopting newer OpenSSL versions. + + + + + + + Support Tcl 8.6 in MSVC builds (Álvaro Herrera) + + + + + + + Sync our copy of the timezone library with IANA release tzcode2017b + (Tom Lane) + + + + This fixes a bug affecting some DST transitions in January 2038. + + + + + + + Update time zone data files to tzdata release 2017b + for DST law changes in Chile, Haiti, and Mongolia, plus historical + corrections for Ecuador, Kazakhstan, Liberia, and Spain. + Switch to numeric abbreviations for numerous time zones in South + America, the Pacific and Indian oceans, and some Asian and Middle + Eastern countries. + + + + The IANA time zone database previously provided textual abbreviations + for all time zones, sometimes making up abbreviations that have little + or no currency among the local population. They are in process of + reversing that policy in favor of using numeric UTC offsets in zones + where there is no evidence of real-world use of an English + abbreviation. At least for the time being, PostgreSQL + will continue to accept such removed abbreviations for timestamp input. + But they will not be shown in the pg_timezone_names + view nor used for output. + + + + + + + + Release 9.6.2 -- cgit v1.2.3 From a9c6d704354bfe91bc389742cb5d331ae4e93831 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 6 May 2017 14:19:47 -0400 Subject: Document current_role. This system function has been there a very long time, but somehow escaped being listed in func.sgml. Fabien Coelho and Tom Lane Discussion: https://postgr.es/m/alpine.DEB.2.20.1705061027580.3896@lancre --- doc/src/sgml/func.sgml | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index f06d0a92c0..629865701f 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -15735,6 +15735,12 @@ SELECT * FROM pg_ls_dir('.') WITH ORDINALITY AS t(ls,n); by the client (might contain more than one statement) + + current_role + name + equivalent to current_user + + current_schema[()] name @@ -15871,8 +15877,11 @@ SELECT * FROM pg_ls_dir('.') WITH ORDINALITY AS t(ls,n); - current_catalog, current_schema, - current_user, session_user, + current_catalog, + current_role, + current_schema, + current_user, + session_user, and user have special syntactic status in SQL: they must be called without trailing parentheses. (In PostgreSQL, parentheses can optionally be used with @@ -15892,6 +15901,10 @@ SELECT * FROM pg_ls_dir('.') WITH ORDINALITY AS t(ls,n); current_query + + current_role + + current_schema @@ -15943,6 +15956,11 @@ SELECT * FROM pg_ls_dir('.') WITH ORDINALITY AS t(ls,n); functions with the attribute SECURITY DEFINER. In Unix parlance, the session user is the real user and the current user is the effective user. + current_role and user are + synonyms for current_user. (The SQL standard draws + a distinction between current_role + and current_user, but PostgreSQL + does not, since it unifies users and roles into a single kind of entity.) -- cgit v1.2.3 From 334b82cd56a65e09154d9f930d35a761a9c5cfab Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 6 May 2017 16:28:20 -0400 Subject: Second pass on 9.6.3 release notes. Improve description of logical decoding snapshot issues, per suggestion from Petr Jelinek. Mention possible need to re-sync logical replicas as a post-upgrade task. Minor copy-editing for some other items. --- doc/src/sgml/release-9.6.sgml | 100 +++++++++++++++++++++++------------------- 1 file changed, 56 insertions(+), 44 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/release-9.6.sgml b/doc/src/sgml/release-9.6.sgml index cdff0849ea..6948153065 100644 --- a/doc/src/sgml/release-9.6.sgml +++ b/doc/src/sgml/release-9.6.sgml @@ -23,7 +23,12 @@ - However, if you are upgrading from a version earlier than 9.6.2, + However, if you are using third-party replication tools that depend + on logical decoding, see the first changelog entry below. + + + + Also, if you are upgrading from a version earlier than 9.6.2, see . @@ -35,6 +40,40 @@ + + Fix possibly-invalid initial snapshot during logical decoding + (Petr Jelinek, Andres Freund) + + + + The initial snapshot created for a logical decoding replication slot + was potentially incorrect. This could cause third-party tools that + use logical decoding to copy incomplete/inconsistent initial data. + This was more likely to happen if the source server was busy at the + time of slot creation, or if another logical slot already existed. + + + + If you are using a replication tool that depends on logical decoding, + and it should have copied a nonempty data set at the start of + replication, it is advisable to recreate the replica after + installing this update, or to verify its contents against the source + server. + + + + + - - Fix possibly-corrupt initial snapshot during logical decoding - (Petr Jelinek, Andres Freund) - - - - If a logical decoding replication slot was created while another slot - already exists, it was initialized with a potentially-corrupted - snapshot, allowing wrong data to be returned during decoding. - The time window during which this snapshot continued to be used - depended on how busy the server was; under low load it would be hard - to see any problem. - - - - - - Prevent delays in postmaster's launching of parallel worker processes - (Tom Lane) + Prevent delays in postmaster's launching of multiple parallel worker + processes (Tom Lane) There could be a significant delay (up to tens of seconds) before - satisfying a query's request for more than one worker process. On most - platforms this required unlucky timing, but on some it was the typical - case. + satisfying a query's request for more than one worker process, or when + multiple queries requested workers simultaneously. On most platforms + this required unlucky timing, but on some it was the typical case. @@ -168,8 +180,8 @@ Branch: REL9_4_STABLE [436b560b8] 2017-04-24 12:16:58 -0400 - Previously, the postmaster partially updated its state as though - the process had been successfully launched, resulting in subsequent + Previously, the postmaster updated portions of its state as though + the process had been launched successfully, resulting in subsequent confusion. @@ -227,7 +239,7 @@ Branch: REL9_5_STABLE [6f0f98bb0] 2017-04-17 15:29:00 -0400 This optimization supposed that retrieving all columns of a tuple is inexpensive, which is true for ordinary Postgres tuples; but it - might not be the case at all for a custom scan provider. + might not be the case for a custom scan provider. @@ -271,7 +283,7 @@ Branch: REL9_2_STABLE [27a8c8033] 2017-02-12 16:05:23 -0500 The command failed if the calling user did not currently have - privileges to use the tablespace an index had been created in. + CREATE privilege for the tablespace containing the index. That behavior seems unhelpful, so skip the check, allowing the index to be rebuilt where it is. @@ -289,7 +301,7 @@ Branch: REL9_2_STABLE [f60f0c8fe] 2017-04-28 14:55:42 -0400 --> Fix ALTER TABLE ... VALIDATE CONSTRAINT to not recurse - to child tables if the constraint is marked NO INHERIT + to child tables when the constraint is marked NO INHERIT (Amit Langote) @@ -404,7 +416,7 @@ Branch: REL9_4_STABLE [8851bcf88] 2017-04-05 23:51:28 -0400 --> Fix integer-overflow problems in interval comparison (Kyotaro - Horiguchi and Tom Lane) + Horiguchi, Tom Lane) @@ -682,9 +694,9 @@ Branch: REL9_6_STABLE [4e8b2fd33] 2017-02-17 15:06:34 -0500 - This resulted in pg_dump always thinking that the - language had no initial privileges; since that's true for most - procedural languages, bad effects from this bug are probably rare. + This resulted in pg_dump always believing that the + language had no initial privileges. Since that's true for most + procedural languages, ill effects from this bug are probably rare. @@ -763,8 +775,8 @@ Branch: REL9_3_STABLE [f6cfc14e5] 2017-03-11 13:33:22 -0800 Branch: REL9_2_STABLE [c4613c3f4] 2017-03-11 13:33:30 -0800 --> - In contrib/dblink, avoid connection leak when establishing - a new unnamed connection (Joe Conway) + In contrib/dblink, avoid leaking the previous unnamed + connection when establishing a new unnamed connection (Joe Conway) -- cgit v1.2.3 From 628462bda908873688ce738a191b470ab769d604 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Sat, 6 May 2017 23:31:54 -0400 Subject: doc PG 10: adjustments to BRIN, WAL, JSON, XML items, syntax Reported-by: Alvaro Herrera --- doc/src/sgml/release-10.sgml | 145 +++++++++++++++++++------------------------ 1 file changed, 64 insertions(+), 81 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/release-10.sgml b/doc/src/sgml/release-10.sgml index dafddfbd17..f4da5b8fcc 100644 --- a/doc/src/sgml/release-10.sgml +++ b/doc/src/sgml/release-10.sgml @@ -458,13 +458,13 @@ Add GUC to limit the number of worker processes that can be used for - parallelism (Julien Rouhaud) + query parallelism (Julien Rouhaud) This can be set lower than to reserve worker processes - for non-parallel purposes. + for purposes other than parallel queries. @@ -477,22 +477,6 @@ - - - - Add full text search support for JSON and JSONB - (Dmitry Dolgov) - - - - This is accessed via ts_headline() and - to_tsvector. RIGHT SECTION? - - - - Cause BRIN index summarization to happen more - aggressively (Álvaro Herrera) + Add option to allow BRIN index summarization to happen + more aggressively (Álvaro Herrera) - Specifically, summarize the previous page range when a new page + Specifically, a new CREATE + INDEX option allows auto-summarizion of the + previous BRIN page range when a new page range is created. @@ -541,14 +527,18 @@ 2017-04-01 [c655899ba] BRIN de-summarization --> - Add function brin_desummarize_range() to remove - BRIN summarization of a specified range (Álvaro + Add functions to remove and re-add BRIN + summarization for BRIN index ranges (Álvaro Herrera) - This allows future BRIN index summarization to be - more compact. CLARIFY + New SQL function brin_summarize_range() + updates BRIN index summarization for a specified + range and brin_desummarize_range() removes it. + This is helpful to update summarization of a range that is now + smaller due to UPDATEs and DELETEs. @@ -860,22 +850,12 @@ - - - - Add pg_sequences view - to show all sequences (Peter Eisentraut) - - - Create a - Sequence metadata includes start, increment, etc, which is now - transactional. Sequence counters are still stored in separate - heap relations. + Sequence metadata includes start, increment, etc, + which is now transactional. Sequence counters are + still stored in separate heap relations. Also add pg_sequences view + to show all sequences. @@ -1167,7 +1149,7 @@ - This proves better security than the existing md5 + This provides better security than the existing md5 negotiation and storage method. @@ -1238,8 +1220,8 @@ 2017-01-04 [6667d9a6d] Re-allow SSL passphrase prompt at server start, but not --> - Allow SSL configuration to be updated at - SIGHUP (Andreas Karlsson, Tom Lane) + Allow SSL configuration to be updated during + configuration reload (Andreas Karlsson, Tom Lane) @@ -1317,23 +1299,6 @@ - - - - Remove orphaned temporary tables more aggressively (Robert Haas, - Tom Lane) - - - - Previously such tables were removed only when necessary. SECTION? - - - @@ -1376,8 +1341,14 @@ 2017-04-05 [00b6b6feb] Allow \-\-with-wal-segsize=n up to n=1024MB --> - Increase the maximum configurable WAL size to 1 - gigabyte (Beena Emerson) + Increase the maximum configurable WAL segment size + to one gigabyte (Beena Emerson) + + + + Larger WAL segment sizes allows for fewer + invocations and fewer + WAL files to manage. @@ -1569,6 +1540,19 @@ + + + + Add XMLTABLE + function that converts XML-formatted data into a row set + (Pavel Stehule, Álvaro Herrera) + + + + + Add full text search support for JSON and JSONB + (Dmitry Dolgov) + + + + This is accessed via ts_headline() and + to_tsvector. + + + - - Add support for converting XML-formatted data into a row - set (Pavel Stehule, Álvaro Herrera) - - - - This is done by referencing the new XMLTABLE - function. - - - - Add log options for pg_ctl wait ( -- cgit v1.2.3 From 86713deecda4ddbf8e339ec48fafa4d8080f6079 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 7 May 2017 14:43:04 -0400 Subject: Third pass on 9.6.3 release notes. Add updates for recent commits. In passing, credit Etsuro Fujita for his work on the postgres_fdw query cancel feature in 9.6; I seem to have missed that in the original drafting of the 9.6 notes. --- doc/src/sgml/release-9.6.sgml | 87 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 85 insertions(+), 2 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/release-9.6.sgml b/doc/src/sgml/release-9.6.sgml index 6948153065..b6cc481494 100644 --- a/doc/src/sgml/release-9.6.sgml +++ b/doc/src/sgml/release-9.6.sgml @@ -245,6 +245,24 @@ Branch: REL9_5_STABLE [6f0f98bb0] 2017-04-17 15:29:00 -0400 + + Use the correct sub-expression when applying a FOR ALL + row-level-security policy (Stephen Frost) + + + + In some cases the WITH CHECK restriction would be applied + when the USING restriction is more appropriate. + + + + + - Improve performance of pg_timezone_names view (Tom Lane) + Improve performance of pg_timezone_names view + (Tom Lane, David Rowley) @@ -826,6 +851,26 @@ Branch: REL9_6_STABLE [86e640a69] 2017-04-26 09:14:21 -0400 + + In contrib/postgres_fdw, + transmit query cancellation requests to the remote server + (Michael Paquier, Etsuro Fujita) + + + + Previously, a local query cancellation request did not cause an + already-sent remote query to terminate early. This is a back-patch + of work originally done for 9.6. + + + + + Sync our copy of the timezone library with IANA release tzcode2017b @@ -910,6 +961,38 @@ Branch: REL9_2_STABLE [c96ccc40e] 2017-05-01 11:54:08 -0400 + + + + Use correct daylight-savings rules for POSIX-style time zone names + in MSVC builds (David Rowley) + + + + The Microsoft MSVC build scripts neglected to install + the posixrules file in the timezone directory tree. + This resulted in the timezone code falling back to its built-in + rule about what DST behavior to assume for a POSIX-style time zone + name. For historical reasons that still corresponds to the DST rules + the USA was using before 2007 (i.e., change on first Sunday in April + and last Sunday in October). With this fix, a POSIX-style zone name + will use the current and historical DST transition dates of + the US/Eastern zone. If you don't want that, remove + the posixrules file, or replace it with a copy of some + other zone file (see ). Note that + due to caching, you may need to restart the server to get such changes + to take effect. + + + @@ -6679,7 +6762,7 @@ This commit is also listed under libpq and PL/pgSQL --> Transmit query cancellation requests to the remote server - (Michael Paquier) + (Michael Paquier, Etsuro Fujita) -- cgit v1.2.3 From 27dae036a5809a61104b7380f0cd98c37b43170f Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 7 May 2017 16:56:02 -0400 Subject: Release notes for 9.6.3, 9.5.7, 9.4.12, 9.3.17, 9.2.21. --- doc/src/sgml/release-9.2.sgml | 325 +++++++++++++++++++++++++++ doc/src/sgml/release-9.3.sgml | 346 +++++++++++++++++++++++++++++ doc/src/sgml/release-9.4.sgml | 435 ++++++++++++++++++++++++++++++++++++ doc/src/sgml/release-9.5.sgml | 496 ++++++++++++++++++++++++++++++++++++++++++ doc/src/sgml/release-9.6.sgml | 50 ----- 5 files changed, 1602 insertions(+), 50 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/release-9.2.sgml b/doc/src/sgml/release-9.2.sgml index b27176bdd6..b2454ad061 100644 --- a/doc/src/sgml/release-9.2.sgml +++ b/doc/src/sgml/release-9.2.sgml @@ -1,6 +1,331 @@ + + Release 9.2.21 + + + Release Date + 2017-05-11 + + + + This release contains a variety of fixes from 9.2.20. + For information about new features in the 9.2 major release, see + . + + + + The PostgreSQL community will stop releasing updates + for the 9.2.X release series in September 2017. + Users are encouraged to update to a newer release branch soon. + + + + Migration to Version 9.2.21 + + + A dump/restore is not required for those running 9.2.X. + + + + However, if you are upgrading from a version earlier than 9.2.20, + see . + + + + + + Changes + + + + + + Fix possible corruption of init forks of unlogged indexes + (Robert Haas, Michael Paquier) + + + + This could result in an unlogged index being set to an invalid state + after a crash and restart. Such a problem would persist until the + index was dropped and rebuilt. + + + + + + Fix incorrect reconstruction of pg_subtrans entries + when a standby server replays a prepared but uncommitted two-phase + transaction (Tom Lane) + + + + In most cases this turned out to have no visible ill effects, but in + corner cases it could result in circular references + in pg_subtrans, potentially causing infinite loops + in queries that examine rows modified by the two-phase transaction. + + + + + + Ensure parsing of queries in extension scripts sees the results of + immediately-preceding DDL (Julien Rouhaud, Tom Lane) + + + + Due to lack of a cache flush step between commands in an extension + script file, non-utility queries might not see the effects of an + immediately preceding catalog change, such as ALTER TABLE + ... RENAME. + + + + + + Skip tablespace privilege checks when ALTER TABLE ... ALTER + COLUMN TYPE rebuilds an existing index (Noah Misch) + + + + The command failed if the calling user did not currently have + CREATE privilege for the tablespace containing the index. + That behavior seems unhelpful, so skip the check, allowing the + index to be rebuilt where it is. + + + + + + Fix ALTER TABLE ... VALIDATE CONSTRAINT to not recurse + to child tables when the constraint is marked NO INHERIT + (Amit Langote) + + + + This fix prevents unwanted constraint does not exist failures + when no matching constraint is present in the child tables. + + + + + + Fix VACUUM to account properly for pages that could not + be scanned due to conflicting page pins (Andrew Gierth) + + + + This tended to lead to underestimation of the number of tuples in + the table. In the worst case of a small heavily-contended + table, VACUUM could incorrectly report that the table + contained no tuples, leading to very bad planning choices. + + + + + + Ensure that bulk-tuple-transfer loops within a hash join are + interruptible by query cancel requests (Tom Lane, Thomas Munro) + + + + + + Fix cursor_to_xml() to produce valid output + with tableforest = false + (Thomas Munro, Peter Eisentraut) + + + + Previously it failed to produce a wrapping <table> + element. + + + + + + Improve performance of pg_timezone_names view + (Tom Lane, David Rowley) + + + + + + Fix sloppy handling of corner-case errors from lseek() + and close() (Tom Lane) + + + + Neither of these system calls are likely to fail in typical situations, + but if they did, fd.c could get quite confused. + + + + + + Fix incorrect check for whether postmaster is running as a Windows + service (Michael Paquier) + + + + This could result in attempting to write to the event log when that + isn't accessible, so that no logging happens at all. + + + + + + Fix ecpg to support COMMIT PREPARED + and ROLLBACK PREPARED (Masahiko Sawada) + + + + + + Fix a double-free error when processing dollar-quoted string literals + in ecpg (Michael Meskes) + + + + + + In pg_dump, fix incorrect schema and owner marking for + comments and security labels of some types of database objects + (Giuseppe Broccolo, Tom Lane) + + + + In simple cases this caused no ill effects; but for example, a + schema-selective restore might omit comments it should include, because + they were not marked as belonging to the schema of their associated + object. + + + + + + Avoid emitting an invalid list file in pg_restore -l + when SQL object names contain newlines (Tom Lane) + + + + Replace newlines by spaces, which is sufficient to make the output + valid for pg_restore -L's purposes. + + + + + + Fix pg_upgrade to transfer comments and security labels + attached to large objects (blobs) (Stephen Frost) + + + + Previously, blobs were correctly transferred to the new database, but + any comments or security labels attached to them were lost. + + + + + + Improve error handling + in contrib/adminpack's pg_file_write() + function (Noah Misch) + + + + Notably, it failed to detect errors reported + by fclose(). + + + + + + In contrib/dblink, avoid leaking the previous unnamed + connection when establishing a new unnamed connection (Joe Conway) + + + + + + Support OpenSSL 1.1.0 (Heikki Linnakangas, Andreas Karlsson, Tom Lane) + + + + This is a back-patch of work previously done in newer branches; + it's needed since many platforms are adopting newer OpenSSL versions. + + + + + + Support Tcl 8.6 in MSVC builds (Álvaro Herrera) + + + + + + Sync our copy of the timezone library with IANA release tzcode2017b + (Tom Lane) + + + + This fixes a bug affecting some DST transitions in January 2038. + + + + + + Update time zone data files to tzdata release 2017b + for DST law changes in Chile, Haiti, and Mongolia, plus historical + corrections for Ecuador, Kazakhstan, Liberia, and Spain. + Switch to numeric abbreviations for numerous time zones in South + America, the Pacific and Indian oceans, and some Asian and Middle + Eastern countries. + + + + The IANA time zone database previously provided textual abbreviations + for all time zones, sometimes making up abbreviations that have little + or no currency among the local population. They are in process of + reversing that policy in favor of using numeric UTC offsets in zones + where there is no evidence of real-world use of an English + abbreviation. At least for the time being, PostgreSQL + will continue to accept such removed abbreviations for timestamp input. + But they will not be shown in the pg_timezone_names + view nor used for output. + + + + + + Use correct daylight-savings rules for POSIX-style time zone names + in MSVC builds (David Rowley) + + + + The Microsoft MSVC build scripts neglected to install + the posixrules file in the timezone directory tree. + This resulted in the timezone code falling back to its built-in + rule about what DST behavior to assume for a POSIX-style time zone + name. For historical reasons that still corresponds to the DST rules + the USA was using before 2007 (i.e., change on first Sunday in April + and last Sunday in October). With this fix, a POSIX-style zone name + will use the current and historical DST transition dates of + the US/Eastern zone. If you don't want that, remove + the posixrules file, or replace it with a copy of some + other zone file (see ). Note that + due to caching, you may need to restart the server to get such changes + to take effect. + + + + + + + + Release 9.2.20 diff --git a/doc/src/sgml/release-9.3.sgml b/doc/src/sgml/release-9.3.sgml index df36947930..44c1e2ac69 100644 --- a/doc/src/sgml/release-9.3.sgml +++ b/doc/src/sgml/release-9.3.sgml @@ -1,6 +1,352 @@ + + Release 9.3.17 + + + Release Date + 2017-05-11 + + + + This release contains a variety of fixes from 9.3.16. + For information about new features in the 9.3 major release, see + . + + + + Migration to Version 9.3.17 + + + A dump/restore is not required for those running 9.3.X. + + + + However, if you are upgrading from a version earlier than 9.3.16, + see . + + + + + + Changes + + + + + + Fix possible corruption of init forks of unlogged indexes + (Robert Haas, Michael Paquier) + + + + This could result in an unlogged index being set to an invalid state + after a crash and restart. Such a problem would persist until the + index was dropped and rebuilt. + + + + + + Fix incorrect reconstruction of pg_subtrans entries + when a standby server replays a prepared but uncommitted two-phase + transaction (Tom Lane) + + + + In most cases this turned out to have no visible ill effects, but in + corner cases it could result in circular references + in pg_subtrans, potentially causing infinite loops + in queries that examine rows modified by the two-phase transaction. + + + + + + Ensure parsing of queries in extension scripts sees the results of + immediately-preceding DDL (Julien Rouhaud, Tom Lane) + + + + Due to lack of a cache flush step between commands in an extension + script file, non-utility queries might not see the effects of an + immediately preceding catalog change, such as ALTER TABLE + ... RENAME. + + + + + + Skip tablespace privilege checks when ALTER TABLE ... ALTER + COLUMN TYPE rebuilds an existing index (Noah Misch) + + + + The command failed if the calling user did not currently have + CREATE privilege for the tablespace containing the index. + That behavior seems unhelpful, so skip the check, allowing the + index to be rebuilt where it is. + + + + + + Fix ALTER TABLE ... VALIDATE CONSTRAINT to not recurse + to child tables when the constraint is marked NO INHERIT + (Amit Langote) + + + + This fix prevents unwanted constraint does not exist failures + when no matching constraint is present in the child tables. + + + + + + Fix VACUUM to account properly for pages that could not + be scanned due to conflicting page pins (Andrew Gierth) + + + + This tended to lead to underestimation of the number of tuples in + the table. In the worst case of a small heavily-contended + table, VACUUM could incorrectly report that the table + contained no tuples, leading to very bad planning choices. + + + + + + Ensure that bulk-tuple-transfer loops within a hash join are + interruptible by query cancel requests (Tom Lane, Thomas Munro) + + + + + + Fix cursor_to_xml() to produce valid output + with tableforest = false + (Thomas Munro, Peter Eisentraut) + + + + Previously it failed to produce a wrapping <table> + element. + + + + + + Improve performance of pg_timezone_names view + (Tom Lane, David Rowley) + + + + + + Fix sloppy handling of corner-case errors from lseek() + and close() (Tom Lane) + + + + Neither of these system calls are likely to fail in typical situations, + but if they did, fd.c could get quite confused. + + + + + + Fix incorrect check for whether postmaster is running as a Windows + service (Michael Paquier) + + + + This could result in attempting to write to the event log when that + isn't accessible, so that no logging happens at all. + + + + + + Fix ecpg to support COMMIT PREPARED + and ROLLBACK PREPARED (Masahiko Sawada) + + + + + + Fix a double-free error when processing dollar-quoted string literals + in ecpg (Michael Meskes) + + + + + + In pg_dump, fix incorrect schema and owner marking for + comments and security labels of some types of database objects + (Giuseppe Broccolo, Tom Lane) + + + + In simple cases this caused no ill effects; but for example, a + schema-selective restore might omit comments it should include, because + they were not marked as belonging to the schema of their associated + object. + + + + + + Avoid emitting an invalid list file in pg_restore -l + when SQL object names contain newlines (Tom Lane) + + + + Replace newlines by spaces, which is sufficient to make the output + valid for pg_restore -L's purposes. + + + + + + Fix pg_upgrade to transfer comments and security labels + attached to large objects (blobs) (Stephen Frost) + + + + Previously, blobs were correctly transferred to the new database, but + any comments or security labels attached to them were lost. + + + + + + Improve error handling + in contrib/adminpack's pg_file_write() + function (Noah Misch) + + + + Notably, it failed to detect errors reported + by fclose(). + + + + + + In contrib/dblink, avoid leaking the previous unnamed + connection when establishing a new unnamed connection (Joe Conway) + + + + + + Fix contrib/pg_trgm's extraction of trigrams from regular + expressions (Tom Lane) + + + + In some cases it would produce a broken data structure that could never + match anything, leading to GIN or GiST indexscans that use a trigram + index not finding any matches to the regular expression. + + + + + + In contrib/postgres_fdw, + transmit query cancellation requests to the remote server + (Michael Paquier, Etsuro Fujita) + + + + Previously, a local query cancellation request did not cause an + already-sent remote query to terminate early. This is a back-patch + of work originally done for 9.6. + + + + + + Support OpenSSL 1.1.0 (Heikki Linnakangas, Andreas Karlsson, Tom Lane) + + + + This is a back-patch of work previously done in newer branches; + it's needed since many platforms are adopting newer OpenSSL versions. + + + + + + Support Tcl 8.6 in MSVC builds (Álvaro Herrera) + + + + + + Sync our copy of the timezone library with IANA release tzcode2017b + (Tom Lane) + + + + This fixes a bug affecting some DST transitions in January 2038. + + + + + + Update time zone data files to tzdata release 2017b + for DST law changes in Chile, Haiti, and Mongolia, plus historical + corrections for Ecuador, Kazakhstan, Liberia, and Spain. + Switch to numeric abbreviations for numerous time zones in South + America, the Pacific and Indian oceans, and some Asian and Middle + Eastern countries. + + + + The IANA time zone database previously provided textual abbreviations + for all time zones, sometimes making up abbreviations that have little + or no currency among the local population. They are in process of + reversing that policy in favor of using numeric UTC offsets in zones + where there is no evidence of real-world use of an English + abbreviation. At least for the time being, PostgreSQL + will continue to accept such removed abbreviations for timestamp input. + But they will not be shown in the pg_timezone_names + view nor used for output. + + + + + + Use correct daylight-savings rules for POSIX-style time zone names + in MSVC builds (David Rowley) + + + + The Microsoft MSVC build scripts neglected to install + the posixrules file in the timezone directory tree. + This resulted in the timezone code falling back to its built-in + rule about what DST behavior to assume for a POSIX-style time zone + name. For historical reasons that still corresponds to the DST rules + the USA was using before 2007 (i.e., change on first Sunday in April + and last Sunday in October). With this fix, a POSIX-style zone name + will use the current and historical DST transition dates of + the US/Eastern zone. If you don't want that, remove + the posixrules file, or replace it with a copy of some + other zone file (see ). Note that + due to caching, you may need to restart the server to get such changes + to take effect. + + + + + + + + Release 9.3.16 diff --git a/doc/src/sgml/release-9.4.sgml b/doc/src/sgml/release-9.4.sgml index 5f0280a18a..fe5ccca536 100644 --- a/doc/src/sgml/release-9.4.sgml +++ b/doc/src/sgml/release-9.4.sgml @@ -1,6 +1,441 @@ + + Release 9.4.12 + + + Release Date + 2017-05-11 + + + + This release contains a variety of fixes from 9.4.11. + For information about new features in the 9.4 major release, see + . + + + + Migration to Version 9.4.12 + + + A dump/restore is not required for those running 9.4.X. + + + + However, if you are using third-party replication tools that depend + on logical decoding, see the first changelog entry below. + + + + Also, if you are upgrading from a version earlier than 9.4.11, + see . + + + + + Changes + + + + + + Fix possibly-invalid initial snapshot during logical decoding + (Petr Jelinek, Andres Freund) + + + + The initial snapshot created for a logical decoding replication slot + was potentially incorrect. This could cause third-party tools that + use logical decoding to copy incomplete/inconsistent initial data. + This was more likely to happen if the source server was busy at the + time of slot creation, or if another logical slot already existed. + + + + If you are using a replication tool that depends on logical decoding, + and it should have copied a nonempty data set at the start of + replication, it is advisable to recreate the replica after + installing this update, or to verify its contents against the source + server. + + + + + + Fix possible corruption of init forks of unlogged indexes + (Robert Haas, Michael Paquier) + + + + This could result in an unlogged index being set to an invalid state + after a crash and restart. Such a problem would persist until the + index was dropped and rebuilt. + + + + + + Fix incorrect reconstruction of pg_subtrans entries + when a standby server replays a prepared but uncommitted two-phase + transaction (Tom Lane) + + + + In most cases this turned out to have no visible ill effects, but in + corner cases it could result in circular references + in pg_subtrans, potentially causing infinite loops + in queries that examine rows modified by the two-phase transaction. + + + + + + Avoid possible crash in walsender due to failure + to initialize a string buffer (Stas Kelvich, Fujii Masao) + + + + + + Fix postmaster's handling of fork() failure for a + background worker process (Tom Lane) + + + + Previously, the postmaster updated portions of its state as though + the process had been launched successfully, resulting in subsequent + confusion. + + + + + + Ensure parsing of queries in extension scripts sees the results of + immediately-preceding DDL (Julien Rouhaud, Tom Lane) + + + + Due to lack of a cache flush step between commands in an extension + script file, non-utility queries might not see the effects of an + immediately preceding catalog change, such as ALTER TABLE + ... RENAME. + + + + + + Skip tablespace privilege checks when ALTER TABLE ... ALTER + COLUMN TYPE rebuilds an existing index (Noah Misch) + + + + The command failed if the calling user did not currently have + CREATE privilege for the tablespace containing the index. + That behavior seems unhelpful, so skip the check, allowing the + index to be rebuilt where it is. + + + + + + Fix ALTER TABLE ... VALIDATE CONSTRAINT to not recurse + to child tables when the constraint is marked NO INHERIT + (Amit Langote) + + + + This fix prevents unwanted constraint does not exist failures + when no matching constraint is present in the child tables. + + + + + + Fix VACUUM to account properly for pages that could not + be scanned due to conflicting page pins (Andrew Gierth) + + + + This tended to lead to underestimation of the number of tuples in + the table. In the worst case of a small heavily-contended + table, VACUUM could incorrectly report that the table + contained no tuples, leading to very bad planning choices. + + + + + + Ensure that bulk-tuple-transfer loops within a hash join are + interruptible by query cancel requests (Tom Lane, Thomas Munro) + + + + + + Fix integer-overflow problems in interval comparison (Kyotaro + Horiguchi, Tom Lane) + + + + The comparison operators for type interval could yield wrong + answers for intervals larger than about 296000 years. Indexes on + columns containing such large values should be reindexed, since they + may be corrupt. + + + + + + Fix cursor_to_xml() to produce valid output + with tableforest = false + (Thomas Munro, Peter Eisentraut) + + + + Previously it failed to produce a wrapping <table> + element. + + + + + + Fix roundoff problems in float8_timestamptz() + and make_interval() (Tom Lane) + + + + These functions truncated, rather than rounded, when converting a + floating-point value to integer microseconds; that could cause + unexpectedly off-by-one results. + + + + + + Improve performance of pg_timezone_names view + (Tom Lane, David Rowley) + + + + + + Reduce memory management overhead for contexts containing many large + blocks (Tom Lane) + + + + + + Fix sloppy handling of corner-case errors from lseek() + and close() (Tom Lane) + + + + Neither of these system calls are likely to fail in typical situations, + but if they did, fd.c could get quite confused. + + + + + + Fix incorrect check for whether postmaster is running as a Windows + service (Michael Paquier) + + + + This could result in attempting to write to the event log when that + isn't accessible, so that no logging happens at all. + + + + + + Fix ecpg to support COMMIT PREPARED + and ROLLBACK PREPARED (Masahiko Sawada) + + + + + + Fix a double-free error when processing dollar-quoted string literals + in ecpg (Michael Meskes) + + + + + + In pg_dump, fix incorrect schema and owner marking for + comments and security labels of some types of database objects + (Giuseppe Broccolo, Tom Lane) + + + + In simple cases this caused no ill effects; but for example, a + schema-selective restore might omit comments it should include, because + they were not marked as belonging to the schema of their associated + object. + + + + + + Avoid emitting an invalid list file in pg_restore -l + when SQL object names contain newlines (Tom Lane) + + + + Replace newlines by spaces, which is sufficient to make the output + valid for pg_restore -L's purposes. + + + + + + Fix pg_upgrade to transfer comments and security labels + attached to large objects (blobs) (Stephen Frost) + + + + Previously, blobs were correctly transferred to the new database, but + any comments or security labels attached to them were lost. + + + + + + Improve error handling + in contrib/adminpack's pg_file_write() + function (Noah Misch) + + + + Notably, it failed to detect errors reported + by fclose(). + + + + + + In contrib/dblink, avoid leaking the previous unnamed + connection when establishing a new unnamed connection (Joe Conway) + + + + + + Fix contrib/pg_trgm's extraction of trigrams from regular + expressions (Tom Lane) + + + + In some cases it would produce a broken data structure that could never + match anything, leading to GIN or GiST indexscans that use a trigram + index not finding any matches to the regular expression. + + + + + + In contrib/postgres_fdw, + transmit query cancellation requests to the remote server + (Michael Paquier, Etsuro Fujita) + + + + Previously, a local query cancellation request did not cause an + already-sent remote query to terminate early. This is a back-patch + of work originally done for 9.6. + + + + + + + Support OpenSSL 1.1.0 (Heikki Linnakangas, Andreas Karlsson, Tom Lane) + + + + This is a back-patch of work previously done in newer branches; + it's needed since many platforms are adopting newer OpenSSL versions. + + + + + + Support Tcl 8.6 in MSVC builds (Álvaro Herrera) + + + + + + Sync our copy of the timezone library with IANA release tzcode2017b + (Tom Lane) + + + + This fixes a bug affecting some DST transitions in January 2038. + + + + + + Update time zone data files to tzdata release 2017b + for DST law changes in Chile, Haiti, and Mongolia, plus historical + corrections for Ecuador, Kazakhstan, Liberia, and Spain. + Switch to numeric abbreviations for numerous time zones in South + America, the Pacific and Indian oceans, and some Asian and Middle + Eastern countries. + + + + The IANA time zone database previously provided textual abbreviations + for all time zones, sometimes making up abbreviations that have little + or no currency among the local population. They are in process of + reversing that policy in favor of using numeric UTC offsets in zones + where there is no evidence of real-world use of an English + abbreviation. At least for the time being, PostgreSQL + will continue to accept such removed abbreviations for timestamp input. + But they will not be shown in the pg_timezone_names + view nor used for output. + + + + + + Use correct daylight-savings rules for POSIX-style time zone names + in MSVC builds (David Rowley) + + + + The Microsoft MSVC build scripts neglected to install + the posixrules file in the timezone directory tree. + This resulted in the timezone code falling back to its built-in + rule about what DST behavior to assume for a POSIX-style time zone + name. For historical reasons that still corresponds to the DST rules + the USA was using before 2007 (i.e., change on first Sunday in April + and last Sunday in October). With this fix, a POSIX-style zone name + will use the current and historical DST transition dates of + the US/Eastern zone. If you don't want that, remove + the posixrules file, or replace it with a copy of some + other zone file (see ). Note that + due to caching, you may need to restart the server to get such changes + to take effect. + + + + + + + + Release 9.4.11 diff --git a/doc/src/sgml/release-9.5.sgml b/doc/src/sgml/release-9.5.sgml index c2c653a9b3..3610e3037e 100644 --- a/doc/src/sgml/release-9.5.sgml +++ b/doc/src/sgml/release-9.5.sgml @@ -1,6 +1,502 @@ + + Release 9.5.7 + + + Release Date + 2017-05-11 + + + + This release contains a variety of fixes from 9.5.6. + For information about new features in the 9.5 major release, see + . + + + + Migration to Version 9.5.7 + + + A dump/restore is not required for those running 9.5.X. + + + + However, if you are using third-party replication tools that depend + on logical decoding, see the first changelog entry below. + + + + Also, if you are upgrading from a version earlier than 9.5.6, + see . + + + + + Changes + + + + + + Fix possibly-invalid initial snapshot during logical decoding + (Petr Jelinek, Andres Freund) + + + + The initial snapshot created for a logical decoding replication slot + was potentially incorrect. This could cause third-party tools that + use logical decoding to copy incomplete/inconsistent initial data. + This was more likely to happen if the source server was busy at the + time of slot creation, or if another logical slot already existed. + + + + If you are using a replication tool that depends on logical decoding, + and it should have copied a nonempty data set at the start of + replication, it is advisable to recreate the replica after + installing this update, or to verify its contents against the source + server. + + + + + + Fix possible corruption of init forks of unlogged indexes + (Robert Haas, Michael Paquier) + + + + This could result in an unlogged index being set to an invalid state + after a crash and restart. Such a problem would persist until the + index was dropped and rebuilt. + + + + + + Fix incorrect reconstruction of pg_subtrans entries + when a standby server replays a prepared but uncommitted two-phase + transaction (Tom Lane) + + + + In most cases this turned out to have no visible ill effects, but in + corner cases it could result in circular references + in pg_subtrans, potentially causing infinite loops + in queries that examine rows modified by the two-phase transaction. + + + + + + Avoid possible crash in walsender due to failure + to initialize a string buffer (Stas Kelvich, Fujii Masao) + + + + + + Fix possible crash when rescanning a nearest-neighbor index-only scan + on a GiST index (Tom Lane) + + + + + + Fix postmaster's handling of fork() failure for a + background worker process (Tom Lane) + + + + Previously, the postmaster updated portions of its state as though + the process had been launched successfully, resulting in subsequent + confusion. + + + + + + + Fix crash or wrong answers when a GROUPING SETS column's + data type is hashable but not sortable (Pavan Deolasee) + + + + + + Avoid applying physical targetlist optimization to custom + scans (Dmitry Ivanov, Tom Lane) + + + + This optimization supposed that retrieving all columns of a tuple + is inexpensive, which is true for ordinary Postgres tuples; but it + might not be the case for a custom scan provider. + + + + + + Use the correct sub-expression when applying a FOR ALL + row-level-security policy (Stephen Frost) + + + + In some cases the WITH CHECK restriction would be applied + when the USING restriction is more appropriate. + + + + + + Ensure parsing of queries in extension scripts sees the results of + immediately-preceding DDL (Julien Rouhaud, Tom Lane) + + + + Due to lack of a cache flush step between commands in an extension + script file, non-utility queries might not see the effects of an + immediately preceding catalog change, such as ALTER TABLE + ... RENAME. + + + + + + Skip tablespace privilege checks when ALTER TABLE ... ALTER + COLUMN TYPE rebuilds an existing index (Noah Misch) + + + + The command failed if the calling user did not currently have + CREATE privilege for the tablespace containing the index. + That behavior seems unhelpful, so skip the check, allowing the + index to be rebuilt where it is. + + + + + + Fix ALTER TABLE ... VALIDATE CONSTRAINT to not recurse + to child tables when the constraint is marked NO INHERIT + (Amit Langote) + + + + This fix prevents unwanted constraint does not exist failures + when no matching constraint is present in the child tables. + + + + + + Avoid dangling pointer in COPY ... TO when row-level + security is active for the source table (Tom Lane) + + + + Usually this had no ill effects, but sometimes it would cause + unexpected errors or crashes. + + + + + + Avoid accessing an already-closed relcache entry in CLUSTER + and VACUUM FULL (Tom Lane) + + + + With some bad luck, this could lead to indexes on the target + relation getting rebuilt with the wrong persistence setting. + + + + + + Fix VACUUM to account properly for pages that could not + be scanned due to conflicting page pins (Andrew Gierth) + + + + This tended to lead to underestimation of the number of tuples in + the table. In the worst case of a small heavily-contended + table, VACUUM could incorrectly report that the table + contained no tuples, leading to very bad planning choices. + + + + + + Ensure that bulk-tuple-transfer loops within a hash join are + interruptible by query cancel requests (Tom Lane, Thomas Munro) + + + + + + Fix integer-overflow problems in interval comparison (Kyotaro + Horiguchi, Tom Lane) + + + + The comparison operators for type interval could yield wrong + answers for intervals larger than about 296000 years. Indexes on + columns containing such large values should be reindexed, since they + may be corrupt. + + + + + + Fix cursor_to_xml() to produce valid output + with tableforest = false + (Thomas Munro, Peter Eisentraut) + + + + Previously it failed to produce a wrapping <table> + element. + + + + + + Fix roundoff problems in float8_timestamptz() + and make_interval() (Tom Lane) + + + + These functions truncated, rather than rounded, when converting a + floating-point value to integer microseconds; that could cause + unexpectedly off-by-one results. + + + + + + Fix pg_get_object_address() to handle members of operator + families correctly (Álvaro Herrera) + + + + + + Improve performance of pg_timezone_names view + (Tom Lane, David Rowley) + + + + + + Reduce memory management overhead for contexts containing many large + blocks (Tom Lane) + + + + + + Fix sloppy handling of corner-case errors from lseek() + and close() (Tom Lane) + + + + Neither of these system calls are likely to fail in typical situations, + but if they did, fd.c could get quite confused. + + + + + + Fix incorrect check for whether postmaster is running as a Windows + service (Michael Paquier) + + + + This could result in attempting to write to the event log when that + isn't accessible, so that no logging happens at all. + + + + + + Fix ecpg to support COMMIT PREPARED + and ROLLBACK PREPARED (Masahiko Sawada) + + + + + + Fix a double-free error when processing dollar-quoted string literals + in ecpg (Michael Meskes) + + + + + + In pg_dump, fix incorrect schema and owner marking for + comments and security labels of some types of database objects + (Giuseppe Broccolo, Tom Lane) + + + + In simple cases this caused no ill effects; but for example, a + schema-selective restore might omit comments it should include, because + they were not marked as belonging to the schema of their associated + object. + + + + + + Avoid emitting an invalid list file in pg_restore -l + when SQL object names contain newlines (Tom Lane) + + + + Replace newlines by spaces, which is sufficient to make the output + valid for pg_restore -L's purposes. + + + + + + Fix pg_upgrade to transfer comments and security labels + attached to large objects (blobs) (Stephen Frost) + + + + Previously, blobs were correctly transferred to the new database, but + any comments or security labels attached to them were lost. + + + + + + Improve error handling + in contrib/adminpack's pg_file_write() + function (Noah Misch) + + + + Notably, it failed to detect errors reported + by fclose(). + + + + + + In contrib/dblink, avoid leaking the previous unnamed + connection when establishing a new unnamed connection (Joe Conway) + + + + + + Fix contrib/pg_trgm's extraction of trigrams from regular + expressions (Tom Lane) + + + + In some cases it would produce a broken data structure that could never + match anything, leading to GIN or GiST indexscans that use a trigram + index not finding any matches to the regular expression. + + + + + + + In contrib/postgres_fdw, + transmit query cancellation requests to the remote server + (Michael Paquier, Etsuro Fujita) + + + + Previously, a local query cancellation request did not cause an + already-sent remote query to terminate early. This is a back-patch + of work originally done for 9.6. + + + + + + Support Tcl 8.6 in MSVC builds (Álvaro Herrera) + + + + + + Sync our copy of the timezone library with IANA release tzcode2017b + (Tom Lane) + + + + This fixes a bug affecting some DST transitions in January 2038. + + + + + + Update time zone data files to tzdata release 2017b + for DST law changes in Chile, Haiti, and Mongolia, plus historical + corrections for Ecuador, Kazakhstan, Liberia, and Spain. + Switch to numeric abbreviations for numerous time zones in South + America, the Pacific and Indian oceans, and some Asian and Middle + Eastern countries. + + + + The IANA time zone database previously provided textual abbreviations + for all time zones, sometimes making up abbreviations that have little + or no currency among the local population. They are in process of + reversing that policy in favor of using numeric UTC offsets in zones + where there is no evidence of real-world use of an English + abbreviation. At least for the time being, PostgreSQL + will continue to accept such removed abbreviations for timestamp input. + But they will not be shown in the pg_timezone_names + view nor used for output. + + + + + + Use correct daylight-savings rules for POSIX-style time zone names + in MSVC builds (David Rowley) + + + + The Microsoft MSVC build scripts neglected to install + the posixrules file in the timezone directory tree. + This resulted in the timezone code falling back to its built-in + rule about what DST behavior to assume for a POSIX-style time zone + name. For historical reasons that still corresponds to the DST rules + the USA was using before 2007 (i.e., change on first Sunday in April + and last Sunday in October). With this fix, a POSIX-style zone name + will use the current and historical DST transition dates of + the US/Eastern zone. If you don't want that, remove + the posixrules file, or replace it with a copy of some + other zone file (see ). Note that + due to caching, you may need to restart the server to get such changes + to take effect. + + + + + + + + Release 9.5.6 diff --git a/doc/src/sgml/release-9.6.sgml b/doc/src/sgml/release-9.6.sgml index b6cc481494..bacbc29d67 100644 --- a/doc/src/sgml/release-9.6.sgml +++ b/doc/src/sgml/release-9.6.sgml @@ -188,17 +188,6 @@ Branch: REL9_4_STABLE [436b560b8] 2017-04-24 12:16:58 -0400 - - Fix crash or wrong answers when a GROUPING SETS column's - data type is hashable but not sortable (Pavan Deolasee) - - - - - - - In contrib/postgres_fdw, - transmit query cancellation requests to the remote server - (Michael Paquier, Etsuro Fujita) - - - - Previously, a local query cancellation request did not cause an - already-sent remote query to terminate early. This is a back-patch - of work originally done for 9.6. - - - - - - - Support OpenSSL 1.1.0 (Heikki Linnakangas, Andreas Karlsson, Tom Lane) - - - - This is a back-patch of work previously done in newer branches; - it's needed since many platforms are adopting newer OpenSSL versions. - - - - - Have VACUUM VERBOSE report - the number of skipped frozen pages and oldest xmin (Masahiko Sawada) + the number of skipped frozen pages and oldest xmin (Masahiko + Sawada, Simon Riggs) -- cgit v1.2.3 From 0170b10dff04e0f50f5522c377a4d10d4424155c Mon Sep 17 00:00:00 2001 From: Noah Misch Date: Mon, 8 May 2017 07:24:24 -0700 Subject: Restore PGREQUIRESSL recognition in libpq. Commit 65c3bf19fd3e1f6a591618e92eb4c54d0b217564 moved handling of the, already then, deprecated requiressl parameter into conninfo_storeval(). The default PGREQUIRESSL environment variable was however lost in the change resulting in a potentially silent accept of a non-SSL connection even when set. Its documentation remained. Restore its implementation. Also amend the documentation to mark PGREQUIRESSL as deprecated for those not following the link to requiressl. Back-patch to 9.3, where commit 65c3bf1 first appeared. Behavior has been more complex when the user provides both deprecated and non-deprecated settings. Before commit 65c3bf1, libpq operated according to the first of these found: requiressl=1 PGREQUIRESSL=1 sslmode=* PGSSLMODE=* (Note requiressl=0 didn't override sslmode=*; it would only suppress PGREQUIRESSL=1 or a previous requiressl=1. PGREQUIRESSL=0 had no effect whatsoever.) Starting with commit 65c3bf1, libpq ignored PGREQUIRESSL, and order of precedence changed to this: last of requiressl=* or sslmode=* PGSSLMODE=* Starting now, adopt the following order of precedence: last of requiressl=* or sslmode=* PGSSLMODE=* PGREQUIRESSL=1 This retains the 65c3bf1 behavior for connection strings that contain both requiressl=* and sslmode=*. It retains the 65c3bf1 change that either connection string option overrides both environment variables. For the first time, PGSSLMODE has precedence over PGREQUIRESSL; this avoids reducing security of "PGREQUIRESSL=1 PGSSLMODE=verify-full" configurations originating under v9.3 and later. Daniel Gustafsson Security: CVE-2017-7485 --- doc/src/sgml/libpq.sgml | 3 +++ src/interfaces/libpq/fe-connect.c | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+) (limited to 'doc/src') diff --git a/doc/src/sgml/libpq.sgml b/doc/src/sgml/libpq.sgml index c2b7abc603..bf95b0bada 100644 --- a/doc/src/sgml/libpq.sgml +++ b/doc/src/sgml/libpq.sgml @@ -7061,6 +7061,9 @@ myEventProc(PGEventId evtId, void *evtInfo, void *passThrough) PGREQUIRESSL behaves the same as the connection parameter. + This environment variable is deprecated in favor of the + PGSSLMODE variable; setting both variables suppresses the + effect of this one. diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index eb5aaf7098..4dc892402d 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -5083,6 +5083,30 @@ conninfo_add_defaults(PQconninfoOption *options, PQExpBuffer errorMessage) } } + /* + * Interpret the deprecated PGREQUIRESSL environment variable. Per + * tradition, translate values starting with "1" to sslmode=require, + * and ignore other values. Given both PGREQUIRESSL=1 and PGSSLMODE, + * PGSSLMODE takes precedence; the opposite was true before v9.3. + */ + if (strcmp(option->keyword, "sslmode") == 0) + { + const char *requiresslenv = getenv("PGREQUIRESSL"); + + if (requiresslenv != NULL && requiresslenv[0] == '1') + { + option->val = strdup("require"); + if (!option->val) + { + if (errorMessage) + printfPQExpBuffer(errorMessage, + libpq_gettext("out of memory\n")); + return false; + } + continue; + } + } + /* * No environment variable specified or the variable isn't set - try * compiled-in default -- cgit v1.2.3 From 3eefc51053f250837c3115c12f8119d16881a2d7 Mon Sep 17 00:00:00 2001 From: Noah Misch Date: Mon, 8 May 2017 07:24:24 -0700 Subject: Match pg_user_mappings limits to information_schema.user_mapping_options. Both views replace the umoptions field with NULL when the user does not meet qualifications to see it. They used different qualifications, and pg_user_mappings documented qualifications did not match its implemented qualifications. Make its documentation and implementation match those of user_mapping_options. One might argue for stronger qualifications, but these have long, documented tenure. pg_user_mappings has always exhibited this problem, so back-patch to 9.2 (all supported versions). Michael Paquier and Feike Steenbergen. Reviewed by Jeff Janes. Reported by Andrew Wheelwright. Security: CVE-2017-7486 --- doc/src/sgml/catalogs.sgml | 7 ++-- src/backend/catalog/system_views.sql | 10 +++--- src/test/regress/expected/foreign_data.out | 54 ++++++++++++++++++++++++++++++ src/test/regress/expected/rules.out | 4 ++- src/test/regress/sql/foreign_data.sql | 15 +++++++++ 5 files changed, 82 insertions(+), 8 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 787fcbd51a..aa5e705e57 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -11084,8 +11084,11 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx User mapping specific options, as keyword=value - strings, if the current user is the owner of the foreign - server, else null + strings. This column will show as null unless the current user + is the user being mapped, or the mapping is for + PUBLIC and the current user is the server + owner, or the current user is a superuser. The intent is + to protect password information stored as user mapping option. diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 421d51db47..5cd176bc38 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -910,11 +910,11 @@ CREATE VIEW pg_user_mappings AS ELSE A.rolname END AS usename, - CASE WHEN pg_has_role(S.srvowner, 'USAGE') OR has_server_privilege(S.oid, 'USAGE') THEN - U.umoptions - ELSE - NULL - END AS umoptions + CASE WHEN (U.umuser <> 0 AND A.rolname = current_user) + OR (U.umuser = 0 AND pg_has_role(S.srvowner, 'USAGE')) + OR (SELECT rolsuper FROM pg_authid WHERE rolname = current_user) + THEN U.umoptions + ELSE NULL END AS umoptions FROM pg_user_mapping U JOIN pg_foreign_server S ON (U.umserver = S.oid) LEFT JOIN pg_authid A ON (A.oid = U.umuser); diff --git a/src/test/regress/expected/foreign_data.out b/src/test/regress/expected/foreign_data.out index 1c7a7593f9..af327d030d 100644 --- a/src/test/regress/expected/foreign_data.out +++ b/src/test/regress/expected/foreign_data.out @@ -1200,7 +1200,61 @@ WARNING: no privileges were granted for "s9" CREATE USER MAPPING FOR current_user SERVER s9; DROP SERVER s9 CASCADE; -- ERROR ERROR: must be owner of foreign server s9 +-- Check visibility of user mapping data +SET ROLE regress_test_role; +CREATE SERVER s10 FOREIGN DATA WRAPPER foo; +CREATE USER MAPPING FOR public SERVER s10 OPTIONS (user 'secret'); +GRANT USAGE ON FOREIGN SERVER s10 TO regress_unprivileged_role; +-- owner of server can see option fields +\deu+ + List of user mappings + Server | User name | FDW Options +--------+---------------------------+------------------- + s10 | public | ("user" 'secret') + s4 | regress_foreign_data_user | + s5 | regress_test_role | (modified '1') + s6 | regress_test_role | + s8 | public | + s8 | regress_foreign_data_user | + s9 | regress_unprivileged_role | + t1 | public | (modified '1') +(8 rows) + +RESET ROLE; +-- superuser can see option fields +\deu+ + List of user mappings + Server | User name | FDW Options +--------+---------------------------+--------------------- + s10 | public | ("user" 'secret') + s4 | regress_foreign_data_user | + s5 | regress_test_role | (modified '1') + s6 | regress_test_role | + s8 | public | + s8 | regress_foreign_data_user | (password 'public') + s9 | regress_unprivileged_role | + t1 | public | (modified '1') +(8 rows) + +-- unprivileged user cannot see option fields +SET ROLE regress_unprivileged_role; +\deu+ + List of user mappings + Server | User name | FDW Options +--------+---------------------------+------------- + s10 | public | + s4 | regress_foreign_data_user | + s5 | regress_test_role | + s6 | regress_test_role | + s8 | public | + s8 | regress_foreign_data_user | + s9 | regress_unprivileged_role | + t1 | public | +(8 rows) + RESET ROLE; +DROP SERVER s10 CASCADE; +NOTICE: drop cascades to user mapping for public on server s10 -- Triggers CREATE FUNCTION dummy_trigger() RETURNS TRIGGER AS $$ BEGIN diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 0165471a4d..259b26b6f7 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2228,7 +2228,9 @@ pg_user_mappings| SELECT u.oid AS umid, ELSE a.rolname END AS usename, CASE - WHEN (pg_has_role(s.srvowner, 'USAGE'::text) OR has_server_privilege(s.oid, 'USAGE'::text)) THEN u.umoptions + WHEN (((u.umuser <> (0)::oid) AND (a.rolname = CURRENT_USER)) OR ((u.umuser = (0)::oid) AND pg_has_role(s.srvowner, 'USAGE'::text)) OR ( SELECT pg_authid.rolsuper + FROM pg_authid + WHERE (pg_authid.rolname = CURRENT_USER))) THEN u.umoptions ELSE NULL::text[] END AS umoptions FROM ((pg_user_mapping u diff --git a/src/test/regress/sql/foreign_data.sql b/src/test/regress/sql/foreign_data.sql index aaf079cf52..ba528b5d36 100644 --- a/src/test/regress/sql/foreign_data.sql +++ b/src/test/regress/sql/foreign_data.sql @@ -493,7 +493,22 @@ ALTER SERVER s9 VERSION '1.2'; -- ERROR GRANT USAGE ON FOREIGN SERVER s9 TO regress_test_role; -- WARNING CREATE USER MAPPING FOR current_user SERVER s9; DROP SERVER s9 CASCADE; -- ERROR + +-- Check visibility of user mapping data +SET ROLE regress_test_role; +CREATE SERVER s10 FOREIGN DATA WRAPPER foo; +CREATE USER MAPPING FOR public SERVER s10 OPTIONS (user 'secret'); +GRANT USAGE ON FOREIGN SERVER s10 TO regress_unprivileged_role; +-- owner of server can see option fields +\deu+ +RESET ROLE; +-- superuser can see option fields +\deu+ +-- unprivileged user cannot see option fields +SET ROLE regress_unprivileged_role; +\deu+ RESET ROLE; +DROP SERVER s10 CASCADE; -- Triggers CREATE FUNCTION dummy_trigger() RETURNS TRIGGER AS $$ -- cgit v1.2.3 From c89d2d0204f25e556e94dabd0fd5174cf6963b1d Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 8 May 2017 12:57:27 -0400 Subject: Last-minute updates for release notes. Security: CVE-2017-7484, CVE-2017-7485, CVE-2017-7486 --- doc/src/sgml/release-9.2.sgml | 125 ++++++++++++++++++++++++++++- doc/src/sgml/release-9.3.sgml | 143 +++++++++++++++++++++++++++++++++- doc/src/sgml/release-9.4.sgml | 145 +++++++++++++++++++++++++++++++++- doc/src/sgml/release-9.5.sgml | 145 +++++++++++++++++++++++++++++++++- doc/src/sgml/release-9.6.sgml | 177 +++++++++++++++++++++++++++++++++++++++++- 5 files changed, 727 insertions(+), 8 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/release-9.2.sgml b/doc/src/sgml/release-9.2.sgml index b2454ad061..6aee919ffe 100644 --- a/doc/src/sgml/release-9.2.sgml +++ b/doc/src/sgml/release-9.2.sgml @@ -29,7 +29,12 @@ - However, if you are upgrading from a version earlier than 9.2.20, + However, if you use foreign data servers that make use of user + passwords for authentication, see the first changelog entry below. + + + + Also, if you are upgrading from a version earlier than 9.2.20, see . @@ -40,6 +45,124 @@ + + + Restrict visibility + of pg_user_mappings.umoptions, to + protect passwords stored as user mapping options + (Michael Paquier, Feike Steenbergen) + + + + The previous coding allowed the owner of a foreign server object, + or anyone he has granted server USAGE permission to, + to see the options for all user mappings associated with that server. + This might well include passwords for other users. + Adjust the view definition to match the behavior of + information_schema.user_mapping_options, namely that + these options are visible to the user being mapped, or if the mapping + is for PUBLIC and the current user is the server + owner, or if the current user is a superuser. + (CVE-2017-7486) + + + + By itself, this patch will only fix the behavior in newly initdb'd + databases. If you wish to apply this change in an existing database, + you will need to do the following: + + + + + + Restart the postmaster after adding allow_system_table_mods + = true to postgresql.conf. (In versions + supporting ALTER SYSTEM, you can use that to make the + configuration change, but you'll still need a restart.) + + + + + + In each database of the cluster, + run the following commands as superuser: + +SET search_path = pg_catalog; +CREATE OR REPLACE VIEW pg_user_mappings AS + SELECT + U.oid AS umid, + S.oid AS srvid, + S.srvname AS srvname, + U.umuser AS umuser, + CASE WHEN U.umuser = 0 THEN + 'public' + ELSE + A.rolname + END AS usename, + CASE WHEN (U.umuser <> 0 AND A.rolname = current_user) + OR (U.umuser = 0 AND pg_has_role(S.srvowner, 'USAGE')) + OR (SELECT rolsuper FROM pg_authid WHERE rolname = current_user) + THEN U.umoptions + ELSE NULL END AS umoptions + FROM pg_user_mapping U + LEFT JOIN pg_authid A ON (A.oid = U.umuser) JOIN + pg_foreign_server S ON (U.umserver = S.oid); + + + + + + + Do not forget to include the template0 + and template1 databases, or the vulnerability will still + exist in databases you create later. To fix template0, + you'll need to temporarily make it accept connections. + In PostgreSQL 9.5 and later, you can use + +ALTER DATABASE template0 WITH ALLOW_CONNECTIONS true; + + and then after fixing template0, undo that with + +ALTER DATABASE template0 WITH ALLOW_CONNECTIONS false; + + In prior versions, instead use + +UPDATE pg_database SET datallowconn = true WHERE datname = 'template0'; +UPDATE pg_database SET datallowconn = false WHERE datname = 'template0'; + + + + + + + Finally, remove the allow_system_table_mods configuration + setting, and again restart the postmaster. + + + + + + + + Prevent exposure of statistical information via leaky operators + (Peter Eisentraut) + + + + Some selectivity estimation functions in the planner will apply + user-defined operators to values obtained + from pg_statistic, such as most common values and + histogram entries. This occurs before table permissions are checked, + so a nefarious user could exploit the behavior to obtain these values + for table columns he does not have permission to read. To fix, + fall back to a default estimate if the operator's implementation + function is not certified leak-proof and the calling user does not have + permission to read the table column whose statistics are needed. + At least one of these criteria is satisfied in most cases in practice. + (CVE-2017-7484) + + + Fix possible corruption of init forks of unlogged indexes diff --git a/doc/src/sgml/release-9.3.sgml b/doc/src/sgml/release-9.3.sgml index 44c1e2ac69..6d620ffacc 100644 --- a/doc/src/sgml/release-9.3.sgml +++ b/doc/src/sgml/release-9.3.sgml @@ -23,7 +23,12 @@ - However, if you are upgrading from a version earlier than 9.3.16, + However, if you use foreign data servers that make use of user + passwords for authentication, see the first changelog entry below. + + + + Also, if you are upgrading from a version earlier than 9.3.16, see . @@ -34,6 +39,142 @@ + + + Restrict visibility + of pg_user_mappings.umoptions, to + protect passwords stored as user mapping options + (Michael Paquier, Feike Steenbergen) + + + + The previous coding allowed the owner of a foreign server object, + or anyone he has granted server USAGE permission to, + to see the options for all user mappings associated with that server. + This might well include passwords for other users. + Adjust the view definition to match the behavior of + information_schema.user_mapping_options, namely that + these options are visible to the user being mapped, or if the mapping + is for PUBLIC and the current user is the server + owner, or if the current user is a superuser. + (CVE-2017-7486) + + + + By itself, this patch will only fix the behavior in newly initdb'd + databases. If you wish to apply this change in an existing database, + you will need to do the following: + + + + + + Restart the postmaster after adding allow_system_table_mods + = true to postgresql.conf. (In versions + supporting ALTER SYSTEM, you can use that to make the + configuration change, but you'll still need a restart.) + + + + + + In each database of the cluster, + run the following commands as superuser: + +SET search_path = pg_catalog; +CREATE OR REPLACE VIEW pg_user_mappings AS + SELECT + U.oid AS umid, + S.oid AS srvid, + S.srvname AS srvname, + U.umuser AS umuser, + CASE WHEN U.umuser = 0 THEN + 'public' + ELSE + A.rolname + END AS usename, + CASE WHEN (U.umuser <> 0 AND A.rolname = current_user) + OR (U.umuser = 0 AND pg_has_role(S.srvowner, 'USAGE')) + OR (SELECT rolsuper FROM pg_authid WHERE rolname = current_user) + THEN U.umoptions + ELSE NULL END AS umoptions + FROM pg_user_mapping U + LEFT JOIN pg_authid A ON (A.oid = U.umuser) JOIN + pg_foreign_server S ON (U.umserver = S.oid); + + + + + + + Do not forget to include the template0 + and template1 databases, or the vulnerability will still + exist in databases you create later. To fix template0, + you'll need to temporarily make it accept connections. + In PostgreSQL 9.5 and later, you can use + +ALTER DATABASE template0 WITH ALLOW_CONNECTIONS true; + + and then after fixing template0, undo that with + +ALTER DATABASE template0 WITH ALLOW_CONNECTIONS false; + + In prior versions, instead use + +UPDATE pg_database SET datallowconn = true WHERE datname = 'template0'; +UPDATE pg_database SET datallowconn = false WHERE datname = 'template0'; + + + + + + + Finally, remove the allow_system_table_mods configuration + setting, and again restart the postmaster. + + + + + + + + Prevent exposure of statistical information via leaky operators + (Peter Eisentraut) + + + + Some selectivity estimation functions in the planner will apply + user-defined operators to values obtained + from pg_statistic, such as most common values and + histogram entries. This occurs before table permissions are checked, + so a nefarious user could exploit the behavior to obtain these values + for table columns he does not have permission to read. To fix, + fall back to a default estimate if the operator's implementation + function is not certified leak-proof and the calling user does not have + permission to read the table column whose statistics are needed. + At least one of these criteria is satisfied in most cases in practice. + (CVE-2017-7484) + + + + + + Restore libpq's recognition of + the PGREQUIRESSL environment variable (Daniel Gustafsson) + + + + Processing of this environment variable was unintentionally dropped + in PostgreSQL 9.3, but its documentation remained. + This creates a security hazard, since users might be relying on the + environment variable to force SSL-encrypted connections, but that + would no longer be guaranteed. Restore handling of the variable, + but give it lower priority than PGSSLMODE, to avoid + breaking configurations that work correctly with post-9.3 code. + (CVE-2017-7485) + + + Fix possible corruption of init forks of unlogged indexes diff --git a/doc/src/sgml/release-9.4.sgml b/doc/src/sgml/release-9.4.sgml index fe5ccca536..ae60b80959 100644 --- a/doc/src/sgml/release-9.4.sgml +++ b/doc/src/sgml/release-9.4.sgml @@ -23,8 +23,13 @@ - However, if you are using third-party replication tools that depend - on logical decoding, see the first changelog entry below. + However, if you use foreign data servers that make use of user + passwords for authentication, see the first changelog entry below. + + + + Also, if you are using third-party replication tools that depend + on logical decoding, see the fourth changelog entry below. @@ -38,6 +43,142 @@ + + + Restrict visibility + of pg_user_mappings.umoptions, to + protect passwords stored as user mapping options + (Michael Paquier, Feike Steenbergen) + + + + The previous coding allowed the owner of a foreign server object, + or anyone he has granted server USAGE permission to, + to see the options for all user mappings associated with that server. + This might well include passwords for other users. + Adjust the view definition to match the behavior of + information_schema.user_mapping_options, namely that + these options are visible to the user being mapped, or if the mapping + is for PUBLIC and the current user is the server + owner, or if the current user is a superuser. + (CVE-2017-7486) + + + + By itself, this patch will only fix the behavior in newly initdb'd + databases. If you wish to apply this change in an existing database, + you will need to do the following: + + + + + + Restart the postmaster after adding allow_system_table_mods + = true to postgresql.conf. (In versions + supporting ALTER SYSTEM, you can use that to make the + configuration change, but you'll still need a restart.) + + + + + + In each database of the cluster, + run the following commands as superuser: + +SET search_path = pg_catalog; +CREATE OR REPLACE VIEW pg_user_mappings AS + SELECT + U.oid AS umid, + S.oid AS srvid, + S.srvname AS srvname, + U.umuser AS umuser, + CASE WHEN U.umuser = 0 THEN + 'public' + ELSE + A.rolname + END AS usename, + CASE WHEN (U.umuser <> 0 AND A.rolname = current_user) + OR (U.umuser = 0 AND pg_has_role(S.srvowner, 'USAGE')) + OR (SELECT rolsuper FROM pg_authid WHERE rolname = current_user) + THEN U.umoptions + ELSE NULL END AS umoptions + FROM pg_user_mapping U + LEFT JOIN pg_authid A ON (A.oid = U.umuser) JOIN + pg_foreign_server S ON (U.umserver = S.oid); + + + + + + + Do not forget to include the template0 + and template1 databases, or the vulnerability will still + exist in databases you create later. To fix template0, + you'll need to temporarily make it accept connections. + In PostgreSQL 9.5 and later, you can use + +ALTER DATABASE template0 WITH ALLOW_CONNECTIONS true; + + and then after fixing template0, undo that with + +ALTER DATABASE template0 WITH ALLOW_CONNECTIONS false; + + In prior versions, instead use + +UPDATE pg_database SET datallowconn = true WHERE datname = 'template0'; +UPDATE pg_database SET datallowconn = false WHERE datname = 'template0'; + + + + + + + Finally, remove the allow_system_table_mods configuration + setting, and again restart the postmaster. + + + + + + + + Prevent exposure of statistical information via leaky operators + (Peter Eisentraut) + + + + Some selectivity estimation functions in the planner will apply + user-defined operators to values obtained + from pg_statistic, such as most common values and + histogram entries. This occurs before table permissions are checked, + so a nefarious user could exploit the behavior to obtain these values + for table columns he does not have permission to read. To fix, + fall back to a default estimate if the operator's implementation + function is not certified leak-proof and the calling user does not have + permission to read the table column whose statistics are needed. + At least one of these criteria is satisfied in most cases in practice. + (CVE-2017-7484) + + + + + + Restore libpq's recognition of + the PGREQUIRESSL environment variable (Daniel Gustafsson) + + + + Processing of this environment variable was unintentionally dropped + in PostgreSQL 9.3, but its documentation remained. + This creates a security hazard, since users might be relying on the + environment variable to force SSL-encrypted connections, but that + would no longer be guaranteed. Restore handling of the variable, + but give it lower priority than PGSSLMODE, to avoid + breaking configurations that work correctly with post-9.3 code. + (CVE-2017-7485) + + + Fix possibly-invalid initial snapshot during logical decoding diff --git a/doc/src/sgml/release-9.5.sgml b/doc/src/sgml/release-9.5.sgml index 3610e3037e..1c2aca0683 100644 --- a/doc/src/sgml/release-9.5.sgml +++ b/doc/src/sgml/release-9.5.sgml @@ -23,8 +23,13 @@ - However, if you are using third-party replication tools that depend - on logical decoding, see the first changelog entry below. + However, if you use foreign data servers that make use of user + passwords for authentication, see the first changelog entry below. + + + + Also, if you are using third-party replication tools that depend + on logical decoding, see the fourth changelog entry below. @@ -38,6 +43,142 @@ + + + Restrict visibility + of pg_user_mappings.umoptions, to + protect passwords stored as user mapping options + (Michael Paquier, Feike Steenbergen) + + + + The previous coding allowed the owner of a foreign server object, + or anyone he has granted server USAGE permission to, + to see the options for all user mappings associated with that server. + This might well include passwords for other users. + Adjust the view definition to match the behavior of + information_schema.user_mapping_options, namely that + these options are visible to the user being mapped, or if the mapping + is for PUBLIC and the current user is the server + owner, or if the current user is a superuser. + (CVE-2017-7486) + + + + By itself, this patch will only fix the behavior in newly initdb'd + databases. If you wish to apply this change in an existing database, + you will need to do the following: + + + + + + Restart the postmaster after adding allow_system_table_mods + = true to postgresql.conf. (In versions + supporting ALTER SYSTEM, you can use that to make the + configuration change, but you'll still need a restart.) + + + + + + In each database of the cluster, + run the following commands as superuser: + +SET search_path = pg_catalog; +CREATE OR REPLACE VIEW pg_user_mappings AS + SELECT + U.oid AS umid, + S.oid AS srvid, + S.srvname AS srvname, + U.umuser AS umuser, + CASE WHEN U.umuser = 0 THEN + 'public' + ELSE + A.rolname + END AS usename, + CASE WHEN (U.umuser <> 0 AND A.rolname = current_user) + OR (U.umuser = 0 AND pg_has_role(S.srvowner, 'USAGE')) + OR (SELECT rolsuper FROM pg_authid WHERE rolname = current_user) + THEN U.umoptions + ELSE NULL END AS umoptions + FROM pg_user_mapping U + LEFT JOIN pg_authid A ON (A.oid = U.umuser) JOIN + pg_foreign_server S ON (U.umserver = S.oid); + + + + + + + Do not forget to include the template0 + and template1 databases, or the vulnerability will still + exist in databases you create later. To fix template0, + you'll need to temporarily make it accept connections. + In PostgreSQL 9.5 and later, you can use + +ALTER DATABASE template0 WITH ALLOW_CONNECTIONS true; + + and then after fixing template0, undo that with + +ALTER DATABASE template0 WITH ALLOW_CONNECTIONS false; + + In prior versions, instead use + +UPDATE pg_database SET datallowconn = true WHERE datname = 'template0'; +UPDATE pg_database SET datallowconn = false WHERE datname = 'template0'; + + + + + + + Finally, remove the allow_system_table_mods configuration + setting, and again restart the postmaster. + + + + + + + + Prevent exposure of statistical information via leaky operators + (Peter Eisentraut) + + + + Some selectivity estimation functions in the planner will apply + user-defined operators to values obtained + from pg_statistic, such as most common values and + histogram entries. This occurs before table permissions are checked, + so a nefarious user could exploit the behavior to obtain these values + for table columns he does not have permission to read. To fix, + fall back to a default estimate if the operator's implementation + function is not certified leak-proof and the calling user does not have + permission to read the table column whose statistics are needed. + At least one of these criteria is satisfied in most cases in practice. + (CVE-2017-7484) + + + + + + Restore libpq's recognition of + the PGREQUIRESSL environment variable (Daniel Gustafsson) + + + + Processing of this environment variable was unintentionally dropped + in PostgreSQL 9.3, but its documentation remained. + This creates a security hazard, since users might be relying on the + environment variable to force SSL-encrypted connections, but that + would no longer be guaranteed. Restore handling of the variable, + but give it lower priority than PGSSLMODE, to avoid + breaking configurations that work correctly with post-9.3 code. + (CVE-2017-7485) + + + Fix possibly-invalid initial snapshot during logical decoding diff --git a/doc/src/sgml/release-9.6.sgml b/doc/src/sgml/release-9.6.sgml index bacbc29d67..e8fb97cb7f 100644 --- a/doc/src/sgml/release-9.6.sgml +++ b/doc/src/sgml/release-9.6.sgml @@ -23,8 +23,13 @@ - However, if you are using third-party replication tools that depend - on logical decoding, see the first changelog entry below. + However, if you use foreign data servers that make use of user + passwords for authentication, see the first changelog entry below. + + + + Also, if you are using third-party replication tools that depend + on logical decoding, see the fourth changelog entry below. @@ -40,6 +45,174 @@ + + Restrict visibility + of pg_user_mappings.umoptions, to + protect passwords stored as user mapping options + (Michael Paquier, Feike Steenbergen) + + + + The previous coding allowed the owner of a foreign server object, + or anyone he has granted server USAGE permission to, + to see the options for all user mappings associated with that server. + This might well include passwords for other users. + Adjust the view definition to match the behavior of + information_schema.user_mapping_options, namely that + these options are visible to the user being mapped, or if the mapping + is for PUBLIC and the current user is the server + owner, or if the current user is a superuser. + (CVE-2017-7486) + + + + By itself, this patch will only fix the behavior in newly initdb'd + databases. If you wish to apply this change in an existing database, + you will need to do the following: + + + + + + Restart the postmaster after adding allow_system_table_mods + = true to postgresql.conf. (In versions + supporting ALTER SYSTEM, you can use that to make the + configuration change, but you'll still need a restart.) + + + + + + In each database of the cluster, + run the following commands as superuser: + +SET search_path = pg_catalog; +CREATE OR REPLACE VIEW pg_user_mappings AS + SELECT + U.oid AS umid, + S.oid AS srvid, + S.srvname AS srvname, + U.umuser AS umuser, + CASE WHEN U.umuser = 0 THEN + 'public' + ELSE + A.rolname + END AS usename, + CASE WHEN (U.umuser <> 0 AND A.rolname = current_user) + OR (U.umuser = 0 AND pg_has_role(S.srvowner, 'USAGE')) + OR (SELECT rolsuper FROM pg_authid WHERE rolname = current_user) + THEN U.umoptions + ELSE NULL END AS umoptions + FROM pg_user_mapping U + LEFT JOIN pg_authid A ON (A.oid = U.umuser) JOIN + pg_foreign_server S ON (U.umserver = S.oid); + + + + + + + Do not forget to include the template0 + and template1 databases, or the vulnerability will still + exist in databases you create later. To fix template0, + you'll need to temporarily make it accept connections. + In PostgreSQL 9.5 and later, you can use + +ALTER DATABASE template0 WITH ALLOW_CONNECTIONS true; + + and then after fixing template0, undo that with + +ALTER DATABASE template0 WITH ALLOW_CONNECTIONS false; + + In prior versions, instead use + +UPDATE pg_database SET datallowconn = true WHERE datname = 'template0'; +UPDATE pg_database SET datallowconn = false WHERE datname = 'template0'; + + + + + + + Finally, remove the allow_system_table_mods configuration + setting, and again restart the postmaster. + + + + + + + + + Prevent exposure of statistical information via leaky operators + (Peter Eisentraut) + + + + Some selectivity estimation functions in the planner will apply + user-defined operators to values obtained + from pg_statistic, such as most common values and + histogram entries. This occurs before table permissions are checked, + so a nefarious user could exploit the behavior to obtain these values + for table columns he does not have permission to read. To fix, + fall back to a default estimate if the operator's implementation + function is not certified leak-proof and the calling user does not have + permission to read the table column whose statistics are needed. + At least one of these criteria is satisfied in most cases in practice. + (CVE-2017-7484) + + + + + + + Restore libpq's recognition of + the PGREQUIRESSL environment variable (Daniel Gustafsson) + + + + Processing of this environment variable was unintentionally dropped + in PostgreSQL 9.3, but its documentation remained. + This creates a security hazard, since users might be relying on the + environment variable to force SSL-encrypted connections, but that + would no longer be guaranteed. Restore handling of the variable, + but give it lower priority than PGSSLMODE, to avoid + breaking configurations that work correctly with post-9.3 code. + (CVE-2017-7485) + + + + + - Allow pg_basebackup to stream transaction log in + Allow pg_basebackup to stream write-ahead log in tar mode (Magnus Hagander) diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index a11f0f8526..b442e7aa88 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -151,7 +151,7 @@ readTimeLineHistory(TimeLineID targetTLI) if (nfields != 3) ereport(FATAL, (errmsg("syntax error in history file: %s", fline), - errhint("Expected a transaction log switchpoint location."))); + errhint("Expected a write-ahead log switchpoint location."))); if (result && tli <= lasttli) ereport(FATAL, diff --git a/src/backend/access/transam/transam.c b/src/backend/access/transam/transam.c index 562b53be9a..968b232364 100644 --- a/src/backend/access/transam/transam.c +++ b/src/backend/access/transam/transam.c @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * transam.c - * postgres transaction log interface routines + * postgres transaction (commit) log interface routines * * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 5d6f8b75b8..bc5fb4bb83 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * xlog.c - * PostgreSQL transaction log manager + * PostgreSQL write-ahead log manager * * * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group @@ -3532,7 +3532,7 @@ XLogFileOpen(XLogSegNo segno) if (fd < 0) ereport(PANIC, (errcode_for_file_access(), - errmsg("could not open transaction log file \"%s\": %m", path))); + errmsg("could not open write-ahead log file \"%s\": %m", path))); return fd; } @@ -3838,7 +3838,7 @@ RemoveOldXlogFiles(XLogSegNo segno, XLogRecPtr PriorRedoPtr, XLogRecPtr endptr) if (xldir == NULL) ereport(ERROR, (errcode_for_file_access(), - errmsg("could not open transaction log directory \"%s\": %m", + errmsg("could not open write-ahead log directory \"%s\": %m", XLOGDIR))); /* @@ -3913,7 +3913,7 @@ RemoveNonParentXlogFiles(XLogRecPtr switchpoint, TimeLineID newTLI) if (xldir == NULL) ereport(ERROR, (errcode_for_file_access(), - errmsg("could not open transaction log directory \"%s\": %m", + errmsg("could not open write-ahead log directory \"%s\": %m", XLOGDIR))); /* @@ -3994,7 +3994,7 @@ RemoveXlogFile(const char *segname, XLogRecPtr PriorRedoPtr, XLogRecPtr endptr) true, recycleSegNo, true)) { ereport(DEBUG2, - (errmsg("recycled transaction log file \"%s\"", + (errmsg("recycled write-ahead log file \"%s\"", segname))); CheckpointStats.ckpt_segs_recycled++; /* Needn't recheck that slot on future iterations */ @@ -4006,7 +4006,7 @@ RemoveXlogFile(const char *segname, XLogRecPtr PriorRedoPtr, XLogRecPtr endptr) int rc; ereport(DEBUG2, - (errmsg("removing transaction log file \"%s\"", + (errmsg("removing write-ahead log file \"%s\"", segname))); #ifdef WIN32 @@ -4026,7 +4026,7 @@ RemoveXlogFile(const char *segname, XLogRecPtr PriorRedoPtr, XLogRecPtr endptr) { ereport(LOG, (errcode_for_file_access(), - errmsg("could not rename old transaction log file \"%s\": %m", + errmsg("could not rename old write-ahead log file \"%s\": %m", path))); return; } @@ -4108,7 +4108,7 @@ CleanupBackupHistory(void) if (xldir == NULL) ereport(ERROR, (errcode_for_file_access(), - errmsg("could not open transaction log directory \"%s\": %m", + errmsg("could not open write-ahead log directory \"%s\": %m", XLOGDIR))); while ((xlde = ReadDir(xldir, XLOGDIR)) != NULL) @@ -4117,9 +4117,8 @@ CleanupBackupHistory(void) { if (XLogArchiveCheckDone(xlde->d_name)) { - ereport(DEBUG2, - (errmsg("removing transaction log backup history file \"%s\"", - xlde->d_name))); + elog(DEBUG2, "removing WAL backup history file \"%s\"", + xlde->d_name); snprintf(path, sizeof(path), XLOGDIR "/%s", xlde->d_name); unlink(path); XLogArchiveCleanup(xlde->d_name); @@ -5074,7 +5073,7 @@ BootStrapXLOG(void) errno = ENOSPC; ereport(PANIC, (errcode_for_file_access(), - errmsg("could not write bootstrap transaction log file: %m"))); + errmsg("could not write bootstrap write-ahead log file: %m"))); } pgstat_report_wait_end(); @@ -5082,13 +5081,13 @@ BootStrapXLOG(void) if (pg_fsync(openLogFile) != 0) ereport(PANIC, (errcode_for_file_access(), - errmsg("could not fsync bootstrap transaction log file: %m"))); + errmsg("could not fsync bootstrap write-ahead log file: %m"))); pgstat_report_wait_end(); if (close(openLogFile)) ereport(PANIC, (errcode_for_file_access(), - errmsg("could not close bootstrap transaction log file: %m"))); + errmsg("could not close bootstrap write-ahead log file: %m"))); openLogFile = -1; @@ -8432,7 +8431,7 @@ LogCheckpointEnd(bool restartpoint) average_usecs = average_sync_time - (uint64) average_secs *1000000; elog(LOG, "%s complete: wrote %d buffers (%.1f%%); " - "%d transaction log file(s) added, %d removed, %d recycled; " + "%d WAL file(s) added, %d removed, %d recycled; " "write=%ld.%03d s, sync=%ld.%03d s, total=%ld.%03d s; " "sync files=%d, longest=%ld.%03d s, average=%ld.%03d s; " "distance=%d kB, estimate=%d kB", @@ -8842,7 +8841,7 @@ CreateCheckPoint(int flags) */ if (shutdown && checkPoint.redo != ProcLastRecPtr) ereport(PANIC, - (errmsg("concurrent transaction log activity while database system is shutting down"))); + (errmsg("concurrent write-ahead log activity while database system is shutting down"))); /* * Remember the prior checkpoint's redo pointer, used later to determine diff --git a/src/backend/access/transam/xlogfuncs.c b/src/backend/access/transam/xlogfuncs.c index aa47b0dcff..8568c8abd6 100644 --- a/src/backend/access/transam/xlogfuncs.c +++ b/src/backend/access/transam/xlogfuncs.c @@ -2,7 +2,7 @@ * * xlogfuncs.c * - * PostgreSQL transaction log manager user interface functions + * PostgreSQL write-ahead log manager user interface functions * * This file contains WAL control and information functions. * diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c index fb7f2e7974..d7f2e55b09 100644 --- a/src/backend/access/transam/xlogutils.c +++ b/src/backend/access/transam/xlogutils.c @@ -2,7 +2,7 @@ * * xlogutils.c * - * PostgreSQL transaction log manager utility routines + * PostgreSQL write-ahead log manager utility routines * * This file contains support routines that are used by XLOG replay functions. * None of this code is used during normal system operation. diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c index d12db0d5a7..a8dc355ead 100644 --- a/src/backend/postmaster/checkpointer.c +++ b/src/backend/postmaster/checkpointer.c @@ -626,9 +626,8 @@ CheckArchiveTimeout(void) * assume nothing happened. */ if ((switchpoint % XLogSegSize) != 0) - ereport(DEBUG1, - (errmsg("transaction log switch forced (archive_timeout=%d)", - XLogArchiveTimeout))); + elog(DEBUG1, "write-ahead log switch forced (archive_timeout=%d)", + XLogArchiveTimeout); } /* diff --git a/src/backend/postmaster/pgarch.c b/src/backend/postmaster/pgarch.c index f3f58bd390..2dce39fdef 100644 --- a/src/backend/postmaster/pgarch.c +++ b/src/backend/postmaster/pgarch.c @@ -482,7 +482,7 @@ pgarch_ArchiverCopyLoop(void) if (++failures >= NUM_ARCHIVE_RETRIES) { ereport(WARNING, - (errmsg("archiving transaction log file \"%s\" failed too many times, will try again later", + (errmsg("archiving write-ahead log file \"%s\" failed too many times, will try again later", xlog))); return; /* give up archiving for now */ } @@ -628,8 +628,7 @@ pgarch_archiveXlog(char *xlog) return false; } - ereport(DEBUG1, - (errmsg("archived transaction log file \"%s\"", xlog))); + elog(DEBUG1, "archived write-ahead log file \"%s\"", xlog); snprintf(activitymsg, sizeof(activitymsg), "last was %s", xlog); set_ps_display(activitymsg, false); diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 01f1c2805f..fdce5524f4 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -5036,7 +5036,7 @@ sigusr1_handler(SIGNAL_ARGS) { /* * Send SIGUSR1 to archiver process, to wake it up and begin archiving - * next transaction log file. + * next WAL file. */ signal_child(PgArchPID, SIGUSR1); } diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index 95da8b7722..62aa40a583 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -563,19 +563,19 @@ exit_nicely(void) if (made_new_xlogdir) { - fprintf(stderr, _("%s: removing transaction log directory \"%s\"\n"), + fprintf(stderr, _("%s: removing WAL directory \"%s\"\n"), progname, xlog_dir); if (!rmtree(xlog_dir, true)) - fprintf(stderr, _("%s: failed to remove transaction log directory\n"), + fprintf(stderr, _("%s: failed to remove WAL directory\n"), progname); } else if (found_existing_xlogdir) { fprintf(stderr, - _("%s: removing contents of transaction log directory \"%s\"\n"), + _("%s: removing contents of WAL directory \"%s\"\n"), progname, xlog_dir); if (!rmtree(xlog_dir, false)) - fprintf(stderr, _("%s: failed to remove contents of transaction log directory\n"), + fprintf(stderr, _("%s: failed to remove contents of WAL directory\n"), progname); } /* otherwise died during startup, do nothing! */ @@ -589,7 +589,7 @@ exit_nicely(void) if (made_new_xlogdir || found_existing_xlogdir) fprintf(stderr, - _("%s: transaction log directory \"%s\" not removed at user's request\n"), + _("%s: WAL directory \"%s\" not removed at user's request\n"), progname, xlog_dir); } @@ -2704,7 +2704,7 @@ create_data_directory(void) } -/* Create transaction log directory, and symlink if required */ +/* Create WAL directory, and symlink if required */ void create_xlog_or_symlink(void) { @@ -2721,7 +2721,7 @@ create_xlog_or_symlink(void) canonicalize_path(xlog_dir); if (!is_absolute_path(xlog_dir)) { - fprintf(stderr, _("%s: transaction log directory location must be an absolute path\n"), progname); + fprintf(stderr, _("%s: WAL directory location must be an absolute path\n"), progname); exit_nicely(); } @@ -2775,8 +2775,8 @@ create_xlog_or_symlink(void) warn_on_mount_point(ret); else fprintf(stderr, - _("If you want to store the transaction log there, either\n" - "remove or empty the directory \"%s\".\n"), + _("If you want to store the WAL there, either remove or empty the directory\n" + "\"%s\".\n"), xlog_dir); exit_nicely(); diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index e2a2ebb30f..c7ddfcbfdf 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -174,19 +174,19 @@ cleanup_directories_atexit(void) if (made_new_xlogdir) { - fprintf(stderr, _("%s: removing transaction log directory \"%s\"\n"), + fprintf(stderr, _("%s: removing WAL directory \"%s\"\n"), progname, xlog_dir); if (!rmtree(xlog_dir, true)) - fprintf(stderr, _("%s: failed to remove transaction log directory\n"), + fprintf(stderr, _("%s: failed to remove WAL directory\n"), progname); } else if (found_existing_xlogdir) { fprintf(stderr, - _("%s: removing contents of transaction log directory \"%s\"\n"), + _("%s: removing contents of WAL directory \"%s\"\n"), progname, xlog_dir); if (!rmtree(xlog_dir, false)) - fprintf(stderr, _("%s: failed to remove contents of transaction log directory\n"), + fprintf(stderr, _("%s: failed to remove contents of WAL directory\n"), progname); } } @@ -199,7 +199,7 @@ cleanup_directories_atexit(void) if (made_new_xlogdir || found_existing_xlogdir) fprintf(stderr, - _("%s: transaction log directory \"%s\" not removed at user's request\n"), + _("%s: WAL directory \"%s\" not removed at user's request\n"), progname, xlog_dir); } @@ -341,7 +341,7 @@ usage(void) " relocate tablespace in OLDDIR to NEWDIR\n")); printf(_(" -X, --wal-method=none|fetch|stream\n" " include required WAL files with specified method\n")); - printf(_(" --waldir=WALDIR location for the transaction log directory\n")); + printf(_(" --waldir=WALDIR location for the write-ahead log directory\n")); printf(_(" -z, --gzip compress tar output\n")); printf(_(" -Z, --compress=0-9 compress tar output with given compression level\n")); printf(_("\nGeneral options:\n")); @@ -414,7 +414,7 @@ reached_end_position(XLogRecPtr segendpos, uint32 timeline, if (sscanf(xlogend, "%X/%X", &hi, &lo) != 2) { fprintf(stderr, - _("%s: could not parse transaction log location \"%s\"\n"), + _("%s: could not parse write-ahead log location \"%s\"\n"), progname, xlogend); exit(1); } @@ -549,7 +549,7 @@ StartLogStreamer(char *startpos, uint32 timeline, char *sysidentifier) if (sscanf(startpos, "%X/%X", &hi, &lo) != 2) { fprintf(stderr, - _("%s: could not parse transaction log location \"%s\"\n"), + _("%s: could not parse write-ahead log location \"%s\"\n"), progname, startpos); disconnect_and_exit(1); } @@ -1404,7 +1404,7 @@ ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum) /* * When streaming WAL, pg_wal (or pg_xlog for pre-9.6 * clusters) will have been created by the wal receiver - * process. Also, when transaction log directory location + * process. Also, when the WAL directory location * was specified, pg_wal (or pg_xlog) has already been * created as a symbolic link before starting the actual * backup. So just ignore creation failures on related @@ -1817,7 +1817,7 @@ BaseBackup(void) MemSet(xlogend, 0, sizeof(xlogend)); if (verbose && includewal != NO_WAL) - fprintf(stderr, _("%s: transaction log start point: %s on timeline %u\n"), + fprintf(stderr, _("%s: write-ahead log start point: %s on timeline %u\n"), progname, xlogstart, starttli); /* @@ -1907,20 +1907,20 @@ BaseBackup(void) if (PQresultStatus(res) != PGRES_TUPLES_OK) { fprintf(stderr, - _("%s: could not get transaction log end position from server: %s"), + _("%s: could not get write-ahead log end position from server: %s"), progname, PQerrorMessage(conn)); disconnect_and_exit(1); } if (PQntuples(res) != 1) { fprintf(stderr, - _("%s: no transaction log end position returned from server\n"), + _("%s: no write-ahead log end position returned from server\n"), progname); disconnect_and_exit(1); } strlcpy(xlogend, PQgetvalue(res, 0, 0), sizeof(xlogend)); if (verbose && includewal != NO_WAL) - fprintf(stderr, _("%s: transaction log end point: %s\n"), progname, xlogend); + fprintf(stderr, _("%s: write-ahead log end point: %s\n"), progname, xlogend); PQclear(res); res = PQgetResult(conn); @@ -1998,7 +1998,7 @@ BaseBackup(void) if (sscanf(xlogend, "%X/%X", &hi, &lo) != 2) { fprintf(stderr, - _("%s: could not parse transaction log location \"%s\"\n"), + _("%s: could not parse write-ahead log location \"%s\"\n"), progname, xlogend); disconnect_and_exit(1); } @@ -2312,7 +2312,7 @@ main(int argc, char **argv) if (format == 't' && includewal == STREAM_WAL && strcmp(basedir, "-") == 0) { fprintf(stderr, - _("%s: cannot stream transaction logs in tar mode to stdout\n"), + _("%s: cannot stream write-ahead logs in tar mode to stdout\n"), progname); fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); @@ -2348,7 +2348,7 @@ main(int argc, char **argv) if (format != 'p') { fprintf(stderr, - _("%s: transaction log directory location can only be specified in plain mode\n"), + _("%s: WAL directory location can only be specified in plain mode\n"), progname); fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); @@ -2359,7 +2359,7 @@ main(int argc, char **argv) canonicalize_path(xlog_dir); if (!is_absolute_path(xlog_dir)) { - fprintf(stderr, _("%s: transaction log directory location must be " + fprintf(stderr, _("%s: WAL directory location must be " "an absolute path\n"), progname); fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); @@ -2393,7 +2393,7 @@ main(int argc, char **argv) exit(1); } - /* Create transaction log symlink, if required */ + /* Create pg_wal symlink, if required */ if (strcmp(xlog_dir, "") != 0) { char *linkloc; diff --git a/src/bin/pg_basebackup/pg_receivewal.c b/src/bin/pg_basebackup/pg_receivewal.c index 09385c5cbf..b11984b088 100644 --- a/src/bin/pg_basebackup/pg_receivewal.c +++ b/src/bin/pg_basebackup/pg_receivewal.c @@ -1,6 +1,6 @@ /*------------------------------------------------------------------------- * - * pg_receivewal.c - receive streaming transaction log data and write it + * pg_receivewal.c - receive streaming WAL data and write it * to a local file. * * Author: Magnus Hagander @@ -71,18 +71,18 @@ static bool stop_streaming(XLogRecPtr segendpos, uint32 timeline, static void usage(void) { - printf(_("%s receives PostgreSQL streaming transaction logs.\n\n"), + printf(_("%s receives PostgreSQL streaming write-ahead logs.\n\n"), progname); printf(_("Usage:\n")); printf(_(" %s [OPTION]...\n"), progname); printf(_("\nOptions:\n")); - printf(_(" -D, --directory=DIR receive transaction log files into this directory\n")); + printf(_(" -D, --directory=DIR receive write-ahead log files into this directory\n")); printf(_(" --if-not-exists do not error if slot already exists when creating a slot\n")); printf(_(" -n, --no-loop do not loop on connection lost\n")); printf(_(" -s, --status-interval=SECS\n" " time between status packets sent to server (default: %d)\n"), (standby_message_timeout / 1000)); printf(_(" -S, --slot=SLOTNAME replication slot to use\n")); - printf(_(" --synchronous flush transaction log immediately after writing\n")); + printf(_(" --synchronous flush write-ahead log immediately after writing\n")); printf(_(" -v, --verbose output verbose messages\n")); printf(_(" -V, --version output version information, then exit\n")); printf(_(" -Z, --compress=0-9 compress logs with given compression level\n")); diff --git a/src/bin/pg_basebackup/receivelog.c b/src/bin/pg_basebackup/receivelog.c index c41bba28cd..0f2845bbc2 100644 --- a/src/bin/pg_basebackup/receivelog.c +++ b/src/bin/pg_basebackup/receivelog.c @@ -1,6 +1,6 @@ /*------------------------------------------------------------------------- * - * receivelog.c - receive transaction log files using the streaming + * receivelog.c - receive WAL files using the streaming * replication protocol. * * Author: Magnus Hagander @@ -116,7 +116,7 @@ open_walfile(StreamCtl *stream, XLogRecPtr startpoint) if (size < 0) { fprintf(stderr, - _("%s: could not get size of transaction log file \"%s\": %s\n"), + _("%s: could not get size of write-ahead log file \"%s\": %s\n"), progname, fn, stream->walmethod->getlasterror()); return false; } @@ -127,7 +127,7 @@ open_walfile(StreamCtl *stream, XLogRecPtr startpoint) if (f == NULL) { fprintf(stderr, - _("%s: could not open existing transaction log file \"%s\": %s\n"), + _("%s: could not open existing write-ahead log file \"%s\": %s\n"), progname, fn, stream->walmethod->getlasterror()); return false; } @@ -136,7 +136,7 @@ open_walfile(StreamCtl *stream, XLogRecPtr startpoint) if (stream->walmethod->sync(f) != 0) { fprintf(stderr, - _("%s: could not sync existing transaction log file \"%s\": %s\n"), + _("%s: could not sync existing write-ahead log file \"%s\": %s\n"), progname, fn, stream->walmethod->getlasterror()); stream->walmethod->close(f, CLOSE_UNLINK); return false; @@ -151,7 +151,7 @@ open_walfile(StreamCtl *stream, XLogRecPtr startpoint) if (errno == 0) errno = ENOSPC; fprintf(stderr, - _("%s: transaction log file \"%s\" has %d bytes, should be 0 or %d\n"), + _("%s: write-ahead log file \"%s\" has %d bytes, should be 0 or %d\n"), progname, fn, (int) size, XLogSegSize); return false; } @@ -164,7 +164,7 @@ open_walfile(StreamCtl *stream, XLogRecPtr startpoint) if (f == NULL) { fprintf(stderr, - _("%s: could not open transaction log file \"%s\": %s\n"), + _("%s: could not open write-ahead log file \"%s\": %s\n"), progname, fn, stream->walmethod->getlasterror()); return false; } @@ -1121,7 +1121,7 @@ ProcessXLogDataMsg(PGconn *conn, StreamCtl *stream, char *copybuf, int len, if (xlogoff != 0) { fprintf(stderr, - _("%s: received transaction log record for offset %u with no file open\n"), + _("%s: received write-ahead log record for offset %u with no file open\n"), progname, xlogoff); return false; } diff --git a/src/bin/pg_basebackup/streamutil.c b/src/bin/pg_basebackup/streamutil.c index 1f2b4068d5..7ea3b0f8ee 100644 --- a/src/bin/pg_basebackup/streamutil.c +++ b/src/bin/pg_basebackup/streamutil.c @@ -282,7 +282,7 @@ RunIdentifySystem(PGconn *conn, char **sysid, TimeLineID *starttli, if (sscanf(PQgetvalue(res, 0, 2), "%X/%X", &hi, &lo) != 2) { fprintf(stderr, - _("%s: could not parse transaction log location \"%s\"\n"), + _("%s: could not parse write-ahead log location \"%s\"\n"), progname, PQgetvalue(res, 0, 2)); PQclear(res); diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c index 73bd9fe6ec..9c3a28beb0 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c @@ -434,7 +434,7 @@ main(int argc, char *argv[]) if (ControlFile.state != DB_SHUTDOWNED && !force) { printf(_("The database server was not shut down cleanly.\n" - "Resetting the transaction log might cause data to be lost.\n" + "Resetting the write-ahead log might cause data to be lost.\n" "If you want to proceed anyway, use -f to force reset.\n")); exit(1); } @@ -447,7 +447,7 @@ main(int argc, char *argv[]) KillExistingArchiveStatus(); WriteEmptyXLOG(); - printf(_("Transaction log reset\n")); + printf(_("Write-ahead log reset\n")); return 0; } @@ -1159,7 +1159,7 @@ WriteEmptyXLOG(void) static void usage(void) { - printf(_("%s resets the PostgreSQL transaction log.\n\n"), progname); + printf(_("%s resets the PostgreSQL write-ahead log.\n\n"), progname); printf(_("Usage:\n %s [OPTION]... DATADIR\n\n"), progname); printf(_("Options:\n")); printf(_(" -c XID,XID set oldest and newest transactions bearing commit timestamp\n")); @@ -1167,7 +1167,7 @@ usage(void) printf(_(" [-D] DATADIR data directory\n")); printf(_(" -e XIDEPOCH set next transaction ID epoch\n")); printf(_(" -f force update to be done\n")); - printf(_(" -l XLOGFILE force minimum WAL starting location for new transaction log\n")); + printf(_(" -l WALFILE force minimum WAL starting location for new write-ahead log\n")); printf(_(" -m MXID,MXID set next and oldest multitransaction ID\n")); printf(_(" -n no update, just show what would be done (for testing)\n")); printf(_(" -o OID set next OID\n")); diff --git a/src/bin/pg_rewind/timeline.c b/src/bin/pg_rewind/timeline.c index f1a792ff6b..3cd5af57f2 100644 --- a/src/bin/pg_rewind/timeline.c +++ b/src/bin/pg_rewind/timeline.c @@ -80,7 +80,7 @@ rewind_parseTimeLineHistory(char *buffer, TimeLineID targetTLI, int *nentries) if (nfields != 3) { fprintf(stderr, _("syntax error in history file: %s\n"), fline); - fprintf(stderr, _("Expected a transaction log switchpoint location.\n")); + fprintf(stderr, _("Expected a write-ahead log switchpoint location.\n")); exit(1); } if (entries && tli <= lasttli) diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index 80c2c33ecb..9491ccb13b 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -680,7 +680,7 @@ XLogDumpDisplayStats(XLogDumpConfig *config, XLogDumpStats *stats) static void usage(void) { - printf(_("%s decodes and displays PostgreSQL transaction logs for debugging.\n\n"), + printf(_("%s decodes and displays PostgreSQL write-ahead logs for debugging.\n\n"), progname); printf(_("Usage:\n")); printf(_(" %s [OPTION]... [STARTSEG [ENDSEG]]\n"), progname); diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index d23aab589e..4d9773459d 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -1,7 +1,7 @@ /* * xlog.h * - * PostgreSQL transaction log manager + * PostgreSQL write-ahead log manager * * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index c09c0f8e5f..6e2bfd0ad0 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -1,7 +1,7 @@ /* * xlog_internal.h * - * PostgreSQL transaction log internal declarations + * PostgreSQL write-ahead log internal declarations * * NOTE: this file is intended to contain declarations useful for * manipulating the XLOG files directly, but it is not supposed to be diff --git a/src/include/access/xlogdefs.h b/src/include/access/xlogdefs.h index 53b60c1dcd..0f07bb2674 100644 --- a/src/include/access/xlogdefs.h +++ b/src/include/access/xlogdefs.h @@ -1,7 +1,7 @@ /* * xlogdefs.h * - * Postgres transaction log manager record pointer and + * Postgres write-ahead log manager record pointer and * timeline number definitions * * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm index e2274681b1..61d2c5fdf5 100644 --- a/src/test/perl/PostgresNode.pm +++ b/src/test/perl/PostgresNode.pm @@ -483,7 +483,7 @@ sub append_conf =item $node->backup(backup_name) Create a hot backup with B in subdirectory B of -B<< $node->backup_dir >>, including the transaction logs. Transaction logs are +B<< $node->backup_dir >>, including the WAL. WAL files fetched at the end of the backup, not streamed. You'll have to configure a suitable B on the @@ -507,7 +507,7 @@ sub backup =item $node->backup_fs_hot(backup_name) Create a backup with a filesystem level copy in subdirectory B of -B<< $node->backup_dir >>, including transaction logs. +B<< $node->backup_dir >>, including WAL. Archiving must be enabled, as B and B are used. This is not checked or enforced. @@ -525,7 +525,7 @@ sub backup_fs_hot =item $node->backup_fs_cold(backup_name) Create a backup with a filesystem level copy in subdirectory B of -B<< $node->backup_dir >>, including transaction logs. The server must be +B<< $node->backup_dir >>, including WAL. The server must be stopped as no attempt to handle concurrent writes is made. Use B or B if you want to back up a running server. -- cgit v1.2.3 From d496a65790734f808789f39e4f63b2790821c2be Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 12 May 2017 13:51:27 -0400 Subject: Standardize "WAL location" terminology Other previously used terms were "WAL position" or "log position". --- doc/src/sgml/config.sgml | 2 +- doc/src/sgml/datatype.sgml | 2 +- doc/src/sgml/logicaldecoding.sgml | 2 +- doc/src/sgml/monitoring.sgml | 22 +++++++++++----------- doc/src/sgml/protocol.sgml | 10 +++++----- doc/src/sgml/ref/pg_waldump.sgml | 4 ++-- doc/src/sgml/wal.sgml | 2 +- src/backend/access/transam/recovery.conf.sample | 2 +- src/backend/access/transam/timeline.c | 4 ++-- src/backend/access/transam/twophase.c | 2 +- src/backend/access/transam/xlog.c | 14 +++++++------- src/backend/replication/walreceiver.c | 4 ++-- src/backend/replication/walsender.c | 24 ++++++++++++------------ src/bin/pg_basebackup/pg_basebackup.c | 2 +- src/bin/pg_basebackup/receivelog.c | 4 ++-- src/bin/pg_rewind/parsexlog.c | 2 +- src/bin/pg_rewind/pg_rewind.c | 6 +++--- src/bin/pg_waldump/pg_waldump.c | 14 +++++++------- src/include/access/timeline.h | 2 +- 19 files changed, 62 insertions(+), 62 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index e297166bd9..b51826dc68 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3312,7 +3312,7 @@ ANY num_sync ( pg_stat_replication view. The standby will report - the last write-ahead log position it has written, the last position it + the last write-ahead log location it has written, the last position it has flushed to disk, and the last position it has applied. This parameter's value is the maximum interval, in seconds, between reports. Updates are diff --git a/doc/src/sgml/datatype.sgml b/doc/src/sgml/datatype.sgml index c96374a201..42b2bb71bb 100644 --- a/doc/src/sgml/datatype.sgml +++ b/doc/src/sgml/datatype.sgml @@ -4635,7 +4635,7 @@ SELECT * FROM pg_attribute standard comparison operators, like = and >. Two LSNs can be subtracted using the - operator; the result is the number of bytes separating - those write-ahead log positions. + those write-ahead log locations. diff --git a/doc/src/sgml/logicaldecoding.sgml b/doc/src/sgml/logicaldecoding.sgml index c2950613fa..8dcfc6c742 100644 --- a/doc/src/sgml/logicaldecoding.sgml +++ b/doc/src/sgml/logicaldecoding.sgml @@ -632,7 +632,7 @@ typedef void (*LogicalDecodeMessageCB) (struct LogicalDecodingContext *ctx, its XID. Note however that it can be NULL when the message is non-transactional and the XID was not assigned yet in the transaction which logged the message. The lsn has WAL - position of the message. The transactional says + location of the message. The transactional says if the message was sent as transactional or not. The prefix is arbitrary null-terminated prefix which can be used for identifying interesting messages for the current diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index de61e64069..79ca45a156 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -1829,7 +1829,7 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i being generated, but would differ when the sender becomes idle. In particular, when the standby has caught up completely, pg_stat_replication shows the time taken to - write, flush and replay the most recent reported WAL position rather than + write, flush and replay the most recent reported WAL location rather than zero as some users might expect. This is consistent with the goal of measuring synchronous commit and transaction visibility delays for recent write transactions. @@ -1865,7 +1865,7 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i receive_start_lsn pg_lsn - First write-ahead log position used when WAL receiver is + First write-ahead log location used when WAL receiver is started @@ -1876,16 +1876,16 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i received_lsn pg_lsn - Last write-ahead log position already received and flushed to - disk, the initial value of this field being the first log position used + Last write-ahead log location already received and flushed to + disk, the initial value of this field being the first log location used when WAL receiver is started received_tli integer - Timeline number of last write-ahead log position received and + Timeline number of last write-ahead log location received and flushed to disk, the initial value of this field being the timeline - number of the first log position used when WAL receiver is started + number of the first log location used when WAL receiver is started @@ -1901,12 +1901,12 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i latest_end_lsn pg_lsn - Last write-ahead log position reported to origin WAL sender + Last write-ahead log location reported to origin WAL sender latest_end_time timestamp with time zone - Time of last write-ahead log position reported to origin WAL sender + Time of last write-ahead log location reported to origin WAL sender slot_name @@ -1967,7 +1967,7 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i received_lsn pg_lsn - Last write-ahead log position received, the initial value of + Last write-ahead log location received, the initial value of this field being 0 @@ -1984,13 +1984,13 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i latest_end_lsn pg_lsn - Last write-ahead log position reported to origin WAL sender + Last write-ahead log location reported to origin WAL sender latest_end_time timestamp with time zone - Time of last write-ahead log position reported to origin WAL + Time of last write-ahead log location reported to origin WAL sender diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index dadaa47d69..d23df0261c 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -1724,7 +1724,7 @@ The commands accepted in walsender mode are: consistent_point (text) - The WAL position at which the slot became consistent. This is the + The WAL location at which the slot became consistent. This is the earliest location from which streaming can start on this replication slot. @@ -1764,7 +1764,7 @@ The commands accepted in walsender mode are: Instructs server to start streaming WAL, starting at - WAL position XXX/XXX. + WAL location XXX/XXX. If TIMELINE option is specified, streaming starts on timeline tli; otherwise, the server's current timeline is selected. The server can @@ -1796,7 +1796,7 @@ The commands accepted in walsender mode are: acknowledges this by also exiting COPY mode, the server sends a result set with one row and two columns, indicating the next timeline in this server's history. The first column is the next timeline's ID (type int8), and the - second column is the WAL position where the switch happened (type text). Usually, + second column is the WAL location where the switch happened (type text). Usually, the switch position is the end of the WAL that was streamed, but there are corner cases where the server can send some WAL from the old timeline that it has not itself replayed before promoting. Finally, the @@ -2115,7 +2115,7 @@ The commands accepted in walsender mode are: Instructs server to start streaming WAL for logical replication, starting - at WAL position XXX/XXX. The server can + at WAL location XXX/XXX. The server can reply with an error, for example if the requested section of WAL has already been recycled. On success, server responds with a CopyBothResponse message, and then starts to stream WAL to the frontend. @@ -2147,7 +2147,7 @@ The commands accepted in walsender mode are: XXX/XXX - The WAL position to begin streaming at. + The WAL location to begin streaming at. diff --git a/doc/src/sgml/ref/pg_waldump.sgml b/doc/src/sgml/ref/pg_waldump.sgml index 4c92eeed68..cff88a4c1e 100644 --- a/doc/src/sgml/ref/pg_waldump.sgml +++ b/doc/src/sgml/ref/pg_waldump.sgml @@ -85,7 +85,7 @@ PostgreSQL documentation - Stop reading at the specified log position, instead of reading to the + Stop reading at the specified WAL location, instead of reading to the end of the log stream. @@ -144,7 +144,7 @@ PostgreSQL documentation - Log position at which to start reading. The default is to start reading + WAL location at which to start reading. The default is to start reading the first valid log record found in the earliest file found. diff --git a/doc/src/sgml/wal.sgml b/doc/src/sgml/wal.sgml index a749b83dc0..940c37b21a 100644 --- a/doc/src/sgml/wal.sgml +++ b/doc/src/sgml/wal.sgml @@ -791,7 +791,7 @@ pg_control. Therefore, at the start of recovery, the server first reads pg_control and then the checkpoint record; then it performs the REDO operation by - scanning forward from the log position indicated in the checkpoint + scanning forward from the log location indicated in the checkpoint record. Because the entire content of data pages is saved in the log on the first page modification after a checkpoint (assuming is not disabled), all pages diff --git a/src/backend/access/transam/recovery.conf.sample b/src/backend/access/transam/recovery.conf.sample index d5cb437e50..de4e38f9fe 100644 --- a/src/backend/access/transam/recovery.conf.sample +++ b/src/backend/access/transam/recovery.conf.sample @@ -67,7 +67,7 @@ # must set a recovery target. # # You may set a recovery target either by transactionId, by name, -# by timestamp or by WAL position (LSN). Recovery may either include or +# by timestamp or by WAL location (LSN). Recovery may either include or # exclude the transaction(s) with the recovery target value (ie, stop either # just after or just before the given target, respectively). # diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index b442e7aa88..8cab8b9aa9 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -15,7 +15,7 @@ * * * parentTLI ID of the parent timeline - * switchpoint XLogRecPtr of the WAL position where the switch happened + * switchpoint XLogRecPtr of the WAL location where the switch happened * reason human-readable explanation of why the timeline was changed * * The fields are separated by tabs. Lines beginning with # are comments, and @@ -278,7 +278,7 @@ findNewestTimeLine(TimeLineID startTLI) * * newTLI: ID of the new timeline * parentTLI: ID of its immediate parent - * switchpoint: XLOG position where the system switched to the new timeline + * switchpoint: WAL location where the system switched to the new timeline * reason: human-readable explanation of why the timeline was switched * * Currently this is only used at the end recovery, and so there are no locking diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index c9fff42991..7bf2555af2 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -2294,7 +2294,7 @@ RecordTransactionAbortPrepared(TransactionId xid, * * Store pointers to the start/end of the WAL record along with the xid in * a gxact entry in shared memory TwoPhaseState structure. If caller - * specifies InvalidXLogRecPtr as WAL position to fetch the two-phase + * specifies InvalidXLogRecPtr as WAL location to fetch the two-phase * data, the entry is marked as located on disk. */ void diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index bc5fb4bb83..b98e37e1d3 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -5643,7 +5643,7 @@ recoveryStopsBefore(XLogReaderState *record) recoveryStopTime = 0; recoveryStopName[0] = '\0'; ereport(LOG, - (errmsg("recovery stopping before WAL position (LSN) \"%X/%X\"", + (errmsg("recovery stopping before WAL location (LSN) \"%X/%X\"", (uint32) (recoveryStopLSN >> 32), (uint32) recoveryStopLSN))); return true; @@ -5800,7 +5800,7 @@ recoveryStopsAfter(XLogReaderState *record) recoveryStopTime = 0; recoveryStopName[0] = '\0'; ereport(LOG, - (errmsg("recovery stopping after WAL position (LSN) \"%X/%X\"", + (errmsg("recovery stopping after WAL location (LSN) \"%X/%X\"", (uint32) (recoveryStopLSN >> 32), (uint32) recoveryStopLSN))); return true; @@ -6323,7 +6323,7 @@ StartupXLOG(void) recoveryTargetName))); else if (recoveryTarget == RECOVERY_TARGET_LSN) ereport(LOG, - (errmsg("starting point-in-time recovery to WAL position (LSN) \"%X/%X\"", + (errmsg("starting point-in-time recovery to WAL location (LSN) \"%X/%X\"", (uint32) (recoveryTargetLSN >> 32), (uint32) recoveryTargetLSN))); else if (recoveryTarget == RECOVERY_TARGET_IMMEDIATE) @@ -7455,7 +7455,7 @@ StartupXLOG(void) exitArchiveRecovery(EndOfLogTLI, EndOfLog); /* - * Prepare to write WAL starting at EndOfLog position, and init xlog + * Prepare to write WAL starting at EndOfLog location, and init xlog * buffer cache using the block containing the last record from the * previous incarnation. */ @@ -10159,7 +10159,7 @@ XLogFileNameP(TimeLineID tli, XLogSegNo segno) * when backup needs to generate tablespace_map file, it is used to * embed escape character before newline character in tablespace path. * - * Returns the minimum WAL position that must be present to restore from this + * Returns the minimum WAL location that must be present to restore from this * backup, and the corresponding timeline ID in *starttli_p. * * Every successfully started non-exclusive backup must be stopped by calling @@ -10669,7 +10669,7 @@ get_backup_status(void) * If labelfile is NULL, this stops an exclusive backup. Otherwise this stops * the non-exclusive backup specified by 'labelfile'. * - * Returns the last WAL position that must be present to restore from this + * Returns the last WAL location that must be present to restore from this * backup, and the corresponding timeline ID in *stoptli_p. * * It is the responsibility of the caller of this function to verify the @@ -11569,7 +11569,7 @@ next_record_is_invalid: } /* - * Open the WAL segment containing WAL position 'RecPtr'. + * Open the WAL segment containing WAL location 'RecPtr'. * * The segment can be fetched via restore_command, or via walreceiver having * streamed the record, or it can already be present in pg_wal. Checking diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c index df93265c20..028170c952 100644 --- a/src/backend/replication/walreceiver.c +++ b/src/backend/replication/walreceiver.c @@ -1090,7 +1090,7 @@ XLogWalRcvFlush(bool dying) } /* - * Send reply message to primary, indicating our current XLOG positions, oldest + * Send reply message to primary, indicating our current WAL locations, oldest * xmin and the current time. * * If 'force' is not set, the message is only sent if enough time has @@ -1125,7 +1125,7 @@ XLogWalRcvSendReply(bool force, bool requestReply) * We can compare the write and flush positions to the last message we * sent without taking any lock, but the apply position requires a spin * lock, so we don't check that unless something else has changed or 10 - * seconds have passed. This means that the apply log position will + * seconds have passed. This means that the apply WAL location will * appear, from the master's point of view, to lag slightly, but since * this is only for reporting purposes and only on idle systems, that's * probably OK. diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index e4e5337d54..a899841d83 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -194,7 +194,7 @@ static volatile sig_atomic_t replication_active = false; static LogicalDecodingContext *logical_decoding_ctx = NULL; static XLogRecPtr logical_startptr = InvalidXLogRecPtr; -/* A sample associating a log position with the time it was written. */ +/* A sample associating a WAL location with the time it was written. */ typedef struct { XLogRecPtr lsn; @@ -340,7 +340,7 @@ static void IdentifySystem(void) { char sysid[32]; - char xpos[MAXFNAMELEN]; + char xloc[MAXFNAMELEN]; XLogRecPtr logptr; char *dbname = NULL; DestReceiver *dest; @@ -367,7 +367,7 @@ IdentifySystem(void) else logptr = GetFlushRecPtr(); - snprintf(xpos, sizeof(xpos), "%X/%X", (uint32) (logptr >> 32), (uint32) logptr); + snprintf(xloc, sizeof(xloc), "%X/%X", (uint32) (logptr >> 32), (uint32) logptr); if (MyDatabaseId != InvalidOid) { @@ -406,8 +406,8 @@ IdentifySystem(void) /* column 2: timeline */ values[1] = Int32GetDatum(ThisTimeLineID); - /* column 3: xlog position */ - values[2] = CStringGetTextDatum(xpos); + /* column 3: wal location */ + values[2] = CStringGetTextDatum(xloc); /* column 4: database name, or NULL if none */ if (dbname) @@ -842,7 +842,7 @@ static void CreateReplicationSlot(CreateReplicationSlotCmd *cmd) { const char *snapshot_name = NULL; - char xpos[MAXFNAMELEN]; + char xloc[MAXFNAMELEN]; char *slot_name; bool reserve_wal = false; CRSSnapshotAction snapshot_action = CRS_EXPORT_SNAPSHOT; @@ -975,7 +975,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) ReplicationSlotSave(); } - snprintf(xpos, sizeof(xpos), "%X/%X", + snprintf(xloc, sizeof(xloc), "%X/%X", (uint32) (MyReplicationSlot->data.confirmed_flush >> 32), (uint32) MyReplicationSlot->data.confirmed_flush); @@ -1008,7 +1008,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd) values[0] = CStringGetTextDatum(slot_name); /* consistent wal location */ - values[1] = CStringGetTextDatum(xpos); + values[1] = CStringGetTextDatum(xloc); /* snapshot name, or NULL if none */ if (snapshot_name != NULL) @@ -1729,7 +1729,7 @@ PhysicalConfirmReceivedLocation(XLogRecPtr lsn) } /* - * Regular reply from standby advising of WAL positions on standby server. + * Regular reply from standby advising of WAL locations on standby server. */ static void ProcessStandbyReplyMessage(void) @@ -2579,7 +2579,7 @@ XLogSendPhysical(void) /* * Record the current system time as an approximation of the time at which - * this WAL position was written for the purposes of lag tracking. + * this WAL location was written for the purposes of lag tracking. * * In theory we could make XLogFlush() record a time in shmem whenever WAL * is flushed and we could get that time as well as the LSN when we call @@ -3353,7 +3353,7 @@ WalSndKeepaliveIfNecessary(TimestampTz now) /* * Record the end of the WAL and the time it was flushed locally, so that - * LagTrackerRead can compute the elapsed time (lag) when this WAL position is + * LagTrackerRead can compute the elapsed time (lag) when this WAL location is * eventually reported to have been written, flushed and applied by the * standby in a reply message. */ @@ -3410,7 +3410,7 @@ LagTrackerWrite(XLogRecPtr lsn, TimestampTz local_flush_time) } /* - * Find out how much time has elapsed between the moment WAL position 'lsn' + * Find out how much time has elapsed between the moment WAL location 'lsn' * (or the highest known earlier LSN) was flushed locally and the time 'now'. * We have a separate read head for each of the reported LSN locations we * receive in replies from standby; 'head' controls which read head is diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index c7ddfcbfdf..b3811bc44e 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -1782,7 +1782,7 @@ BaseBackup(void) } /* - * Get the starting xlog position + * Get the starting WAL location */ res = PQgetResult(conn); if (PQresultStatus(res) != PGRES_TUPLES_OK) diff --git a/src/bin/pg_basebackup/receivelog.c b/src/bin/pg_basebackup/receivelog.c index 0f2845bbc2..1b79c00275 100644 --- a/src/bin/pg_basebackup/receivelog.c +++ b/src/bin/pg_basebackup/receivelog.c @@ -438,7 +438,7 @@ CheckServerVersionForStreaming(PGconn *conn) * If 'synchronous' is true, the received WAL is flushed as soon as written, * otherwise only when the WAL file is closed. * - * Note: The log position *must* be at a log segment start! + * Note: The WAL location *must* be at a log segment start! */ bool ReceiveXlogStream(PGconn *conn, StreamCtl *stream) @@ -733,7 +733,7 @@ ReadEndOfStreamingResult(PGresult *res, XLogRecPtr *startpos, uint32 *timeline) * 4 | 0/9949AE0 * * next_tli is the timeline ID of the next timeline after the one that - * just finished streaming. next_tli_startpos is the XLOG position where + * just finished streaming. next_tli_startpos is the WAL location where * the server switched to it. *---------- */ diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index a7f6fe2df3..1e3d329705 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -149,7 +149,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex) } /* - * Find the previous checkpoint preceding given WAL position. + * Find the previous checkpoint preceding given WAL location. */ void findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2014feea40..622dc6d135 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -231,7 +231,7 @@ main(int argc, char **argv) else { findCommonAncestorTimeline(&divergerec, &lastcommontliIndex); - printf(_("servers diverged at WAL position %X/%X on timeline %u\n"), + printf(_("servers diverged at WAL location %X/%X on timeline %u\n"), (uint32) (divergerec >> 32), (uint32) divergerec, targetHistory[lastcommontliIndex].tli); @@ -415,9 +415,9 @@ sanityChecks(void) } /* - * Find minimum from two XLOG positions assuming InvalidXLogRecPtr means + * Find minimum from two WAL locations assuming InvalidXLogRecPtr means * infinity as src/include/access/timeline.h states. This routine should - * be used only when comparing XLOG positions related to history files. + * be used only when comparing WAL locations related to history files. */ static XLogRecPtr MinXLogRecPtr(XLogRecPtr a, XLogRecPtr b) diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index 9491ccb13b..77b36f60e1 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -686,7 +686,7 @@ usage(void) printf(_(" %s [OPTION]... [STARTSEG [ENDSEG]]\n"), progname); printf(_("\nOptions:\n")); printf(_(" -b, --bkp-details output detailed information about backup blocks\n")); - printf(_(" -e, --end=RECPTR stop reading at log position RECPTR\n")); + printf(_(" -e, --end=RECPTR stop reading at WAL location RECPTR\n")); printf(_(" -f, --follow keep retrying after reaching end of WAL\n")); printf(_(" -n, --limit=N number of records to display\n")); printf(_(" -p, --path=PATH directory in which to find log segment files or a\n" @@ -694,7 +694,7 @@ usage(void) " (default: current directory, ./pg_wal, PGDATA/pg_wal)\n")); printf(_(" -r, --rmgr=RMGR only show records generated by resource manager RMGR\n" " use --rmgr=list to list valid resource manager names\n")); - printf(_(" -s, --start=RECPTR start reading at log position RECPTR\n")); + printf(_(" -s, --start=RECPTR start reading at WAL location RECPTR\n")); printf(_(" -t, --timeline=TLI timeline from which to read log records\n" " (default: 1 or the value used in STARTSEG)\n")); printf(_(" -V, --version output version information, then exit\n")); @@ -775,7 +775,7 @@ main(int argc, char **argv) case 'e': if (sscanf(optarg, "%X/%X", &xlogid, &xrecoff) != 2) { - fprintf(stderr, _("%s: could not parse end log position \"%s\"\n"), + fprintf(stderr, _("%s: could not parse end WAL location \"%s\"\n"), progname, optarg); goto bad_argument; } @@ -829,7 +829,7 @@ main(int argc, char **argv) case 's': if (sscanf(optarg, "%X/%X", &xlogid, &xrecoff) != 2) { - fprintf(stderr, _("%s: could not parse start log position \"%s\"\n"), + fprintf(stderr, _("%s: could not parse start WAL location \"%s\"\n"), progname, optarg); goto bad_argument; } @@ -929,7 +929,7 @@ main(int argc, char **argv) else if (!XLByteInSeg(private.startptr, segno)) { fprintf(stderr, - _("%s: start log position %X/%X is not inside file \"%s\"\n"), + _("%s: start WAL location %X/%X is not inside file \"%s\"\n"), progname, (uint32) (private.startptr >> 32), (uint32) private.startptr, @@ -973,7 +973,7 @@ main(int argc, char **argv) private.endptr != (segno + 1) * XLogSegSize) { fprintf(stderr, - _("%s: end log position %X/%X is not inside file \"%s\"\n"), + _("%s: end WAL location %X/%X is not inside file \"%s\"\n"), progname, (uint32) (private.endptr >> 32), (uint32) private.endptr, @@ -985,7 +985,7 @@ main(int argc, char **argv) /* we don't know what to print */ if (XLogRecPtrIsInvalid(private.startptr)) { - fprintf(stderr, _("%s: no start log position given.\n"), progname); + fprintf(stderr, _("%s: no start WAL location given\n"), progname); goto bad_argument; } diff --git a/src/include/access/timeline.h b/src/include/access/timeline.h index 05363c6842..c1911feb16 100644 --- a/src/include/access/timeline.h +++ b/src/include/access/timeline.h @@ -17,7 +17,7 @@ /* * A list of these structs describes the timeline history of the server. Each * TimeLineHistoryEntry represents a piece of WAL belonging to the history, - * from newest to oldest. All WAL positions between 'begin' and 'end' belong to + * from newest to oldest. All WAL locations between 'begin' and 'end' belong to * the timeline represented by the entry. Together the 'begin' and 'end' * pointers of all the entries form a contiguous line from beginning of time * to infinity. -- cgit v1.2.3 From 46052d9ef314deafa8c94ac7fda4a2811db0679e Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 12 May 2017 13:53:24 -0400 Subject: Replace another "transaction log" with "write-ahead log" Reported-by: Dagfinn Ilmari Mannsåker --- doc/src/sgml/func.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 0527641b7c..5f47c59f8a 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -18628,7 +18628,7 @@ postgres=# select pg_start_backup('label_goes_here'); pg_current_wal_insert_lsn displays the current write-ahead log insertion location and pg_current_wal_flush_lsn displays the current write-ahead log flush location. The insertion location is the logical - end of the transaction log at any instant, while the write location is the end of + end of the write-ahead log at any instant, while the write location is the end of what has actually been written out from the server's internal buffers and flush location is the location guaranteed to be written to durable storage. The write location is the end of what can be examined from outside the server, and is usually -- cgit v1.2.3 From bc085205c8a425fcaa54e27c6dcd83101130439b Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Fri, 12 May 2017 14:59:23 -0300 Subject: Change CREATE STATISTICS syntax Previously, we had the WITH clause in the middle of the command, where you'd specify both generic options as well as statistic types. Few people liked this, so this commit changes it to remove the WITH keyword from that clause and makes it accept statistic types only. (We currently don't have any generic options, but if we invent in the future, we will gain a new WITH clause, probably at the end of the command). Also, the column list is now specified without parens, which makes the whole command look more similar to a SELECT command. This change will let us expand the command to supporting expressions (not just columns names) as well as multiple tables and their join conditions. Tom added lots of code comments and fixed some parts of the CREATE STATISTICS reference page, too; more changes in this area are forthcoming. He also fixed a potential problem in the alter_generic regression test, reducing verbosity on a cascaded drop to avoid dependency on message ordering, as we do in other tests. Tom also closed a security bug: we documented that table ownership was required in order to create a statistics object on it, but didn't actually implement it. Implement tab-completion for statistics objects. This can stand some more improvement. Authors: Alvaro Herrera, with lots of cleanup by Tom Lane Discussion: https://postgr.es/m/20170420212426.ltvgyhnefvhixm6i@alvherre.pgsql --- doc/src/sgml/perform.sgml | 8 +- doc/src/sgml/planstats.sgml | 4 +- doc/src/sgml/ref/alter_statistics.sgml | 28 +++---- doc/src/sgml/ref/create_statistics.sgml | 85 +++++++++------------ doc/src/sgml/ref/drop_statistics.sgml | 16 ++-- src/backend/commands/statscmds.c | 109 +++++++++++++++++++-------- src/backend/nodes/copyfuncs.c | 6 +- src/backend/nodes/equalfuncs.c | 6 +- src/backend/nodes/outfuncs.c | 6 +- src/backend/parser/gram.y | 42 +++++------ src/backend/utils/adt/ruleutils.c | 10 +-- src/bin/pg_dump/t/002_pg_dump.pl | 10 +-- src/bin/psql/describe.c | 14 ++-- src/bin/psql/tab-complete.c | 38 +++++++++- src/include/nodes/parsenodes.h | 8 +- src/test/regress/expected/alter_generic.out | 52 ++----------- src/test/regress/expected/object_address.out | 2 +- src/test/regress/expected/stats_ext.out | 85 ++++++++++++--------- src/test/regress/sql/alter_generic.sql | 12 +-- src/test/regress/sql/object_address.sql | 2 +- src/test/regress/sql/stats_ext.sql | 64 ++++++++-------- 21 files changed, 321 insertions(+), 286 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/perform.sgml b/doc/src/sgml/perform.sgml index b10b734b90..32e17ee5f8 100644 --- a/doc/src/sgml/perform.sgml +++ b/doc/src/sgml/perform.sgml @@ -1132,8 +1132,8 @@ WHERE tablename = 'road'; To inspect functional dependencies on a statistics stts, you may do this: -CREATE STATISTICS stts WITH (dependencies) - ON (zip, city) FROM zipcodes; +CREATE STATISTICS stts (dependencies) + ON zip, city FROM zipcodes; ANALYZE zipcodes; SELECT stxname, stxkeys, stxdependencies FROM pg_statistic_ext @@ -1219,8 +1219,8 @@ EXPLAIN (ANALYZE, TIMING OFF) SELECT * FROM t WHERE a = 1 AND b = 10; Continuing the above example, the n-distinct coefficients in a ZIP code table may look like the following: -CREATE STATISTICS stts2 WITH (ndistinct) - ON (zip, state, city) FROM zipcodes; +CREATE STATISTICS stts2 (ndistinct) + ON zip, state, city FROM zipcodes; ANALYZE zipcodes; SELECT stxkeys AS k, stxndistinct AS nd FROM pg_statistic_ext diff --git a/doc/src/sgml/planstats.sgml b/doc/src/sgml/planstats.sgml index 11580bfd22..ef847b9633 100644 --- a/doc/src/sgml/planstats.sgml +++ b/doc/src/sgml/planstats.sgml @@ -526,7 +526,7 @@ EXPLAIN (ANALYZE, TIMING OFF) SELECT * FROM t WHERE a = 1 AND b = 1; multivariate statistics on the two columns: -CREATE STATISTICS stts WITH (dependencies) ON (a, b) FROM t; +CREATE STATISTICS stts (dependencies) ON a, b FROM t; ANALYZE t; EXPLAIN (ANALYZE, TIMING OFF) SELECT * FROM t WHERE a = 1 AND b = 1; QUERY PLAN @@ -569,7 +569,7 @@ EXPLAIN (ANALYZE, TIMING OFF) SELECT COUNT(*) FROM t GROUP BY a, b; calculation, the estimate is much improved: DROP STATISTICS stts; -CREATE STATISTICS stts WITH (dependencies, ndistinct) ON (a, b) FROM t; +CREATE STATISTICS stts (dependencies, ndistinct) ON a, b FROM t; ANALYZE t; EXPLAIN (ANALYZE, TIMING OFF) SELECT COUNT(*) FROM t GROUP BY a, b; QUERY PLAN diff --git a/doc/src/sgml/ref/alter_statistics.sgml b/doc/src/sgml/ref/alter_statistics.sgml index 3e4d28614a..4f25669852 100644 --- a/doc/src/sgml/ref/alter_statistics.sgml +++ b/doc/src/sgml/ref/alter_statistics.sgml @@ -17,7 +17,7 @@ PostgreSQL documentation ALTER STATISTICS - change the definition of a extended statistics + change the definition of an extended statistics object @@ -34,19 +34,20 @@ ALTER STATISTICS name SET SCHEMA ALTER STATISTICS changes the parameters of an existing - extended statistics. Any parameters not specifically set in the + extended statistics object. Any parameters not specifically set in the ALTER STATISTICS command retain their prior settings. - You must own the statistics to use ALTER STATISTICS. - To change a statistics' schema, you must also have CREATE - privilege on the new schema. + You must own the statistics object to use ALTER STATISTICS. + To change a statistics object's schema, you must also + have CREATE privilege on the new schema. To alter the owner, you must also be a direct or indirect member of the new owning role, and that role must have CREATE privilege on - the statistics' schema. (These restrictions enforce that altering the owner - doesn't do anything you couldn't do by dropping and recreating the statistics. - However, a superuser can alter ownership of any statistics anyway.) + the statistics object's schema. (These restrictions enforce that altering + the owner doesn't do anything you couldn't do by dropping and recreating + the statistics object. However, a superuser can alter ownership of any + statistics object anyway.) @@ -59,7 +60,8 @@ ALTER STATISTICS name SET SCHEMA name - The name (optionally schema-qualified) of the statistics to be altered. + The name (optionally schema-qualified) of the statistics object to be + altered. @@ -68,7 +70,7 @@ ALTER STATISTICS name SET SCHEMA new_owner - The user name of the new owner of the statistics. + The user name of the new owner of the statistics object. @@ -77,7 +79,7 @@ ALTER STATISTICS name SET SCHEMA new_name - The new name for the statistics. + The new name for the statistics object. @@ -86,7 +88,7 @@ ALTER STATISTICS name SET SCHEMA new_schema - The new schema for the statistics. + The new schema for the statistics object. @@ -99,7 +101,7 @@ ALTER STATISTICS name SET SCHEMA Compatibility - There's no ALTER STATISTICS command in the SQL standard. + There is no ALTER STATISTICS command in the SQL standard. diff --git a/doc/src/sgml/ref/create_statistics.sgml b/doc/src/sgml/ref/create_statistics.sgml index edbcf5840b..92ee4e4efa 100644 --- a/doc/src/sgml/ref/create_statistics.sgml +++ b/doc/src/sgml/ref/create_statistics.sgml @@ -22,8 +22,8 @@ PostgreSQL documentation CREATE STATISTICS [ IF NOT EXISTS ] statistics_name - WITH ( option [= value] [, ... ] ) - ON ( column_name, column_name [, ...]) + [ ( statistic_type [, ... ] ) ] + ON column_name, column_name [, ...] FROM table_name @@ -34,17 +34,17 @@ CREATE STATISTICS [ IF NOT EXISTS ] statistics_na CREATE STATISTICS will create a new extended statistics - object on the specified table, foreign table or materialized view. - The statistics will be created in the current database and - will be owned by the user issuing the command. + object tracking data about the specified table, foreign table or + materialized view. The statistics object will be created in the current + database and will be owned by the user issuing the command. If a schema name is given (for example, CREATE STATISTICS - myschema.mystat ...) then the statistics is created in the specified - schema. Otherwise it is created in the current schema. The name of - the statistics must be distinct from the name of any other statistics in the - same schema. + myschema.mystat ...) then the statistics object is created in the + specified schema. Otherwise it is created in the current schema. + The name of the statistics object must be distinct from the name of any + other statistics object in the same schema. @@ -57,10 +57,10 @@ CREATE STATISTICS [ IF NOT EXISTS ] statistics_na IF NOT EXISTS - Do not throw an error if a statistics with the same name already exists. - A notice is issued in this case. Note that only the name of the - statistics object is considered here. The definition of the statistics is - not considered. + Do not throw an error if a statistics object with the same name already + exists. A notice is issued in this case. Note that only the name of + the statistics object is considered here, not the details of its + definition. @@ -69,67 +69,45 @@ CREATE STATISTICS [ IF NOT EXISTS ] statistics_na statistics_name - The name (optionally schema-qualified) of the statistics to be created. + The name (optionally schema-qualified) of the statistics object to be + created. - column_name + statistic_type - The name of a column to be included in the statistics. + A statistic type to be computed in this statistics object. Currently + supported types are ndistinct, which enables + n-distinct coefficient tracking, + and dependencies, which enables functional + dependencies. - table_name - - - The name (optionally schema-qualified) of the table the statistics should - be created on. - - - - - - - - Parameters - - - statistics parameters - - - - The WITH clause can specify options - for the statistics. Available options are listed below. - - - - - - dependencies (boolean) + column_name - Enables functional dependencies for the statistics. + The name of a table column to be included in the statistics object. - ndistinct (boolean) + table_name - Enables ndistinct coefficients for the statistics. + The name (optionally schema-qualified) of the table containing the + column(s) the statistics are computed on. - - - + @@ -158,7 +136,7 @@ CREATE TABLE t1 ( INSERT INTO t1 SELECT i/100, i/500 FROM generate_series(1,1000000) s(i); -CREATE STATISTICS s1 WITH (dependencies) ON (a, b) FROM t1; +CREATE STATISTICS s1 (dependencies) ON a, b FROM t1; ANALYZE t1; @@ -168,6 +146,11 @@ EXPLAIN ANALYZE SELECT * FROM t1 WHERE (a = 1) AND (b = 0); -- invalid combination of values EXPLAIN ANALYZE SELECT * FROM t1 WHERE (a = 1) AND (b = 1); + + Without functional-dependency statistics, the planner would make the + same estimate of the number of matching rows for these two queries. + With such statistics, it is able to tell that one case has matches + and the other does not. @@ -176,7 +159,7 @@ EXPLAIN ANALYZE SELECT * FROM t1 WHERE (a = 1) AND (b = 1); Compatibility - There's no CREATE STATISTICS command in the SQL standard. + There is no CREATE STATISTICS command in the SQL standard. diff --git a/doc/src/sgml/ref/drop_statistics.sgml b/doc/src/sgml/ref/drop_statistics.sgml index 98c338182b..ef659fca61 100644 --- a/doc/src/sgml/ref/drop_statistics.sgml +++ b/doc/src/sgml/ref/drop_statistics.sgml @@ -29,9 +29,9 @@ DROP STATISTICS [ IF EXISTS ] name Description - DROP STATISTICS removes statistics from the database. - Only the statistics owner, the schema owner, and superuser can drop a - statistics. + DROP STATISTICS removes statistics object(s) from the + database. Only the statistics object's owner, the schema owner, or a + superuser can drop a statistics object. @@ -44,8 +44,8 @@ DROP STATISTICS [ IF EXISTS ] name IF EXISTS - Do not throw an error if the statistics do not exist. A notice is - issued in this case. + Do not throw an error if the statistics object does not exist. A notice + is issued in this case. @@ -54,7 +54,7 @@ DROP STATISTICS [ IF EXISTS ] name name - The name (optionally schema-qualified) of the statistics to drop. + The name (optionally schema-qualified) of the statistics object to drop. @@ -66,7 +66,7 @@ DROP STATISTICS [ IF EXISTS ] name Examples - To destroy two statistics objects on different schemas, without failing + To destroy two statistics objects in different schemas, without failing if they don't exist: @@ -82,7 +82,7 @@ DROP STATISTICS IF EXISTS Compatibility - There's no DROP STATISTICS command in the SQL standard. + There is no DROP STATISTICS command in the SQL standard. diff --git a/src/backend/commands/statscmds.c b/src/backend/commands/statscmds.c index 0b9c33e30a..662b4fa15d 100644 --- a/src/backend/commands/statscmds.c +++ b/src/backend/commands/statscmds.c @@ -60,7 +60,7 @@ CreateStatistics(CreateStatsStmt *stmt) bool nulls[Natts_pg_statistic_ext]; int2vector *stxkeys; Relation statrel; - Relation rel; + Relation rel = NULL; Oid relid; ObjectAddress parentobject, childobject; @@ -71,7 +71,7 @@ CreateStatistics(CreateStatsStmt *stmt) bool build_dependencies; bool requested_type = false; int i; - ListCell *l; + ListCell *cell; Assert(IsA(stmt, CreateStatsStmt)); @@ -101,35 +101,81 @@ CreateStatistics(CreateStatsStmt *stmt) } /* - * CREATE STATISTICS will influence future execution plans but does not - * interfere with currently executing plans. So it should be enough to - * take only ShareUpdateExclusiveLock on relation, conflicting with - * ANALYZE and other DDL that sets statistical information, but not with - * normal queries. + * Examine the FROM clause. Currently, we only allow it to be a single + * simple table, but later we'll probably allow multiple tables and JOIN + * syntax. The grammar is already prepared for that, so we have to check + * here that what we got is what we can support. */ - rel = relation_openrv(stmt->relation, ShareUpdateExclusiveLock); - relid = RelationGetRelid(rel); - - if (rel->rd_rel->relkind != RELKIND_RELATION && - rel->rd_rel->relkind != RELKIND_MATVIEW && - rel->rd_rel->relkind != RELKIND_FOREIGN_TABLE && - rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE) + if (list_length(stmt->relations) != 1) ereport(ERROR, - (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("relation \"%s\" is not a table, foreign table, or materialized view", - RelationGetRelationName(rel)))); + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("only a single relation is allowed in CREATE STATISTICS"))); + + foreach(cell, stmt->relations) + { + Node *rln = (Node *) lfirst(cell); + + if (!IsA(rln, RangeVar)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("only a single relation is allowed in CREATE STATISTICS"))); + + /* + * CREATE STATISTICS will influence future execution plans but does + * not interfere with currently executing plans. So it should be + * enough to take only ShareUpdateExclusiveLock on relation, + * conflicting with ANALYZE and other DDL that sets statistical + * information, but not with normal queries. + */ + rel = relation_openrv((RangeVar *) rln, ShareUpdateExclusiveLock); + + /* Restrict to allowed relation types */ + if (rel->rd_rel->relkind != RELKIND_RELATION && + rel->rd_rel->relkind != RELKIND_MATVIEW && + rel->rd_rel->relkind != RELKIND_FOREIGN_TABLE && + rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("relation \"%s\" is not a table, foreign table, or materialized view", + RelationGetRelationName(rel)))); + + /* You must own the relation to create stats on it */ + if (!pg_class_ownercheck(RelationGetRelid(rel), GetUserId())) + aclcheck_error(ACLCHECK_NOT_OWNER, ACL_KIND_CLASS, + RelationGetRelationName(rel)); + } + + Assert(rel); + relid = RelationGetRelid(rel); /* - * Transform column names to array of attnums. While at it, enforce some - * constraints. + * Currently, we only allow simple column references in the expression + * list. That will change someday, and again the grammar already supports + * it so we have to enforce restrictions here. For now, we can convert + * the expression list to a simple array of attnums. While at it, enforce + * some constraints. */ - foreach(l, stmt->keys) + foreach(cell, stmt->exprs) { - char *attname = strVal(lfirst(l)); + Node *expr = (Node *) lfirst(cell); + ColumnRef *cref; + char *attname; HeapTuple atttuple; Form_pg_attribute attForm; TypeCacheEntry *type; + if (!IsA(expr, ColumnRef)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("only simple column references are allowed in CREATE STATISTICS"))); + cref = (ColumnRef *) expr; + + if (list_length(cref->fields) != 1) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("only simple column references are allowed in CREATE STATISTICS"))); + attname = strVal((Value *) linitial(cref->fields)); + atttuple = SearchSysCacheAttName(relid, attname); if (!HeapTupleIsValid(atttuple)) ereport(ERROR, @@ -194,30 +240,29 @@ CreateStatistics(CreateStatsStmt *stmt) stxkeys = buildint2vector(attnums, numcols); /* - * Parse the statistics options. Currently only statistics types are - * recognized. + * Parse the statistics types. */ build_ndistinct = false; build_dependencies = false; - foreach(l, stmt->options) + foreach(cell, stmt->stat_types) { - DefElem *opt = (DefElem *) lfirst(l); + char *type = strVal((Value *) lfirst(cell)); - if (strcmp(opt->defname, "ndistinct") == 0) + if (strcmp(type, "ndistinct") == 0) { - build_ndistinct = defGetBoolean(opt); + build_ndistinct = true; requested_type = true; } - else if (strcmp(opt->defname, "dependencies") == 0) + else if (strcmp(type, "dependencies") == 0) { - build_dependencies = defGetBoolean(opt); + build_dependencies = true; requested_type = true; } else ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("unrecognized STATISTICS option \"%s\"", - opt->defname))); + errmsg("unrecognized statistics type \"%s\"", + type))); } /* If no statistic type was specified, build them all. */ if (!requested_type) @@ -268,6 +313,8 @@ CreateStatistics(CreateStatsStmt *stmt) /* * Add a dependency on the table, so that stats get dropped on DROP TABLE. + * + * XXX don't we need dependencies on the specific columns, instead? */ ObjectAddressSet(parentobject, RelationRelationId, relid); ObjectAddressSet(childobject, StatisticExtRelationId, statoid); diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index 2d2a9d00b7..d13a6fc03f 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -3389,9 +3389,9 @@ _copyCreateStatsStmt(const CreateStatsStmt *from) CreateStatsStmt *newnode = makeNode(CreateStatsStmt); COPY_NODE_FIELD(defnames); - COPY_NODE_FIELD(relation); - COPY_NODE_FIELD(keys); - COPY_NODE_FIELD(options); + COPY_NODE_FIELD(stat_types); + COPY_NODE_FIELD(exprs); + COPY_NODE_FIELD(relations); COPY_SCALAR_FIELD(if_not_exists); return newnode; diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index b5459cd726..c9a8c34892 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -1349,9 +1349,9 @@ static bool _equalCreateStatsStmt(const CreateStatsStmt *a, const CreateStatsStmt *b) { COMPARE_NODE_FIELD(defnames); - COMPARE_NODE_FIELD(relation); - COMPARE_NODE_FIELD(keys); - COMPARE_NODE_FIELD(options); + COMPARE_NODE_FIELD(stat_types); + COMPARE_NODE_FIELD(exprs); + COMPARE_NODE_FIELD(relations); COMPARE_SCALAR_FIELD(if_not_exists); return true; diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 98f67681a7..3d5b09aeee 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -2639,9 +2639,9 @@ _outCreateStatsStmt(StringInfo str, const CreateStatsStmt *node) WRITE_NODE_TYPE("CREATESTATSSTMT"); WRITE_NODE_FIELD(defnames); - WRITE_NODE_FIELD(relation); - WRITE_NODE_FIELD(keys); - WRITE_NODE_FIELD(options); + WRITE_NODE_FIELD(stat_types); + WRITE_NODE_FIELD(exprs); + WRITE_NODE_FIELD(relations); WRITE_BOOL_FIELD(if_not_exists); } diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index d04bb7ea3e..28223311e6 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -3835,31 +3835,29 @@ ExistingIndex: USING INDEX index_name { $$ = $3; } /***************************************************************************** * * QUERY : - * CREATE STATISTICS stats_name WITH (options) ON (columns) FROM relname + * CREATE STATISTICS stats_name [(stat types)] + * ON expression-list FROM from_list + * + * Note: the expectation here is that the clauses after ON are a subset of + * SELECT syntax, allowing for expressions and joined tables, and probably + * someday a WHERE clause. Much less than that is currently implemented, + * but the grammar accepts it and then we'll throw FEATURE_NOT_SUPPORTED + * errors as necessary at execution. * *****************************************************************************/ - -CreateStatsStmt: CREATE STATISTICS any_name opt_reloptions ON '(' columnList ')' FROM qualified_name - { - CreateStatsStmt *n = makeNode(CreateStatsStmt); - n->defnames = $3; - n->relation = $10; - n->keys = $7; - n->options = $4; - n->if_not_exists = false; - $$ = (Node *)n; - } - | CREATE STATISTICS IF_P NOT EXISTS any_name opt_reloptions ON '(' columnList ')' FROM qualified_name - { - CreateStatsStmt *n = makeNode(CreateStatsStmt); - n->defnames = $6; - n->relation = $13; - n->keys = $10; - n->options = $7; - n->if_not_exists = true; - $$ = (Node *)n; - } +CreateStatsStmt: + CREATE opt_if_not_exists STATISTICS any_name + opt_name_list ON expr_list FROM from_list + { + CreateStatsStmt *n = makeNode(CreateStatsStmt); + n->defnames = $4; + n->stat_types = $5; + n->exprs = $7; + n->relations = $9; + n->if_not_exists = $2; + $$ = (Node *)n; + } ; /***************************************************************************** diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c index cbde1fff01..983b9800cc 100644 --- a/src/backend/utils/adt/ruleutils.c +++ b/src/backend/utils/adt/ruleutils.c @@ -1504,15 +1504,15 @@ pg_get_statisticsext_worker(Oid statextid, bool missing_ok) } /* - * If any option is disabled, then we'll need to append a WITH clause to - * show which options are enabled. We omit the WITH clause on purpose + * If any option is disabled, then we'll need to append the types clause + * to show which options are enabled. We omit the types clause on purpose * when all options are enabled, so a pg_dump/pg_restore will create all * statistics types on a newer postgres version, if the statistics had all * options enabled on the original version. */ if (!ndistinct_enabled || !dependencies_enabled) { - appendStringInfoString(&buf, " WITH ("); + appendStringInfoString(&buf, " ("); if (ndistinct_enabled) appendStringInfoString(&buf, "ndistinct"); else if (dependencies_enabled) @@ -1521,7 +1521,7 @@ pg_get_statisticsext_worker(Oid statextid, bool missing_ok) appendStringInfoChar(&buf, ')'); } - appendStringInfoString(&buf, " ON ("); + appendStringInfoString(&buf, " ON "); for (colno = 0; colno < statextrec->stxkeys.dim1; colno++) { @@ -1536,7 +1536,7 @@ pg_get_statisticsext_worker(Oid statextid, bool missing_ok) appendStringInfoString(&buf, quote_identifier(attname)); } - appendStringInfo(&buf, ") FROM %s", + appendStringInfo(&buf, " FROM %s", generate_relation_name(statextrec->stxrelid, NIL)); ReleaseSysCache(statexttup); diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl index b62299929f..9bd400e067 100644 --- a/src/bin/pg_dump/t/002_pg_dump.pl +++ b/src/bin/pg_dump/t/002_pg_dump.pl @@ -4955,9 +4955,9 @@ qr/CREATE TRANSFORM FOR integer LANGUAGE sql \(FROM SQL WITH FUNCTION pg_catalog catch_all => 'CREATE ... commands', create_order => 97, create_sql => 'CREATE STATISTICS dump_test.test_ext_stats_no_options - ON (col1, col2) FROM dump_test.test_fifth_table', + ON col1, col2 FROM dump_test.test_fifth_table', regexp => qr/^ - \QCREATE STATISTICS dump_test.test_ext_stats_no_options ON (col1, col2) FROM test_fifth_table;\E + \QCREATE STATISTICS dump_test.test_ext_stats_no_options ON col1, col2 FROM test_fifth_table;\E /xms, like => { binary_upgrade => 1, @@ -4988,10 +4988,10 @@ qr/CREATE TRANSFORM FOR integer LANGUAGE sql \(FROM SQL WITH FUNCTION pg_catalog all_runs => 1, catch_all => 'CREATE ... commands', create_order => 97, - create_sql => 'CREATE STATISTICS dump_test.test_ext_stats_using - WITH (ndistinct) ON (col1, col2) FROM dump_test.test_fifth_table', + create_sql => 'CREATE STATISTICS dump_test.test_ext_stats_opts + (ndistinct) ON col1, col2 FROM dump_test.test_fifth_table', regexp => qr/^ - \QCREATE STATISTICS dump_test.test_ext_stats_using WITH (ndistinct) ON (col1, col2) FROM test_fifth_table;\E + \QCREATE STATISTICS dump_test.test_ext_stats_opts (ndistinct) ON col1, col2 FROM test_fifth_table;\E /xms, like => { binary_upgrade => 1, diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index 13395f5ca6..386af6168f 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -2355,8 +2355,9 @@ describeOneTableDetails(const char *schemaname, { printfPQExpBuffer(&buf, "SELECT oid, " + "stxrelid::pg_catalog.regclass, " "stxnamespace::pg_catalog.regnamespace AS nsp, " - "stxname, stxkeys,\n" + "stxname,\n" " (SELECT pg_catalog.string_agg(pg_catalog.quote_ident(attname),', ')\n" " FROM pg_catalog.unnest(stxkeys) s(attnum)\n" " JOIN pg_catalog.pg_attribute a ON (stxrelid = a.attrelid AND\n" @@ -2385,9 +2386,9 @@ describeOneTableDetails(const char *schemaname, printfPQExpBuffer(&buf, " "); /* statistics name (qualified with namespace) */ - appendPQExpBuffer(&buf, "\"%s.%s\" WITH (", - PQgetvalue(result, i, 1), - PQgetvalue(result, i, 2)); + appendPQExpBuffer(&buf, "\"%s\".\"%s\" (", + PQgetvalue(result, i, 2), + PQgetvalue(result, i, 3)); /* options */ if (strcmp(PQgetvalue(result, i, 5), "t") == 0) @@ -2401,8 +2402,9 @@ describeOneTableDetails(const char *schemaname, appendPQExpBuffer(&buf, "%sdependencies", gotone ? ", " : ""); } - appendPQExpBuffer(&buf, ") ON (%s)", - PQgetvalue(result, i, 4)); + appendPQExpBuffer(&buf, ") ON %s FROM %s", + PQgetvalue(result, i, 4), + PQgetvalue(result, i, 1)); printTableAddFooter(&cont, buf.data); } diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index 92abcc3ac3..09fb30f270 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -616,6 +616,21 @@ static const SchemaQuery Query_for_list_of_matviews = { NULL }; +static const SchemaQuery Query_for_list_of_statistics = { + /* catname */ + "pg_catalog.pg_statistic_ext s", + /* selcondition */ + NULL, + /* viscondition */ + NULL, + /* namespace */ + "s.stxnamespace", + /* result */ + "pg_catalog.quote_ident(s.stxname)", + /* qualresult */ + NULL +}; + /* * Queries to get lists of names of various kinds of things, possibly @@ -1023,6 +1038,7 @@ static const pgsql_thing_t words_after_create[] = { {"SCHEMA", Query_for_list_of_schemas}, {"SEQUENCE", NULL, &Query_for_list_of_sequences}, {"SERVER", Query_for_list_of_servers}, + {"STATISTICS", NULL, &Query_for_list_of_statistics}, {"SUBSCRIPTION", Query_for_list_of_subscriptions}, {"SYSTEM", NULL, NULL, THING_NO_CREATE | THING_NO_DROP}, {"TABLE", NULL, &Query_for_list_of_tables}, @@ -1782,6 +1798,10 @@ psql_completion(const char *text, int start, int end) else if (Matches5("ALTER", "RULE", MatchAny, "ON", MatchAny)) COMPLETE_WITH_CONST("RENAME TO"); + /* ALTER STATISTICS */ + else if (Matches3("ALTER", "STATISTICS", MatchAny)) + COMPLETE_WITH_LIST3("OWNER TO", "RENAME TO", "SET SCHEMA"); + /* ALTER TRIGGER , add ON */ else if (Matches3("ALTER", "TRIGGER", MatchAny)) COMPLETE_WITH_CONST("ON"); @@ -2118,7 +2138,8 @@ psql_completion(const char *text, int start, int end) {"ACCESS METHOD", "CAST", "COLLATION", "CONVERSION", "DATABASE", "EVENT TRIGGER", "EXTENSION", "FOREIGN DATA WRAPPER", "FOREIGN TABLE", - "SERVER", "INDEX", "LANGUAGE", "POLICY", "PUBLICATION", "RULE", "SCHEMA", "SEQUENCE", "SUBSCRIPTION", + "SERVER", "INDEX", "LANGUAGE", "POLICY", "PUBLICATION", "RULE", + "SCHEMA", "SEQUENCE", "STATISTICS", "SUBSCRIPTION", "TABLE", "TYPE", "VIEW", "MATERIALIZED VIEW", "COLUMN", "AGGREGATE", "FUNCTION", "OPERATOR", "TRIGGER", "CONSTRAINT", "DOMAIN", "LARGE OBJECT", "TABLESPACE", "TEXT SEARCH", "ROLE", NULL}; @@ -2380,6 +2401,19 @@ psql_completion(const char *text, int start, int end) else if (Matches3("CREATE", "SERVER", MatchAny)) COMPLETE_WITH_LIST3("TYPE", "VERSION", "FOREIGN DATA WRAPPER"); +/* CREATE STATISTICS */ + else if (Matches3("CREATE", "STATISTICS", MatchAny)) + COMPLETE_WITH_LIST2("(", "ON"); + else if (Matches4("CREATE", "STATISTICS", MatchAny, "(")) + COMPLETE_WITH_LIST2("ndistinct", "dependencies"); + else if (HeadMatches3("CREATE", "STATISTICS", MatchAny) && + previous_words[0][0] == '(' && + previous_words[0][strlen(previous_words[0]) - 1] == ')') + COMPLETE_WITH_CONST("ON"); + else if (HeadMatches3("CREATE", "STATISTICS", MatchAny) && + TailMatches1("FROM")) + COMPLETE_WITH_SCHEMA_QUERY(Query_for_list_of_tables, NULL); + /* CREATE TABLE --- is allowed inside CREATE SCHEMA, so use TailMatches */ /* Complete "CREATE TEMP/TEMPORARY" with the possible temp objects */ else if (TailMatches2("CREATE", "TEMP|TEMPORARY")) @@ -2585,7 +2619,7 @@ psql_completion(const char *text, int start, int end) /* DROP */ /* Complete DROP object with CASCADE / RESTRICT */ else if (Matches3("DROP", - "COLLATION|CONVERSION|DOMAIN|EXTENSION|LANGUAGE|PUBLICATION|SCHEMA|SEQUENCE|SERVER|SUBSCRIPTION|TABLE|TYPE|VIEW", + "COLLATION|CONVERSION|DOMAIN|EXTENSION|LANGUAGE|PUBLICATION|SCHEMA|SEQUENCE|SERVER|SUBSCRIPTION|STATISTICS|TABLE|TYPE|VIEW", MatchAny) || Matches4("DROP", "ACCESS", "METHOD", MatchAny) || (Matches4("DROP", "AGGREGATE|FUNCTION", MatchAny, MatchAny) && diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 46c23c2530..d396be382b 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -2689,10 +2689,10 @@ typedef struct CreateStatsStmt { NodeTag type; List *defnames; /* qualified name (list of Value strings) */ - RangeVar *relation; /* relation to build statistics on */ - List *keys; /* String nodes naming referenced columns */ - List *options; /* list of DefElem */ - bool if_not_exists; /* do nothing if statistics already exists */ + List *stat_types; /* stat types (list of Value strings) */ + List *exprs; /* expressions to build statistics on */ + List *relations; /* rels to build stats on (list of RangeVar) */ + bool if_not_exists; /* do nothing if stats name already exists */ } CreateStatsStmt; /* ---------------------- diff --git a/src/test/regress/expected/alter_generic.out b/src/test/regress/expected/alter_generic.out index a81a4edfb2..28e69166be 100644 --- a/src/test/regress/expected/alter_generic.out +++ b/src/test/regress/expected/alter_generic.out @@ -501,8 +501,8 @@ DROP OPERATOR FAMILY alt_opf18 USING btree; -- SET SESSION AUTHORIZATION regress_alter_user1; CREATE TABLE alt_regress_1 (a INTEGER, b INTEGER); -CREATE STATISTICS alt_stat1 ON (a, b) FROM alt_regress_1; -CREATE STATISTICS alt_stat2 ON (a, b) FROM alt_regress_1; +CREATE STATISTICS alt_stat1 ON a, b FROM alt_regress_1; +CREATE STATISTICS alt_stat2 ON a, b FROM alt_regress_1; ALTER STATISTICS alt_stat1 RENAME TO alt_stat2; -- failed (name conflict) ERROR: statistics "alt_stat2" already exists in schema "alt_nsp1" ALTER STATISTICS alt_stat1 RENAME TO alt_stat3; -- failed (name conflict) @@ -511,8 +511,9 @@ ERROR: must be member of role "regress_alter_user2" ALTER STATISTICS alt_stat2 OWNER TO regress_alter_user3; -- OK ALTER STATISTICS alt_stat2 SET SCHEMA alt_nsp2; -- OK SET SESSION AUTHORIZATION regress_alter_user2; -CREATE STATISTICS alt_stat1 ON (a, b) FROM alt_regress_1; -CREATE STATISTICS alt_stat2 ON (a, b) FROM alt_regress_1; +CREATE TABLE alt_regress_2 (a INTEGER, b INTEGER); +CREATE STATISTICS alt_stat1 ON a, b FROM alt_regress_2; +CREATE STATISTICS alt_stat2 ON a, b FROM alt_regress_2; ALTER STATISTICS alt_stat3 RENAME TO alt_stat4; -- failed (not owner) ERROR: must be owner of statistics alt_stat3 ALTER STATISTICS alt_stat1 RENAME TO alt_stat4; -- OK @@ -672,54 +673,13 @@ SELECT nspname, prsname --- --- Cleanup resources --- +set client_min_messages to warning; -- suppress cascade notices DROP FOREIGN DATA WRAPPER alt_fdw2 CASCADE; -NOTICE: drop cascades to server alt_fserv2 DROP FOREIGN DATA WRAPPER alt_fdw3 CASCADE; -NOTICE: drop cascades to server alt_fserv3 DROP LANGUAGE alt_lang2 CASCADE; DROP LANGUAGE alt_lang3 CASCADE; -DROP LANGUAGE alt_lang4 CASCADE; -ERROR: language "alt_lang4" does not exist DROP SCHEMA alt_nsp1 CASCADE; -NOTICE: drop cascades to 27 other objects -DETAIL: drop cascades to function alt_func3(integer) -drop cascades to function alt_agg3(integer) -drop cascades to function alt_func4(integer) -drop cascades to function alt_func2(integer) -drop cascades to function alt_agg4(integer) -drop cascades to function alt_agg2(integer) -drop cascades to conversion alt_conv3 -drop cascades to conversion alt_conv4 -drop cascades to conversion alt_conv2 -drop cascades to operator @+@(integer,integer) -drop cascades to operator @-@(integer,integer) -drop cascades to operator family alt_opf3 for access method hash -drop cascades to operator family alt_opc1 for access method hash -drop cascades to operator family alt_opc2 for access method hash -drop cascades to operator family alt_opf4 for access method hash -drop cascades to operator family alt_opf2 for access method hash -drop cascades to table alt_regress_1 -drop cascades to text search dictionary alt_ts_dict3 -drop cascades to text search dictionary alt_ts_dict4 -drop cascades to text search dictionary alt_ts_dict2 -drop cascades to text search configuration alt_ts_conf3 -drop cascades to text search configuration alt_ts_conf4 -drop cascades to text search configuration alt_ts_conf2 -drop cascades to text search template alt_ts_temp3 -drop cascades to text search template alt_ts_temp2 -drop cascades to text search parser alt_ts_prs3 -drop cascades to text search parser alt_ts_prs2 DROP SCHEMA alt_nsp2 CASCADE; -NOTICE: drop cascades to 9 other objects -DETAIL: drop cascades to function alt_nsp2.alt_func2(integer) -drop cascades to function alt_nsp2.alt_agg2(integer) -drop cascades to conversion alt_conv2 -drop cascades to operator alt_nsp2.@-@(integer,integer) -drop cascades to operator family alt_nsp2.alt_opf2 for access method hash -drop cascades to text search dictionary alt_ts_dict2 -drop cascades to text search configuration alt_ts_conf2 -drop cascades to text search template alt_ts_temp2 -drop cascades to text search parser alt_ts_prs2 DROP USER regress_alter_user1; DROP USER regress_alter_user2; DROP USER regress_alter_user3; diff --git a/src/test/regress/expected/object_address.out b/src/test/regress/expected/object_address.out index 700f261827..3ca5c764ca 100644 --- a/src/test/regress/expected/object_address.out +++ b/src/test/regress/expected/object_address.out @@ -39,7 +39,7 @@ CREATE TRANSFORM FOR int LANGUAGE SQL ( CREATE PUBLICATION addr_pub FOR TABLE addr_nsp.gentable; CREATE SUBSCRIPTION addr_sub CONNECTION '' PUBLICATION bar WITH (connect = false, slot_name = NONE); WARNING: tables were not subscribed, you will have to run ALTER SUBSCRIPTION ... REFRESH PUBLICATION to subscribe the tables -CREATE STATISTICS addr_nsp.gentable_stat ON (a,b) FROM addr_nsp.gentable; +CREATE STATISTICS addr_nsp.gentable_stat ON a, b FROM addr_nsp.gentable; -- test some error cases SELECT pg_get_object_address('stone', '{}', '{}'); ERROR: unrecognized object type "stone" diff --git a/src/test/regress/expected/stats_ext.out b/src/test/regress/expected/stats_ext.out index 92ac84ac67..4ccdf21a01 100644 --- a/src/test/regress/expected/stats_ext.out +++ b/src/test/regress/expected/stats_ext.out @@ -5,24 +5,49 @@ SET max_parallel_workers = 0; SET max_parallel_workers_per_gather = 0; SET work_mem = '128kB'; +-- Verify failures +CREATE STATISTICS tst; +ERROR: syntax error at or near ";" +LINE 1: CREATE STATISTICS tst; + ^ +CREATE STATISTICS tst ON a, b; +ERROR: syntax error at or near ";" +LINE 1: CREATE STATISTICS tst ON a, b; + ^ +CREATE STATISTICS tst FROM sometab; +ERROR: syntax error at or near "FROM" +LINE 1: CREATE STATISTICS tst FROM sometab; + ^ +CREATE STATISTICS tst ON a, b FROM nonexistant; +ERROR: relation "nonexistant" does not exist +CREATE STATISTICS tst ON a, b FROM pg_class; +ERROR: column "a" referenced in statistics does not exist +CREATE STATISTICS tst ON relname, relname, relnatts FROM pg_class; +ERROR: duplicate column name in statistics definition +CREATE STATISTICS tst ON relnatts + relpages FROM pg_class; +ERROR: only simple column references are allowed in CREATE STATISTICS +CREATE STATISTICS tst ON (relpages, reltuples) FROM pg_class; +ERROR: only simple column references are allowed in CREATE STATISTICS +CREATE STATISTICS tst (unrecognized) ON relname, relnatts FROM pg_class; +ERROR: unrecognized statistics type "unrecognized" -- Ensure stats are dropped sanely CREATE TABLE ab1 (a INTEGER, b INTEGER, c INTEGER); -CREATE STATISTICS ab1_a_b_stats ON (a, b) FROM ab1; +CREATE STATISTICS ab1_a_b_stats ON a, b FROM ab1; DROP STATISTICS ab1_a_b_stats; CREATE SCHEMA regress_schema_2; -CREATE STATISTICS regress_schema_2.ab1_a_b_stats ON (a, b) FROM ab1; +CREATE STATISTICS regress_schema_2.ab1_a_b_stats ON a, b FROM ab1; -- Let's also verify the pg_get_statisticsextdef output looks sane. SELECT pg_get_statisticsextdef(oid) FROM pg_statistic_ext WHERE stxname = 'ab1_a_b_stats'; - pg_get_statisticsextdef ---------------------------------------------------------------------- - CREATE STATISTICS regress_schema_2.ab1_a_b_stats ON (a, b) FROM ab1 + pg_get_statisticsextdef +------------------------------------------------------------------- + CREATE STATISTICS regress_schema_2.ab1_a_b_stats ON a, b FROM ab1 (1 row) DROP STATISTICS regress_schema_2.ab1_a_b_stats; -- Ensure statistics are dropped when columns are -CREATE STATISTICS ab1_b_c_stats ON (b, c) FROM ab1; -CREATE STATISTICS ab1_a_b_c_stats ON (a, b, c) FROM ab1; -CREATE STATISTICS ab1_a_b_stats ON (a, b) FROM ab1; +CREATE STATISTICS ab1_b_c_stats ON b, c FROM ab1; +CREATE STATISTICS ab1_a_b_c_stats ON a, b, c FROM ab1; +CREATE STATISTICS ab1_a_b_stats ON a, b FROM ab1; ALTER TABLE ab1 DROP COLUMN a; \d ab1 Table "public.ab1" @@ -31,14 +56,14 @@ ALTER TABLE ab1 DROP COLUMN a; b | integer | | | c | integer | | | Statistics: - "public.ab1_b_c_stats" WITH (ndistinct, dependencies) ON (b, c) + "public"."ab1_b_c_stats" (ndistinct, dependencies) ON b, c FROM ab1 DROP TABLE ab1; -- Ensure things work sanely with SET STATISTICS 0 CREATE TABLE ab1 (a INTEGER, b INTEGER); ALTER TABLE ab1 ALTER a SET STATISTICS 0; INSERT INTO ab1 SELECT a, a%23 FROM generate_series(1, 1000) a; -CREATE STATISTICS ab1_a_b_stats ON (a, b) FROM ab1; +CREATE STATISTICS ab1_a_b_stats ON a, b FROM ab1; ANALYZE ab1; WARNING: extended statistics "public.ab1_a_b_stats" could not be collected for relation public.ab1 ALTER TABLE ab1 ALTER a SET STATISTICS -1; @@ -60,24 +85,24 @@ CREATE SERVER extstats_dummy_srv FOREIGN DATA WRAPPER extstats_dummy_fdw; CREATE FOREIGN TABLE tststats.f (a int, b int, c text) SERVER extstats_dummy_srv; CREATE TABLE tststats.pt (a int, b int, c text) PARTITION BY RANGE (a, b); CREATE TABLE tststats.pt1 PARTITION OF tststats.pt FOR VALUES FROM (-10, -10) TO (10, 10); -CREATE STATISTICS tststats.s1 ON (a, b) FROM tststats.t; -CREATE STATISTICS tststats.s2 ON (a, b) FROM tststats.ti; +CREATE STATISTICS tststats.s1 ON a, b FROM tststats.t; +CREATE STATISTICS tststats.s2 ON a, b FROM tststats.ti; ERROR: relation "ti" is not a table, foreign table, or materialized view -CREATE STATISTICS tststats.s3 ON (a, b) FROM tststats.s; +CREATE STATISTICS tststats.s3 ON a, b FROM tststats.s; ERROR: relation "s" is not a table, foreign table, or materialized view -CREATE STATISTICS tststats.s4 ON (a, b) FROM tststats.v; +CREATE STATISTICS tststats.s4 ON a, b FROM tststats.v; ERROR: relation "v" is not a table, foreign table, or materialized view -CREATE STATISTICS tststats.s5 ON (a, b) FROM tststats.mv; -CREATE STATISTICS tststats.s6 ON (a, b) FROM tststats.ty; +CREATE STATISTICS tststats.s5 ON a, b FROM tststats.mv; +CREATE STATISTICS tststats.s6 ON a, b FROM tststats.ty; ERROR: relation "ty" is not a table, foreign table, or materialized view -CREATE STATISTICS tststats.s7 ON (a, b) FROM tststats.f; -CREATE STATISTICS tststats.s8 ON (a, b) FROM tststats.pt; -CREATE STATISTICS tststats.s9 ON (a, b) FROM tststats.pt1; +CREATE STATISTICS tststats.s7 ON a, b FROM tststats.f; +CREATE STATISTICS tststats.s8 ON a, b FROM tststats.pt; +CREATE STATISTICS tststats.s9 ON a, b FROM tststats.pt1; DO $$ DECLARE relname text := reltoastrelid::regclass FROM pg_class WHERE oid = 'tststats.t'::regclass; BEGIN - EXECUTE 'CREATE STATISTICS tststats.s10 ON (a, b) FROM ' || relname; + EXECUTE 'CREATE STATISTICS tststats.s10 ON a, b FROM ' || relname; EXCEPTION WHEN wrong_object_type THEN RAISE NOTICE 'stats on toast table not created'; END; @@ -158,20 +183,8 @@ EXPLAIN (COSTS off) -> Seq Scan on ndistinct (5 rows) --- unknown column -CREATE STATISTICS s10 ON (unknown_column) FROM ndistinct; -ERROR: column "unknown_column" referenced in statistics does not exist --- single column -CREATE STATISTICS s10 ON (a) FROM ndistinct; -ERROR: extended statistics require at least 2 columns --- single column, duplicated -CREATE STATISTICS s10 ON (a,a) FROM ndistinct; -ERROR: duplicate column name in statistics definition --- two columns, one duplicated -CREATE STATISTICS s10 ON (a, a, b) FROM ndistinct; -ERROR: duplicate column name in statistics definition -- correct command -CREATE STATISTICS s10 ON (a, b, c) FROM ndistinct; +CREATE STATISTICS s10 ON a, b, c FROM ndistinct; ANALYZE ndistinct; SELECT stxkind, stxndistinct FROM pg_statistic_ext WHERE stxrelid = 'ndistinct'::regclass; @@ -352,7 +365,6 @@ EXPLAIN (COSTS off) -> Seq Scan on ndistinct (3 rows) -DROP TABLE ndistinct; -- functional dependencies tests CREATE TABLE functional_dependencies ( filler1 TEXT, @@ -389,7 +401,7 @@ EXPLAIN (COSTS OFF) (2 rows) -- create statistics -CREATE STATISTICS func_deps_stat WITH (dependencies) ON (a, b, c) FROM functional_dependencies; +CREATE STATISTICS func_deps_stat (dependencies) ON a, b, c FROM functional_dependencies; ANALYZE functional_dependencies; EXPLAIN (COSTS OFF) SELECT * FROM functional_dependencies WHERE a = 1 AND b = '1'; @@ -432,7 +444,7 @@ EXPLAIN (COSTS OFF) (2 rows) -- create statistics -CREATE STATISTICS func_deps_stat WITH (dependencies) ON (a, b, c) FROM functional_dependencies; +CREATE STATISTICS func_deps_stat (dependencies) ON a, b, c FROM functional_dependencies; ANALYZE functional_dependencies; EXPLAIN (COSTS OFF) SELECT * FROM functional_dependencies WHERE a = 1 AND b = '1'; @@ -456,4 +468,3 @@ EXPLAIN (COSTS OFF) (5 rows) RESET random_page_cost; -DROP TABLE functional_dependencies; diff --git a/src/test/regress/sql/alter_generic.sql b/src/test/regress/sql/alter_generic.sql index 88e8d7eb86..342f82856e 100644 --- a/src/test/regress/sql/alter_generic.sql +++ b/src/test/regress/sql/alter_generic.sql @@ -438,8 +438,8 @@ DROP OPERATOR FAMILY alt_opf18 USING btree; -- SET SESSION AUTHORIZATION regress_alter_user1; CREATE TABLE alt_regress_1 (a INTEGER, b INTEGER); -CREATE STATISTICS alt_stat1 ON (a, b) FROM alt_regress_1; -CREATE STATISTICS alt_stat2 ON (a, b) FROM alt_regress_1; +CREATE STATISTICS alt_stat1 ON a, b FROM alt_regress_1; +CREATE STATISTICS alt_stat2 ON a, b FROM alt_regress_1; ALTER STATISTICS alt_stat1 RENAME TO alt_stat2; -- failed (name conflict) ALTER STATISTICS alt_stat1 RENAME TO alt_stat3; -- failed (name conflict) @@ -448,8 +448,9 @@ ALTER STATISTICS alt_stat2 OWNER TO regress_alter_user3; -- OK ALTER STATISTICS alt_stat2 SET SCHEMA alt_nsp2; -- OK SET SESSION AUTHORIZATION regress_alter_user2; -CREATE STATISTICS alt_stat1 ON (a, b) FROM alt_regress_1; -CREATE STATISTICS alt_stat2 ON (a, b) FROM alt_regress_1; +CREATE TABLE alt_regress_2 (a INTEGER, b INTEGER); +CREATE STATISTICS alt_stat1 ON a, b FROM alt_regress_2; +CREATE STATISTICS alt_stat2 ON a, b FROM alt_regress_2; ALTER STATISTICS alt_stat3 RENAME TO alt_stat4; -- failed (not owner) ALTER STATISTICS alt_stat1 RENAME TO alt_stat4; -- OK @@ -572,12 +573,13 @@ SELECT nspname, prsname --- --- Cleanup resources --- +set client_min_messages to warning; -- suppress cascade notices + DROP FOREIGN DATA WRAPPER alt_fdw2 CASCADE; DROP FOREIGN DATA WRAPPER alt_fdw3 CASCADE; DROP LANGUAGE alt_lang2 CASCADE; DROP LANGUAGE alt_lang3 CASCADE; -DROP LANGUAGE alt_lang4 CASCADE; DROP SCHEMA alt_nsp1 CASCADE; DROP SCHEMA alt_nsp2 CASCADE; diff --git a/src/test/regress/sql/object_address.sql b/src/test/regress/sql/object_address.sql index 8a738e20d6..f25ed735e1 100644 --- a/src/test/regress/sql/object_address.sql +++ b/src/test/regress/sql/object_address.sql @@ -41,7 +41,7 @@ CREATE TRANSFORM FOR int LANGUAGE SQL ( TO SQL WITH FUNCTION int4recv(internal)); CREATE PUBLICATION addr_pub FOR TABLE addr_nsp.gentable; CREATE SUBSCRIPTION addr_sub CONNECTION '' PUBLICATION bar WITH (connect = false, slot_name = NONE); -CREATE STATISTICS addr_nsp.gentable_stat ON (a,b) FROM addr_nsp.gentable; +CREATE STATISTICS addr_nsp.gentable_stat ON a, b FROM addr_nsp.gentable; -- test some error cases SELECT pg_get_object_address('stone', '{}', '{}'); diff --git a/src/test/regress/sql/stats_ext.sql b/src/test/regress/sql/stats_ext.sql index 72c7659c4b..4050f33c08 100644 --- a/src/test/regress/sql/stats_ext.sql +++ b/src/test/regress/sql/stats_ext.sql @@ -7,13 +7,24 @@ SET max_parallel_workers = 0; SET max_parallel_workers_per_gather = 0; SET work_mem = '128kB'; +-- Verify failures +CREATE STATISTICS tst; +CREATE STATISTICS tst ON a, b; +CREATE STATISTICS tst FROM sometab; +CREATE STATISTICS tst ON a, b FROM nonexistant; +CREATE STATISTICS tst ON a, b FROM pg_class; +CREATE STATISTICS tst ON relname, relname, relnatts FROM pg_class; +CREATE STATISTICS tst ON relnatts + relpages FROM pg_class; +CREATE STATISTICS tst ON (relpages, reltuples) FROM pg_class; +CREATE STATISTICS tst (unrecognized) ON relname, relnatts FROM pg_class; + -- Ensure stats are dropped sanely CREATE TABLE ab1 (a INTEGER, b INTEGER, c INTEGER); -CREATE STATISTICS ab1_a_b_stats ON (a, b) FROM ab1; +CREATE STATISTICS ab1_a_b_stats ON a, b FROM ab1; DROP STATISTICS ab1_a_b_stats; CREATE SCHEMA regress_schema_2; -CREATE STATISTICS regress_schema_2.ab1_a_b_stats ON (a, b) FROM ab1; +CREATE STATISTICS regress_schema_2.ab1_a_b_stats ON a, b FROM ab1; -- Let's also verify the pg_get_statisticsextdef output looks sane. SELECT pg_get_statisticsextdef(oid) FROM pg_statistic_ext WHERE stxname = 'ab1_a_b_stats'; @@ -21,9 +32,9 @@ SELECT pg_get_statisticsextdef(oid) FROM pg_statistic_ext WHERE stxname = 'ab1_a DROP STATISTICS regress_schema_2.ab1_a_b_stats; -- Ensure statistics are dropped when columns are -CREATE STATISTICS ab1_b_c_stats ON (b, c) FROM ab1; -CREATE STATISTICS ab1_a_b_c_stats ON (a, b, c) FROM ab1; -CREATE STATISTICS ab1_a_b_stats ON (a, b) FROM ab1; +CREATE STATISTICS ab1_b_c_stats ON b, c FROM ab1; +CREATE STATISTICS ab1_a_b_c_stats ON a, b, c FROM ab1; +CREATE STATISTICS ab1_a_b_stats ON a, b FROM ab1; ALTER TABLE ab1 DROP COLUMN a; \d ab1 DROP TABLE ab1; @@ -32,7 +43,7 @@ DROP TABLE ab1; CREATE TABLE ab1 (a INTEGER, b INTEGER); ALTER TABLE ab1 ALTER a SET STATISTICS 0; INSERT INTO ab1 SELECT a, a%23 FROM generate_series(1, 1000) a; -CREATE STATISTICS ab1_a_b_stats ON (a, b) FROM ab1; +CREATE STATISTICS ab1_a_b_stats ON a, b FROM ab1; ANALYZE ab1; ALTER TABLE ab1 ALTER a SET STATISTICS -1; -- partial analyze doesn't build stats either @@ -55,20 +66,20 @@ CREATE FOREIGN TABLE tststats.f (a int, b int, c text) SERVER extstats_dummy_srv CREATE TABLE tststats.pt (a int, b int, c text) PARTITION BY RANGE (a, b); CREATE TABLE tststats.pt1 PARTITION OF tststats.pt FOR VALUES FROM (-10, -10) TO (10, 10); -CREATE STATISTICS tststats.s1 ON (a, b) FROM tststats.t; -CREATE STATISTICS tststats.s2 ON (a, b) FROM tststats.ti; -CREATE STATISTICS tststats.s3 ON (a, b) FROM tststats.s; -CREATE STATISTICS tststats.s4 ON (a, b) FROM tststats.v; -CREATE STATISTICS tststats.s5 ON (a, b) FROM tststats.mv; -CREATE STATISTICS tststats.s6 ON (a, b) FROM tststats.ty; -CREATE STATISTICS tststats.s7 ON (a, b) FROM tststats.f; -CREATE STATISTICS tststats.s8 ON (a, b) FROM tststats.pt; -CREATE STATISTICS tststats.s9 ON (a, b) FROM tststats.pt1; +CREATE STATISTICS tststats.s1 ON a, b FROM tststats.t; +CREATE STATISTICS tststats.s2 ON a, b FROM tststats.ti; +CREATE STATISTICS tststats.s3 ON a, b FROM tststats.s; +CREATE STATISTICS tststats.s4 ON a, b FROM tststats.v; +CREATE STATISTICS tststats.s5 ON a, b FROM tststats.mv; +CREATE STATISTICS tststats.s6 ON a, b FROM tststats.ty; +CREATE STATISTICS tststats.s7 ON a, b FROM tststats.f; +CREATE STATISTICS tststats.s8 ON a, b FROM tststats.pt; +CREATE STATISTICS tststats.s9 ON a, b FROM tststats.pt1; DO $$ DECLARE relname text := reltoastrelid::regclass FROM pg_class WHERE oid = 'tststats.t'::regclass; BEGIN - EXECUTE 'CREATE STATISTICS tststats.s10 ON (a, b) FROM ' || relname; + EXECUTE 'CREATE STATISTICS tststats.s10 ON a, b FROM ' || relname; EXCEPTION WHEN wrong_object_type THEN RAISE NOTICE 'stats on toast table not created'; END; @@ -113,20 +124,8 @@ EXPLAIN (COSTS off) EXPLAIN (COSTS off) SELECT COUNT(*) FROM ndistinct GROUP BY b, c, d; --- unknown column -CREATE STATISTICS s10 ON (unknown_column) FROM ndistinct; - --- single column -CREATE STATISTICS s10 ON (a) FROM ndistinct; - --- single column, duplicated -CREATE STATISTICS s10 ON (a,a) FROM ndistinct; - --- two columns, one duplicated -CREATE STATISTICS s10 ON (a, a, b) FROM ndistinct; - -- correct command -CREATE STATISTICS s10 ON (a, b, c) FROM ndistinct; +CREATE STATISTICS s10 ON a, b, c FROM ndistinct; ANALYZE ndistinct; @@ -202,8 +201,6 @@ EXPLAIN (COSTS off) EXPLAIN (COSTS off) SELECT COUNT(*) FROM ndistinct GROUP BY a, d; -DROP TABLE ndistinct; - -- functional dependencies tests CREATE TABLE functional_dependencies ( filler1 TEXT, @@ -233,7 +230,7 @@ EXPLAIN (COSTS OFF) SELECT * FROM functional_dependencies WHERE a = 1 AND b = '1' AND c = 1; -- create statistics -CREATE STATISTICS func_deps_stat WITH (dependencies) ON (a, b, c) FROM functional_dependencies; +CREATE STATISTICS func_deps_stat (dependencies) ON a, b, c FROM functional_dependencies; ANALYZE functional_dependencies; @@ -259,7 +256,7 @@ EXPLAIN (COSTS OFF) SELECT * FROM functional_dependencies WHERE a = 1 AND b = '1' AND c = 1; -- create statistics -CREATE STATISTICS func_deps_stat WITH (dependencies) ON (a, b, c) FROM functional_dependencies; +CREATE STATISTICS func_deps_stat (dependencies) ON a, b, c FROM functional_dependencies; ANALYZE functional_dependencies; @@ -270,4 +267,3 @@ EXPLAIN (COSTS OFF) SELECT * FROM functional_dependencies WHERE a = 1 AND b = '1' AND c = 1; RESET random_page_cost; -DROP TABLE functional_dependencies; -- cgit v1.2.3 From 9ed74fd463ede1db4ce829f9ff461d0b7f28f1f3 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Fri, 12 May 2017 18:31:55 -0400 Subject: doc: update markup for release note "release date" block This has to be backpatched to all supported releases so release markup added to HEAD and copied to back branches matches the existing markup. Reported-by: Peter Eisentraut Discussion: 2b8a2552-fffa-f7c8-97c5-14db47a87731@2ndquadrant.com Author: initial patch and sample markup by Peter Eisentraut Backpatch-through: 9.2 --- doc/src/sgml/release-10.sgml | 9 +- doc/src/sgml/release-7.4.sgml | 248 ++++++++++----------- doc/src/sgml/release-8.0.sgml | 216 +++++++++---------- doc/src/sgml/release-8.1.sgml | 192 ++++++++--------- doc/src/sgml/release-8.2.sgml | 192 ++++++++--------- doc/src/sgml/release-8.3.sgml | 192 ++++++++--------- doc/src/sgml/release-8.4.sgml | 184 ++++++++-------- doc/src/sgml/release-9.0.sgml | 192 ++++++++--------- doc/src/sgml/release-9.1.sgml | 200 ++++++++--------- doc/src/sgml/release-9.2.sgml | 176 +++++++-------- doc/src/sgml/release-9.3.sgml | 144 ++++++------- doc/src/sgml/release-9.4.sgml | 104 ++++----- doc/src/sgml/release-9.5.sgml | 64 +++--- doc/src/sgml/release-9.6.sgml | 32 +-- doc/src/sgml/release-old.sgml | 488 +++++++++++++++++++++--------------------- 15 files changed, 1316 insertions(+), 1317 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/release-10.sgml b/doc/src/sgml/release-10.sgml index 402fdb44fa..6497641ba6 100644 --- a/doc/src/sgml/release-10.sgml +++ b/doc/src/sgml/release-10.sgml @@ -4,11 +4,10 @@ Release 10 - - Release Date - 2017-09-XX - CURRENT AS OF 2017-04-22 - + + Release date: + 2017-09-XX (CURRENT AS OF 2017-04-22) + Overview diff --git a/doc/src/sgml/release-7.4.sgml b/doc/src/sgml/release-7.4.sgml index b4a217af2c..bc4f4e18d0 100644 --- a/doc/src/sgml/release-7.4.sgml +++ b/doc/src/sgml/release-7.4.sgml @@ -4,10 +4,10 @@ Release 7.4.30 - - Release Date - 2010-10-04 - + + Release date: + 2010-10-04 + This release contains a variety of fixes from 7.4.29. @@ -138,10 +138,10 @@ Release 7.4.29 - - Release Date - 2010-05-17 - + + Release date: + 2010-05-17 + This release contains a variety of fixes from 7.4.28. @@ -282,10 +282,10 @@ Release 7.4.28 - - Release Date - 2010-03-15 - + + Release date: + 2010-03-15 + This release contains a variety of fixes from 7.4.27. @@ -401,10 +401,10 @@ Release 7.4.27 - - Release Date - 2009-12-14 - + + Release date: + 2009-12-14 + This release contains a variety of fixes from 7.4.26. @@ -521,10 +521,10 @@ Release 7.4.26 - - Release Date - 2009-09-09 - + + Release date: + 2009-09-09 + This release contains a variety of fixes from 7.4.25. @@ -651,10 +651,10 @@ Release 7.4.25 - - Release Date - 2009-03-16 - + + Release date: + 2009-03-16 + This release contains a variety of fixes from 7.4.24. @@ -731,10 +731,10 @@ Release 7.4.24 - - Release Date - 2009-02-02 - + + Release date: + 2009-02-02 + This release contains a variety of fixes from 7.4.23. @@ -817,10 +817,10 @@ Release 7.4.23 - - Release Date - 2008-11-03 - + + Release date: + 2008-11-03 + This release contains a variety of fixes from 7.4.22. @@ -901,10 +901,10 @@ Release 7.4.22 - - Release Date - 2008-09-22 - + + Release date: + 2008-09-22 + This release contains a variety of fixes from 7.4.21. @@ -977,10 +977,10 @@ Release 7.4.21 - - Release Date - 2008-06-12 - + + Release date: + 2008-06-12 + This release contains one serious bug fix over 7.4.20. @@ -1031,10 +1031,10 @@ Release 7.4.20 - - Release Date - never released - + + Release date: + never released + This release contains a variety of fixes from 7.4.19. @@ -1176,10 +1176,10 @@ Release 7.4.19 - - Release Date - 2008-01-07 - + + Release date: + 2008-01-07 + This release contains a variety of fixes from 7.4.18, @@ -1330,10 +1330,10 @@ Release 7.4.18 - - Release Date - 2007-09-17 - + + Release date: + 2007-09-17 + This release contains fixes from 7.4.17. @@ -1406,10 +1406,10 @@ Release 7.4.17 - - Release Date - 2007-04-23 - + + Release date: + 2007-04-23 + This release contains fixes from 7.4.16, @@ -1477,10 +1477,10 @@ Release 7.4.16 - - Release Date - 2007-02-05 - + + Release date: + 2007-02-05 + This release contains a variety of fixes from 7.4.15, including @@ -1548,10 +1548,10 @@ Release 7.4.15 - - Release Date - 2007-01-08 - + + Release date: + 2007-01-08 + This release contains a variety of fixes from 7.4.14. @@ -1637,10 +1637,10 @@ Release 7.4.14 - - Release Date - 2006-10-16 - + + Release date: + 2006-10-16 + This release contains a variety of fixes from 7.4.13. @@ -1684,10 +1684,10 @@ ANYARRAY Release 7.4.13 - - Release Date - 2006-05-23 - + + Release date: + 2006-05-23 + This release contains a variety of fixes from 7.4.12, @@ -1791,10 +1791,10 @@ Fuhr) Release 7.4.12 - - Release Date - 2006-02-14 - + + Release date: + 2006-02-14 + This release contains a variety of fixes from 7.4.11. @@ -1854,10 +1854,10 @@ and isinf during configure (Tom) Release 7.4.11 - - Release Date - 2006-01-09 - + + Release date: + 2006-01-09 + This release contains a variety of fixes from 7.4.10. @@ -1921,10 +1921,10 @@ what's actually returned by the query (Joe) Release 7.4.10 - - Release Date - 2005-12-12 - + + Release date: + 2005-12-12 + This release contains a variety of fixes from 7.4.9. @@ -1974,10 +1974,10 @@ table has been dropped Release 7.4.9 - - Release Date - 2005-10-04 - + + Release date: + 2005-10-04 + This release contains a variety of fixes from 7.4.8. @@ -2043,10 +2043,10 @@ code Release 7.4.8 - - Release Date - 2005-05-09 - + + Release date: + 2005-05-09 + This release contains a variety of fixes from 7.4.7, including several @@ -2227,10 +2227,10 @@ holder of the lock released it within a very narrow window. Release 7.4.7 - - Release Date - 2005-01-31 - + + Release date: + 2005-01-31 + This release contains a variety of fixes from 7.4.6, including several @@ -2287,10 +2287,10 @@ GMT Release 7.4.6 - - Release Date - 2004-10-22 - + + Release date: + 2004-10-22 + This release contains a variety of fixes from 7.4.5. @@ -2359,10 +2359,10 @@ ECPG prepare statement Release 7.4.5 - - Release Date - 2004-08-18 - + + Release date: + 2004-08-18 + This release contains one serious bug fix over 7.4.4. @@ -2397,10 +2397,10 @@ still worth a re-release. The bug does not exist in pre-7.4 releases. Release 7.4.4 - - Release Date - 2004-08-16 - + + Release date: + 2004-08-16 + This release contains a variety of fixes from 7.4.3. @@ -2449,10 +2449,10 @@ aggregate plan Release 7.4.3 - - Release Date - 2004-06-14 - + + Release date: + 2004-06-14 + This release contains a variety of fixes from 7.4.2. @@ -2507,10 +2507,10 @@ names from outer query levels. Release 7.4.2 - - Release Date - 2004-03-08 - + + Release date: + 2004-03-08 + This release contains a variety of fixes from 7.4.1. @@ -2650,10 +2650,10 @@ inconveniences associated with the i/I problem. Release 7.4.1 - - Release Date - 2003-12-22 - + + Release date: + 2003-12-22 + This release contains a variety of fixes from 7.4. @@ -2764,10 +2764,10 @@ DROP SCHEMA information_schema CASCADE; Release 7.4 - - Release Date - 2003-11-17 - + + Release date: + 2003-11-17 + Overview diff --git a/doc/src/sgml/release-8.0.sgml b/doc/src/sgml/release-8.0.sgml index 6224187ab4..0f43e24b1d 100644 --- a/doc/src/sgml/release-8.0.sgml +++ b/doc/src/sgml/release-8.0.sgml @@ -4,10 +4,10 @@ Release 8.0.26 - - Release Date - 2010-10-04 - + + Release date: + 2010-10-04 + This release contains a variety of fixes from 8.0.25. @@ -208,10 +208,10 @@ Release 8.0.25 - - Release Date - 2010-05-17 - + + Release date: + 2010-05-17 + This release contains a variety of fixes from 8.0.24. @@ -368,10 +368,10 @@ Release 8.0.24 - - Release Date - 2010-03-15 - + + Release date: + 2010-03-15 + This release contains a variety of fixes from 8.0.23. @@ -545,10 +545,10 @@ Release 8.0.23 - - Release Date - 2009-12-14 - + + Release date: + 2009-12-14 + This release contains a variety of fixes from 8.0.22. @@ -700,10 +700,10 @@ Release 8.0.22 - - Release Date - 2009-09-09 - + + Release date: + 2009-09-09 + This release contains a variety of fixes from 8.0.21. @@ -864,10 +864,10 @@ Release 8.0.21 - - Release Date - 2009-03-16 - + + Release date: + 2009-03-16 + This release contains a variety of fixes from 8.0.20. @@ -944,10 +944,10 @@ Release 8.0.20 - - Release Date - 2009-02-02 - + + Release date: + 2009-02-02 + This release contains a variety of fixes from 8.0.19. @@ -1030,10 +1030,10 @@ Release 8.0.19 - - Release Date - 2008-11-03 - + + Release date: + 2008-11-03 + This release contains a variety of fixes from 8.0.18. @@ -1141,10 +1141,10 @@ Release 8.0.18 - - Release Date - 2008-09-22 - + + Release date: + 2008-09-22 + This release contains a variety of fixes from 8.0.17. @@ -1275,10 +1275,10 @@ Release 8.0.17 - - Release Date - 2008-06-12 - + + Release date: + 2008-06-12 + This release contains one serious bug fix over 8.0.16. @@ -1329,10 +1329,10 @@ Release 8.0.16 - - Release Date - never released - + + Release date: + never released + This release contains a variety of fixes from 8.0.15. @@ -1566,10 +1566,10 @@ Release 8.0.15 - - Release Date - 2008-01-07 - + + Release date: + 2008-01-07 + This release contains a variety of fixes from 8.0.14, @@ -1782,10 +1782,10 @@ Release 8.0.14 - - Release Date - 2007-09-17 - + + Release date: + 2007-09-17 + This release contains a variety of fixes from 8.0.13. @@ -1890,10 +1890,10 @@ Release 8.0.13 - - Release Date - 2007-04-23 - + + Release date: + 2007-04-23 + This release contains a variety of fixes from 8.0.12, @@ -1967,10 +1967,10 @@ Release 8.0.12 - - Release Date - 2007-02-07 - + + Release date: + 2007-02-07 + This release contains one fix from 8.0.11. @@ -2009,10 +2009,10 @@ Release 8.0.11 - - Release Date - 2007-02-05 - + + Release date: + 2007-02-05 + This release contains a variety of fixes from 8.0.10, including @@ -2080,10 +2080,10 @@ Release 8.0.10 - - Release Date - 2007-01-08 - + + Release date: + 2007-01-08 + This release contains a variety of fixes from 8.0.9. @@ -2199,10 +2199,10 @@ Release 8.0.9 - - Release Date - 2006-10-16 - + + Release date: + 2006-10-16 + This release contains a variety of fixes from 8.0.8. @@ -2255,10 +2255,10 @@ Wieland) Release 8.0.8 - - Release Date - 2006-05-23 - + + Release date: + 2006-05-23 + This release contains a variety of fixes from 8.0.7, @@ -2369,10 +2369,10 @@ Fuhr) Release 8.0.7 - - Release Date - 2006-02-14 - + + Release date: + 2006-02-14 + This release contains a variety of fixes from 8.0.6. @@ -2468,10 +2468,10 @@ and isinf during configure (Tom) Release 8.0.6 - - Release Date - 2006-01-09 - + + Release date: + 2006-01-09 + This release contains a variety of fixes from 8.0.5. @@ -2553,10 +2553,10 @@ what's actually returned by the query (Joe) Release 8.0.5 - - Release Date - 2005-12-12 - + + Release date: + 2005-12-12 + This release contains a variety of fixes from 8.0.4. @@ -2635,10 +2635,10 @@ to subquery results Release 8.0.4 - - Release Date - 2005-10-04 - + + Release date: + 2005-10-04 + This release contains a variety of fixes from 8.0.3. @@ -2732,10 +2732,10 @@ code Release 8.0.3 - - Release Date - 2005-05-09 - + + Release date: + 2005-05-09 + This release contains a variety of fixes from 8.0.2, including several @@ -2852,10 +2852,10 @@ data types Release 8.0.2 - - Release Date - 2005-04-07 - + + Release date: + 2005-04-07 + This release contains a variety of fixes from 8.0.1. @@ -3009,10 +3009,10 @@ addresses in INET data types (Tom) Release 8.0.1 - - Release Date - 2005-01-31 - + + Release date: + 2005-01-31 + This release contains a variety of fixes from 8.0.0, including several @@ -3078,10 +3078,10 @@ typedefs (Michael) Release 8.0 - - Release Date - 2005-01-19 - + + Release date: + 2005-01-19 + Overview diff --git a/doc/src/sgml/release-8.1.sgml b/doc/src/sgml/release-8.1.sgml index 49a4fe2ad8..d48bccd17d 100644 --- a/doc/src/sgml/release-8.1.sgml +++ b/doc/src/sgml/release-8.1.sgml @@ -4,10 +4,10 @@ Release 8.1.23 - - Release Date - 2010-12-16 - + + Release date: + 2010-12-16 + This release contains a variety of fixes from 8.1.22. @@ -223,10 +223,10 @@ Release 8.1.22 - - Release Date - 2010-10-04 - + + Release date: + 2010-10-04 + This release contains a variety of fixes from 8.1.21. @@ -441,10 +441,10 @@ Release 8.1.21 - - Release Date - 2010-05-17 - + + Release date: + 2010-05-17 + This release contains a variety of fixes from 8.1.20. @@ -595,10 +595,10 @@ Release 8.1.20 - - Release Date - 2010-03-15 - + + Release date: + 2010-03-15 + This release contains a variety of fixes from 8.1.19. @@ -785,10 +785,10 @@ Release 8.1.19 - - Release Date - 2009-12-14 - + + Release date: + 2009-12-14 + This release contains a variety of fixes from 8.1.18. @@ -966,10 +966,10 @@ Release 8.1.18 - - Release Date - 2009-09-09 - + + Release date: + 2009-09-09 + This release contains a variety of fixes from 8.1.17. @@ -1130,10 +1130,10 @@ Release 8.1.17 - - Release Date - 2009-03-16 - + + Release date: + 2009-03-16 + This release contains a variety of fixes from 8.1.16. @@ -1253,10 +1253,10 @@ Release 8.1.16 - - Release Date - 2009-02-02 - + + Release date: + 2009-02-02 + This release contains a variety of fixes from 8.1.15. @@ -1373,10 +1373,10 @@ Release 8.1.15 - - Release Date - 2008-11-03 - + + Release date: + 2008-11-03 + This release contains a variety of fixes from 8.1.14. @@ -1525,10 +1525,10 @@ Release 8.1.14 - - Release Date - 2008-09-22 - + + Release date: + 2008-09-22 + This release contains a variety of fixes from 8.1.13. @@ -1701,10 +1701,10 @@ Release 8.1.13 - - Release Date - 2008-06-12 - + + Release date: + 2008-06-12 + This release contains one serious and one minor bug fix over 8.1.12. @@ -1768,10 +1768,10 @@ Release 8.1.12 - - Release Date - never released - + + Release date: + never released + This release contains a variety of fixes from 8.1.11. @@ -2018,10 +2018,10 @@ Release 8.1.11 - - Release Date - 2008-01-07 - + + Release date: + 2008-01-07 + This release contains a variety of fixes from 8.1.10, @@ -2270,10 +2270,10 @@ Release 8.1.10 - - Release Date - 2007-09-17 - + + Release date: + 2007-09-17 + This release contains a variety of fixes from 8.1.9. @@ -2391,10 +2391,10 @@ Release 8.1.9 - - Release Date - 2007-04-23 - + + Release date: + 2007-04-23 + This release contains a variety of fixes from 8.1.8, @@ -2482,10 +2482,10 @@ Release 8.1.8 - - Release Date - 2007-02-07 - + + Release date: + 2007-02-07 + This release contains one fix from 8.1.7. @@ -2524,10 +2524,10 @@ Release 8.1.7 - - Release Date - 2007-02-05 - + + Release date: + 2007-02-05 + This release contains a variety of fixes from 8.1.6, including @@ -2626,10 +2626,10 @@ Release 8.1.6 - - Release Date - 2007-01-08 - + + Release date: + 2007-01-08 + This release contains a variety of fixes from 8.1.5. @@ -2776,10 +2776,10 @@ Release 8.1.5 - - Release Date - 2006-10-16 - + + Release date: + 2006-10-16 + This release contains a variety of fixes from 8.1.4. @@ -2853,10 +2853,10 @@ compilers (Hiroshi Saito) Release 8.1.4 - - Release Date - 2006-05-23 - + + Release date: + 2006-05-23 + This release contains a variety of fixes from 8.1.3, @@ -3007,10 +3007,10 @@ documented (Tom) Release 8.1.3 - - Release Date - 2006-02-14 - + + Release date: + 2006-02-14 + This release contains a variety of fixes from 8.1.2, @@ -3129,10 +3129,10 @@ creation (Tom) Release 8.1.2 - - Release Date - 2006-01-09 - + + Release date: + 2006-01-09 + This release contains a variety of fixes from 8.1.1. @@ -3232,10 +3232,10 @@ what's actually returned by the query (Joe) Release 8.1.1 - - Release Date - 2005-12-12 - + + Release date: + 2005-12-12 + This release contains a variety of fixes from 8.1.0. @@ -3324,10 +3324,10 @@ DISTINCT query Release 8.1 - - Release Date - 2005-11-08 - + + Release date: + 2005-11-08 + Overview diff --git a/doc/src/sgml/release-8.2.sgml b/doc/src/sgml/release-8.2.sgml index acb7b810c6..c00cbd3467 100644 --- a/doc/src/sgml/release-8.2.sgml +++ b/doc/src/sgml/release-8.2.sgml @@ -4,10 +4,10 @@ Release 8.2.23 - - Release Date - 2011-12-05 - + + Release date: + 2011-12-05 + This release contains a variety of fixes from 8.2.22. @@ -232,10 +232,10 @@ Release 8.2.22 - - Release Date - 2011-09-26 - + + Release date: + 2011-09-26 + This release contains a variety of fixes from 8.2.21. @@ -539,10 +539,10 @@ Release 8.2.21 - - Release Date - 2011-04-18 - + + Release date: + 2011-04-18 + This release contains a variety of fixes from 8.2.20. @@ -671,10 +671,10 @@ Release 8.2.20 - - Release Date - 2011-01-31 - + + Release date: + 2011-01-31 + This release contains a variety of fixes from 8.2.19. @@ -803,10 +803,10 @@ Release 8.2.19 - - Release Date - 2010-12-16 - + + Release date: + 2010-12-16 + This release contains a variety of fixes from 8.2.18. @@ -1047,10 +1047,10 @@ Release 8.2.18 - - Release Date - 2010-10-04 - + + Release date: + 2010-10-04 + This release contains a variety of fixes from 8.2.17. @@ -1333,10 +1333,10 @@ Release 8.2.17 - - Release Date - 2010-05-17 - + + Release date: + 2010-05-17 + This release contains a variety of fixes from 8.2.16. @@ -1534,10 +1534,10 @@ Release 8.2.16 - - Release Date - 2010-03-15 - + + Release date: + 2010-03-15 + This release contains a variety of fixes from 8.2.15. @@ -1805,10 +1805,10 @@ Release 8.2.15 - - Release Date - 2009-12-14 - + + Release date: + 2009-12-14 + This release contains a variety of fixes from 8.2.14. @@ -2049,10 +2049,10 @@ Release 8.2.14 - - Release Date - 2009-09-09 - + + Release date: + 2009-09-09 + This release contains a variety of fixes from 8.2.13. @@ -2279,10 +2279,10 @@ Release 8.2.13 - - Release Date - 2009-03-16 - + + Release date: + 2009-03-16 + This release contains a variety of fixes from 8.2.12. @@ -2445,10 +2445,10 @@ Release 8.2.12 - - Release Date - 2009-02-02 - + + Release date: + 2009-02-02 + This release contains a variety of fixes from 8.2.11. @@ -2624,10 +2624,10 @@ Release 8.2.11 - - Release Date - 2008-11-03 - + + Release date: + 2008-11-03 + This release contains a variety of fixes from 8.2.10. @@ -2808,10 +2808,10 @@ Release 8.2.10 - - Release Date - 2008-09-22 - + + Release date: + 2008-09-22 + This release contains a variety of fixes from 8.2.9. @@ -3040,10 +3040,10 @@ Release 8.2.9 - - Release Date - 2008-06-12 - + + Release date: + 2008-06-12 + This release contains one serious and one minor bug fix over 8.2.8. @@ -3107,10 +3107,10 @@ Release 8.2.8 - - Release Date - never released - + + Release date: + never released + This release contains a variety of fixes from 8.2.7. @@ -3302,10 +3302,10 @@ Release 8.2.7 - - Release Date - 2008-03-17 - + + Release date: + 2008-03-17 + This release contains a variety of fixes from 8.2.6. @@ -3571,10 +3571,10 @@ Release 8.2.6 - - Release Date - 2008-01-07 - + + Release date: + 2008-01-07 + This release contains a variety of fixes from 8.2.5, @@ -3862,10 +3862,10 @@ Release 8.2.5 - - Release Date - 2007-09-17 - + + Release date: + 2007-09-17 + This release contains a variety of fixes from 8.2.4. @@ -4039,10 +4039,10 @@ Release 8.2.4 - - Release Date - 2007-04-23 - + + Release date: + 2007-04-23 + This release contains a variety of fixes from 8.2.3, @@ -4183,10 +4183,10 @@ Release 8.2.3 - - Release Date - 2007-02-07 - + + Release date: + 2007-02-07 + This release contains two fixes from 8.2.2. @@ -4229,10 +4229,10 @@ Release 8.2.2 - - Release Date - 2007-02-05 - + + Release date: + 2007-02-05 + This release contains a variety of fixes from 8.2.1, including @@ -4387,10 +4387,10 @@ Release 8.2.1 - - Release Date - 2007-01-08 - + + Release date: + 2007-01-08 + This release contains a variety of fixes from 8.2. @@ -4522,10 +4522,10 @@ Release 8.2 - - Release Date - 2006-12-05 - + + Release date: + 2006-12-05 + Overview diff --git a/doc/src/sgml/release-8.3.sgml b/doc/src/sgml/release-8.3.sgml index 76ab36e39c..a82410d057 100644 --- a/doc/src/sgml/release-8.3.sgml +++ b/doc/src/sgml/release-8.3.sgml @@ -4,10 +4,10 @@ Release 8.3.23 - - Release Date - 2013-02-07 - + + Release date: + 2013-02-07 + This release contains a variety of fixes from 8.3.22. @@ -173,10 +173,10 @@ Release 8.3.22 - - Release Date - 2012-12-06 - + + Release date: + 2012-12-06 + This release contains a variety of fixes from 8.3.21. @@ -469,10 +469,10 @@ Release 8.3.21 - - Release Date - 2012-09-24 - + + Release date: + 2012-09-24 + This release contains a variety of fixes from 8.3.20. @@ -579,10 +579,10 @@ Release 8.3.20 - - Release Date - 2012-08-17 - + + Release date: + 2012-08-17 + This release contains a variety of fixes from 8.3.19. @@ -802,10 +802,10 @@ Release 8.3.19 - - Release Date - 2012-06-04 - + + Release date: + 2012-06-04 + This release contains a variety of fixes from 8.3.18. @@ -1031,10 +1031,10 @@ Release 8.3.18 - - Release Date - 2012-02-27 - + + Release date: + 2012-02-27 + This release contains a variety of fixes from 8.3.17. @@ -1319,10 +1319,10 @@ Release 8.3.17 - - Release Date - 2011-12-05 - + + Release date: + 2011-12-05 + This release contains a variety of fixes from 8.3.16. @@ -1586,10 +1586,10 @@ Release 8.3.16 - - Release Date - 2011-09-26 - + + Release date: + 2011-09-26 + This release contains a variety of fixes from 8.3.15. @@ -1957,10 +1957,10 @@ Release 8.3.15 - - Release Date - 2011-04-18 - + + Release date: + 2011-04-18 + This release contains a variety of fixes from 8.3.14. @@ -2120,10 +2120,10 @@ Release 8.3.14 - - Release Date - 2011-01-31 - + + Release date: + 2011-01-31 + This release contains a variety of fixes from 8.3.13. @@ -2252,10 +2252,10 @@ Release 8.3.13 - - Release Date - 2010-12-16 - + + Release date: + 2010-12-16 + This release contains a variety of fixes from 8.3.12. @@ -2530,10 +2530,10 @@ Release 8.3.12 - - Release Date - 2010-10-04 - + + Release date: + 2010-10-04 + This release contains a variety of fixes from 8.3.11. @@ -2893,10 +2893,10 @@ Release 8.3.11 - - Release Date - 2010-05-17 - + + Release date: + 2010-05-17 + This release contains a variety of fixes from 8.3.10. @@ -3122,10 +3122,10 @@ Release 8.3.10 - - Release Date - 2010-03-15 - + + Release date: + 2010-03-15 + This release contains a variety of fixes from 8.3.9. @@ -3449,10 +3449,10 @@ Release 8.3.9 - - Release Date - 2009-12-14 - + + Release date: + 2009-12-14 + This release contains a variety of fixes from 8.3.8. @@ -3782,10 +3782,10 @@ Release 8.3.8 - - Release Date - 2009-09-09 - + + Release date: + 2009-09-09 + This release contains a variety of fixes from 8.3.7. @@ -4072,10 +4072,10 @@ Release 8.3.7 - - Release Date - 2009-03-16 - + + Release date: + 2009-03-16 + This release contains a variety of fixes from 8.3.6. @@ -4295,10 +4295,10 @@ Release 8.3.6 - - Release Date - 2009-02-02 - + + Release date: + 2009-02-02 + This release contains a variety of fixes from 8.3.5. @@ -4589,10 +4589,10 @@ Release 8.3.5 - - Release Date - 2008-11-03 - + + Release date: + 2008-11-03 + This release contains a variety of fixes from 8.3.4. @@ -4844,10 +4844,10 @@ Release 8.3.4 - - Release Date - 2008-09-22 - + + Release date: + 2008-09-22 + This release contains a variety of fixes from 8.3.3. @@ -5198,10 +5198,10 @@ Release 8.3.3 - - Release Date - 2008-06-12 - + + Release date: + 2008-06-12 + This release contains one serious and one minor bug fix over 8.3.2. @@ -5265,10 +5265,10 @@ Release 8.3.2 - - Release Date - never released - + + Release date: + never released + This release contains a variety of fixes from 8.3.1. @@ -5630,10 +5630,10 @@ Release 8.3.1 - - Release Date - 2008-03-17 - + + Release date: + 2008-03-17 + This release contains a variety of fixes from 8.3.0. @@ -5943,10 +5943,10 @@ Release 8.3 - - Release Date - 2008-02-04 - + + Release date: + 2008-02-04 + Overview diff --git a/doc/src/sgml/release-8.4.sgml b/doc/src/sgml/release-8.4.sgml index dfe4787757..16004edb74 100644 --- a/doc/src/sgml/release-8.4.sgml +++ b/doc/src/sgml/release-8.4.sgml @@ -4,10 +4,10 @@ Release 8.4.22 - - Release Date - 2014-07-24 - + + Release date: + 2014-07-24 + This release contains a variety of fixes from 8.4.21. @@ -323,10 +323,10 @@ Release 8.4.21 - - Release Date - 2014-03-20 - + + Release date: + 2014-03-20 + This release contains a variety of fixes from 8.4.20. @@ -442,10 +442,10 @@ Release 8.4.20 - - Release Date - 2014-02-20 - + + Release date: + 2014-02-20 + This release contains a variety of fixes from 8.4.19. @@ -901,10 +901,10 @@ Release 8.4.19 - - Release Date - 2013-12-05 - + + Release date: + 2013-12-05 + This release contains a variety of fixes from 8.4.18. @@ -1102,10 +1102,10 @@ Release 8.4.18 - - Release Date - 2013-10-10 - + + Release date: + 2013-10-10 + This release contains a variety of fixes from 8.4.17. @@ -1302,10 +1302,10 @@ Release 8.4.17 - - Release Date - 2013-04-04 - + + Release date: + 2013-04-04 + This release contains a variety of fixes from 8.4.16. @@ -1531,10 +1531,10 @@ Release 8.4.16 - - Release Date - 2013-02-07 - + + Release date: + 2013-02-07 + This release contains a variety of fixes from 8.4.15. @@ -1741,10 +1741,10 @@ Release 8.4.15 - - Release Date - 2012-12-06 - + + Release date: + 2012-12-06 + This release contains a variety of fixes from 8.4.14. @@ -2043,10 +2043,10 @@ Release 8.4.14 - - Release Date - 2012-09-24 - + + Release date: + 2012-09-24 + This release contains a variety of fixes from 8.4.13. @@ -2159,10 +2159,10 @@ Release 8.4.13 - - Release Date - 2012-08-17 - + + Release date: + 2012-08-17 + This release contains a variety of fixes from 8.4.12. @@ -2393,10 +2393,10 @@ Release 8.4.12 - - Release Date - 2012-06-04 - + + Release date: + 2012-06-04 + This release contains a variety of fixes from 8.4.11. @@ -2678,10 +2678,10 @@ Release 8.4.11 - - Release Date - 2012-02-27 - + + Release date: + 2012-02-27 + This release contains a variety of fixes from 8.4.10. @@ -3040,10 +3040,10 @@ Release 8.4.10 - - Release Date - 2011-12-05 - + + Release date: + 2011-12-05 + This release contains a variety of fixes from 8.4.9. @@ -3355,10 +3355,10 @@ Release 8.4.9 - - Release Date - 2011-09-26 - + + Release date: + 2011-09-26 + This release contains a variety of fixes from 8.4.8. @@ -3880,10 +3880,10 @@ Release 8.4.8 - - Release Date - 2011-04-18 - + + Release date: + 2011-04-18 + This release contains a variety of fixes from 8.4.7. @@ -4125,10 +4125,10 @@ Release 8.4.7 - - Release Date - 2011-01-31 - + + Release date: + 2011-01-31 + This release contains a variety of fixes from 8.4.6. @@ -4257,10 +4257,10 @@ Release 8.4.6 - - Release Date - 2010-12-16 - + + Release date: + 2010-12-16 + This release contains a variety of fixes from 8.4.5. @@ -4568,10 +4568,10 @@ Release 8.4.5 - - Release Date - 2010-10-04 - + + Release date: + 2010-10-04 + This release contains a variety of fixes from 8.4.4. @@ -5090,10 +5090,10 @@ Release 8.4.4 - - Release Date - 2010-05-17 - + + Release date: + 2010-05-17 + This release contains a variety of fixes from 8.4.3. @@ -5381,10 +5381,10 @@ Release 8.4.3 - - Release Date - 2010-03-15 - + + Release date: + 2010-03-15 + This release contains a variety of fixes from 8.4.2. @@ -5848,10 +5848,10 @@ Release 8.4.2 - - Release Date - 2009-12-14 - + + Release date: + 2009-12-14 + This release contains a variety of fixes from 8.4.1. @@ -6379,10 +6379,10 @@ WITH w AS (SELECT * FROM foo) SELECT * FROM w, bar ... FOR UPDATE Release 8.4.1 - - Release Date - 2009-09-09 - + + Release date: + 2009-09-09 + This release contains a variety of fixes from 8.4. @@ -6678,10 +6678,10 @@ WITH w AS (SELECT * FROM foo) SELECT * FROM w, bar ... FOR UPDATE Release 8.4 - - Release Date - 2009-07-01 - + + Release date: + 2009-07-01 + Overview diff --git a/doc/src/sgml/release-9.0.sgml b/doc/src/sgml/release-9.0.sgml index 08bc339e2e..e7d2ffddaf 100644 --- a/doc/src/sgml/release-9.0.sgml +++ b/doc/src/sgml/release-9.0.sgml @@ -4,10 +4,10 @@ Release 9.0.23 - - Release Date - 2015-10-08 - + + Release date: + 2015-10-08 + This release contains a variety of fixes from 9.0.22. @@ -501,10 +501,10 @@ Release 9.0.22 - - Release Date - 2015-06-12 - + + Release date: + 2015-06-12 + This release contains a small number of fixes from 9.0.21. @@ -575,10 +575,10 @@ Release 9.0.21 - - Release Date - 2015-06-04 - + + Release date: + 2015-06-04 + This release contains a small number of fixes from 9.0.20. @@ -669,10 +669,10 @@ Release 9.0.20 - - Release Date - 2015-05-22 - + + Release date: + 2015-05-22 + This release contains a variety of fixes from 9.0.19. @@ -1113,10 +1113,10 @@ Release 9.0.19 - - Release Date - 2015-02-05 - + + Release date: + 2015-02-05 + This release contains a variety of fixes from 9.0.18. @@ -1839,10 +1839,10 @@ Release 9.0.18 - - Release Date - 2014-07-24 - + + Release date: + 2014-07-24 + This release contains a variety of fixes from 9.0.17. @@ -2185,10 +2185,10 @@ Release 9.0.17 - - Release Date - 2014-03-20 - + + Release date: + 2014-03-20 + This release contains a variety of fixes from 9.0.16. @@ -2338,10 +2338,10 @@ Release 9.0.16 - - Release Date - 2014-02-20 - + + Release date: + 2014-02-20 + This release contains a variety of fixes from 9.0.15. @@ -2839,10 +2839,10 @@ Release 9.0.15 - - Release Date - 2013-12-05 - + + Release date: + 2013-12-05 + This release contains a variety of fixes from 9.0.14. @@ -3077,10 +3077,10 @@ Release 9.0.14 - - Release Date - 2013-10-10 - + + Release date: + 2013-10-10 + This release contains a variety of fixes from 9.0.13. @@ -3343,10 +3343,10 @@ Release 9.0.13 - - Release Date - 2013-04-04 - + + Release date: + 2013-04-04 + This release contains a variety of fixes from 9.0.12. @@ -3640,10 +3640,10 @@ Release 9.0.12 - - Release Date - 2013-02-07 - + + Release date: + 2013-02-07 + This release contains a variety of fixes from 9.0.11. @@ -3902,10 +3902,10 @@ Release 9.0.11 - - Release Date - 2012-12-06 - + + Release date: + 2012-12-06 + This release contains a variety of fixes from 9.0.10. @@ -4280,10 +4280,10 @@ Release 9.0.10 - - Release Date - 2012-09-24 - + + Release date: + 2012-09-24 + This release contains a variety of fixes from 9.0.9. @@ -4422,10 +4422,10 @@ Release 9.0.9 - - Release Date - 2012-08-17 - + + Release date: + 2012-08-17 + This release contains a variety of fixes from 9.0.8. @@ -4728,10 +4728,10 @@ Release 9.0.8 - - Release Date - 2012-06-04 - + + Release date: + 2012-06-04 + This release contains a variety of fixes from 9.0.7. @@ -5048,10 +5048,10 @@ Release 9.0.7 - - Release Date - 2012-02-27 - + + Release date: + 2012-02-27 + This release contains a variety of fixes from 9.0.6. @@ -5564,10 +5564,10 @@ Release 9.0.6 - - Release Date - 2011-12-05 - + + Release date: + 2011-12-05 + This release contains a variety of fixes from 9.0.5. @@ -5948,10 +5948,10 @@ Release 9.0.5 - - Release Date - 2011-09-26 - + + Release date: + 2011-09-26 + This release contains a variety of fixes from 9.0.4. @@ -6598,10 +6598,10 @@ Release 9.0.4 - - Release Date - 2011-04-18 - + + Release date: + 2011-04-18 + This release contains a variety of fixes from 9.0.3. @@ -6939,10 +6939,10 @@ Release 9.0.3 - - Release Date - 2011-01-31 - + + Release date: + 2011-01-31 + This release contains a variety of fixes from 9.0.2. @@ -7115,10 +7115,10 @@ Release 9.0.2 - - Release Date - 2010-12-16 - + + Release date: + 2010-12-16 + This release contains a variety of fixes from 9.0.1. @@ -7562,10 +7562,10 @@ Release 9.0.1 - - Release Date - 2010-10-04 - + + Release date: + 2010-10-04 + This release contains a variety of fixes from 9.0.0. @@ -7704,10 +7704,10 @@ Release 9.0 - - Release Date - 2010-09-20 - + + Release date: + 2010-09-20 + Overview diff --git a/doc/src/sgml/release-9.1.sgml b/doc/src/sgml/release-9.1.sgml index 2d0540c39e..0454f849d4 100644 --- a/doc/src/sgml/release-9.1.sgml +++ b/doc/src/sgml/release-9.1.sgml @@ -4,10 +4,10 @@ Release 9.1.24 - - Release Date - 2016-10-27 - + + Release date: + 2016-10-27 + This release contains a variety of fixes from 9.1.23. @@ -214,10 +214,10 @@ Release 9.1.23 - - Release Date - 2016-08-11 - + + Release date: + 2016-08-11 + This release contains a variety of fixes from 9.1.22. @@ -556,10 +556,10 @@ Branch: REL9_1_STABLE [354b3a3ac] 2016-06-19 14:01:17 -0400 Release 9.1.22 - - Release Date - 2016-05-12 - + + Release date: + 2016-05-12 + This release contains a variety of fixes from 9.1.21. @@ -721,10 +721,10 @@ Branch: REL9_1_STABLE [354b3a3ac] 2016-06-19 14:01:17 -0400 Release 9.1.21 - - Release Date - 2016-03-31 - + + Release date: + 2016-03-31 + This release contains a variety of fixes from 9.1.20. @@ -927,10 +927,10 @@ Branch: REL9_1_STABLE [354b3a3ac] 2016-06-19 14:01:17 -0400 Release 9.1.20 - - Release Date - 2016-02-11 - + + Release date: + 2016-02-11 + This release contains a variety of fixes from 9.1.19. @@ -1440,10 +1440,10 @@ Branch: REL9_1_STABLE [354b3a3ac] 2016-06-19 14:01:17 -0400 Release 9.1.19 - - Release Date - 2015-10-08 - + + Release date: + 2015-10-08 + This release contains a variety of fixes from 9.1.18. @@ -2001,10 +2001,10 @@ Branch: REL9_0_STABLE [9d6af7367] 2015-08-15 11:02:34 -0400 Release 9.1.18 - - Release Date - 2015-06-12 - + + Release date: + 2015-06-12 + This release contains a small number of fixes from 9.1.17. @@ -2069,10 +2069,10 @@ Branch: REL9_0_STABLE [9d6af7367] 2015-08-15 11:02:34 -0400 Release 9.1.17 - - Release Date - 2015-06-04 - + + Release date: + 2015-06-04 + This release contains a small number of fixes from 9.1.16. @@ -2157,10 +2157,10 @@ Branch: REL9_0_STABLE [9d6af7367] 2015-08-15 11:02:34 -0400 Release 9.1.16 - - Release Date - 2015-05-22 - + + Release date: + 2015-05-22 + This release contains a variety of fixes from 9.1.15. @@ -2683,10 +2683,10 @@ Branch: REL9_0_STABLE [9d6af7367] 2015-08-15 11:02:34 -0400 Release 9.1.15 - - Release Date - 2015-02-05 - + + Release date: + 2015-02-05 + This release contains a variety of fixes from 9.1.14. @@ -3493,10 +3493,10 @@ Branch: REL9_0_STABLE [9d6af7367] 2015-08-15 11:02:34 -0400 Release 9.1.14 - - Release Date - 2014-07-24 - + + Release date: + 2014-07-24 + This release contains a variety of fixes from 9.1.13. @@ -3876,10 +3876,10 @@ Branch: REL9_0_STABLE [9d6af7367] 2015-08-15 11:02:34 -0400 Release 9.1.13 - - Release Date - 2014-03-20 - + + Release date: + 2014-03-20 + This release contains a variety of fixes from 9.1.12. @@ -4043,10 +4043,10 @@ Branch: REL9_0_STABLE [9d6af7367] 2015-08-15 11:02:34 -0400 Release 9.1.12 - - Release Date - 2014-02-20 - + + Release date: + 2014-02-20 + This release contains a variety of fixes from 9.1.11. @@ -4593,10 +4593,10 @@ Branch: REL9_0_STABLE [9d6af7367] 2015-08-15 11:02:34 -0400 Release 9.1.11 - - Release Date - 2013-12-05 - + + Release date: + 2013-12-05 + This release contains a variety of fixes from 9.1.10. @@ -4850,10 +4850,10 @@ Branch: REL9_0_STABLE [9d6af7367] 2015-08-15 11:02:34 -0400 Release 9.1.10 - - Release Date - 2013-10-10 - + + Release date: + 2013-10-10 + This release contains a variety of fixes from 9.1.9. @@ -5185,10 +5185,10 @@ Branch: REL9_0_STABLE [9d6af7367] 2015-08-15 11:02:34 -0400 Release 9.1.9 - - Release Date - 2013-04-04 - + + Release date: + 2013-04-04 + This release contains a variety of fixes from 9.1.8. @@ -5519,10 +5519,10 @@ Branch: REL9_0_STABLE [9d6af7367] 2015-08-15 11:02:34 -0400 Release 9.1.8 - - Release Date - 2013-02-07 - + + Release date: + 2013-02-07 + This release contains a variety of fixes from 9.1.7. @@ -5841,10 +5841,10 @@ Branch: REL9_0_STABLE [9d6af7367] 2015-08-15 11:02:34 -0400 Release 9.1.7 - - Release Date - 2012-12-06 - + + Release date: + 2012-12-06 + This release contains a variety of fixes from 9.1.6. @@ -6304,10 +6304,10 @@ Branch: REL9_0_STABLE [9d6af7367] 2015-08-15 11:02:34 -0400 Release 9.1.6 - - Release Date - 2012-09-24 - + + Release date: + 2012-09-24 + This release contains a variety of fixes from 9.1.5. @@ -6556,10 +6556,10 @@ Branch: REL9_0_STABLE [9d6af7367] 2015-08-15 11:02:34 -0400 Release 9.1.5 - - Release Date - 2012-08-17 - + + Release date: + 2012-08-17 + This release contains a variety of fixes from 9.1.4. @@ -6943,10 +6943,10 @@ Branch: REL9_0_STABLE [9d6af7367] 2015-08-15 11:02:34 -0400 Release 9.1.4 - - Release Date - 2012-06-04 - + + Release date: + 2012-06-04 + This release contains a variety of fixes from 9.1.3. @@ -7433,10 +7433,10 @@ Branch: REL9_0_STABLE [9d6af7367] 2015-08-15 11:02:34 -0400 Release 9.1.3 - - Release Date - 2012-02-27 - + + Release date: + 2012-02-27 + This release contains a variety of fixes from 9.1.2. @@ -8070,10 +8070,10 @@ Branch: REL9_0_STABLE [9d6af7367] 2015-08-15 11:02:34 -0400 Release 9.1.2 - - Release Date - 2011-12-05 - + + Release date: + 2011-12-05 + This release contains a variety of fixes from 9.1.1. @@ -8717,10 +8717,10 @@ Branch: REL9_0_STABLE [9d6af7367] 2015-08-15 11:02:34 -0400 Release 9.1.1 - - Release Date - 2011-09-26 - + + Release date: + 2011-09-26 + This release contains a small number of fixes from 9.1.0. @@ -8785,10 +8785,10 @@ Branch: REL9_0_STABLE [9d6af7367] 2015-08-15 11:02:34 -0400 Release 9.1 - - Release Date - 2011-09-12 - + + Release date: + 2011-09-12 + Overview diff --git a/doc/src/sgml/release-9.2.sgml b/doc/src/sgml/release-9.2.sgml index 6aee919ffe..1f1cc12b61 100644 --- a/doc/src/sgml/release-9.2.sgml +++ b/doc/src/sgml/release-9.2.sgml @@ -4,10 +4,10 @@ Release 9.2.21 - - Release Date - 2017-05-11 - + + Release date: + 2017-05-11 + This release contains a variety of fixes from 9.2.20. @@ -452,10 +452,10 @@ UPDATE pg_database SET datallowconn = false WHERE datname = 'template0'; Release 9.2.20 - - Release Date - 2017-02-09 - + + Release date: + 2017-02-09 + This release contains a variety of fixes from 9.2.19. @@ -864,10 +864,10 @@ Branch: REL9_2_STABLE [38bec1805] 2017-01-25 07:02:25 +0900 Release 9.2.19 - - Release Date - 2016-10-27 - + + Release date: + 2016-10-27 + This release contains a variety of fixes from 9.2.18. @@ -1130,10 +1130,10 @@ Branch: REL9_2_STABLE [38bec1805] 2017-01-25 07:02:25 +0900 Release 9.2.18 - - Release Date - 2016-08-11 - + + Release date: + 2016-08-11 + This release contains a variety of fixes from 9.2.17. @@ -1471,10 +1471,10 @@ Branch: REL9_2_STABLE [38bec1805] 2017-01-25 07:02:25 +0900 Release 9.2.17 - - Release Date - 2016-05-12 - + + Release date: + 2016-05-12 + This release contains a variety of fixes from 9.2.16. @@ -1667,10 +1667,10 @@ Branch: REL9_2_STABLE [38bec1805] 2017-01-25 07:02:25 +0900 Release 9.2.16 - - Release Date - 2016-03-31 - + + Release date: + 2016-03-31 + This release contains a variety of fixes from 9.2.15. @@ -1873,10 +1873,10 @@ Branch: REL9_2_STABLE [38bec1805] 2017-01-25 07:02:25 +0900 Release 9.2.15 - - Release Date - 2016-02-11 - + + Release date: + 2016-02-11 + This release contains a variety of fixes from 9.2.14. @@ -2417,10 +2417,10 @@ Branch: REL9_2_STABLE [38bec1805] 2017-01-25 07:02:25 +0900 Release 9.2.14 - - Release Date - 2015-10-08 - + + Release date: + 2015-10-08 + This release contains a variety of fixes from 9.2.13. @@ -3014,10 +3014,10 @@ Branch: REL9_2_STABLE [e90a629e1] 2015-09-22 14:58:38 -0700 Release 9.2.13 - - Release Date - 2015-06-12 - + + Release date: + 2015-06-12 + This release contains a small number of fixes from 9.2.12. @@ -3082,10 +3082,10 @@ Branch: REL9_2_STABLE [e90a629e1] 2015-09-22 14:58:38 -0700 Release 9.2.12 - - Release Date - 2015-06-04 - + + Release date: + 2015-06-04 + This release contains a small number of fixes from 9.2.11. @@ -3177,10 +3177,10 @@ Branch: REL9_2_STABLE [e90a629e1] 2015-09-22 14:58:38 -0700 Release 9.2.11 - - Release Date - 2015-05-22 - + + Release date: + 2015-05-22 + This release contains a variety of fixes from 9.2.10. @@ -3759,10 +3759,10 @@ Branch: REL9_2_STABLE [6b700301c] 2015-02-17 16:03:00 +0100 Release 9.2.10 - - Release Date - 2015-02-05 - + + Release date: + 2015-02-05 + This release contains a variety of fixes from 9.2.9. @@ -4676,10 +4676,10 @@ Branch: REL9_2_STABLE [6b700301c] 2015-02-17 16:03:00 +0100 Release 9.2.9 - - Release Date - 2014-07-24 - + + Release date: + 2014-07-24 + This release contains a variety of fixes from 9.2.8. @@ -5157,10 +5157,10 @@ Branch: REL9_2_STABLE [6b700301c] 2015-02-17 16:03:00 +0100 Release 9.2.8 - - Release Date - 2014-03-20 - + + Release date: + 2014-03-20 + This release contains a variety of fixes from 9.2.7. @@ -5346,10 +5346,10 @@ Branch: REL9_2_STABLE [6b700301c] 2015-02-17 16:03:00 +0100 Release 9.2.7 - - Release Date - 2014-02-20 - + + Release date: + 2014-02-20 + This release contains a variety of fixes from 9.2.6. @@ -5953,10 +5953,10 @@ Branch: REL9_2_STABLE [6b700301c] 2015-02-17 16:03:00 +0100 Release 9.2.6 - - Release Date - 2013-12-05 - + + Release date: + 2013-12-05 + This release contains a variety of fixes from 9.2.5. @@ -6290,10 +6290,10 @@ Branch: REL9_2_STABLE [6b700301c] 2015-02-17 16:03:00 +0100 Release 9.2.5 - - Release Date - 2013-10-10 - + + Release date: + 2013-10-10 + This release contains a variety of fixes from 9.2.4. @@ -6709,10 +6709,10 @@ Branch: REL9_2_STABLE [6b700301c] 2015-02-17 16:03:00 +0100 Release 9.2.4 - - Release Date - 2013-04-04 - + + Release date: + 2013-04-04 + This release contains a variety of fixes from 9.2.3. @@ -7116,10 +7116,10 @@ Branch: REL9_2_STABLE [6b700301c] 2015-02-17 16:03:00 +0100 Release 9.2.3 - - Release Date - 2013-02-07 - + + Release date: + 2013-02-07 + This release contains a variety of fixes from 9.2.2. @@ -7567,10 +7567,10 @@ Branch: REL9_2_STABLE [6b700301c] 2015-02-17 16:03:00 +0100 Release 9.2.2 - - Release Date - 2012-12-06 - + + Release date: + 2012-12-06 + This release contains a variety of fixes from 9.2.1. @@ -8291,10 +8291,10 @@ Branch: REL9_2_STABLE [6b700301c] 2015-02-17 16:03:00 +0100 Release 9.2.1 - - Release Date - 2012-09-24 - + + Release date: + 2012-09-24 + This release contains a variety of fixes from 9.2.0. @@ -8459,10 +8459,10 @@ Branch: REL9_2_STABLE [6b700301c] 2015-02-17 16:03:00 +0100 Release 9.2 - - Release Date - 2012-09-10 - + + Release date: + 2012-09-10 + Overview diff --git a/doc/src/sgml/release-9.3.sgml b/doc/src/sgml/release-9.3.sgml index 6d620ffacc..86c1ae6d3f 100644 --- a/doc/src/sgml/release-9.3.sgml +++ b/doc/src/sgml/release-9.3.sgml @@ -4,10 +4,10 @@ Release 9.3.17 - - Release Date - 2017-05-11 - + + Release date: + 2017-05-11 + This release contains a variety of fixes from 9.3.16. @@ -491,10 +491,10 @@ UPDATE pg_database SET datallowconn = false WHERE datname = 'template0'; Release 9.3.16 - - Release Date - 2017-02-09 - + + Release date: + 2017-02-09 + This release contains a variety of fixes from 9.3.15. @@ -939,10 +939,10 @@ UPDATE pg_database SET datallowconn = false WHERE datname = 'template0'; Release 9.3.15 - - Release Date - 2016-10-27 - + + Release date: + 2016-10-27 + This release contains a variety of fixes from 9.3.14. @@ -1268,10 +1268,10 @@ UPDATE pg_database SET datallowconn = false WHERE datname = 'template0'; Release 9.3.14 - - Release Date - 2016-08-11 - + + Release date: + 2016-08-11 + This release contains a variety of fixes from 9.3.13. @@ -1758,10 +1758,10 @@ UPDATE pg_database SET datallowconn = false WHERE datname = 'template0'; Release 9.3.13 - - Release Date - 2016-05-12 - + + Release date: + 2016-05-12 + This release contains a variety of fixes from 9.3.12. @@ -1988,10 +1988,10 @@ Branch: REL9_2_STABLE [37f30b251] 2016-04-18 13:19:52 -0400 Release 9.3.12 - - Release Date - 2016-03-31 - + + Release date: + 2016-03-31 + This release contains a variety of fixes from 9.3.11. @@ -2207,10 +2207,10 @@ Branch: REL9_2_STABLE [37f30b251] 2016-04-18 13:19:52 -0400 Release 9.3.11 - - Release Date - 2016-02-11 - + + Release date: + 2016-02-11 + This release contains a variety of fixes from 9.3.10. @@ -2830,10 +2830,10 @@ Branch: REL9_2_STABLE [37f30b251] 2016-04-18 13:19:52 -0400 Release 9.3.10 - - Release Date - 2015-10-08 - + + Release date: + 2015-10-08 + This release contains a variety of fixes from 9.3.9. @@ -3459,10 +3459,10 @@ Branch: REL9_2_STABLE [37f30b251] 2016-04-18 13:19:52 -0400 Release 9.3.9 - - Release Date - 2015-06-12 - + + Release date: + 2015-06-12 + This release contains a small number of fixes from 9.3.8. @@ -3608,10 +3608,10 @@ Branch: REL9_2_STABLE [37f30b251] 2016-04-18 13:19:52 -0400 Release 9.3.8 - - Release Date - 2015-06-04 - + + Release date: + 2015-06-04 + This release contains a small number of fixes from 9.3.7. @@ -3716,10 +3716,10 @@ Branch: REL9_0_STABLE [4dddf8552] 2015-05-21 20:41:55 -0400 Release 9.3.7 - - Release Date - 2015-05-22 - + + Release date: + 2015-05-22 + This release contains a variety of fixes from 9.3.6. @@ -4306,10 +4306,10 @@ Branch: REL9_0_STABLE [4dddf8552] 2015-05-21 20:41:55 -0400 Release 9.3.6 - - Release Date - 2015-02-05 - + + Release date: + 2015-02-05 + This release contains a variety of fixes from 9.3.5. @@ -6191,10 +6191,10 @@ Branch: REL9_0_STABLE [b6391f587] 2014-10-04 14:18:43 -0400 Release 9.3.5 - - Release Date - 2014-07-24 - + + Release date: + 2014-07-24 + This release contains a variety of fixes from 9.3.4. @@ -7311,10 +7311,10 @@ Branch: REL8_4_STABLE [c51da696b] 2014-07-19 15:01:45 -0400 Release 9.3.4 - - Release Date - 2014-03-20 - + + Release date: + 2014-03-20 + This release contains a variety of fixes from 9.3.3. @@ -7773,10 +7773,10 @@ Branch: REL8_4_STABLE [6e6c2c2e1] 2014-03-15 13:36:57 -0400 Release 9.3.3 - - Release Date - 2014-02-20 - + + Release date: + 2014-02-20 + This release contains a variety of fixes from 9.3.2. @@ -9235,10 +9235,10 @@ Branch: REL8_4_STABLE [c0c2d62ac] 2014-02-14 21:59:56 -0500 Release 9.3.2 - - Release Date - 2013-12-05 - + + Release date: + 2013-12-05 + This release contains a variety of fixes from 9.3.1. @@ -9714,10 +9714,10 @@ SELECT ... FROM tab1 x CROSS JOIN (tab2 x CROSS JOIN tab3 y) z Release 9.3.1 - - Release Date - 2013-10-10 - + + Release date: + 2013-10-10 + This release contains a variety of fixes from 9.3.0. @@ -9817,10 +9817,10 @@ ALTER EXTENSION hstore UPDATE; Release 9.3 - - Release Date - 2013-09-09 - + + Release date: + 2013-09-09 + Overview diff --git a/doc/src/sgml/release-9.4.sgml b/doc/src/sgml/release-9.4.sgml index ae60b80959..3317c11fc2 100644 --- a/doc/src/sgml/release-9.4.sgml +++ b/doc/src/sgml/release-9.4.sgml @@ -4,10 +4,10 @@ Release 9.4.12 - - Release Date - 2017-05-11 - + + Release date: + 2017-05-11 + This release contains a variety of fixes from 9.4.11. @@ -580,10 +580,10 @@ Branch: REL9_2_STABLE [fb50c38e9] 2017-04-17 13:52:42 -0400 Release 9.4.11 - - Release Date - 2017-02-09 - + + Release date: + 2017-02-09 + This release contains a variety of fixes from 9.4.10. @@ -1113,10 +1113,10 @@ Branch: REL9_2_STABLE [fb50c38e9] 2017-04-17 13:52:42 -0400 Release 9.4.10 - - Release Date - 2016-10-27 - + + Release date: + 2016-10-27 + This release contains a variety of fixes from 9.4.9. @@ -1579,10 +1579,10 @@ Branch: REL9_4_STABLE [10ad15f48] 2016-09-01 11:45:16 -0400 Release 9.4.9 - - Release Date - 2016-08-11 - + + Release date: + 2016-08-11 + This release contains a variety of fixes from 9.4.8. @@ -2101,10 +2101,10 @@ Branch: REL9_1_STABLE [de887cc8a] 2016-05-25 19:39:49 -0400 Release 9.4.8 - - Release Date - 2016-05-12 - + + Release date: + 2016-05-12 + This release contains a variety of fixes from 9.4.7. @@ -2364,10 +2364,10 @@ Branch: REL9_1_STABLE [de887cc8a] 2016-05-25 19:39:49 -0400 Release 9.4.7 - - Release Date - 2016-03-31 - + + Release date: + 2016-03-31 + This release contains a variety of fixes from 9.4.6. @@ -2635,10 +2635,10 @@ Branch: REL9_1_STABLE [de887cc8a] 2016-05-25 19:39:49 -0400 Release 9.4.6 - - Release Date - 2016-02-11 - + + Release date: + 2016-02-11 + This release contains a variety of fixes from 9.4.5. @@ -3732,10 +3732,10 @@ Branch: REL9_1_STABLE [386dcd539] 2015-12-11 19:08:40 -0500 Release 9.4.5 - - Release Date - 2015-10-08 - + + Release date: + 2015-10-08 + This release contains a variety of fixes from 9.4.4. @@ -5255,10 +5255,10 @@ Branch: REL9_0_STABLE [47ac95f37] 2015-10-02 19:16:37 -0400 Release 9.4.4 - - Release Date - 2015-06-12 - + + Release date: + 2015-06-12 + This release contains a small number of fixes from 9.4.3. @@ -5436,10 +5436,10 @@ Branch: REL9_3_STABLE [d3fdec6ae] 2015-06-03 11:58:47 -0400 Release 9.4.3 - - Release Date - 2015-06-04 - + + Release date: + 2015-06-04 + This release contains a small number of fixes from 9.4.2. @@ -5579,10 +5579,10 @@ Branch: REL9_0_STABLE [b06649b7f] 2015-05-26 22:15:00 -0400 Release 9.4.2 - - Release Date - 2015-05-22 - + + Release date: + 2015-05-22 + This release contains a variety of fixes from 9.4.1. @@ -6860,10 +6860,10 @@ Branch: REL9_0_STABLE [3c3749a3b] 2015-05-15 19:36:20 -0400 Release 9.4.1 - - Release Date - 2015-02-05 - + + Release date: + 2015-02-05 + This release contains a variety of fixes from 9.4.0. @@ -7673,10 +7673,10 @@ Branch: REL9_4_STABLE [c2b06ab17] 2015-01-30 22:45:58 -0500 Release 9.4 - - Release Date - 2014-12-18 - + + Release date: + 2014-12-18 + Overview diff --git a/doc/src/sgml/release-9.5.sgml b/doc/src/sgml/release-9.5.sgml index 1c2aca0683..14a9b9d3b0 100644 --- a/doc/src/sgml/release-9.5.sgml +++ b/doc/src/sgml/release-9.5.sgml @@ -4,10 +4,10 @@ Release 9.5.7 - - Release Date - 2017-05-11 - + + Release date: + 2017-05-11 + This release contains a variety of fixes from 9.5.6. @@ -641,10 +641,10 @@ Branch: REL9_3_STABLE [3aa16b117] 2017-05-06 22:17:35 -0400 Release 9.5.6 - - Release Date - 2017-02-09 - + + Release date: + 2017-02-09 + This release contains a variety of fixes from 9.5.5. @@ -1303,10 +1303,10 @@ Branch: REL9_2_STABLE [60314e28e] 2016-12-13 19:08:09 -0600 Release 9.5.5 - - Release Date - 2016-10-27 - + + Release date: + 2016-10-27 + This release contains a variety of fixes from 9.5.4. @@ -2127,10 +2127,10 @@ Branch: REL9_1_STABLE [380dad29d] 2016-09-02 17:29:32 -0400 Release 9.5.4 - - Release Date - 2016-08-11 - + + Release date: + 2016-08-11 + This release contains a variety of fixes from 9.5.3. @@ -3200,10 +3200,10 @@ Branch: REL9_1_STABLE [a44388ffe] 2016-08-05 12:59:02 -0400 Release 9.5.3 - - Release Date - 2016-05-12 - + + Release date: + 2016-05-12 + This release contains a variety of fixes from 9.5.2. @@ -3701,10 +3701,10 @@ Branch: REL9_1_STABLE [bfc39da64] 2016-05-05 20:09:32 -0400 Release 9.5.2 - - Release Date - 2016-03-31 - + + Release date: + 2016-03-31 + This release contains a variety of fixes from 9.5.1. @@ -4412,10 +4412,10 @@ Branch: REL9_1_STABLE [e5fd35cc5] 2016-03-25 19:03:54 -0400 Release 9.5.1 - - Release Date - 2016-02-11 - + + Release date: + 2016-02-11 + This release contains a variety of fixes from 9.5.0. @@ -4860,10 +4860,10 @@ Branch: REL9_1_STABLE [6887d72d0] 2016-02-05 10:59:39 -0500 Release 9.5 - - Release Date - 2016-01-07 - + + Release date: + 2016-01-07 + Overview diff --git a/doc/src/sgml/release-9.6.sgml b/doc/src/sgml/release-9.6.sgml index e8fb97cb7f..764f812d60 100644 --- a/doc/src/sgml/release-9.6.sgml +++ b/doc/src/sgml/release-9.6.sgml @@ -4,10 +4,10 @@ Release 9.6.3 - - Release Date - 2017-05-11 - + + Release date: + 2017-05-11 + This release contains a variety of fixes from 9.6.2. @@ -1124,10 +1124,10 @@ Branch: REL9_2_STABLE [82e7d3dfd] 2017-05-07 11:57:41 -0400 Release 9.6.2 - - Release Date - 2017-02-09 - + + Release date: + 2017-02-09 + This release contains a variety of fixes from 9.6.1. @@ -2541,10 +2541,10 @@ Branch: REL9_2_STABLE [ef878cc2c] 2017-01-30 11:41:09 -0500 Release 9.6.1 - - Release Date - 2016-10-27 - + + Release date: + 2016-10-27 + This release contains a variety of fixes from 9.6.0. @@ -3136,10 +3136,10 @@ Branch: REL9_1_STABLE [22cf97635] 2016-10-19 17:57:06 -0400 Release 9.6 - - Release Date - 2016-09-29 - + + Release date: + 2016-09-29 + Overview diff --git a/doc/src/sgml/release-old.sgml b/doc/src/sgml/release-old.sgml index 037b1e8e0c..d4de6b1357 100644 --- a/doc/src/sgml/release-old.sgml +++ b/doc/src/sgml/release-old.sgml @@ -4,10 +4,10 @@ Release 7.3.21 - - Release Date - 2008-01-07 - + + Release date: + 2008-01-07 + This release contains a variety of fixes from 7.3.20, @@ -116,10 +116,10 @@ Release 7.3.20 - - Release Date - 2007-09-17 - + + Release date: + 2007-09-17 + This release contains fixes from 7.3.19. @@ -177,10 +177,10 @@ Release 7.3.19 - - Release Date - 2007-04-23 - + + Release date: + 2007-04-23 + This release contains fixes from 7.3.18, @@ -233,10 +233,10 @@ Release 7.3.18 - - Release Date - 2007-02-05 - + + Release date: + 2007-02-05 + This release contains a variety of fixes from 7.3.17, including @@ -295,10 +295,10 @@ Release 7.3.17 - - Release Date - 2007-01-08 - + + Release date: + 2007-01-08 + This release contains a variety of fixes from 7.3.16. @@ -351,10 +351,10 @@ Release 7.3.16 - - Release Date - 2006-10-16 - + + Release date: + 2006-10-16 + This release contains a variety of fixes from 7.3.15. @@ -393,10 +393,10 @@ Release 7.3.15 - - Release Date - 2006-05-23 - + + Release date: + 2006-05-23 + This release contains a variety of fixes from 7.3.14, @@ -485,10 +485,10 @@ Fuhr) Release 7.3.14 - - Release Date - 2006-02-14 - + + Release date: + 2006-02-14 + This release contains a variety of fixes from 7.3.13. @@ -542,10 +542,10 @@ and isinf during configure (Tom) Release 7.3.13 - - Release Date - 2006-01-09 - + + Release date: + 2006-01-09 + This release contains a variety of fixes from 7.3.12. @@ -604,10 +604,10 @@ what's actually returned by the query (Joe) Release 7.3.12 - - Release Date - 2005-12-12 - + + Release date: + 2005-12-12 + This release contains a variety of fixes from 7.3.11. @@ -651,10 +651,10 @@ table has been dropped Release 7.3.11 - - Release Date - 2005-10-04 - + + Release date: + 2005-10-04 + This release contains a variety of fixes from 7.3.10. @@ -702,10 +702,10 @@ the variable is of pass-by-reference type Release 7.3.10 - - Release Date - 2005-05-09 - + + Release date: + 2005-05-09 + This release contains a variety of fixes from 7.3.9, including several @@ -828,10 +828,10 @@ month-related formats Release 7.3.9 - - Release Date - 2005-01-31 - + + Release date: + 2005-01-31 + This release contains a variety of fixes from 7.3.8, including several @@ -884,10 +884,10 @@ datestyles Release 7.3.8 - - Release Date - 2004-10-22 - + + Release date: + 2004-10-22 + This release contains a variety of fixes from 7.3.7. @@ -934,10 +934,10 @@ concern since there is no reason for non-developers to use this script anyway. Release 7.3.7 - - Release Date - 2004-08-16 - + + Release date: + 2004-08-16 + This release contains one critical fix over 7.3.6, and some minor items. @@ -974,10 +974,10 @@ since PostgreSQL 7.1. Release 7.3.6 - - Release Date - 2004-03-02 - + + Release date: + 2004-03-02 + This release contains a variety of fixes from 7.3.5. @@ -1039,10 +1039,10 @@ operations on bytea columns (Joe) Release 7.3.5 - - Release Date - 2003-12-03 - + + Release date: + 2003-12-03 + This has a variety of fixes from 7.3.4. @@ -1090,10 +1090,10 @@ operations on bytea columns (Joe) Release 7.3.4 - - Release Date - 2003-07-24 - + + Release date: + 2003-07-24 + This has a variety of fixes from 7.3.3. @@ -1130,10 +1130,10 @@ operations on bytea columns (Joe) Release 7.3.3 - - Release Date - 2003-05-22 - + + Release date: + 2003-05-22 + This release contains a variety of fixes for version 7.3.2. @@ -1240,10 +1240,10 @@ operations on bytea columns (Joe) Release 7.3.2 - - Release Date - 2003-02-04 - + + Release date: + 2003-02-04 + This release contains a variety of fixes for version 7.3.1. @@ -1301,10 +1301,10 @@ operations on bytea columns (Joe) Release 7.3.1 - - Release Date - 2002-12-18 - + + Release date: + 2002-12-18 + This release contains a variety of fixes for version 7.3. @@ -1351,10 +1351,10 @@ operations on bytea columns (Joe) Release 7.3 - - Release Date - 2002-11-27 - + + Release date: + 2002-11-27 + Overview @@ -1987,10 +1987,10 @@ operations on bytea columns (Joe) Release 7.2.8 - - Release Date - 2005-05-09 - + + Release date: + 2005-05-09 + This release contains a variety of fixes from 7.2.7, including one @@ -2038,10 +2038,10 @@ month-related formats Release 7.2.7 - - Release Date - 2005-01-31 - + + Release date: + 2005-01-31 + This release contains a variety of fixes from 7.2.6, including several @@ -2086,10 +2086,10 @@ datestyles Release 7.2.6 - - Release Date - 2004-10-22 - + + Release date: + 2004-10-22 + This release contains a variety of fixes from 7.2.5. @@ -2137,10 +2137,10 @@ concern since there is no reason for non-developers to use this script anyway. Release 7.2.5 - - Release Date - 2004-08-16 - + + Release date: + 2004-08-16 + This release contains a variety of fixes from 7.2.4. @@ -2180,10 +2180,10 @@ since PostgreSQL 7.1. Release 7.2.4 - - Release Date - 2003-01-30 - + + Release date: + 2003-01-30 + This release contains a variety of fixes for version 7.2.3, @@ -2219,10 +2219,10 @@ since PostgreSQL 7.1. Release 7.2.3 - - Release Date - 2002-10-01 - + + Release date: + 2002-10-01 + This release contains a variety of fixes for version 7.2.2, @@ -2256,10 +2256,10 @@ since PostgreSQL 7.1. Release 7.2.2 - - Release Date - 2002-08-23 - + + Release date: + 2002-08-23 + This release contains a variety of fixes for version 7.2.1. @@ -2299,10 +2299,10 @@ since PostgreSQL 7.1. Release 7.2.1 - - Release Date - 2002-03-21 - + + Release date: + 2002-03-21 + This release contains a variety of fixes for version 7.2. @@ -2345,10 +2345,10 @@ since PostgreSQL 7.1. Release 7.2 - - Release Date - 2002-02-04 - + + Release date: + 2002-02-04 + Overview @@ -2901,10 +2901,10 @@ since PostgreSQL 7.1. Release 7.1.3 - - Release Date - 2001-08-15 - + + Release date: + 2001-08-15 + Migration to Version 7.1.3 @@ -2939,10 +2939,10 @@ Cygwin build (Jason Tishler) Release 7.1.2 - - Release Date - 2001-05-11 - + + Release date: + 2001-05-11 + This has one fix from 7.1.1. @@ -2977,10 +2977,10 @@ pg_dump cleanups Release 7.1.1 - - Release Date - 2001-05-05 - + + Release date: + 2001-05-05 + This has a variety of fixes from 7.1. @@ -3024,10 +3024,10 @@ Python fixes (Darcy) Release 7.1 - - Release Date - 2001-04-13 - + + Release date: + 2001-04-13 + This release focuses on removing limitations that have existed in the @@ -3322,10 +3322,10 @@ New FreeBSD tools ipc_check, start-scripts/freebsd Release 7.0.3 - - Release Date - 2000-11-11 - + + Release date: + 2000-11-11 + This has a variety of fixes from 7.0.2. @@ -3394,10 +3394,10 @@ Fix for crash of backend, on abort (Tom) Release 7.0.2 - - Release Date - 2000-06-05 - + + Release date: + 2000-06-05 + This is a repackaging of 7.0.1 with added documentation. @@ -3428,10 +3428,10 @@ Added documentation to tarball. Release 7.0.1 - - Release Date - 2000-06-01 - + + Release date: + 2000-06-01 + This is a cleanup release for 7.0. @@ -3483,10 +3483,10 @@ ecpg changes (Michael) Release 7.0 - - Release Date - 2000-05-08 - + + Release date: + 2000-05-08 + This release contains improvements in many areas, demonstrating @@ -3952,10 +3952,10 @@ New multibyte encodings Release 6.5.3 - - Release Date - 1999-10-13 - + + Release date: + 1999-10-13 + This is basically a cleanup release for 6.5.2. We have added a new @@ -3988,10 +3988,10 @@ Fix dumping rules on inherited tables Release 6.5.2 - - Release Date - 1999-09-15 - + + Release date: + 1999-09-15 + This is basically a cleanup release for 6.5.1. We have fixed a variety of @@ -4045,10 +4045,10 @@ Updated version of pgaccess 0.98 Release 6.5.1 - - Release Date - 1999-07-15 - + + Release date: + 1999-07-15 + This is basically a cleanup release for 6.5. We have fixed a variety of @@ -4100,10 +4100,10 @@ Add Win1250 (Czech) support (Pavel Behal) Release 6.5 - - Release Date - 1999-06-09 - + + Release date: + 1999-06-09 + This release marks a major step in the development team's mastery of the source @@ -4500,10 +4500,10 @@ New install commands for plpgsql(Jan) Release 6.4.2 - - Release Date - 1998-12-20 - + + Release date: + 1998-12-20 + The 6.4.1 release was improperly packaged. This also has one additional @@ -4535,10 +4535,10 @@ Fix for datetime constant problem on some platforms(Thomas) Release 6.4.1 - - Release Date - 1998-12-18 - + + Release date: + 1998-12-18 + This is basically a cleanup release for 6.4. We have fixed a variety of @@ -4599,10 +4599,10 @@ Upgrade to PyGreSQL 2.2(D'Arcy) Release 6.4 - - Release Date - 1998-10-30 - + + Release date: + 1998-10-30 + There are many new features and improvements in this release. @@ -4896,10 +4896,10 @@ new Makefile.shlib for shared library configuration(Tom) Release 6.3.2 - - Release Date - 1998-04-07 - + + Release date: + 1998-04-07 + This is a bug-fix release for 6.3.x. @@ -4966,10 +4966,10 @@ ASSERT fixes(Bruce) Release 6.3.1 - - Release Date - 1998-03-23 - + + Release date: + 1998-03-23 + Summary: @@ -5052,10 +5052,10 @@ Better identify tcl and tk libs and includes(Bruce) Release 6.3 - - Release Date - 1998-03-01 - + + Release date: + 1998-03-01 + There are many new features and improvements in this release. @@ -5364,10 +5364,10 @@ Remove un-needed malloc() calls and replace with palloc()(Bruce) Release 6.2.1 - - Release Date - 1997-10-17 - + + Release date: + 1997-10-17 + 6.2.1 is a bug-fix and usability release on 6.2. @@ -5446,10 +5446,10 @@ Trigger function for inserting user names for INSERT/UPDATE(Brook Milligan) Release 6.2 - - Release Date - 1997-10-02 - + + Release date: + 1997-10-02 + A dump/restore is required for those wishing to migrate data from @@ -5599,10 +5599,10 @@ SPI and Trigger programming guides (Vadim & D'Arcy) Release 6.1.1 - - Release Date - 1997-07-22 - + + Release date: + 1997-07-22 + Migration from version 6.1 to version 6.1.1 @@ -5644,10 +5644,10 @@ pg_dumpall now returns proper status, portability fix(Bruce) Release 6.1 - - Release Date - 1997-06-08 - + + Release date: + 1997-06-08 + The regression tests have been adapted and extensively modified for the @@ -5816,10 +5816,10 @@ DG/UX, Ultrix, IRIX, AIX portability fixes Release 6.0 - - Release Date - 1997-01-29 - + + Release date: + 1997-01-29 + A dump/restore is required for those wishing to migrate data from @@ -5960,10 +5960,10 @@ Unused/uninitialized variables corrected Release 1.09 - - Release Date - 1996-11-04 - + + Release date: + 1996-11-04 + Sorry, we didn't keep track of changes from 1.02 to 1.09. Some of @@ -5975,10 +5975,10 @@ releases. Release 1.02 - - Release Date - 1996-08-01 - + + Release date: + 1996-08-01 + Migration from version 1.02 to version 1.02.1 @@ -6124,10 +6124,10 @@ Contributors (apologies to any missed) Release 1.01 - - Release Date - 1996-02-23 - + + Release date: + 1996-02-23 + @@ -6316,10 +6316,10 @@ Bug fixes: Release 1.0 - - Release Date - 1995-09-05 - + + Release date: + 1995-09-05 + Changes @@ -6373,10 +6373,10 @@ Bug fixes: <productname>Postgres95</productname> Release 0.03 - - Release Date - 1995-07-21 - + + Release date: + 1995-07-21 + Changes @@ -6495,10 +6495,10 @@ New documentation: <productname>Postgres95</productname> Release 0.02 - - Release Date - 1995-05-25 - + + Release date: + 1995-05-25 + Changes @@ -6546,10 +6546,10 @@ The following bugs have been fixed in postgres95-beta-0.02: <productname>Postgres95</productname> Release 0.01 - - Release Date - 1995-05-01 - + + Release date: + 1995-05-01 + Initial release. -- cgit v1.2.3 From d99d58cdc8c0b5b50ee92995e8575c100b1a458a Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Sat, 13 May 2017 01:05:48 -0300 Subject: Complete tab completion for DROP STATISTICS Tab-completing DROP STATISTICS would only work if you started writing the schema name containing the statistics object, because the visibility clause was missing. To add it, we need to add SQL-callable support for testing visibility of a statistics object, like all other object types already have. Discussion: https://postgr.es/m/22676.1494557205@sss.pgh.pa.us --- doc/src/sgml/func.sgml | 9 +++++ src/backend/catalog/namespace.c | 78 +++++++++++++++++++++++++++++++++++++ src/backend/utils/cache/lsyscache.c | 1 + src/bin/psql/tab-complete.c | 2 +- src/include/catalog/catversion.h | 2 +- src/include/catalog/namespace.h | 4 +- src/include/catalog/pg_proc.h | 2 + 7 files changed, 95 insertions(+), 3 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 5f47c59f8a..d55656769b 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -16680,6 +16680,12 @@ SELECT relname FROM pg_class WHERE pg_table_is_visible(oid); boolean is operator family visible in search path + + pg_statistic_ext_is_visible(stat_oid) + + boolean + is statistics object visible in search path + pg_table_is_visible(table_oid) @@ -16738,6 +16744,9 @@ SELECT relname FROM pg_class WHERE pg_table_is_visible(oid); pg_opfamily_is_visible + + pg_statistic_ext_is_visible + pg_table_is_visible diff --git a/src/backend/catalog/namespace.c b/src/backend/catalog/namespace.c index e521bd908a..5b339222af 100644 --- a/src/backend/catalog/namespace.c +++ b/src/backend/catalog/namespace.c @@ -34,6 +34,7 @@ #include "catalog/pg_operator.h" #include "catalog/pg_opfamily.h" #include "catalog/pg_proc.h" +#include "catalog/pg_statistic_ext.h" #include "catalog/pg_ts_config.h" #include "catalog/pg_ts_dict.h" #include "catalog/pg_ts_parser.h" @@ -2141,6 +2142,72 @@ get_statistics_oid(List *names, bool missing_ok) return stats_oid; } +/* + * StatisticsObjIsVisible + * Determine whether a statistics object (identified by OID) is visible in + * the current search path. Visible means "would be found by searching + * for the unqualified statistics object name". + */ +bool +StatisticsObjIsVisible(Oid relid) +{ + HeapTuple stxtup; + Form_pg_statistic_ext stxform; + Oid stxnamespace; + bool visible; + + stxtup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(relid)); + if (!HeapTupleIsValid(stxtup)) + elog(ERROR, "cache lookup failed for statistics object %u", relid); + stxform = (Form_pg_statistic_ext) GETSTRUCT(stxtup); + + recomputeNamespacePath(); + + /* + * Quick check: if it ain't in the path at all, it ain't visible. Items in + * the system namespace are surely in the path and so we needn't even do + * list_member_oid() for them. + */ + stxnamespace = stxform->stxnamespace; + if (stxnamespace != PG_CATALOG_NAMESPACE && + !list_member_oid(activeSearchPath, stxnamespace)) + visible = false; + else + { + /* + * If it is in the path, it might still not be visible; it could be + * hidden by another statistics object of the same name earlier in the + * path. So we must do a slow check for conflicting objects. + */ + char *stxname = NameStr(stxform->stxname); + ListCell *l; + + visible = false; + foreach(l, activeSearchPath) + { + Oid namespaceId = lfirst_oid(l); + + if (namespaceId == stxnamespace) + { + /* Found it first in path */ + visible = true; + break; + } + if (SearchSysCacheExists2(STATEXTNAMENSP, + PointerGetDatum(stxname), + ObjectIdGetDatum(namespaceId))) + { + /* Found something else first in path */ + break; + } + } + } + + ReleaseSysCache(stxtup); + + return visible; +} + /* * get_ts_parser_oid - find a TS parser by possibly qualified name * @@ -4235,6 +4302,17 @@ pg_conversion_is_visible(PG_FUNCTION_ARGS) PG_RETURN_BOOL(ConversionIsVisible(oid)); } +Datum +pg_statistic_ext_is_visible(PG_FUNCTION_ARGS) +{ + Oid oid = PG_GETARG_OID(0); + + if (!SearchSysCacheExists1(STATEXTOID, ObjectIdGetDatum(oid))) + PG_RETURN_NULL(); + + PG_RETURN_BOOL(StatisticsObjIsVisible(oid)); +} + Datum pg_ts_parser_is_visible(PG_FUNCTION_ARGS) { diff --git a/src/backend/utils/cache/lsyscache.c b/src/backend/utils/cache/lsyscache.c index 236d876b1c..720e540276 100644 --- a/src/backend/utils/cache/lsyscache.c +++ b/src/backend/utils/cache/lsyscache.c @@ -32,6 +32,7 @@ #include "catalog/pg_proc.h" #include "catalog/pg_range.h" #include "catalog/pg_statistic.h" +#include "catalog/pg_statistic_ext.h" #include "catalog/pg_transform.h" #include "catalog/pg_type.h" #include "miscadmin.h" diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index 09fb30f270..ae3730257d 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -622,7 +622,7 @@ static const SchemaQuery Query_for_list_of_statistics = { /* selcondition */ NULL, /* viscondition */ - NULL, + "pg_catalog.pg_statistic_ext_is_visible(s.oid)", /* namespace */ "s.stxnamespace", /* result */ diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 85ee2f6bde..8e5b95a5ee 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201705121 +#define CATALOG_VERSION_NO 201705122 #endif diff --git a/src/include/catalog/namespace.h b/src/include/catalog/namespace.h index 35e0e2b089..959ee4c50e 100644 --- a/src/include/catalog/namespace.h +++ b/src/include/catalog/namespace.h @@ -92,6 +92,9 @@ extern bool CollationIsVisible(Oid collid); extern Oid ConversionGetConid(const char *conname); extern bool ConversionIsVisible(Oid conid); +extern Oid get_statistics_oid(List *names, bool missing_ok); +extern bool StatisticsObjIsVisible(Oid stxid); + extern Oid get_ts_parser_oid(List *names, bool missing_ok); extern bool TSParserIsVisible(Oid prsId); @@ -141,7 +144,6 @@ extern Oid get_collation_oid(List *collname, bool missing_ok); extern Oid get_conversion_oid(List *conname, bool missing_ok); extern Oid FindDefaultConversionProc(int32 for_encoding, int32 to_encoding); -extern Oid get_statistics_oid(List *names, bool missing_ok); /* initialization & transaction cleanup code */ extern void InitializeSearchPath(void); diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index 77d8ed5184..8d84d9a890 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -3181,6 +3181,8 @@ DATA(insert OID = 3829 ( pg_opfamily_is_visible PGNSP PGUID 12 10 0 0 0 f f f f DESCR("is opfamily visible in search path?"); DATA(insert OID = 2093 ( pg_conversion_is_visible PGNSP PGUID 12 10 0 0 0 f f f f t f s s 1 0 16 "26" _null_ _null_ _null_ _null_ _null_ pg_conversion_is_visible _null_ _null_ _null_ )); DESCR("is conversion visible in search path?"); +DATA(insert OID = 3403 ( pg_statistic_ext_is_visible PGNSP PGUID 12 10 0 0 0 f f f f t f s s 1 0 16 "26" _null_ _null_ _null_ _null_ _null_ pg_statistic_ext_is_visible _null_ _null_ _null_ )); +DESCR("is statistics object visible in search path?"); DATA(insert OID = 3756 ( pg_ts_parser_is_visible PGNSP PGUID 12 10 0 0 0 f f f f t f s s 1 0 16 "26" _null_ _null_ _null_ _null_ _null_ pg_ts_parser_is_visible _null_ _null_ _null_ )); DESCR("is text search parser visible in search path?"); DATA(insert OID = 3757 ( pg_ts_dict_is_visible PGNSP PGUID 12 10 0 0 0 f f f f t f s s 1 0 16 "26" _null_ _null_ _null_ _null_ _null_ pg_ts_dict_is_visible _null_ _null_ _null_ )); -- cgit v1.2.3 From f04c9a61468904b6815b2bc73a48878817766e0e Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 14 May 2017 10:54:47 -0400 Subject: Standardize terminology for pg_statistic_ext entries. Consistently refer to such an entry as a "statistics object", not just "statistics" or "extended statistics". Previously we had a mismash of terms, accompanied by utter confusion as to whether the term was singular or plural. That's not only grating (at least to the ear of a native English speaker) but could be outright misleading, eg in error messages that seemed to be referring to multiple objects where only one could be meant. This commit fixes the code and a lot of comments (though I may have missed a few). I also renamed two new SQL functions, pg_get_statisticsextdef -> pg_get_statisticsobjdef pg_statistic_ext_is_visible -> pg_statistics_obj_is_visible to conform better with this terminology. I have not touched the SGML docs other than fixing those function names; the docs certainly need work but it seems like a separable task. Discussion: https://postgr.es/m/22676.1494557205@sss.pgh.pa.us --- doc/src/sgml/func.sgml | 12 ++++---- src/backend/catalog/aclchk.c | 13 ++++---- src/backend/catalog/namespace.c | 8 ++--- src/backend/catalog/objectaddress.c | 15 +++++----- src/backend/commands/alter.c | 2 +- src/backend/commands/dropcmds.c | 2 +- src/backend/commands/statscmds.c | 18 ++++++------ src/backend/optimizer/util/plancat.c | 6 ++-- src/backend/statistics/README | 9 +++--- src/backend/statistics/README.dependencies | 2 +- src/backend/statistics/dependencies.c | 14 ++++----- src/backend/statistics/extended_stats.c | 44 +++++++++++++++------------- src/backend/statistics/mvdistinct.c | 7 +++-- src/backend/utils/adt/ruleutils.c | 16 +++++----- src/backend/utils/adt/selfuncs.c | 16 +++++----- src/backend/utils/cache/relcache.c | 4 +-- src/bin/pg_dump/pg_dump.c | 2 +- src/bin/psql/describe.c | 4 +-- src/bin/psql/tab-complete.c | 2 +- src/include/catalog/catversion.h | 2 +- src/include/catalog/namespace.h | 2 +- src/include/catalog/pg_proc.h | 6 ++-- src/include/catalog/pg_statistic_ext.h | 10 ++++--- src/include/utils/acl.h | 2 +- src/test/regress/expected/alter_generic.out | 10 +++---- src/test/regress/expected/object_address.out | 4 +-- src/test/regress/expected/rules.out | 4 +-- src/test/regress/expected/stats_ext.out | 14 ++++----- src/test/regress/sql/object_address.sql | 2 +- src/test/regress/sql/rules.sql | 2 +- src/test/regress/sql/stats_ext.sql | 4 +-- 31 files changed, 133 insertions(+), 125 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index d55656769b..14aae736c3 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -16681,7 +16681,7 @@ SELECT relname FROM pg_class WHERE pg_table_is_visible(oid); is operator family visible in search path - pg_statistic_ext_is_visible(stat_oid) + pg_statistics_obj_is_visible(stat_oid) boolean is statistics object visible in search path @@ -16745,7 +16745,7 @@ SELECT relname FROM pg_class WHERE pg_table_is_visible(oid); pg_opfamily_is_visible - pg_statistic_ext_is_visible + pg_statistics_obj_is_visible pg_table_is_visible @@ -16836,7 +16836,7 @@ SELECT pg_type_is_visible('myschema.widget'::regtype); - pg_get_statisticsextdef + pg_get_statisticsobjdef @@ -17009,9 +17009,9 @@ SELECT pg_type_is_visible('myschema.widget'::regtype); uses - pg_get_statisticsextdef(statext_oid) + pg_get_statisticsobjdef(statobj_oid) text - get CREATE STATISTICS command for extended statistics objects + get CREATE STATISTICS command for extended statistics object pg_get_triggerdef(trigger_oid) @@ -17158,7 +17158,7 @@ SELECT pg_type_is_visible('myschema.widget'::regtype); pg_get_constraintdef, pg_get_indexdef, pg_get_ruledef, - pg_get_statisticsextdef, and + pg_get_statisticsobjdef, and pg_get_triggerdef, respectively reconstruct the creating command for a constraint, index, rule, extended statistics object, or trigger. (Note that this is a decompiled reconstruction, not the diff --git a/src/backend/catalog/aclchk.c b/src/backend/catalog/aclchk.c index 32989df2b5..387a3be701 100644 --- a/src/backend/catalog/aclchk.c +++ b/src/backend/catalog/aclchk.c @@ -3321,7 +3321,7 @@ static const char *const no_priv_msg[MAX_ACL_KIND] = /* ACL_KIND_CONVERSION */ gettext_noop("permission denied for conversion %s"), /* ACL_KIND_STATISTICS */ - gettext_noop("permission denied for statistics %s"), + gettext_noop("permission denied for statistics object %s"), /* ACL_KIND_TABLESPACE */ gettext_noop("permission denied for tablespace %s"), /* ACL_KIND_TSDICTIONARY */ @@ -3373,7 +3373,7 @@ static const char *const not_owner_msg[MAX_ACL_KIND] = /* ACL_KIND_CONVERSION */ gettext_noop("must be owner of conversion %s"), /* ACL_KIND_STATISTICS */ - gettext_noop("must be owner of statistics %s"), + gettext_noop("must be owner of statistics object %s"), /* ACL_KIND_TABLESPACE */ gettext_noop("must be owner of tablespace %s"), /* ACL_KIND_TSDICTIONARY */ @@ -3490,7 +3490,7 @@ pg_aclmask(AclObjectKind objkind, Oid table_oid, AttrNumber attnum, Oid roleid, case ACL_KIND_NAMESPACE: return pg_namespace_aclmask(table_oid, roleid, mask, how); case ACL_KIND_STATISTICS: - elog(ERROR, "grantable rights not supported for statistics"); + elog(ERROR, "grantable rights not supported for statistics objects"); /* not reached, but keep compiler quiet */ return ACL_NO_RIGHTS; case ACL_KIND_TABLESPACE: @@ -5130,10 +5130,10 @@ pg_subscription_ownercheck(Oid sub_oid, Oid roleid) } /* - * Ownership check for a extended statistics (specified by OID). + * Ownership check for a statistics object (specified by OID). */ bool -pg_statistics_ownercheck(Oid stat_oid, Oid roleid) +pg_statistics_object_ownercheck(Oid stat_oid, Oid roleid) { HeapTuple tuple; Oid ownerId; @@ -5146,7 +5146,8 @@ pg_statistics_ownercheck(Oid stat_oid, Oid roleid) if (!HeapTupleIsValid(tuple)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), - errmsg("statistics with OID %u do not exist", stat_oid))); + errmsg("statistics object with OID %u does not exist", + stat_oid))); ownerId = ((Form_pg_statistic_ext) GETSTRUCT(tuple))->stxowner; diff --git a/src/backend/catalog/namespace.c b/src/backend/catalog/namespace.c index 5b339222af..2a33eb73fa 100644 --- a/src/backend/catalog/namespace.c +++ b/src/backend/catalog/namespace.c @@ -2087,12 +2087,12 @@ ConversionIsVisible(Oid conid) } /* - * get_statistics_oid - find a statistics by possibly qualified name + * get_statistics_object_oid - find a statistics object by possibly qualified name * * If not found, returns InvalidOid if missing_ok, else throws error */ Oid -get_statistics_oid(List *names, bool missing_ok) +get_statistics_object_oid(List *names, bool missing_ok) { char *schemaname; char *stats_name; @@ -2136,7 +2136,7 @@ get_statistics_oid(List *names, bool missing_ok) if (!OidIsValid(stats_oid) && !missing_ok) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), - errmsg("statistics \"%s\" do not exist", + errmsg("statistics object \"%s\" does not exist", NameListToString(names)))); return stats_oid; @@ -4303,7 +4303,7 @@ pg_conversion_is_visible(PG_FUNCTION_ARGS) } Datum -pg_statistic_ext_is_visible(PG_FUNCTION_ARGS) +pg_statistics_obj_is_visible(PG_FUNCTION_ARGS) { Oid oid = PG_GETARG_OID(0); diff --git a/src/backend/catalog/objectaddress.c b/src/backend/catalog/objectaddress.c index a373df743b..e3efb98b1d 100644 --- a/src/backend/catalog/objectaddress.c +++ b/src/backend/catalog/objectaddress.c @@ -712,7 +712,7 @@ static const struct object_type_map }, /* OBJECT_STATISTIC_EXT */ { - "statistics", OBJECT_STATISTIC_EXT + "statistics object", OBJECT_STATISTIC_EXT } }; @@ -993,8 +993,8 @@ get_object_address(ObjectType objtype, Node *object, break; case OBJECT_STATISTIC_EXT: address.classId = StatisticExtRelationId; - address.objectId = get_statistics_oid(castNode(List, object), - missing_ok); + address.objectId = get_statistics_object_oid(castNode(List, object), + missing_ok); address.objectSubId = 0; break; default: @@ -2398,7 +2398,7 @@ check_object_ownership(Oid roleid, ObjectType objtype, ObjectAddress address, errmsg("must be superuser"))); break; case OBJECT_STATISTIC_EXT: - if (!pg_statistics_ownercheck(address.objectId, roleid)) + if (!pg_statistics_object_ownercheck(address.objectId, roleid)) aclcheck_error_type(ACLCHECK_NOT_OWNER, address.objectId); break; default: @@ -3907,11 +3907,12 @@ getObjectTypeDescription(const ObjectAddress *object) break; case OCLASS_STATISTIC_EXT: - appendStringInfoString(&buffer, "statistics"); + appendStringInfoString(&buffer, "statistics object"); break; default: - appendStringInfo(&buffer, "unrecognized %u", object->classId); + appendStringInfo(&buffer, "unrecognized object class %u", + object->classId); break; } @@ -4946,7 +4947,7 @@ getObjectIdentityParts(const ObjectAddress *object, tup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(object->objectId)); if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for statistics %u", + elog(ERROR, "cache lookup failed for statistics object %u", object->objectId); formStatistic = (Form_pg_statistic_ext) GETSTRUCT(tup); schema = get_namespace_name_or_temp(formStatistic->stxnamespace); diff --git a/src/backend/commands/alter.c b/src/backend/commands/alter.c index 2c6435b759..bc897e409c 100644 --- a/src/backend/commands/alter.c +++ b/src/backend/commands/alter.c @@ -123,7 +123,7 @@ report_namespace_conflict(Oid classId, const char *name, Oid nspOid) break; case StatisticExtRelationId: Assert(OidIsValid(nspOid)); - msgfmt = gettext_noop("statistics \"%s\" already exists in schema \"%s\""); + msgfmt = gettext_noop("statistics object \"%s\" already exists in schema \"%s\""); break; case TSParserRelationId: Assert(OidIsValid(nspOid)); diff --git a/src/backend/commands/dropcmds.c b/src/backend/commands/dropcmds.c index c18bc319be..a1a64fa8c9 100644 --- a/src/backend/commands/dropcmds.c +++ b/src/backend/commands/dropcmds.c @@ -289,7 +289,7 @@ does_not_exist_skipping(ObjectType objtype, Node *object) case OBJECT_STATISTIC_EXT: if (!schema_does_not_exist_skipping(castNode(List, object), &msg, &name)) { - msg = gettext_noop("extended statistics \"%s\" do not exist, skipping"); + msg = gettext_noop("statistics object \"%s\" does not exist, skipping"); name = NameListToString(castNode(List, object)); } break; diff --git a/src/backend/commands/statscmds.c b/src/backend/commands/statscmds.c index db5f4278d5..cab54c962c 100644 --- a/src/backend/commands/statscmds.c +++ b/src/backend/commands/statscmds.c @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * statscmds.c - * Commands for creating and altering extended statistics + * Commands for creating and altering extended statistics objects * * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California @@ -64,7 +64,7 @@ CreateStatistics(CreateStatsStmt *stmt) Oid relid; ObjectAddress parentobject, myself; - Datum types[2]; /* one for each possible type of statistics */ + Datum types[2]; /* one for each possible type of statistic */ int ntypes; ArrayType *stxkind; bool build_ndistinct; @@ -80,7 +80,7 @@ CreateStatistics(CreateStatsStmt *stmt) namestrcpy(&stxname, namestr); /* - * Deal with the possibility that the named statistics already exist. + * Deal with the possibility that the statistics object already exists. */ if (SearchSysCacheExists2(STATEXTNAMENSP, NameGetDatum(&stxname), @@ -90,14 +90,14 @@ CreateStatistics(CreateStatsStmt *stmt) { ereport(NOTICE, (errcode(ERRCODE_DUPLICATE_OBJECT), - errmsg("statistics \"%s\" already exist, skipping", + errmsg("statistics object \"%s\" already exists, skipping", namestr))); return InvalidObjectAddress; } ereport(ERROR, (errcode(ERRCODE_DUPLICATE_OBJECT), - errmsg("statistics \"%s\" already exist", namestr))); + errmsg("statistics object \"%s\" already exists", namestr))); } /* @@ -263,7 +263,7 @@ CreateStatistics(CreateStatsStmt *stmt) else ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("unrecognized statistics type \"%s\"", + errmsg("unrecognized statistic type \"%s\"", type))); } /* If no statistic type was specified, build them all. */ @@ -307,7 +307,7 @@ CreateStatistics(CreateStatsStmt *stmt) relation_close(statrel, RowExclusiveLock); /* - * Invalidate relcache so that others see the new statistics. + * Invalidate relcache so that others see the new statistics object. */ CacheInvalidateRelcache(rel); @@ -346,7 +346,7 @@ CreateStatistics(CreateStatsStmt *stmt) } /* - * Guts of statistics deletion. + * Guts of statistics object deletion. */ void RemoveStatisticsById(Oid statsOid) @@ -365,7 +365,7 @@ RemoveStatisticsById(Oid statsOid) tup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(statsOid)); if (!HeapTupleIsValid(tup)) /* should not happen */ - elog(ERROR, "cache lookup failed for statistics %u", statsOid); + elog(ERROR, "cache lookup failed for statistics object %u", statsOid); statext = (Form_pg_statistic_ext) GETSTRUCT(tup); relid = statext->stxrelid; diff --git a/src/backend/optimizer/util/plancat.c b/src/backend/optimizer/util/plancat.c index 9207c8d809..2a5ec181de 100644 --- a/src/backend/optimizer/util/plancat.c +++ b/src/backend/optimizer/util/plancat.c @@ -1285,13 +1285,13 @@ get_relation_statistics(RelOptInfo *rel, Relation relation) htup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(statOid)); if (!htup) - elog(ERROR, "cache lookup failed for statistics %u", statOid); + elog(ERROR, "cache lookup failed for statistics object %u", statOid); staForm = (Form_pg_statistic_ext) GETSTRUCT(htup); /* * First, build the array of columns covered. This is ultimately - * wasted if no stats are actually built, but it doesn't seem worth - * troubling over that case. + * wasted if no stats within the object have actually been built, but + * it doesn't seem worth troubling over that case. */ for (i = 0; i < staForm->stxkeys.dim1; i++) keys = bms_add_member(keys, staForm->stxkeys.values[i]); diff --git a/src/backend/statistics/README b/src/backend/statistics/README index af7651127e..a8f00a590e 100644 --- a/src/backend/statistics/README +++ b/src/backend/statistics/README @@ -12,7 +12,7 @@ hopefully improving the estimates and producing better plans. Types of statistics ------------------- -There are two kinds of extended statistics: +There are currently two kinds of extended statistics: (a) ndistinct coefficients @@ -36,7 +36,7 @@ Complex clauses We also support estimating more complex clauses - essentially AND/OR clauses with (Var op Const) as leaves, as long as all the referenced attributes are -covered by a single statistics. +covered by a single statistics object. For example this condition @@ -59,7 +59,7 @@ Selectivity estimation Throughout the planner clauselist_selectivity() still remains in charge of most selectivity estimate requests. clauselist_selectivity() can be instructed to try to make use of any extended statistics on the given RelOptInfo, which -it will do, if: +it will do if: (a) An actual valid RelOptInfo was given. Join relations are passed in as NULL, therefore are invalid. @@ -77,6 +77,7 @@ performing estimations knows which clauses are to be skipped. Size of sample in ANALYZE ------------------------- + When performing ANALYZE, the number of rows to sample is determined as (300 * statistics_target) @@ -93,4 +94,4 @@ those are not necessarily limited by statistics_target. This however merits further discussion, because collecting the sample is quite expensive and increasing it further would make ANALYZE even more painful. Judging by the experiments with the current implementation, the fixed size -seems to work reasonably well for now, so we leave this as a future work. +seems to work reasonably well for now, so we leave this as future work. diff --git a/src/backend/statistics/README.dependencies b/src/backend/statistics/README.dependencies index 7bc2533dc6..59f9d57657 100644 --- a/src/backend/statistics/README.dependencies +++ b/src/backend/statistics/README.dependencies @@ -48,7 +48,7 @@ rendering the approach mostly useless even for slightly noisy data sets, or result in sudden changes in behavior depending on minor differences between samples provided to ANALYZE. -For this reason, the statistics implements "soft" functional dependencies, +For this reason, extended statistics implement "soft" functional dependencies, associating each functional dependency with a degree of validity (a number between 0 and 1). This degree is then used to combine selectivities in a smooth manner. diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c index fe9a9ef5de..0e71f058ad 100644 --- a/src/backend/statistics/dependencies.c +++ b/src/backend/statistics/dependencies.c @@ -342,8 +342,8 @@ dependency_degree(int numrows, HeapTuple *rows, int k, AttrNumber *dependency, * detects functional dependencies between groups of columns * * Generates all possible subsets of columns (variations) and computes - * the degree of validity for each one. For example with a statistic on - * three columns (a,b,c) there are 9 possible dependencies + * the degree of validity for each one. For example when creating statistics + * on three columns (a,b,c) there are 9 possible dependencies * * two columns three columns * ----------- ------------- @@ -383,8 +383,8 @@ statext_dependencies_build(int numrows, HeapTuple *rows, Bitmapset *attrs, /* * We'll try build functional dependencies starting from the smallest ones * covering just 2 columns, to the largest ones, covering all columns - * included in the statistics. We start from the smallest ones because we - * want to be able to skip already implied ones. + * included in the statistics object. We start from the smallest ones + * because we want to be able to skip already implied ones. */ for (k = 2; k <= numattrs; k++) { @@ -644,7 +644,7 @@ staext_dependencies_load(Oid mvoid) HeapTuple htup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(mvoid)); if (!HeapTupleIsValid(htup)) - elog(ERROR, "cache lookup failed for extended statistics %u", mvoid); + elog(ERROR, "cache lookup failed for statistics object %u", mvoid); deps = SysCacheGetAttr(STATEXTOID, htup, Anum_pg_statistic_ext_stxdependencies, &isnull); @@ -975,7 +975,7 @@ dependencies_clauselist_selectivity(PlannerInfo *root, return 1.0; } - /* find the best suited statistics for these attnums */ + /* find the best suited statistics object for these attnums */ stat = choose_best_statistics(rel->statlist, clauses_attnums, STATS_EXT_DEPENDENCIES); @@ -986,7 +986,7 @@ dependencies_clauselist_selectivity(PlannerInfo *root, return 1.0; } - /* load the dependency items stored in the statistics */ + /* load the dependency items stored in the statistics object */ dependencies = staext_dependencies_load(stat->statOid); /* diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c index b334140c48..3f74cee05f 100644 --- a/src/backend/statistics/extended_stats.c +++ b/src/backend/statistics/extended_stats.c @@ -3,7 +3,7 @@ * extended_stats.c * POSTGRES extended statistics * - * Generic code supporting statistic objects created via CREATE STATISTICS. + * Generic code supporting statistics objects created via CREATE STATISTICS. * * * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group @@ -35,14 +35,15 @@ /* - * Used internally to refer to an individual pg_statistic_ext entry. + * Used internally to refer to an individual statistics object, i.e., + * a pg_statistic_ext entry. */ typedef struct StatExtEntry { Oid statOid; /* OID of pg_statistic_ext entry */ - char *schema; /* statistics schema */ - char *name; /* statistics name */ - Bitmapset *columns; /* attribute numbers covered by the statistics */ + char *schema; /* statistics object's schema */ + char *name; /* statistics object's name */ + Bitmapset *columns; /* attribute numbers covered by the object */ List *types; /* 'char' list of enabled statistic kinds */ } StatExtEntry; @@ -59,8 +60,8 @@ static void statext_store(Relation pg_stext, Oid relid, * Compute requested extended stats, using the rows sampled for the plain * (single-column) stats. * - * This fetches a list of stats from pg_statistic_ext, computes the stats - * and serializes them back into the catalog (as bytea values). + * This fetches a list of stats types from pg_statistic_ext, computes the + * requested stats, and serializes them back into the catalog. */ void BuildRelationExtStatistics(Relation onerel, double totalrows, @@ -98,7 +99,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows, { ereport(WARNING, (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), - errmsg("extended statistics \"%s.%s\" could not be collected for relation %s.%s", + errmsg("statistics object \"%s.%s\" could not be computed for relation \"%s.%s\"", stat->schema, stat->name, get_namespace_name(onerel->rd_rel->relnamespace), RelationGetRelationName(onerel)), @@ -110,7 +111,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows, Assert(bms_num_members(stat->columns) >= 2 && bms_num_members(stat->columns) <= STATS_MAX_DIMENSIONS); - /* compute statistic of each type */ + /* compute statistic of each requested type */ foreach(lc2, stat->types) { char t = (char) lfirst_int(lc2); @@ -160,7 +161,7 @@ statext_is_kind_built(HeapTuple htup, char type) } /* - * Return a list (of StatExtEntry) of statistics for the given relation. + * Return a list (of StatExtEntry) of statistics objects for the given relation. */ static List * fetch_statentries_for_relation(Relation pg_statext, Oid relid) @@ -171,7 +172,7 @@ fetch_statentries_for_relation(Relation pg_statext, Oid relid) List *result = NIL; /* - * Prepare to scan pg_statistic_ext for entries having indrelid = this + * Prepare to scan pg_statistic_ext for entries having stxrelid = this * rel. */ ScanKeyInit(&skey, @@ -329,7 +330,7 @@ statext_store(Relation pg_stext, Oid statOid, /* there should already be a pg_statistic_ext tuple */ oldtup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(statOid)); if (!HeapTupleIsValid(oldtup)) - elog(ERROR, "cache lookup failed for extended statistics %u", statOid); + elog(ERROR, "cache lookup failed for statistics object %u", statOid); /* replace it */ stup = heap_modify_tuple(oldtup, @@ -433,7 +434,7 @@ multi_sort_compare_dims(int start, int end, /* * has_stats_of_kind - * Check that the list contains statistic of a given kind + * Check whether the list contains statistic of a given kind */ bool has_stats_of_kind(List *stats, char requiredkind) @@ -458,11 +459,12 @@ has_stats_of_kind(List *stats, char requiredkind) * there's no match. * * The current selection criteria is very simple - we choose the statistics - * referencing the most attributes with the least keys. + * object referencing the most of the requested attributes, breaking ties + * in favor of objects with fewer keys overall. * - * XXX if multiple statistics exists of the same size matching the same number - * of keys, then the statistics which are chosen depend on the order that they - * appear in the stats list. Perhaps this needs to be more definitive. + * XXX if multiple statistics objects tie on both criteria, then which object + * is chosen depends on the order that they appear in the stats list. Perhaps + * further tiebreakers are needed. */ StatisticExtInfo * choose_best_statistics(List *stats, Bitmapset *attnums, char requiredkind) @@ -479,7 +481,7 @@ choose_best_statistics(List *stats, Bitmapset *attnums, char requiredkind) int numkeys; Bitmapset *matched; - /* skip statistics that are not the correct type */ + /* skip statistics that are not of the correct type */ if (info->kind != requiredkind) continue; @@ -495,9 +497,9 @@ choose_best_statistics(List *stats, Bitmapset *attnums, char requiredkind) numkeys = bms_num_members(info->keys); /* - * Use these statistics when it increases the number of matched - * clauses or when it matches the same number of attributes but these - * stats have fewer keys than any previous match. + * Use this object when it increases the number of matched clauses or + * when it matches the same number of attributes but these stats have + * fewer keys than any previous match. */ if (num_matched > best_num_matched || (num_matched == best_num_matched && numkeys < best_match_keys)) diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c index f67f576236..47b2490abb 100644 --- a/src/backend/statistics/mvdistinct.c +++ b/src/backend/statistics/mvdistinct.c @@ -9,7 +9,7 @@ * The multivariate ndistinct coefficients address this by storing ndistinct * estimates for combinations of the user-specified columns. So for example * given a statistics object on three columns (a,b,c), this module estimates - * and store n-distinct for (a,b), (a,c), (b,c) and (a,b,c). The per-column + * and stores n-distinct for (a,b), (a,c), (b,c) and (a,b,c). The per-column * estimates are already available in pg_statistic. * * @@ -18,6 +18,7 @@ * * IDENTIFICATION * src/backend/statistics/mvdistinct.c + * *------------------------------------------------------------------------- */ #include "postgres.h" @@ -131,13 +132,13 @@ statext_ndistinct_load(Oid mvoid) htup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(mvoid)); if (!htup) - elog(ERROR, "cache lookup failed for statistics %u", mvoid); + elog(ERROR, "cache lookup failed for statistics object %u", mvoid); ndist = SysCacheGetAttr(STATEXTOID, htup, Anum_pg_statistic_ext_stxndistinct, &isnull); if (isnull) elog(ERROR, - "requested statistic kind %c not yet built for statistics %u", + "requested statistic kind %c is not yet built for statistics object %u", STATS_EXT_NDISTINCT, mvoid); ReleaseSysCache(htup); diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c index c9bded082e..43b1475035 100644 --- a/src/backend/utils/adt/ruleutils.c +++ b/src/backend/utils/adt/ruleutils.c @@ -319,7 +319,7 @@ static char *pg_get_indexdef_worker(Oid indexrelid, int colno, const Oid *excludeOps, bool attrsOnly, bool showTblSpc, int prettyFlags, bool missing_ok); -static char *pg_get_statisticsext_worker(Oid statextid, bool missing_ok); +static char *pg_get_statisticsobj_worker(Oid statextid, bool missing_ok); static char *pg_get_partkeydef_worker(Oid relid, int prettyFlags, bool attrsOnly, bool missing_ok); static char *pg_get_constraintdef_worker(Oid constraintId, bool fullCommand, @@ -1425,16 +1425,16 @@ pg_get_indexdef_worker(Oid indexrelid, int colno, } /* - * pg_get_statisticsextdef + * pg_get_statisticsobjdef * Get the definition of an extended statistics object */ Datum -pg_get_statisticsextdef(PG_FUNCTION_ARGS) +pg_get_statisticsobjdef(PG_FUNCTION_ARGS) { Oid statextid = PG_GETARG_OID(0); char *res; - res = pg_get_statisticsext_worker(statextid, true); + res = pg_get_statisticsobj_worker(statextid, true); if (res == NULL) PG_RETURN_NULL(); @@ -1446,7 +1446,7 @@ pg_get_statisticsextdef(PG_FUNCTION_ARGS) * Internal workhorse to decompile an extended statistics object. */ static char * -pg_get_statisticsext_worker(Oid statextid, bool missing_ok) +pg_get_statisticsobj_worker(Oid statextid, bool missing_ok) { Form_pg_statistic_ext statextrec; HeapTuple statexttup; @@ -1467,7 +1467,7 @@ pg_get_statisticsext_worker(Oid statextid, bool missing_ok) { if (missing_ok) return NULL; - elog(ERROR, "cache lookup failed for extended statistics %u", statextid); + elog(ERROR, "cache lookup failed for statistics object %u", statextid); } statextrec = (Form_pg_statistic_ext) GETSTRUCT(statexttup); @@ -1480,8 +1480,7 @@ pg_get_statisticsext_worker(Oid statextid, bool missing_ok) NameStr(statextrec->stxname))); /* - * Lookup the stxkind column so that we know how to handle the WITH - * clause. + * Decode the stxkind column so that we know which stats types to print. */ datum = SysCacheGetAttr(STATEXTOID, statexttup, Anum_pg_statistic_ext_stxkind, &isnull); @@ -1518,7 +1517,6 @@ pg_get_statisticsext_worker(Oid statextid, bool missing_ok) appendStringInfoString(&buf, "ndistinct"); else if (dependencies_enabled) appendStringInfoString(&buf, "dependencies"); - appendStringInfoChar(&buf, ')'); } diff --git a/src/backend/utils/adt/selfuncs.c b/src/backend/utils/adt/selfuncs.c index 6c4cef9fb3..7028d6387c 100644 --- a/src/backend/utils/adt/selfuncs.c +++ b/src/backend/utils/adt/selfuncs.c @@ -3707,25 +3707,27 @@ estimate_multivariate_ndistinct(PlannerInfo *root, RelOptInfo *rel, { StatisticExtInfo *info = (StatisticExtInfo *) lfirst(lc); Bitmapset *shared; + int nshared; /* skip statistics of other kinds */ if (info->kind != STATS_EXT_NDISTINCT) continue; - /* compute attnums shared by the vars and the statistic */ + /* compute attnums shared by the vars and the statistics object */ shared = bms_intersect(info->keys, attnums); + nshared = bms_num_members(shared); /* - * Does this statistics matches more columns than the currently - * best statistic? If so, use this one instead. + * Does this statistics object match more columns than the currently + * best object? If so, use this one instead. * - * XXX This should break ties using name of the statistic, or - * something like that, to make the outcome stable. + * XXX This should break ties using name of the object, or something + * like that, to make the outcome stable. */ - if (bms_num_members(shared) > nmatches) + if (nshared > nmatches) { statOid = info->statOid; - nmatches = bms_num_members(shared); + nmatches = nshared; matched = shared; } } diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c index c3721d9e43..0cd6289f91 100644 --- a/src/backend/utils/cache/relcache.c +++ b/src/backend/utils/cache/relcache.c @@ -4462,7 +4462,7 @@ RelationGetIndexList(Relation relation) /* * RelationGetStatExtList - * get a list of OIDs of extended statistics on this relation + * get a list of OIDs of statistics objects on this relation * * The statistics list is created only if someone requests it, in a way * similar to RelationGetIndexList(). We scan pg_statistic_ext to find @@ -4470,7 +4470,7 @@ RelationGetIndexList(Relation relation) * won't have to compute it again. Note that shared cache inval of a * relcache entry will delete the old list and set rd_statvalid to 0, * so that we must recompute the statistics list on next request. This - * handles creation or deletion of a statistic. + * handles creation or deletion of a statistics object. * * The returned list is guaranteed to be sorted in order by OID, although * this is not currently needed. diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index b26358c770..b6794d06de 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -6713,7 +6713,7 @@ getExtendedStatistics(Archive *fout, TableInfo tblinfo[], int numTables) "tableoid, " "oid, " "stxname, " - "pg_catalog.pg_get_statisticsextdef(oid) AS stxdef " + "pg_catalog.pg_get_statisticsobjdef(oid) AS stxdef " "FROM pg_statistic_ext " "WHERE stxrelid = '%u' " "ORDER BY stxname", tbinfo->dobj.catId.oid); diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index cb17407036..b9d395b4b4 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -2402,7 +2402,7 @@ describeOneTableDetails(const char *schemaname, if (tuples > 0) { - printTableAddFooter(&cont, _("Statistics:")); + printTableAddFooter(&cont, _("Statistics objects:")); for (i = 0; i < tuples; i++) { @@ -2410,7 +2410,7 @@ describeOneTableDetails(const char *schemaname, printfPQExpBuffer(&buf, " "); - /* statistics name (qualified with namespace) */ + /* statistics object name (qualified with namespace) */ appendPQExpBuffer(&buf, "\"%s\".\"%s\" (", PQgetvalue(result, i, 2), PQgetvalue(result, i, 3)); diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index ae3730257d..b9e3491aec 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -622,7 +622,7 @@ static const SchemaQuery Query_for_list_of_statistics = { /* selcondition */ NULL, /* viscondition */ - "pg_catalog.pg_statistic_ext_is_visible(s.oid)", + "pg_catalog.pg_statistics_obj_is_visible(s.oid)", /* namespace */ "s.stxnamespace", /* result */ diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 7adfd328f6..06fff799af 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201705131 +#define CATALOG_VERSION_NO 201705141 #endif diff --git a/src/include/catalog/namespace.h b/src/include/catalog/namespace.h index 959ee4c50e..5294a52984 100644 --- a/src/include/catalog/namespace.h +++ b/src/include/catalog/namespace.h @@ -92,7 +92,7 @@ extern bool CollationIsVisible(Oid collid); extern Oid ConversionGetConid(const char *conname); extern bool ConversionIsVisible(Oid conid); -extern Oid get_statistics_oid(List *names, bool missing_ok); +extern Oid get_statistics_object_oid(List *names, bool missing_ok); extern bool StatisticsObjIsVisible(Oid stxid); extern Oid get_ts_parser_oid(List *names, bool missing_ok); diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index 74346963d9..5c38ab55ec 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -1987,9 +1987,9 @@ DESCR("select statement of a view"); DATA(insert OID = 1642 ( pg_get_userbyid PGNSP PGUID 12 1 0 0 0 f f f f t f s s 1 0 19 "26" _null_ _null_ _null_ _null_ _null_ pg_get_userbyid _null_ _null_ _null_ )); DESCR("role name by OID (with fallback)"); DATA(insert OID = 1643 ( pg_get_indexdef PGNSP PGUID 12 1 0 0 0 f f f f t f s s 1 0 25 "26" _null_ _null_ _null_ _null_ _null_ pg_get_indexdef _null_ _null_ _null_ )); -DESCR("extended statistics description"); -DATA(insert OID = 3415 ( pg_get_statisticsextdef PGNSP PGUID 12 1 0 0 0 f f f f t f s s 1 0 25 "26" _null_ _null_ _null_ _null_ _null_ pg_get_statisticsextdef _null_ _null_ _null_ )); DESCR("index description"); +DATA(insert OID = 3415 ( pg_get_statisticsobjdef PGNSP PGUID 12 1 0 0 0 f f f f t f s s 1 0 25 "26" _null_ _null_ _null_ _null_ _null_ pg_get_statisticsobjdef _null_ _null_ _null_ )); +DESCR("extended statistics object description"); DATA(insert OID = 3352 ( pg_get_partkeydef PGNSP PGUID 12 1 0 0 0 f f f f t f s s 1 0 25 "26" _null_ _null_ _null_ _null_ _null_ pg_get_partkeydef _null_ _null_ _null_ )); DESCR("partition key description"); DATA(insert OID = 3408 ( pg_get_partition_constraintdef PGNSP PGUID 12 1 0 0 0 f f f f t f s s 1 0 25 "26" _null_ _null_ _null_ _null_ _null_ pg_get_partition_constraintdef _null_ _null_ _null_ )); @@ -3183,7 +3183,7 @@ DATA(insert OID = 3829 ( pg_opfamily_is_visible PGNSP PGUID 12 10 0 0 0 f f f f DESCR("is opfamily visible in search path?"); DATA(insert OID = 2093 ( pg_conversion_is_visible PGNSP PGUID 12 10 0 0 0 f f f f t f s s 1 0 16 "26" _null_ _null_ _null_ _null_ _null_ pg_conversion_is_visible _null_ _null_ _null_ )); DESCR("is conversion visible in search path?"); -DATA(insert OID = 3403 ( pg_statistic_ext_is_visible PGNSP PGUID 12 10 0 0 0 f f f f t f s s 1 0 16 "26" _null_ _null_ _null_ _null_ _null_ pg_statistic_ext_is_visible _null_ _null_ _null_ )); +DATA(insert OID = 3403 ( pg_statistics_obj_is_visible PGNSP PGUID 12 10 0 0 0 f f f f t f s s 1 0 16 "26" _null_ _null_ _null_ _null_ _null_ pg_statistics_obj_is_visible _null_ _null_ _null_ )); DESCR("is statistics object visible in search path?"); DATA(insert OID = 3756 ( pg_ts_parser_is_visible PGNSP PGUID 12 10 0 0 0 f f f f t f s s 1 0 16 "26" _null_ _null_ _null_ _null_ _null_ pg_ts_parser_is_visible _null_ _null_ _null_ )); DESCR("is text search parser visible in search path?"); diff --git a/src/include/catalog/pg_statistic_ext.h b/src/include/catalog/pg_statistic_ext.h index b0fda076fe..f08379699e 100644 --- a/src/include/catalog/pg_statistic_ext.h +++ b/src/include/catalog/pg_statistic_ext.h @@ -30,11 +30,13 @@ CATALOG(pg_statistic_ext,3381) { - /* These fields form the unique key for the entry: */ Oid stxrelid; /* relation containing attributes */ - NameData stxname; /* statistics name */ - Oid stxnamespace; /* OID of namespace containing this statistics */ - Oid stxowner; /* statistics owner */ + + /* These two fields form the unique key for the entry: */ + NameData stxname; /* statistics object name */ + Oid stxnamespace; /* OID of statistics object's namespace */ + + Oid stxowner; /* statistics object's owner */ /* * variable-length fields start here, but we allow direct access to diff --git a/src/include/utils/acl.h b/src/include/utils/acl.h index c957d8e170..2487020ec1 100644 --- a/src/include/utils/acl.h +++ b/src/include/utils/acl.h @@ -327,7 +327,7 @@ extern bool pg_event_trigger_ownercheck(Oid et_oid, Oid roleid); extern bool pg_extension_ownercheck(Oid ext_oid, Oid roleid); extern bool pg_publication_ownercheck(Oid pub_oid, Oid roleid); extern bool pg_subscription_ownercheck(Oid sub_oid, Oid roleid); -extern bool pg_statistics_ownercheck(Oid stat_oid, Oid roleid); +extern bool pg_statistics_object_ownercheck(Oid stat_oid, Oid roleid); extern bool has_createrole_privilege(Oid roleid); extern bool has_bypassrls_privilege(Oid roleid); diff --git a/src/test/regress/expected/alter_generic.out b/src/test/regress/expected/alter_generic.out index 28e69166be..62347bc47e 100644 --- a/src/test/regress/expected/alter_generic.out +++ b/src/test/regress/expected/alter_generic.out @@ -504,7 +504,7 @@ CREATE TABLE alt_regress_1 (a INTEGER, b INTEGER); CREATE STATISTICS alt_stat1 ON a, b FROM alt_regress_1; CREATE STATISTICS alt_stat2 ON a, b FROM alt_regress_1; ALTER STATISTICS alt_stat1 RENAME TO alt_stat2; -- failed (name conflict) -ERROR: statistics "alt_stat2" already exists in schema "alt_nsp1" +ERROR: statistics object "alt_stat2" already exists in schema "alt_nsp1" ALTER STATISTICS alt_stat1 RENAME TO alt_stat3; -- failed (name conflict) ALTER STATISTICS alt_stat2 OWNER TO regress_alter_user2; -- failed (no role membership) ERROR: must be member of role "regress_alter_user2" @@ -515,16 +515,16 @@ CREATE TABLE alt_regress_2 (a INTEGER, b INTEGER); CREATE STATISTICS alt_stat1 ON a, b FROM alt_regress_2; CREATE STATISTICS alt_stat2 ON a, b FROM alt_regress_2; ALTER STATISTICS alt_stat3 RENAME TO alt_stat4; -- failed (not owner) -ERROR: must be owner of statistics alt_stat3 +ERROR: must be owner of statistics object alt_stat3 ALTER STATISTICS alt_stat1 RENAME TO alt_stat4; -- OK ALTER STATISTICS alt_stat3 OWNER TO regress_alter_user2; -- failed (not owner) -ERROR: must be owner of statistics alt_stat3 +ERROR: must be owner of statistics object alt_stat3 ALTER STATISTICS alt_stat2 OWNER TO regress_alter_user3; -- failed (no role membership) ERROR: must be member of role "regress_alter_user3" ALTER STATISTICS alt_stat3 SET SCHEMA alt_nsp2; -- failed (not owner) -ERROR: must be owner of statistics alt_stat3 +ERROR: must be owner of statistics object alt_stat3 ALTER STATISTICS alt_stat2 SET SCHEMA alt_nsp2; -- failed (name conflict) -ERROR: statistics "alt_stat2" already exists in schema "alt_nsp2" +ERROR: statistics object "alt_stat2" already exists in schema "alt_nsp2" RESET SESSION AUTHORIZATION; SELECT nspname, stxname, rolname FROM pg_statistic_ext s, pg_namespace n, pg_authid a diff --git a/src/test/regress/expected/object_address.out b/src/test/regress/expected/object_address.out index 3ca5c764ca..8c56512007 100644 --- a/src/test/regress/expected/object_address.out +++ b/src/test/regress/expected/object_address.out @@ -411,7 +411,7 @@ WITH objects (type, name, args) AS (VALUES ('publication', '{addr_pub}', '{}'), ('publication relation', '{addr_nsp, gentable}', '{addr_pub}'), ('subscription', '{addr_sub}', '{}'), - ('statistics', '{addr_nsp, gentable_stat}', '{}') + ('statistics object', '{addr_nsp, gentable_stat}', '{}') ) SELECT (pg_identify_object(addr1.classid, addr1.objid, addr1.objsubid)).*, -- test roundtrip through pg_identify_object_as_address @@ -459,7 +459,7 @@ SELECT (pg_identify_object(addr1.classid, addr1.objid, addr1.objsubid)).*, trigger | | | t on addr_nsp.gentable | t operator family | pg_catalog | integer_ops | pg_catalog.integer_ops USING btree | t policy | | | genpol on addr_nsp.gentable | t - statistics | addr_nsp | gentable_stat | addr_nsp.gentable_stat | t + statistics object | addr_nsp | gentable_stat | addr_nsp.gentable_stat | t collation | pg_catalog | "default" | pg_catalog."default" | t transform | | | for integer on language sql | t text search dictionary | addr_nsp | addr_ts_dict | addr_nsp.addr_ts_dict | t diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index a4b2d8635d..912360d70a 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -3166,8 +3166,8 @@ SELECT pg_get_ruledef(0); (1 row) -SELECT pg_get_statisticsextdef(0); - pg_get_statisticsextdef +SELECT pg_get_statisticsobjdef(0); + pg_get_statisticsobjdef ------------------------- (1 row) diff --git a/src/test/regress/expected/stats_ext.out b/src/test/regress/expected/stats_ext.out index 13780897c5..564521957f 100644 --- a/src/test/regress/expected/stats_ext.out +++ b/src/test/regress/expected/stats_ext.out @@ -29,16 +29,16 @@ ERROR: only simple column references are allowed in CREATE STATISTICS CREATE STATISTICS tst ON (relpages, reltuples) FROM pg_class; ERROR: only simple column references are allowed in CREATE STATISTICS CREATE STATISTICS tst (unrecognized) ON relname, relnatts FROM pg_class; -ERROR: unrecognized statistics type "unrecognized" +ERROR: unrecognized statistic type "unrecognized" -- Ensure stats are dropped sanely CREATE TABLE ab1 (a INTEGER, b INTEGER, c INTEGER); CREATE STATISTICS ab1_a_b_stats ON a, b FROM ab1; DROP STATISTICS ab1_a_b_stats; CREATE SCHEMA regress_schema_2; CREATE STATISTICS regress_schema_2.ab1_a_b_stats ON a, b FROM ab1; --- Let's also verify the pg_get_statisticsextdef output looks sane. -SELECT pg_get_statisticsextdef(oid) FROM pg_statistic_ext WHERE stxname = 'ab1_a_b_stats'; - pg_get_statisticsextdef +-- Let's also verify the pg_get_statisticsobjdef output looks sane. +SELECT pg_get_statisticsobjdef(oid) FROM pg_statistic_ext WHERE stxname = 'ab1_a_b_stats'; + pg_get_statisticsobjdef ------------------------------------------------------------------- CREATE STATISTICS regress_schema_2.ab1_a_b_stats ON a, b FROM ab1 (1 row) @@ -55,7 +55,7 @@ ALTER TABLE ab1 DROP COLUMN a; --------+---------+-----------+----------+--------- b | integer | | | c | integer | | | -Statistics: +Statistics objects: "public"."ab1_b_c_stats" (ndistinct, dependencies) ON b, c FROM ab1 -- Ensure statistics are dropped when table is @@ -77,11 +77,11 @@ ALTER TABLE ab1 ALTER a SET STATISTICS 0; INSERT INTO ab1 SELECT a, a%23 FROM generate_series(1, 1000) a; CREATE STATISTICS ab1_a_b_stats ON a, b FROM ab1; ANALYZE ab1; -WARNING: extended statistics "public.ab1_a_b_stats" could not be collected for relation public.ab1 +WARNING: statistics object "public.ab1_a_b_stats" could not be computed for relation "public.ab1" ALTER TABLE ab1 ALTER a SET STATISTICS -1; -- partial analyze doesn't build stats either ANALYZE ab1 (a); -WARNING: extended statistics "public.ab1_a_b_stats" could not be collected for relation public.ab1 +WARNING: statistics object "public.ab1_a_b_stats" could not be computed for relation "public.ab1" ANALYZE ab1; DROP TABLE ab1; -- Verify supported object types for extended statistics diff --git a/src/test/regress/sql/object_address.sql b/src/test/regress/sql/object_address.sql index f25ed735e1..ca7ed46e41 100644 --- a/src/test/regress/sql/object_address.sql +++ b/src/test/regress/sql/object_address.sql @@ -187,7 +187,7 @@ WITH objects (type, name, args) AS (VALUES ('publication', '{addr_pub}', '{}'), ('publication relation', '{addr_nsp, gentable}', '{addr_pub}'), ('subscription', '{addr_sub}', '{}'), - ('statistics', '{addr_nsp, gentable_stat}', '{}') + ('statistics object', '{addr_nsp, gentable_stat}', '{}') ) SELECT (pg_identify_object(addr1.classid, addr1.objid, addr1.objsubid)).*, -- test roundtrip through pg_identify_object_as_address diff --git a/src/test/regress/sql/rules.sql b/src/test/regress/sql/rules.sql index 98a8a692d8..aada114ab2 100644 --- a/src/test/regress/sql/rules.sql +++ b/src/test/regress/sql/rules.sql @@ -1155,7 +1155,7 @@ SELECT pg_get_constraintdef(0); SELECT pg_get_functiondef(0); SELECT pg_get_indexdef(0); SELECT pg_get_ruledef(0); -SELECT pg_get_statisticsextdef(0); +SELECT pg_get_statisticsobjdef(0); SELECT pg_get_triggerdef(0); SELECT pg_get_viewdef(0); SELECT pg_get_function_arguments(0); diff --git a/src/test/regress/sql/stats_ext.sql b/src/test/regress/sql/stats_ext.sql index 09b7d8e102..a9dda051c2 100644 --- a/src/test/regress/sql/stats_ext.sql +++ b/src/test/regress/sql/stats_ext.sql @@ -26,8 +26,8 @@ DROP STATISTICS ab1_a_b_stats; CREATE SCHEMA regress_schema_2; CREATE STATISTICS regress_schema_2.ab1_a_b_stats ON a, b FROM ab1; --- Let's also verify the pg_get_statisticsextdef output looks sane. -SELECT pg_get_statisticsextdef(oid) FROM pg_statistic_ext WHERE stxname = 'ab1_a_b_stats'; +-- Let's also verify the pg_get_statisticsobjdef output looks sane. +SELECT pg_get_statisticsobjdef(oid) FROM pg_statistic_ext WHERE stxname = 'ab1_a_b_stats'; DROP STATISTICS regress_schema_2.ab1_a_b_stats; -- cgit v1.2.3 From 93ece9cc887239deef6539d607063d98aa03aff3 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 14 May 2017 19:15:52 -0400 Subject: Edit SGML documentation related to extended statistics. Use the "statistics object" terminology uniformly here too. Assorted copy-editing. Put new catalogs.sgml sections into alphabetical order. --- doc/src/sgml/catalogs.sgml | 432 +++++++++++++++++--------------- doc/src/sgml/perform.sgml | 214 +++++++++------- doc/src/sgml/planstats.sgml | 42 ++-- doc/src/sgml/ref/create_statistics.sgml | 43 ++-- 4 files changed, 397 insertions(+), 334 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index aa5e705e57..b2fae027f5 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -221,13 +221,13 @@ - pg_pltemplate - template data for procedural languages + pg_partitioned_table + information about partition key of tables - pg_partitioned_table - information about partition key of tables + pg_pltemplate + template data for procedural languages @@ -4271,108 +4271,6 @@ SCRAM-SHA-256$<iteration count>:<salt>< - - <structname>pg_statistic_ext</structname> - - - pg_statistic_ext - - - - The catalog pg_statistic_ext - holds extended planner statistics. - - - - <structname>pg_statistic_ext</> Columns - - - - - Name - Type - References - Description - - - - - - - stxrelid - oid - pg_class.oid - The table that the described columns belongs to - - - - stxname - name - - Name of the statistic. - - - - stxnamespace - oid - pg_namespace.oid - - The OID of the namespace that contains this statistic - - - - - stxowner - oid - pg_authid.oid - Owner of the statistic - - - - stxkeys - int2vector - pg_attribute.attnum - - This is an array of values that indicate which table columns this - statistic covers. For example a value of 1 3 would - mean that the first and the third table columns make up the statistic key. - - - - - stxkind - char[] - - - An array with the modes of the enabled statistic types. Valid values - are: - d for ndistinct coefficients, - f for functional dependencies. - - - - - stxndistinct - pg_ndistinct - - - N-distinct coefficients, serialized as pg_ndistinct type. - - - - - stxdependencies - pg_dependencies - - - Functional dependencies, serialized as pg_dependencies type. - - - - - -
    -     <structname>pg_namespace</structname> @@ -4790,6 +4688,111 @@ SCRAM-SHA-256$<iteration count>:<salt>< + + <structname>pg_partitioned_table</structname> + + + pg_partitioned_table + + + + The catalog pg_partitioned_table stores + information about how tables are partitioned. + + + + <structname>pg_partitioned_table</> Columns + + + + + Name + Type + References + Description + + + + + + + partrelid + oid + pg_class.oid + The OID of the pg_class entry for this partitioned table + + + + partstrat + char + + + Partitioning strategy; l = list partitioned table, + r = range partitioned table + + + + + partnatts + int2 + + The number of columns in partition key + + + + partattrs + int2vector + pg_attribute.attnum + + This is an array of partnatts values that + indicate which table columns are part of the partition key. For + example, a value of 1 3 would mean that the first + and the third table columns make up the partition key. A zero in this + array indicates that the corresponding partition key column is an + expression, rather than a simple column reference. + + + + + partclass + oidvector + pg_opclass.oid + + For each column in the partition key, this contains the OID of the + operator class to use. See + pg_opclass for details. + + + + + partcollation + oidvector + pg_opclass.oid + + For each column in the partition key, this contains the OID of the + the collation to use for partitioning. + + + + + partexprs + pg_node_tree + + + Expression trees (in nodeToString() + representation) for partition key columns that are not simple column + references. This is a list with one element for each zero + entry in partattrs. Null if all partition key columns + are simple references. + + + + + +
    +     + + <structname>pg_pltemplate</structname> @@ -4896,109 +4899,6 @@ SCRAM-SHA-256$<iteration count>:<salt>< - - <structname>pg_partitioned_table</structname> - - - pg_partitioned_table - - - - The catalog pg_partitioned_table stores - information about how tables are partitioned. - - - - <structname>pg_partitioned_table</> Columns - - - - - Name - Type - References - Description - - - - - - - partrelid - oid - pg_class.oid - The OID of the pg_class entry for this partitioned table - - - - partstrat - char - - - Partitioning strategy; l = list partitioned table, - r = range partitioned table - - - - - partnatts - int2 - - The number of columns in partition key - - - - partattrs - int2vector - pg_attribute.attnum - - This is an array of partnatts values that - indicate which table columns are part of the partition key. For - example, a value of 1 3 would mean that the first - and the third table columns make up the partition key. A zero in this - array indicates that the corresponding partition key column is an - expression, rather than a simple column reference. - - - - - partclass - oidvector - pg_opclass.oid - - For each column in the partition key, this contains the OID of the - operator class to use. See - pg_opclass for details. - - - - - partcollation - oidvector - pg_opclass.oid - - For each column in the partition key, this contains the OID of the - the collation to use for partitioning. - - - - - partexprs - pg_node_tree - - - Expression trees (in nodeToString() - representation) for partition key columns that are not simple column - references. This is a list with one element for each zero - entry in partattrs. Null if all partition key columns - are simple references. - - - - - -
    -     <structname>pg_policy</structname> @@ -6466,6 +6366,120 @@ SCRAM-SHA-256$<iteration count>:<salt>< + + <structname>pg_statistic_ext</structname> + + + pg_statistic_ext + + + + The catalog pg_statistic_ext + holds extended planner statistics. + Each row in this catalog corresponds to a statistics object + created with . + + + + <structname>pg_statistic_ext</> Columns + + + + + Name + Type + References + Description + + + + + + + stxrelid + oid + pg_class.oid + Table containing the columns described by this object + + + + stxname + name + + Name of the statistics object + + + + stxnamespace + oid + pg_namespace.oid + + The OID of the namespace that contains this statistics object + + + + + stxowner + oid + pg_authid.oid + Owner of the statistics object + + + + stxkeys + int2vector + pg_attribute.attnum + + An array of attribute numbers, indicating which table columns are + covered by this statistics object; + for example a value of 1 3 would + mean that the first and the third table columns are covered + + + + + stxkind + char[] + + + An array containing codes for the enabled statistic types; + valid values are: + d for n-distinct statistics, + f for functional dependency statistics + + + + + stxndistinct + pg_ndistinct + + + N-distinct counts, serialized as pg_ndistinct type + + + + + stxdependencies + pg_dependencies + + + Functional dependency statistics, serialized + as pg_dependencies type + + + + + +
    + + + The stxkind field is filled at creation of the + statistics object, indicating which statistic type(s) are desired. + The fields after it are initially NULL and are filled only when the + corresponding statistic has been computed by ANALYZE. + +     + <structname>pg_subscription</structname> diff --git a/doc/src/sgml/perform.sgml b/doc/src/sgml/perform.sgml index 32e17ee5f8..b4b8f8dcb8 100644 --- a/doc/src/sgml/perform.sgml +++ b/doc/src/sgml/perform.sgml @@ -1071,25 +1071,41 @@ WHERE tablename = 'road'; are independent of each other, an assumption that does not hold when column values are correlated. Regular statistics, because of their per-individual-column nature, - do not capture the knowledge of cross-column correlation; - multivariate statistics can be used to instruct - the server to obtain statistics across such a set of columns, - which are later used by the query optimizer - to determine cardinality and selectivity - of clauses involving those columns. - Multivariate statistics are currently the only use of - extended statistics. + cannot capture any knowledge about cross-column correlation. + However, PostgreSQL has the ability to compute + multivariate statistics, which can capture + such information.     - Extended statistics are created using + Because the number of possible column combinations is very large, + it's impractical to compute multivariate statistics automatically. + Instead, extended statistics objects, more often + called just statistics objects, can be created to instruct + the server to obtain statistics across interesting sets of columns. + + + + Statistics objects are created using , which see for more details. - Data collection is deferred until the next ANALYZE - on the table, after which the stored values can be examined in the + Creation of such an object merely creates a catalog entry expressing + interest in the statistics. Actual data collection is performed + by ANALYZE (either a manual command, or background + auto-analyze). The collected values can be examined in the pg_statistic_ext catalog. + + ANALYZE computes extended statistics based on the same + sample of table rows that it takes for computing regular single-column + statistics. Since the sample size is increased by increasing the + statistics target for the table or any of its columns (as described in + the previous section), a larger statistics target will normally result in + more accurate extended statistics, as well as more time spent calculating + them. + + The following subsections describe the types of extended statistics that are currently supported. @@ -1099,142 +1115,162 @@ WHERE tablename = 'road'; Functional Dependencies - The simplest type of extended statistics are functional dependencies, - a concept used in definitions of database normal forms. - Put simply, it is said that column b is functionally - dependent on column a if knowledge of the value of - a is sufficient to determine the value of b. - In normalized databases, functional dependencies are allowed only on - primary keys and superkeys. However, many data sets are in practice not - fully normalized for various reasons; intentional denormalization for - performance reasons is a common example. + The simplest type of extended statistics tracks functional + dependencies, a concept used in definitions of database normal forms. + We say that column b is functionally dependent on + column a if knowledge of the value of + a is sufficient to determine the value + of b, that is there are no two rows having the same value + of a but different values of b. + In a fully normalized database, functional dependencies should exist + only on primary keys and superkeys. However, in practice many data sets + are not fully normalized for various reasons; intentional + denormalization for performance reasons is a common example. + Even in a fully normalized database, there may be partial correlation + between some columns, which can be expressed as partial functional + dependency. - The existance of functional dependencies directly affects the accuracy - of estimates in certain queries. - The reason is that conditions on the dependent columns do not - restrict the result set, but the query planner (lacking functional - dependency knowledge) considers them independent, resulting in - underestimates. - To inform the planner about the functional dependencies, we collect - measurements of dependency during ANALYZE. Assessing - the degree of dependency between all sets of columns would be - prohibitively expensive, so the search is limited to potential - dependencies defined using the dependencies option of - extended statistics. It is advisable to create - dependencies statistics if and only if functional - dependencies actually exist, to avoid unnecessary overhead on both - ANALYZE and query planning. + The existence of functional dependencies directly affects the accuracy + of estimates in certain queries. If a query contains conditions on + both the independent and the dependent column(s), the + conditions on the dependent columns do not further reduce the result + size; but without knowledge of the functional dependency, the query + planner will assume that the conditions are independent, resulting + in underestimating the result size. - To inspect functional dependencies on a statistics - stts, you may do this: + To inform the planner about functional dependencies, ANALYZE + can collect measurements of cross-column dependency. Assessing the + degree of dependency between all sets of columns would be prohibitively + expensive, so data collection is limited to those groups of columns + appearing together in a statistics object defined with + the dependencies option. It is advisable to create + dependencies statistics only for column groups that are + strongly correlated, to avoid unnecessary overhead in both + ANALYZE and later query planning. + + + + Here is an example of collecting functional-dependency statistics: -CREATE STATISTICS stts (dependencies) - ON zip, city FROM zipcodes; +CREATE STATISTICS stts (dependencies) ON zip, city FROM zipcodes; + ANALYZE zipcodes; + SELECT stxname, stxkeys, stxdependencies FROM pg_statistic_ext - WHERE stxname = 'stts'; + WHERE stxname = 'stts'; stxname | stxkeys | stxdependencies ---------+---------+------------------------------------------ stts | 1 5 | {"1 => 5": 1.000000, "5 => 1": 0.423130} (1 row) - where it can be seen that column 1 (a zip code) fully determines column + Here it can be seen that column 1 (zip code) fully determines column 5 (city) so the coefficient is 1.0, while city only determines zip code about 42% of the time, meaning that there are many cities (58%) that are represented by more than a single ZIP code. - When computing the selectivity, the planner inspects all conditions and - attempts to identify which conditions are already implied by other - conditions. The selectivity estimates from any redundant conditions are - ignored from a selectivity point of view. In the example query above, - the selectivity estimates for either of the conditions may be eliminated, - thus improving the overall estimate. + When computing the selectivity for a query involving functionally + dependent columns, the planner adjusts the per-condition selectivity + estimates using the dependency coefficients so as not to produce + an underestimate. Limitations of Functional Dependencies - Functional dependencies are a very simple type of statistics, and - as such have several limitations. The first limitation is that they - only work with simple equality conditions, comparing columns and constant - values. It's not possible to use them to eliminate equality conditions - comparing two columns or a column to an expression, range clauses, - LIKE or any other type of conditions. + Functional dependencies are currently only applied when considering + simple equality conditions that compare columns to constant values. + They are not used to improve estimates for equality conditions + comparing two columns or comparing a column to an expression, nor for + range clauses, LIKE or any other type of condition. - When eliminating the implied conditions, the planner assumes that the - conditions are compatible. Consider the following example, where - this assumption does not hold: - + When estimating with functional dependencies, the planner assumes that + conditions on the involved columns are compatible and hence redundant. + If they are incompatible, the correct estimate would be zero rows, but + that possibility is not considered. For example, given a query like -EXPLAIN (ANALYZE, TIMING OFF) SELECT * FROM t WHERE a = 1 AND b = 10; - QUERY PLAN ------------------------------------------------------------------------------ - Seq Scan on t (cost=0.00..195.00 rows=100 width=8) (actual rows=0 loops=1) - Filter: ((a = 1) AND (b = 10)) - Rows Removed by Filter: 10000 +SELECT * FROM zipcodes WHERE city = 'San Francisco' AND zip = '94105'; - - While there are no rows with such combination of values, the planner - is unable to verify whether the values match — it only knows that - the columns are functionally dependent. + the planner will disregard the city clause as not + changing the selectivity, which is correct. However, it will make + the same assumption about + +SELECT * FROM zipcodes WHERE city = 'San Francisco' AND zip = '90210'; + + even though there will really be zero rows satisfying this query. + Functional dependency statistics do not provide enough information + to conclude that, however. - This assumption is related to queries executed on the database; in many - cases, it's actually satisfied (e.g. when the GUI only allows selecting - compatible values). But if that's not the case, functional dependencies - may not be a viable option. + In many practical situations, this assumption is usually satisfied; + for example, there might be a GUI in the application that only allows + selecting compatible city and zipcode values to use in a query. + But if that's not the case, functional dependencies may not be a viable + option. - Multivariate N-Distinct Coefficients + Multivariate N-Distinct Counts Single-column statistics store the number of distinct values in each - column. Estimates of the number of distinct values on more than one - column (for example, for GROUP BY a, b) are + column. Estimates of the number of distinct values when combining more + than one column (for example, for GROUP BY a, b) are frequently wrong when the planner only has single-column statistical - data, however, causing it to select bad plans. - In order to improve n-distinct estimation when multiple columns are - grouped together, the ndistinct option of extended statistics - can be used, which instructs ANALYZE to collect n-distinct - estimates for all possible combinations of two or more columns of the set - of columns in the statistics object (the per-column estimates are already - available in pg_statistic). + data, causing it to select bad plans. + + + + To improve such estimates, ANALYZE can collect n-distinct + statistics for groups of columns. As before, it's impractical to do + this for every possible column grouping, so data is collected only for + those groups of columns appearing together in a statistics object + defined with the ndistinct option. Data will be collected + for each possible combination of two or more columns from the set of + listed columns. - Continuing the above example, the n-distinct coefficients in a ZIP - code table may look like the following: + Continuing the previous example, the n-distinct counts in a + table of ZIP codes might look like the following: -CREATE STATISTICS stts2 (ndistinct) - ON zip, state, city FROM zipcodes; +CREATE STATISTICS stts2 (ndistinct) ON zip, state, city FROM zipcodes; + ANALYZE zipcodes; + SELECT stxkeys AS k, stxndistinct AS nd FROM pg_statistic_ext - WHERE stxname = 'stts2'; + WHERE stxname = 'stts2'; -[ RECORD 1 ]-------------------------------------------------------- k | 1 2 5 nd | {"1, 2": 33178, "1, 5": 33178, "2, 5": 27435, "1, 2, 5": 33178} (1 row) - which indicates that there are three combinations of columns that + This indicates that there are three combinations of columns that have 33178 distinct values: ZIP code and state; ZIP code and city; and ZIP code, city and state (the fact that they are all equal is - expected given the nature of ZIP-code data). On the other hand, - the combination of city and state only has 27435 distinct values. + expected given that ZIP code alone is unique in this table). On the + other hand, the combination of city and state has only 27435 distinct + values. + + + + It's advisable to create ndistinct statistics objects only + on combinations of columns that are actually used for grouping, and + for which misestimation of the number of groups is resulting in bad + plans. Otherwise, the ANALYZE cycles are just wasted.     diff --git a/doc/src/sgml/planstats.sgml b/doc/src/sgml/planstats.sgml index ef847b9633..8caf297f85 100644 --- a/doc/src/sgml/planstats.sgml +++ b/doc/src/sgml/planstats.sgml @@ -456,10 +456,11 @@ rows = (outer_cardinality * inner_cardinality) * selectivity - Functional dependencies + Functional Dependencies + - Multivariate correlation can be seen with a very simple data set — a - table with two columns, both containing the same values: + Multivariate correlation can be demonstrated with a very simple data set + — a table with two columns, both containing the same values: CREATE TABLE t (a INT, b INT); @@ -501,8 +502,8 @@ EXPLAIN (ANALYZE, TIMING OFF) SELECT * FROM t WHERE a = 1; number of rows, we see that the estimate is very accurate (in fact exact, as the table is very small). Changing the WHERE to use the b column, an identical - plan is generated. Observe what happens if we apply the same - condition on both columns combining them with AND: + plan is generated. But observe what happens if we apply the same + condition on both columns, combining them with AND: EXPLAIN (ANALYZE, TIMING OFF) SELECT * FROM t WHERE a = 1 AND b = 1; @@ -514,15 +515,16 @@ EXPLAIN (ANALYZE, TIMING OFF) SELECT * FROM t WHERE a = 1 AND b = 1; The planner estimates the selectivity for each condition individually, - arriving to the 1% estimates as above, and then multiplies them, getting - the final 0.01% estimate. The actual figures, however, - show that this results in a significant underestimate, as the actual - number of rows matching the conditions (100) is two orders of magnitude - higher than the estimated value. + arriving at the same 1% estimates as above. Then it assumes that the + conditions are independent, and so it multiplies their selectivities, + producing a final selectivity estimate of just 0.01%. + This is a significant underestimate, as the actual number of rows + matching the conditions (100) is two orders of magnitude higher. - This problem can be fixed by applying functional-dependency + This problem can be fixed by creating a statistics object that + directs ANALYZE to calculate functional-dependency multivariate statistics on the two columns: @@ -539,13 +541,15 @@ EXPLAIN (ANALYZE, TIMING OFF) SELECT * FROM t WHERE a = 1 AND b = 1; - Multivariate N-Distinct coefficients + Multivariate N-Distinct Counts + - A similar problem occurs with estimation of the cardinality of distinct - elements, used to determine the number of groups that would be generated - by a GROUP BY clause. When GROUP BY - lists a single column, the n-distinct estimate (which can be seen as the - number of rows returned by the aggregate execution node) is very accurate: + A similar problem occurs with estimation of the cardinality of sets of + multiple columns, such as the number of groups that would be generated by + a GROUP BY clause. When GROUP BY + lists a single column, the n-distinct estimate (which is visible as the + estimated number of rows returned by the HashAggregate node) is very + accurate: EXPLAIN (ANALYZE, TIMING OFF) SELECT COUNT(*) FROM t GROUP BY a; QUERY PLAN @@ -565,8 +569,8 @@ EXPLAIN (ANALYZE, TIMING OFF) SELECT COUNT(*) FROM t GROUP BY a, b; Group Key: a, b -> Seq Scan on t (cost=0.00..145.00 rows=10000 width=8) (actual rows=10000 loops=1) - By dropping the existing statistics and re-creating it to include n-distinct - calculation, the estimate is much improved: + By redefining the statistics object to include n-distinct counts for the + two columns, the estimate is much improved: DROP STATISTICS stts; CREATE STATISTICS stts (dependencies, ndistinct) ON a, b FROM t; diff --git a/doc/src/sgml/ref/create_statistics.sgml b/doc/src/sgml/ref/create_statistics.sgml index 92ee4e4efa..854746de24 100644 --- a/doc/src/sgml/ref/create_statistics.sgml +++ b/doc/src/sgml/ref/create_statistics.sgml @@ -79,11 +79,13 @@ CREATE STATISTICS [ IF NOT EXISTS ] statistics_na statistic_type - A statistic type to be computed in this statistics object. Currently - supported types are ndistinct, which enables - n-distinct coefficient tracking, - and dependencies, which enables functional - dependencies. + A statistic type to be computed in this statistics object. + Currently supported types are + ndistinct, which enables n-distinct statistics, and + dependencies, which enables functional + dependency statistics. + For more information, see + and . @@ -92,7 +94,8 @@ CREATE STATISTICS [ IF NOT EXISTS ] statistics_na column_name - The name of a table column to be included in the statistics object. + The name of a table column to be covered by the computed statistics. + At least two column names must be given. @@ -114,7 +117,9 @@ CREATE STATISTICS [ IF NOT EXISTS ] statistics_na Notes - You must be the owner of a table to create or change statistics on it. + You must be the owner of a table to create a statistics object + reading it. Once created, however, the ownership of the statistics + object is independent of the underlying table(s). @@ -124,8 +129,8 @@ CREATE STATISTICS [ IF NOT EXISTS ] statistics_na Create table t1 with two functionally dependent columns, i.e. knowledge of a value in the first column is sufficient for determining the - value in the other column. Then functional dependencies are built on those - columns: + value in the other column. Then functional dependency statistics are built + on those columns: CREATE TABLE t1 ( @@ -136,21 +141,25 @@ CREATE TABLE t1 ( INSERT INTO t1 SELECT i/100, i/500 FROM generate_series(1,1000000) s(i); +ANALYZE t1; + +-- the number of matching rows will be drastically underestimated: +EXPLAIN ANALYZE SELECT * FROM t1 WHERE (a = 1) AND (b = 0); + CREATE STATISTICS s1 (dependencies) ON a, b FROM t1; ANALYZE t1; --- valid combination of values +-- now the rowcount estimate is more accurate: EXPLAIN ANALYZE SELECT * FROM t1 WHERE (a = 1) AND (b = 0); - --- invalid combination of values -EXPLAIN ANALYZE SELECT * FROM t1 WHERE (a = 1) AND (b = 1); - Without functional-dependency statistics, the planner would make the - same estimate of the number of matching rows for these two queries. - With such statistics, it is able to tell that one case has matches - and the other does not. + Without functional-dependency statistics, the planner would assume + that the two WHERE conditions are independent, and would + multiply their selectivities together to arrive at a much-too-small + rowcount estimate. + With such statistics, the planner recognizes that the WHERE + conditions are redundant and does not underestimate the rowcount. -- cgit v1.2.3 From b91e5b4684d840c903a78e86700b9d906033c2ad Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Sun, 14 May 2017 22:45:11 -0400 Subject: doc: update PG 10 release notes for recent changes --- doc/src/sgml/release-10.sgml | 72 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 2 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/release-10.sgml b/doc/src/sgml/release-10.sgml index 6497641ba6..4f953dc5d6 100644 --- a/doc/src/sgml/release-10.sgml +++ b/doc/src/sgml/release-10.sgml @@ -126,6 +126,28 @@ + + + + Rename WAL-related functions and views to use lsn + instead of location (David Rowley) + + + + + + + RenameWAL-related functions and views to use lsn + instead of location (David Rowley) + + + + + Remove the ability to store unencrypted passwords on the server + (Heikki Linnakangas) + + + + The server-side variable + no longer supports off or plain. + The UNENCRYPTED option is no longer supported for + CREATE/ALTER USER ... PASSSWORD. Similarly, the + + + + + Add function PQencryptPasswordConn() + to allow creation of more types of encrypted passwords on the + client-side (Michael Paquier, Heikki Linnakangas) + + + + Previously only MD5 passwords could be created using PQencryptPassword(). + This new function can also create SCRAM-SHA-256 passwords. + + + Push aggregates to foreign data wrapper servers, where possible @@ -2858,7 +2925,8 @@ from the foreign data wrapper server, and offloads aggregate computation from the requesting server. The postgres_fdw is able to - perform this optimization. + perform this optimization. There are also improvements in + pushing down joins involving extensions. -- cgit v1.2.3 From de7cca982ca253236051c2a7f6477bee5ce4b045 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Sun, 14 May 2017 23:15:43 -0400 Subject: doc: update the "current as of" date in the PG 10 release notes --- doc/src/sgml/release-10.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/release-10.sgml b/doc/src/sgml/release-10.sgml index 4f953dc5d6..eff1fbc719 100644 --- a/doc/src/sgml/release-10.sgml +++ b/doc/src/sgml/release-10.sgml @@ -6,7 +6,7 @@ Release date: - 2017-09-XX (CURRENT AS OF 2017-04-22) + 2017-09-XX (CURRENT AS OF 2017-05-14) -- cgit v1.2.3 From f8dc1985fd390774aab4ab0ba71036d6d5e631a9 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Tue, 9 May 2017 23:35:31 -0400 Subject: Fix ALTER SEQUENCE locking In 1753b1b027035029c2a2a1649065762fafbf63f3, the pg_sequence system catalog was introduced. This made sequence metadata changes transactional, while the actual sequence values are still behaving nontransactionally. This requires some refinement in how ALTER SEQUENCE, which operates on both, locks the sequence and the catalog. The main problems were: - Concurrent ALTER SEQUENCE causes "tuple concurrently updated" error, caused by updates to pg_sequence catalog. - Sequence WAL writes and catalog updates are not protected by same lock, which could lead to inconsistent recovery order. - nextval() disregarding uncommitted ALTER SEQUENCE changes. To fix, nextval() and friends now lock the sequence using RowExclusiveLock instead of AccessShareLock. ALTER SEQUENCE locks the sequence using ShareRowExclusiveLock. This means that nextval() and ALTER SEQUENCE block each other, and ALTER SEQUENCE on the same sequence blocks itself. (This was already the case previously for the OWNER TO, RENAME, and SET SCHEMA variants.) Also, rearrange some code so that the entire AlterSequence is protected by the lock on the sequence. As an exception, use reduced locking for ALTER SEQUENCE ... RESTART. Since that is basically a setval(), it does not require the full locking of other ALTER SEQUENCE actions. So check whether we are only running a RESTART and run with less locking if so. Reviewed-by: Michael Paquier Reported-by: Jason Petersen Reported-by: Andres Freund --- doc/src/sgml/ref/alter_sequence.sgml | 7 +++ src/backend/commands/sequence.c | 69 +++++++++++++++------- src/test/isolation/expected/sequence-ddl.out | 85 ++++++++++++++++++++++++++++ src/test/isolation/isolation_schedule | 1 + src/test/isolation/specs/sequence-ddl.spec | 39 +++++++++++++ 5 files changed, 180 insertions(+), 21 deletions(-) create mode 100644 src/test/isolation/expected/sequence-ddl.out create mode 100644 src/test/isolation/specs/sequence-ddl.spec (limited to 'doc/src') diff --git a/doc/src/sgml/ref/alter_sequence.sgml b/doc/src/sgml/ref/alter_sequence.sgml index 3fb3400d6b..30e5316b8c 100644 --- a/doc/src/sgml/ref/alter_sequence.sgml +++ b/doc/src/sgml/ref/alter_sequence.sgml @@ -304,6 +304,13 @@ ALTER SEQUENCE [ IF EXISTS ] name S 8.3, it sometimes did.)     + + ALTER SEQUENCE blocks + concurrent nextval, currval, + lastval, and setval calls, except + if only the RESTART clause is used. + + For historical reasons, ALTER TABLE can be used with sequences too; but the only variants of ALTER TABLE diff --git a/src/backend/commands/sequence.c b/src/backend/commands/sequence.c index 4a56926b53..0f7cf1dce8 100644 --- a/src/backend/commands/sequence.c +++ b/src/backend/commands/sequence.c @@ -93,11 +93,12 @@ static HTAB *seqhashtab = NULL; /* hash table for SeqTable items */ static SeqTableData *last_used_seq = NULL; static void fill_seq_with_data(Relation rel, HeapTuple tuple); -static Relation open_share_lock(SeqTable seq); +static Relation lock_and_open_sequence(SeqTable seq); static void create_seq_hashtable(void); static void init_sequence(Oid relid, SeqTable *p_elm, Relation *p_rel); static Form_pg_sequence_data read_seq_tuple(Relation rel, Buffer *buf, HeapTuple seqdatatuple); +static LOCKMODE alter_sequence_get_lock_level(List *options); static void init_params(ParseState *pstate, List *options, bool for_identity, bool isInit, Form_pg_sequence seqform, @@ -427,7 +428,9 @@ AlterSequence(ParseState *pstate, AlterSeqStmt *stmt) HeapTuple tuple; /* Open and lock sequence. */ - relid = RangeVarGetRelid(stmt->sequence, AccessShareLock, stmt->missing_ok); + relid = RangeVarGetRelid(stmt->sequence, + alter_sequence_get_lock_level(stmt->options), + stmt->missing_ok); if (relid == InvalidOid) { ereport(NOTICE, @@ -443,12 +446,6 @@ AlterSequence(ParseState *pstate, AlterSeqStmt *stmt) aclcheck_error(ACLCHECK_NOT_OWNER, ACL_KIND_CLASS, stmt->sequence->relname); - /* lock page' buffer and read tuple into new sequence structure */ - seqdata = read_seq_tuple(seqrel, &buf, &seqdatatuple); - - /* Copy old sequence data into workspace */ - memcpy(&newseqdata, seqdata, sizeof(FormData_pg_sequence_data)); - rel = heap_open(SequenceRelationId, RowExclusiveLock); tuple = SearchSysCacheCopy1(SEQRELID, ObjectIdGetDatum(relid)); @@ -458,6 +455,12 @@ AlterSequence(ParseState *pstate, AlterSeqStmt *stmt) seqform = (Form_pg_sequence) GETSTRUCT(tuple); + /* lock page's buffer and read tuple into new sequence structure */ + seqdata = read_seq_tuple(seqrel, &buf, &seqdatatuple); + + /* Copy old sequence data into workspace */ + memcpy(&newseqdata, seqdata, sizeof(FormData_pg_sequence_data)); + /* Check and set new values */ init_params(pstate, stmt->options, stmt->for_identity, false, seqform, &changed_seqform, &newseqdata, &owned_by); @@ -508,12 +511,12 @@ AlterSequence(ParseState *pstate, AlterSeqStmt *stmt) ObjectAddressSet(address, RelationRelationId, relid); - relation_close(seqrel, NoLock); - if (changed_seqform) CatalogTupleUpdate(rel, &tuple->t_self, tuple); heap_close(rel, RowExclusiveLock); + relation_close(seqrel, NoLock); + return address; } @@ -594,7 +597,7 @@ nextval_internal(Oid relid, bool check_permissions) bool cycle; bool logit = false; - /* open and AccessShareLock sequence */ + /* open and lock sequence */ init_sequence(relid, &elm, &seqrel); if (check_permissions && @@ -829,7 +832,7 @@ currval_oid(PG_FUNCTION_ARGS) SeqTable elm; Relation seqrel; - /* open and AccessShareLock sequence */ + /* open and lock sequence */ init_sequence(relid, &elm, &seqrel); if (pg_class_aclcheck(elm->relid, GetUserId(), @@ -869,7 +872,7 @@ lastval(PG_FUNCTION_ARGS) (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("lastval is not yet defined in this session"))); - seqrel = open_share_lock(last_used_seq); + seqrel = lock_and_open_sequence(last_used_seq); /* nextval() must have already been called for this sequence */ Assert(last_used_seq->last_valid); @@ -913,7 +916,7 @@ do_setval(Oid relid, int64 next, bool iscalled) int64 maxv, minv; - /* open and AccessShareLock sequence */ + /* open and lock sequence */ init_sequence(relid, &elm, &seqrel); if (pg_class_aclcheck(elm->relid, GetUserId(), ACL_UPDATE) != ACLCHECK_OK) @@ -1042,15 +1045,15 @@ setval3_oid(PG_FUNCTION_ARGS) /* - * Open the sequence and acquire AccessShareLock if needed + * Open the sequence and acquire lock if needed * * If we haven't touched the sequence already in this transaction, - * we need to acquire AccessShareLock. We arrange for the lock to + * we need to acquire a lock. We arrange for the lock to * be owned by the top transaction, so that we don't need to do it * more than once per xact. */ static Relation -open_share_lock(SeqTable seq) +lock_and_open_sequence(SeqTable seq) { LocalTransactionId thislxid = MyProc->lxid; @@ -1063,7 +1066,7 @@ open_share_lock(SeqTable seq) PG_TRY(); { CurrentResourceOwner = TopTransactionResourceOwner; - LockRelationOid(seq->relid, AccessShareLock); + LockRelationOid(seq->relid, RowExclusiveLock); } PG_CATCH(); { @@ -1078,7 +1081,7 @@ open_share_lock(SeqTable seq) seq->lxid = thislxid; } - /* We now know we have AccessShareLock, and can safely open the rel */ + /* We now know we have the lock, and can safely open the rel */ return relation_open(seq->relid, NoLock); } @@ -1135,7 +1138,7 @@ init_sequence(Oid relid, SeqTable *p_elm, Relation *p_rel) /* * Open the sequence relation. */ - seqrel = open_share_lock(elm); + seqrel = lock_and_open_sequence(elm); if (seqrel->rd_rel->relkind != RELKIND_SEQUENCE) ereport(ERROR, @@ -1216,6 +1219,30 @@ read_seq_tuple(Relation rel, Buffer *buf, HeapTuple seqdatatuple) return seq; } +/* + * Check the sequence options list and return the appropriate lock level for + * ALTER SEQUENCE. + * + * Most sequence option changes require a self-exclusive lock and should block + * concurrent nextval() et al. But RESTART does not, because it's not + * transactional. Also take a lower lock if no option at all is present. + */ +static LOCKMODE +alter_sequence_get_lock_level(List *options) +{ + ListCell *option; + + foreach(option, options) + { + DefElem *defel = (DefElem *) lfirst(option); + + if (strcmp(defel->defname, "restart") != 0) + return ShareRowExclusiveLock; + } + + return RowExclusiveLock; +} + /* * init_params: process the options list of CREATE or ALTER SEQUENCE, and * store the values into appropriate fields of seqform, for changes that go @@ -1850,7 +1877,7 @@ pg_sequence_last_value(PG_FUNCTION_ARGS) bool is_called; int64 result; - /* open and AccessShareLock sequence */ + /* open and lock sequence */ init_sequence(relid, &elm, &seqrel); if (pg_class_aclcheck(relid, GetUserId(), ACL_SELECT | ACL_USAGE) != ACLCHECK_OK) diff --git a/src/test/isolation/expected/sequence-ddl.out b/src/test/isolation/expected/sequence-ddl.out new file mode 100644 index 0000000000..6b7119738f --- /dev/null +++ b/src/test/isolation/expected/sequence-ddl.out @@ -0,0 +1,85 @@ +Parsed test spec with 2 sessions + +starting permutation: s1alter s1commit s2nv +step s1alter: ALTER SEQUENCE seq1 MAXVALUE 10; +step s1commit: COMMIT; +step s2nv: SELECT nextval('seq1') FROM generate_series(1, 15); +ERROR: nextval: reached maximum value of sequence "seq1" (10) + +starting permutation: s1alter s2nv s1commit +step s1alter: ALTER SEQUENCE seq1 MAXVALUE 10; +step s2nv: SELECT nextval('seq1') FROM generate_series(1, 15); +step s1commit: COMMIT; +step s2nv: <... completed> +error in steps s1commit s2nv: ERROR: nextval: reached maximum value of sequence "seq1" (10) + +starting permutation: s2begin s2nv s1alter2 s2commit s1commit +step s2begin: BEGIN; +step s2nv: SELECT nextval('seq1') FROM generate_series(1, 15); +nextval + +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +step s1alter2: ALTER SEQUENCE seq1 MAXVALUE 20; +step s2commit: COMMIT; +step s1alter2: <... completed> +step s1commit: COMMIT; + +starting permutation: s1restart s2nv s1commit +step s1restart: ALTER SEQUENCE seq1 RESTART WITH 5; +step s2nv: SELECT nextval('seq1') FROM generate_series(1, 15); +nextval + +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +step s1commit: COMMIT; + +starting permutation: s2begin s2nv s1restart s2commit s1commit +step s2begin: BEGIN; +step s2nv: SELECT nextval('seq1') FROM generate_series(1, 15); +nextval + +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +step s1restart: ALTER SEQUENCE seq1 RESTART WITH 5; +step s2commit: COMMIT; +step s1commit: COMMIT; diff --git a/src/test/isolation/isolation_schedule b/src/test/isolation/isolation_schedule index fc1918dedc..32c965b2a0 100644 --- a/src/test/isolation/isolation_schedule +++ b/src/test/isolation/isolation_schedule @@ -58,6 +58,7 @@ test: alter-table-1 test: alter-table-2 test: alter-table-3 test: create-trigger +test: sequence-ddl test: async-notify test: vacuum-reltuples test: timeouts diff --git a/src/test/isolation/specs/sequence-ddl.spec b/src/test/isolation/specs/sequence-ddl.spec new file mode 100644 index 0000000000..42ee3b0615 --- /dev/null +++ b/src/test/isolation/specs/sequence-ddl.spec @@ -0,0 +1,39 @@ +# Test sequence usage and concurrent sequence DDL + +setup +{ + CREATE SEQUENCE seq1; +} + +teardown +{ + DROP SEQUENCE seq1; +} + +session "s1" +setup { BEGIN; } +step "s1alter" { ALTER SEQUENCE seq1 MAXVALUE 10; } +step "s1alter2" { ALTER SEQUENCE seq1 MAXVALUE 20; } +step "s1restart" { ALTER SEQUENCE seq1 RESTART WITH 5; } +step "s1commit" { COMMIT; } + +session "s2" +step "s2begin" { BEGIN; } +step "s2nv" { SELECT nextval('seq1') FROM generate_series(1, 15); } +step "s2commit" { COMMIT; } + +permutation "s1alter" "s1commit" "s2nv" + +# Prior to PG10, the s2nv would see the uncommitted s1alter change, +# but now it waits. +permutation "s1alter" "s2nv" "s1commit" + +# nextval doesn't release lock until transaction end, so s1alter2 has +# to wait for s2commit. +permutation "s2begin" "s2nv" "s1alter2" "s2commit" "s1commit" + +# RESTART is nontransactional, so s2nv sees it right away +permutation "s1restart" "s2nv" "s1commit" + +# RESTART does not wait +permutation "s2begin" "s2nv" "s1restart" "s2commit" "s1commit" -- cgit v1.2.3 From 4b99d32b2b0de97063b85a0ea69d482d8a4bf075 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Mon, 15 May 2017 12:09:19 -0400 Subject: doc: Remove unused file sql.sgml has not been part of the documentation since forever, so it's pointless to keep it around. --- doc/src/sgml/filelist.sgml | 3 - doc/src/sgml/sql.sgml | 2148 -------------------------------------------- 2 files changed, 2151 deletions(-) delete mode 100644 doc/src/sgml/sql.sgml (limited to 'doc/src') diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml index 2ec83af58e..b914086009 100644 --- a/doc/src/sgml/filelist.sgml +++ b/doc/src/sgml/filelist.sgml @@ -12,9 +12,6 @@ - - - diff --git a/doc/src/sgml/sql.sgml b/doc/src/sgml/sql.sgml deleted file mode 100644 index 57396d7c24..0000000000 --- a/doc/src/sgml/sql.sgml +++ /dev/null @@ -1,2148 +0,0 @@ - - - - SQL - - - - This chapter introduces the mathematical concepts behind - relational databases. It is not required reading, so if you bog - down or want to get straight to some simple examples feel free to - jump ahead to the next chapter and come back when you have more - time and patience. This stuff is supposed to be fun! - - - - This material originally appeared as a part of - Stefan Simkovics' Master's Thesis - (). - - - - - SQL has become the most popular relational query - language. - The name SQL is an abbreviation for - Structured Query Language. - In 1974 Donald Chamberlin and others defined the - language SEQUEL (Structured English Query - Language) at IBM - Research. This language was first implemented in an IBM - prototype called SEQUEL-XRM in 1974-75. In 1976-77 a revised version - of SEQUEL called SEQUEL/2 was defined and the name was changed to - SQL - subsequently. - - - - A new prototype called System R was developed by IBM in 1977. System R - implemented a large subset of SEQUEL/2 (now SQL) - and a number of - changes were made to SQL during the project. - System R was installed in - a number of user sites, both internal IBM sites and also some selected - customer sites. Thanks to the success and acceptance of System R at - those user sites IBM started to develop commercial products that - implemented the SQL language based on the System - R technology. - - - - Over the next years IBM and also a number of other vendors announced - SQL products such as - SQL/DS (IBM), - DB2 (IBM), - ORACLE (Oracle Corp.), - DG/SQL (Data General Corp.), - and SYBASE (Sybase Inc.). - - - - SQL is also an official standard now. In 1982 - the American National - Standards Institute (ANSI) chartered its - Database Committee X3H2 to - develop a proposal for a standard relational language. This proposal - was ratified in 1986 and consisted essentially of the IBM dialect of - SQL. In 1987 this ANSI - standard was also accepted as an international - standard by the International Organization for Standardization - (ISO). - This original standard version of SQL is often - referred to, - informally, as SQL/86. In 1989 the original - standard was extended - and this new standard is often, again informally, referred to as - SQL/89. Also in 1989, a related standard called - Database Language Embedded SQL - (ESQL) was developed. - - - - The ISO and ANSI committees - have been working for many years on the - definition of a greatly expanded version of the original standard, - referred to informally as SQL2 - or SQL/92. This version became a - ratified standard - International Standard ISO/IEC 9075:1992, - Database Language SQL - in late 1992. - SQL/92 is the version - normally meant when people refer to the SQL - standard. A detailed - description of SQL/92 is given in - . At the time of - writing this document a new standard informally referred to - as SQL3 - is under development. It is planned to make SQL - a Turing-complete - language, i.e., all computable queries (e.g., recursive queries) will be - possible. This has now been completed as SQL:2003. - - - - The Relational Data Model - - - As mentioned before, SQL is a relational - language. That means it is - based on the relational data model - first published by E.F. Codd in - 1970. We will give a formal description of the relational model - later (in - ) - but first we want to have a look at it from a more intuitive - point of view. - - - - A relational database is a database that is - perceived by its - users as a collection of tables (and - nothing else but tables). - A table consists of rows and columns where each row represents a - record and each column represents an attribute of the records - contained in the table. - - shows an example of a database consisting of three tables: - - - - - SUPPLIER is a table storing the number - (SNO), the name (SNAME) and the city (CITY) of a supplier. - - - - - - PART is a table storing the number (PNO) the name (PNAME) and - the price (PRICE) of a part. - - - - - - SELLS stores information about which part (PNO) is sold by which - supplier (SNO). - It serves in a sense to connect the other two tables together. - - - - - - The Suppliers and Parts Database - -SUPPLIER: SELLS: - SNO | SNAME | CITY SNO | PNO -----+---------+-------- -----+----- - 1 | Smith | London 1 | 1 - 2 | Jones | Paris 1 | 2 - 3 | Adams | Vienna 2 | 4 - 4 | Blake | Rome 3 | 1 - 3 | 3 - 4 | 2 -PART: 4 | 3 - PNO | PNAME | PRICE 4 | 4 -----+---------+--------- - 1 | Screw | 10 - 2 | Nut | 8 - 3 | Bolt | 15 - 4 | Cam | 25 - - - - - - The tables PART and SUPPLIER can be regarded as - entities and - SELLS can be regarded as a relationship - between a particular - part and a particular supplier. - - - - As we will see later, SQL operates on tables - like the ones just - defined but before that we will study the theory of the relational - model. - - - - - Relational Data Model Formalities - - - The mathematical concept underlying the relational model is the - set-theoretic relation which is a subset of - the Cartesian - product of a list of domains. This set-theoretic relation gives - the model its name (do not confuse it with the relationship from the - Entity-Relationship model). - Formally a domain is simply a set of - values. For example the set of integers is a domain. Also the set of - character strings of length 20 and the real numbers are examples of - domains. - - - - - The Cartesian product of domains - D1, - D2, - ... - Dk, - written - D1 × - D2 × - ... × - Dk - is the set of all k-tuples - v1, - v2, - ... - vk, - such that - v1 ∈ - D1, - v2 ∈ - D2, - ... - vk ∈ - Dk. - - - - For example, when we have - - k=2, - D1={0,1} and - D2={a,b,c} then - D1 × - D2 is - {(0,a),(0,b),(0,c),(1,a),(1,b),(1,c)}. - - - - - A Relation is any subset of the Cartesian product of one or more - domains: R ⊆ - D1 × - D2 × - ... × - Dk. - - - - For example {(0,a),(0,b),(1,a)} is a relation; - it is in fact a subset of - D1 × - D2 - mentioned above. - - - - The members of a relation are called tuples. Each relation of some - Cartesian product - D1 × - D2 × - ... × - Dk - is said to have arity k and is therefore a set - of k-tuples. - - - - A relation can be viewed as a table (as we already did, remember - where - every tuple is represented by a row and every column corresponds to - one component of a tuple. Giving names (called attributes) to the - columns leads to the definition of a - relation scheme. - - - - - A relation scheme R is a - finite set of attributes - A1, - A2, - ... - Ak. - There is a domain - Di, - for each attribute - Ai, - 1 <= i <= k, - where the values of the attributes are taken from. We often write - a relation scheme as - R(A1, - A2, - ... - Ak). - - - - A relation scheme is just a kind of template - whereas a relation is an instance of a - relation - scheme. The relation consists of tuples (and can - therefore be - viewed as a table); not so the relation scheme. - - - - - - Domains vs. Data Types - - - We often talked about domains - in the last section. Recall that a - domain is, formally, just a set of values (e.g., the set of integers or - the real numbers). In terms of database systems we often talk of - data types instead of domains. - When we define a table we have to make - a decision about which attributes to include. Additionally we - have to decide which kind of data is going to be stored as - attribute values. For example the values of - SNAME from the table - SUPPLIER will be character strings, - whereas SNO will store - integers. We define this by assigning a data type to each - attribute. The type of SNAME will be - VARCHAR(20) (this is the SQL type - for character strings of length <= 20), - the type of SNO will be - INTEGER. With the assignment of a data type we also - have selected - a domain for an attribute. The domain of - SNAME is the set of all - character strings of length <= 20, - the domain of SNO is the set of - all integer numbers. - - - - - - Operations in the Relational Data Model - - - In the previous section - () - we defined the mathematical notion of - the relational model. Now we know how the data can be stored using a - relational data model but we do not know what to do with all these - tables to retrieve something from the database yet. For example somebody - could ask for the names of all suppliers that sell the part - 'Screw'. Therefore two rather different kinds of notations for - expressing operations on relations have been defined: - - - - - The Relational Algebra which is an - algebraic notation, - where queries are expressed by applying specialized operators to the - relations. - - - - - - The Relational Calculus which is a - logical notation, - where queries are expressed by formulating some logical restrictions - that the tuples in the answer must satisfy. - - - - - - - Relational Algebra - - - The Relational Algebra was introduced by - E. F. Codd in 1972. It consists of a set of operations on relations: - - - - - SELECT (σ): extracts tuples from - a relation that - satisfy a given restriction. Let R be a - table that contains an attribute - A. -σA=a(R) = {t ∈ R ∣ t(A) = a} - where t denotes a - tuple of R and t(A) - denotes the value of attribute A of - tuple t. - - - - - - PROJECT (π): extracts specified - attributes (columns) from a - relation. Let R be a relation - that contains an attribute X. - πX(R) = {t(X) ∣ t ∈ R}, - where t(X) denotes the value of - attribute X of tuple t. - - - - - - PRODUCT (×): builds the Cartesian product of two - relations. Let R be a table with arity - k1 and let - S be a table with - arity k2. - R × S - is the set of all - k1 - + k2-tuples - whose first k1 - components form a tuple in R and whose last - k2 components form a - tuple in S. - - - - - - UNION (∪): builds the set-theoretic union of two - tables. Given the tables R and - S (both must have the same arity), - the union R ∪ S - is the set of tuples that are in R - or S or both. - - - - - - INTERSECT (∩): builds the set-theoretic intersection of two - tables. Given the tables R and - S, - R ∩ S is the - set of tuples - that are in R and in - S. - We again require that R and - S have the - same arity. - - - - - - DIFFERENCE (− or ∖): builds the set difference of - two tables. Let R and S - again be two tables with the same - arity. R - S - is the set of tuples in R but not in - S. - - - - - - JOIN (∏): connects two tables by their common - attributes. Let R be a table with the - attributes A,B - and C and - let S be a table with the attributes - C,D - and E. There is one - attribute common to both relations, - the attribute C. - - R ∏ S = πR.A,R.B,R.C,S.D,S.E(σR.C=S.C(R × S)). - What are we doing here? We first calculate the Cartesian - product - R × S. - Then we select those tuples whose values for the common - attribute C are equal - (σR.C = S.C). - Now we have a table - that contains the attribute C - two times and we correct this by - projecting out the duplicate column. - - - - An Inner Join - - - Let's have a look at the tables that are produced by evaluating the steps - necessary for a join. - Let the following two tables be given: - - -R: S: - A | B | C C | D | E ----+---+--- ---+---+--- - 1 | 2 | 3 3 | a | b - 4 | 5 | 6 6 | c | d - 7 | 8 | 9 - - - - - - First we calculate the Cartesian product - R × S and - get: - - -R x S: - A | B | R.C | S.C | D | E ----+---+-----+-----+---+--- - 1 | 2 | 3 | 3 | a | b - 1 | 2 | 3 | 6 | c | d - 4 | 5 | 6 | 3 | a | b - 4 | 5 | 6 | 6 | c | d - 7 | 8 | 9 | 3 | a | b - 7 | 8 | 9 | 6 | c | d - - - - - After the selection - σR.C=S.C(R × S) - we get: - - - A | B | R.C | S.C | D | E ----+---+-----+-----+---+--- - 1 | 2 | 3 | 3 | a | b - 4 | 5 | 6 | 6 | c | d - - - - - To remove the duplicate column - S.C - we project it out by the following operation: - πR.A,R.B,R.C,S.D,S.E(σR.C=S.C(R × S)) - and get: - - - A | B | C | D | E ----+---+---+---+--- - 1 | 2 | 3 | a | b - 4 | 5 | 6 | c | d - - - - - - - DIVIDE (÷): Let R be a table - with the attributes A, B, C, and D and let - S be a table with the attributes - C and D. - Then we define the division as: - - -R ÷ S = {t ∣ ∀ ts ∈ S ∃ tr ∈ R - - - such that -tr(A,B)=t∧tr(C,D)=ts} - where - tr(x,y) - denotes a - tuple of table R that consists only of - the components x and y. - Note that the tuple t only consists of the - components A and - B of relation R. - - - - Given the following tables - - -R: S: - A | B | C | D C | D ----+---+---+--- ---+--- - a | b | c | d c | d - a | b | e | f e | f - b | c | e | f - e | d | c | d - e | d | e | f - a | b | d | e - - - R ÷ S - is derived as - - - A | B ----+--- - a | b - e | d - - - - - - - - For a more detailed description and definition of the relational - algebra refer to [] or - []. - - - - A Query Using Relational Algebra - - Recall that we formulated all those relational operators to be able to - retrieve data from the database. Let's return to our example from - the previous - section () - where someone wanted to know the names of all - suppliers that sell the part Screw. - This question can be answered - using relational algebra by the following operation: - - -πSUPPLIER.SNAME(σPART.PNAME='Screw'(SUPPLIER ∏ SELLS ∏ PART)) - - - - - We call such an operation a query. If we evaluate the above query - against the our example tables - () - we will obtain the following result: - - - SNAME -------- - Smith - Adams - - - - - - - Relational Calculus - - - The relational calculus is based on the - first order logic. There are - two variants of the relational calculus: - - - - - The Domain Relational Calculus - (DRC), where variables - stand for components (attributes) of the tuples. - - - - - - The Tuple Relational Calculus - (TRC), where variables stand for tuples. - - - - - - - We want to discuss the tuple relational calculus only because it is - the one underlying the most relational languages. For a detailed - discussion on DRC (and also - TRC) see - - or - . - - - - - Tuple Relational Calculus - - - The queries used in TRC are of the following - form: - - -x(A) ∣ F(x) - - - where x is a tuple variable - A is a set of attributes and F is a - formula. The resulting relation consists of all tuples - t(A) that satisfy F(t). - - - - If we want to answer the question from example - - using TRC we formulate the following query: - - -{x(SNAME) ∣ x ∈ SUPPLIER ∧ - ∃ y ∈ SELLS ∃ z ∈ PART (y(SNO)=x(SNO) ∧ - z(PNO)=y(PNO) ∧ - z(PNAME)='Screw')} - - - - - Evaluating the query against the tables from - - again leads to the same result - as in - . - - - - - Relational Algebra vs. Relational Calculus - - - The relational algebra and the relational calculus have the same - expressive power; i.e., all queries that - can be formulated using relational algebra can also be formulated - using the relational calculus and vice versa. - This was first proved by E. F. Codd in - 1972. This proof is based on an algorithm (Codd's reduction - algorithm) by which an arbitrary expression of the relational - calculus can be reduced to a semantically equivalent expression of - relational algebra. For a more detailed discussion on that refer to - - and - . - - - - It is sometimes said that languages based on the relational - calculus are higher level or more - declarative than languages based on relational algebra - because the algebra (partially) specifies the order of operations - while the calculus leaves it to a compiler or interpreter to - determine the most efficient order of evaluation. - - - - - - The <acronym>SQL</acronym> Language - - - As is the case with most modern relational languages, - SQL is based on the tuple - relational calculus. As a result every query that can be formulated - using the tuple relational calculus (or equivalently, relational - algebra) can also be formulated using - SQL. There are, however, - capabilities beyond the scope of relational algebra or calculus. Here - is a list of some additional features provided by - SQL that are not - part of relational algebra or calculus: - - - - - Commands for insertion, deletion or modification of data. - - - - - - Arithmetic capability: In SQL it is possible - to involve - arithmetic operations as well as comparisons, e.g.: - - -A < B + 3. - - - Note - that + or other arithmetic operators appear neither in relational - algebra nor in relational calculus. - - - - - - Assignment and Print Commands: It is possible to print a - relation constructed by a query and to assign a computed relation to a - relation name. - - - - - - Aggregate Functions: Operations such as - average, sum, - max, etc. can be applied to columns of a - relation to - obtain a single quantity. - - - - - - - Select - - - The most often used command in SQL is the - SELECT statement, - used to retrieve data. The syntax is: - - -SELECT [ ALL | DISTINCT [ ON ( expression [, ...] ) ] ] - * | expression [ [ AS ] output_name ] [, ...] - [ INTO [ TEMPORARY | TEMP ] [ TABLE ] new_table ] - [ FROM from_item [, ...] ] - [ WHERE condition ] - [ GROUP BY expression [, ...] ] - [ HAVING condition [, ...] ] - [ { UNION | INTERSECT | EXCEPT } [ ALL ] select ] - [ ORDER BY expression [ ASC | DESC | USING operator ] [ NULLS { FIRST | LAST } ] [, ...] ] - [ LIMIT { count | ALL } ] - [ OFFSET start ] - [ FOR { UPDATE | SHARE } [ OF table_name [, ...] ] [ NOWAIT | SKIP LOCKED ] [...] ] - - - - - Now we will illustrate the complex syntax of the - SELECT statement with various examples. The - tables used for the examples are defined in . - - - - Simple Selects - - - Here are some simple examples using a SELECT statement: - - - Simple Query with Qualification - - To retrieve all tuples from table PART where the attribute PRICE is - greater than 10 we formulate the following query: - - -SELECT * FROM PART - WHERE PRICE > 10; - - - and get the table: - - - PNO | PNAME | PRICE ------+---------+-------- - 3 | Bolt | 15 - 4 | Cam | 25 - - - - - Using * in the SELECT statement - will deliver all attributes from the table. If we want to retrieve - only the attributes PNAME and PRICE from table PART we use the - statement: - - -SELECT PNAME, PRICE - FROM PART - WHERE PRICE > 10; - - - In this case the result is: - - - PNAME | PRICE - --------+-------- - Bolt | 15 - Cam | 25 - - - Note that the SQL SELECT - corresponds to the projection in relational algebra - not to the selection (see for more details). - - - - The qualifications in the WHERE clause can also be logically connected - using the keywords OR, AND, and NOT: - - -SELECT PNAME, PRICE - FROM PART - WHERE PNAME = 'Bolt' AND - (PRICE = 0 OR PRICE <= 15); - - - will lead to the result: - - - PNAME | PRICE ---------+-------- - Bolt | 15 - - - - - Arithmetic operations can be used in the target list and in the WHERE - clause. For example if we want to know how much it would cost if we - take two pieces of a part we could use the following query: - - -SELECT PNAME, PRICE * 2 AS DOUBLE - FROM PART - WHERE PRICE * 2 < 50; - - - and we get: - - - PNAME | DOUBLE ---------+--------- - Screw | 20 - Nut | 16 - Bolt | 30 - - - Note that the word DOUBLE after the keyword AS is the new title of the - second column. This technique can be used for every element of the - target list to assign a new title to the resulting - column. This new title - is often referred to as alias. The alias cannot be used throughout the - rest of the query. - - - - - - - Joins - - - The following example shows how joins are - realized in SQL. - - - - To join the three tables SUPPLIER, PART and SELLS over their common - attributes we formulate the following statement: - - -SELECT S.SNAME, P.PNAME - FROM SUPPLIER S, PART P, SELLS SE - WHERE S.SNO = SE.SNO AND - P.PNO = SE.PNO; - - - and get the following table as a result: - - - SNAME | PNAME --------+------- - Smith | Screw - Smith | Nut - Jones | Cam - Adams | Screw - Adams | Bolt - Blake | Nut - Blake | Bolt - Blake | Cam - - - - - In the FROM clause we introduced an alias name for every relation - because there are common named attributes (SNO and PNO) among the - relations. Now we can distinguish between the common named attributes - by simply prefixing the attribute name with the alias name followed by - a dot. The join is calculated in the same way as shown in - . - First the Cartesian product - - SUPPLIER × PART × SELLS - - is derived. Now only those tuples satisfying the - conditions given in the WHERE clause are selected (i.e., the common - named attributes have to be equal). Finally we project out all - columns but S.SNAME and P.PNAME. - - - - Another way to perform joins is to use the SQL JOIN syntax as follows: - -SELECT sname, pname from supplier - JOIN sells USING (sno) - JOIN part USING (pno); - - giving again: - - sname | pname --------+------- - Smith | Screw - Adams | Screw - Smith | Nut - Blake | Nut - Adams | Bolt - Blake | Bolt - Jones | Cam - Blake | Cam -(8 rows) - - - - - A joined table, created using JOIN syntax, is a table reference list - item that occurs in a FROM clause and before any WHERE, GROUP BY, - or HAVING clause. Other table references, including table names or - other JOIN clauses, can be included in the FROM clause if separated - by commas. JOINed tables are logically like any other - table listed in the FROM clause. - - - - SQL JOINs come in two main types, CROSS JOINs (unqualified joins) - and qualified JOINs. Qualified joins can be further - subdivided based on the way in which the join condition - is specified (ON, USING, or NATURAL) and the way in which it is - applied (INNER or OUTER join). - - - - Join Types - - CROSS JOIN - - - T1 - CROSS JOIN - T2 - - - - A cross join takes two tables T1 and T2 having N and M rows - respectively, and returns a joined table containing all - N*M possible joined rows. For each row R1 of T1, each row - R2 of T2 is joined with R1 to yield a joined table row JR - consisting of all fields in R1 and R2. A CROSS JOIN is - equivalent to an INNER JOIN ON TRUE. - - - - - - Qualified JOINs - - - - T1 - NATURAL - - INNER - - - LEFT - RIGHT - FULL - - OUTER - - - JOIN - T2 - - ON search condition - USING ( join column list ) - - - - - A qualified JOIN must specify its join condition - by providing one (and only one) of NATURAL, ON, or - USING. The ON clause - takes a search condition, - which is the same as in a WHERE clause. The USING - clause takes a comma-separated list of column names, - which the joined tables must have in common, and joins - the tables on equality of those columns. NATURAL is - shorthand for a USING clause that lists all the common - column names of the two tables. A side-effect of both - USING and NATURAL is that only one copy of each joined - column is emitted into the result table (compare the - relational-algebra definition of JOIN, shown earlier). - - - - - - - - INNER - JOIN - - - - - For each row R1 of T1, the joined table has a row for each row - in T2 that satisfies the join condition with R1. - - - - The words INNER and OUTER are optional for all JOINs. - INNER is the default. LEFT, RIGHT, and FULL imply an - OUTER JOIN. - - - - - - - - LEFT - OUTER - JOIN - - - - - First, an INNER JOIN is performed. - Then, for each row in T1 that does not satisfy the join - condition with any row in T2, an additional joined row is - returned with null fields in the columns from T2. - - - - The joined table unconditionally has a row for each row in T1. - - - - - - - - RIGHT - OUTER - JOIN - - - - - First, an INNER JOIN is performed. - Then, for each row in T2 that does not satisfy the join - condition with any row in T1, an additional joined row is - returned with null fields in the columns from T1. - - - - The joined table unconditionally has a row for each row in T2. - - - - - - - - FULL - OUTER - JOIN - - - - - First, an INNER JOIN is performed. - Then, for each row in T1 that does not satisfy the join - condition with any row in T2, an additional joined row is - returned with null fields in the columns from T2. - Also, for each row in T2 that does not satisfy the join - condition with any row in T1, an additional joined row is - returned with null fields in the columns from T1. - - - - The joined table unconditionally has a row for every row of T1 - and a row for every row of T2. - - - - - - - - - - - - - JOINs of all types can be chained together or nested where either or both of - T1 and - T2 can be JOINed tables. - Parenthesis can be used around JOIN clauses to control the order - of JOINs which are otherwise processed left to right. - - - - - - Aggregate Functions - - - SQL provides aggregate functions such as AVG, - COUNT, SUM, MIN, and MAX. The argument(s) of an aggregate function - are evaluated at each row that satisfies the WHERE - clause, and the aggregate function is calculated over this set - of input values. Normally, an aggregate delivers a single - result for a whole SELECT statement. But if - grouping is specified in the query, then a separate calculation - is done over the rows of each group, and an aggregate result is - delivered per group (see next section). - - - Aggregates - - - If we want to know the average cost of all parts in table PART we use - the following query: - - -SELECT AVG(PRICE) AS AVG_PRICE - FROM PART; - - - - - The result is: - - - AVG_PRICE ------------ - 14.5 - - - - - If we want to know how many parts are defined in table PART we use - the statement: - - -SELECT COUNT(PNO) - FROM PART; - - - and get: - - - COUNT -------- - 4 - - - - - - - - - Aggregation by Groups - - - SQL allows one to partition the tuples of a table - into groups. Then the - aggregate functions described above can be applied to the groups — - i.e., the value of the aggregate function is no longer calculated over - all the values of the specified column but over all values of a - group. Thus the aggregate function is evaluated separately for every - group. - - - - The partitioning of the tuples into groups is done by using the - keywords GROUP BY followed by a list of - attributes that define the - groups. If we have - GROUP BY A1, ⃛, Ak - we partition - the relation into groups, such that two tuples are in the same group - if and only if they agree on all the attributes - A1, ⃛, Ak. - - - Aggregates - - If we want to know how many parts are sold by every supplier we - formulate the query: - - -SELECT S.SNO, S.SNAME, COUNT(SE.PNO) - FROM SUPPLIER S, SELLS SE - WHERE S.SNO = SE.SNO - GROUP BY S.SNO, S.SNAME; - - - and get: - - - SNO | SNAME | COUNT ------+-------+------- - 1 | Smith | 2 - 2 | Jones | 1 - 3 | Adams | 2 - 4 | Blake | 3 - - - - - Now let's have a look of what is happening here. - First the join of the - tables SUPPLIER and SELLS is derived: - - - S.SNO | S.SNAME | SE.PNO --------+---------+-------- - 1 | Smith | 1 - 1 | Smith | 2 - 2 | Jones | 4 - 3 | Adams | 1 - 3 | Adams | 3 - 4 | Blake | 2 - 4 | Blake | 3 - 4 | Blake | 4 - - - - - Next we partition the tuples into groups by putting all tuples - together that agree on both attributes S.SNO and S.SNAME: - - - S.SNO | S.SNAME | SE.PNO --------+---------+-------- - 1 | Smith | 1 - | 2 --------------------------- - 2 | Jones | 4 --------------------------- - 3 | Adams | 1 - | 3 --------------------------- - 4 | Blake | 2 - | 3 - | 4 - - - - - In our example we got four groups and now we can apply the aggregate - function COUNT to every group leading to the final result of the query - given above. - - - - - - Note that for a query using GROUP BY and aggregate - functions to make sense, the target list can only refer directly to - the attributes being grouped by. Other attributes can only be used - inside the arguments of aggregate functions. Otherwise there would - not be a unique value to associate with the other attributes. - - - - Also observe that it makes no sense to ask for an aggregate of - an aggregate, e.g., AVG(MAX(sno)), because a - SELECT only does one pass of grouping and - aggregation. You can get a result of this kind by using a - temporary table or a sub-SELECT in the FROM clause to do the - first level of aggregation. - - - - - Having - - - The HAVING clause works much like the WHERE clause and is used to - consider only those groups satisfying the qualification given in the - HAVING clause. Essentially, WHERE filters out unwanted input rows - before grouping and aggregation are done, whereas HAVING filters out - unwanted group rows post-GROUP. Therefore, WHERE cannot refer to the - results of aggregate functions. On the other hand, there's no point - in writing a HAVING condition that doesn't involve an aggregate - function! If your condition doesn't involve aggregates, you might - as well write it in WHERE, and thereby avoid the computation of - aggregates for groups that you're just going to throw away anyway. - - - Having - - - If we want only those suppliers selling more than one part we use the - query: - - -SELECT S.SNO, S.SNAME, COUNT(SE.PNO) - FROM SUPPLIER S, SELLS SE - WHERE S.SNO = SE.SNO - GROUP BY S.SNO, S.SNAME - HAVING COUNT(SE.PNO) > 1; - - - and get: - - - SNO | SNAME | COUNT ------+-------+------- - 1 | Smith | 2 - 3 | Adams | 2 - 4 | Blake | 3 - - - - - - - - Subqueries - - - In the WHERE and HAVING clauses the use of subqueries (subselects) is - allowed in every place where a value is expected. In this case the - value must be derived by evaluating the subquery first. The usage of - subqueries extends the expressive power of - SQL. - - - Subselect - - - If we want to know all parts having a greater price than the part - named 'Screw' we use the query: - - -SELECT * - FROM PART - WHERE PRICE > (SELECT PRICE FROM PART - WHERE PNAME='Screw'); - - - - - The result is: - - - PNO | PNAME | PRICE ------+---------+-------- - 3 | Bolt | 15 - 4 | Cam | 25 - - - - - When we look at the above query we can see the keyword - SELECT two times. The first one at the - beginning of the query - we will refer to it as outer - SELECT - and the one in the WHERE clause which - begins a nested query - we will refer to it as inner - SELECT. For every tuple of the outer - SELECT the inner SELECT has - to be evaluated. After every evaluation we know the price of the - tuple named 'Screw' and we can check if the price of the actual - tuple is greater. (Actually, in this example the inner query need - only be evaluated once, since it does not depend on the state of - the outer query.) - - - - If we want to know all suppliers that do not sell any part - (e.g., to be able to remove these suppliers from the database) we use: - - -SELECT * - FROM SUPPLIER S - WHERE NOT EXISTS - (SELECT * FROM SELLS SE - WHERE SE.SNO = S.SNO); - - - - - In our example the result will be empty because every supplier - sells at least one part. Note that we use S.SNO from the outer - SELECT within the WHERE clause of the inner - SELECT. Here the subquery must be evaluated - afresh for each tuple from the outer query, i.e., the value for - S.SNO is always taken from the current tuple of the outer - SELECT. - - - - - - - Subqueries in FROM - - - A somewhat different way of using subqueries is to put them in the - FROM clause. This is a useful feature because a subquery of this - kind can output multiple columns and rows, whereas a subquery used - in an expression must deliver just a single result. It also lets - us get more than one round of grouping/aggregation without resorting - to a temporary table. - - - Subselect in FROM - - - If we want to know the highest average part price among all our - suppliers, we cannot write MAX(AVG(PRICE)), but we can write: - - -SELECT MAX(subtable.avgprice) - FROM (SELECT AVG(P.PRICE) AS avgprice - FROM SUPPLIER S, PART P, SELLS SE - WHERE S.SNO = SE.SNO AND - P.PNO = SE.PNO - GROUP BY S.SNO) subtable; - - - The subquery returns one row per supplier (because of its GROUP BY) - and then we aggregate over those rows in the outer query. - - - - - - - Union, Intersect, Except - - - These operations calculate the union, intersection and set theoretic - difference of the tuples derived by two subqueries. - - - Union, Intersect, Except - - - The following query is an example for UNION: - - -SELECT S.SNO, S.SNAME, S.CITY - FROM SUPPLIER S - WHERE S.SNAME = 'Jones' -UNION - SELECT S.SNO, S.SNAME, S.CITY - FROM SUPPLIER S - WHERE S.SNAME = 'Adams'; - - -gives the result: - - - SNO | SNAME | CITY ------+-------+-------- - 2 | Jones | Paris - 3 | Adams | Vienna - - - - - Here is an example for INTERSECT: - - -SELECT S.SNO, S.SNAME, S.CITY - FROM SUPPLIER S - WHERE S.SNO > 1 -INTERSECT - SELECT S.SNO, S.SNAME, S.CITY - FROM SUPPLIER S - WHERE S.SNO < 3; - - - gives the result: - - - SNO | SNAME | CITY ------+-------+-------- - 2 | Jones | Paris - - - The only tuple returned by both parts of the query is the one having SNO=2. - - - - Finally an example for EXCEPT: - - -SELECT S.SNO, S.SNAME, S.CITY - FROM SUPPLIER S - WHERE S.SNO > 1 -EXCEPT - SELECT S.SNO, S.SNAME, S.CITY - FROM SUPPLIER S - WHERE S.SNO > 3; - - - gives the result: - - - SNO | SNAME | CITY ------+-------+-------- - 2 | Jones | Paris - 3 | Adams | Vienna - - - - - - - - - Data Definition - - - There is a set of commands used for data definition included in the - SQL language. - - - - Create Table - - - The most fundamental command for data definition is the - one that creates a new relation (a new table). The syntax of the - CREATE TABLE command is: - - -CREATE TABLE table_name - (name_of_attr_1 type_of_attr_1 - [, name_of_attr_2 type_of_attr_2 - [, ...]]); - - - - Table Creation - - - To create the tables defined in - the - following SQL statements are used: - - -CREATE TABLE SUPPLIER - (SNO INTEGER, - SNAME VARCHAR(20), - CITY VARCHAR(20)); - - - -CREATE TABLE PART - (PNO INTEGER, - PNAME VARCHAR(20), - PRICE DECIMAL(4 , 2)); - - - -CREATE TABLE SELLS - (SNO INTEGER, - PNO INTEGER); - - - - - - - - Data Types in <acronym>SQL</acronym> - - - The following is a list of some data types that are supported by - SQL: - - - - - INTEGER: signed fullword binary integer (31 bits precision). - - - - - - SMALLINT: signed halfword binary integer (15 bits precision). - - - - - - DECIMAL (p[,q]): - signed packed decimal number of up to - p - digits, with - q - digits to the right of the decimal point. - If q - is omitted it is assumed to be 0. - - - - - - FLOAT: signed doubleword floating point number. - - - - - - VARCHAR(n): - varying length character string of maximum length - n. - - - - - - CHAR(n): - fixed length character string of length - n. - - - - - - - - - Create Index - - - Indexes are used to speed up access to a relation. If a relation R - has an index on attribute A then we can - retrieve all tuples t - having - t(A) = a - in time roughly proportional to the number of such - tuples t - rather than in time proportional to the size of R. - - - - To create an index in SQL - the CREATE INDEX command is used. The syntax is: - - -CREATE INDEX index_name - ON table_name ( name_of_attribute ); - - - - - - Create Index - - - To create an index named I on attribute SNAME of relation SUPPLIER - we use the following statement: - - -CREATE INDEX I ON SUPPLIER (SNAME); - - - - - The created index is maintained automatically, i.e., whenever a new - tuple is inserted into the relation SUPPLIER the index I is - adapted. Note that the only changes a user can perceive when an - index is present are increased speed for SELECT - and decreases in speed of updates. - - - - - - - Create View - - - A view can be regarded as a virtual table, - i.e., a table that - does not physically exist in the database - but looks to the user - as if it does. By contrast, when we talk of a - base table there is - really a physically stored counterpart of each row of the table - somewhere in the physical storage. - - - - Views do not have their own, physically separate, distinguishable - stored data. Instead, the system stores the definition of the - view (i.e., the rules about how to access physically stored base - tables in order to materialize the view) somewhere in the system - catalogs (see - ). For a - discussion on different techniques to implement views refer to - - SIM98. - - - - In SQL the CREATE VIEW - command is used to define a view. The syntax - is: - - -CREATE VIEW view_name - AS select_stmt - - - where select_stmt - is a valid select statement as defined - in . - Note that select_stmt is - not executed when the view is created. It is just stored in the - system catalogs - and is executed whenever a query against the view is made. - - - - Let the following view definition be given (we use - the tables from - again): - - -CREATE VIEW London_Suppliers - AS SELECT S.SNAME, P.PNAME - FROM SUPPLIER S, PART P, SELLS SE - WHERE S.SNO = SE.SNO AND - P.PNO = SE.PNO AND - S.CITY = 'London'; - - - - - Now we can use this virtual relation - London_Suppliers as - if it were another base table: - - -SELECT * FROM London_Suppliers - WHERE PNAME = 'Screw'; - - - which will return the following table: - - - SNAME | PNAME --------+------- - Smith | Screw - - - - - To calculate this result the database system has to do a - hidden - access to the base tables SUPPLIER, SELLS and PART first. It - does so by executing the query given in the view definition against - those base tables. After that the additional qualifications - (given in the - query against the view) can be applied to obtain the resulting - table. - - - - - Drop Table, Drop Index, Drop View - - - To destroy a table (including all tuples stored in that table) the - DROP TABLE command is used: - - -DROP TABLE table_name; - - - - - To destroy the SUPPLIER table use the following statement: - - -DROP TABLE SUPPLIER; - - - - - The DROP INDEX command is used to destroy an index: - - -DROP INDEX index_name; - - - - - Finally to destroy a given view use the command DROP - VIEW: - - -DROP VIEW view_name; - - - - - - - Data Manipulation - - - Insert Into - - - Once a table is created (see - ), it can be filled - with tuples using the command INSERT INTO. - The syntax is: - - -INSERT INTO table_name (name_of_attr_1 - [, name_of_attr_2 [, ...]]) - VALUES (val_attr_1 [, val_attr_2 [, ...]]); - - - - - To insert the first tuple into the relation SUPPLIER (from - ) we use the - following statement: - - -INSERT INTO SUPPLIER (SNO, SNAME, CITY) - VALUES (1, 'Smith', 'London'); - - - - - To insert the first tuple into the relation SELLS we use: - - -INSERT INTO SELLS (SNO, PNO) - VALUES (1, 1); - - - - - - Update - - - To change one or more attribute values of tuples in a relation the - UPDATE command is used. The syntax is: - - -UPDATE table_name - SET name_of_attr_1 = value_1 - [, ... [, name_of_attr_k = value_k]] - WHERE condition; - - - - - To change the value of attribute PRICE of the part 'Screw' in the - relation PART we use: - - -UPDATE PART - SET PRICE = 15 - WHERE PNAME = 'Screw'; - - - - - The new value of attribute PRICE of the tuple whose name is 'Screw' is - now 15. - - - - - Delete - - - To delete a tuple from a particular table use the command DELETE - FROM. The syntax is: - - -DELETE FROM table_name - WHERE condition; - - - - - To delete the supplier called 'Smith' of the table SUPPLIER the - following statement is used: - - -DELETE FROM SUPPLIER - WHERE SNAME = 'Smith'; - - - - - - - System Catalogs - - - In every SQL database system - system catalogs are used to keep - track of which tables, views indexes etc. are defined in the - database. These system catalogs can be queried as if they were normal - relations. For example there is one catalog used for the definition of - views. This catalog stores the query from the view definition. Whenever - a query against a view is made, the system first gets the - view definition query out of the catalog - and materializes the view - before proceeding with the user query (see - - - for a more detailed - description). For more information about system catalogs refer to - . - - - - - Embedded <acronym>SQL</acronym> - - - In this section we will sketch how SQL can be - embedded into a host language (e.g., C). - There are two main reasons why we want to use SQL - from a host language: - - - - - There are queries that cannot be formulated using pure SQL - (i.e., recursive queries). To be able to perform such queries we need a - host language with a greater expressive power than - SQL. - - - - - - We simply want to access a database from some application that - is written in the host language (e.g., a ticket reservation system - with a graphical user interface is written in C and the information - about which tickets are still left is stored in a database that can be - accessed using embedded SQL). - - - - - - - A program using embedded SQL - in a host language consists of statements - of the host language and of - embedded SQL - (ESQL) statements. Every ESQL - statement begins with the keywords EXEC SQL. - The ESQL statements are - transformed to statements of the host language - by a precompiler - (which usually inserts - calls to library routines that perform the various SQL - commands). - - - - When we look at the examples throughout - we - realize that the result of the queries is very often a set of - tuples. Most host languages are not designed to operate on sets so we - need a mechanism to access every single tuple of the set of tuples - returned by a SELECT statement. This mechanism can be provided by - declaring a cursor. - After that we can use the FETCH command to - retrieve a tuple and set the cursor to the next tuple. - - - - For a detailed discussion on embedded SQL - refer to - , - , - or - . - - - - -- cgit v1.2.3 From 1aedcf98181602890ea8899202a7143543f9785a Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 15 May 2017 15:48:23 -0400 Subject: git-ignore intermediate files from new docs toolchain. Building PDFs with the new toolchain creates *.fo temporary files. --- doc/src/sgml/.gitignore | 2 ++ 1 file changed, 2 insertions(+) (limited to 'doc/src') diff --git a/doc/src/sgml/.gitignore b/doc/src/sgml/.gitignore index a74513837f..a72b7ccb06 100644 --- a/doc/src/sgml/.gitignore +++ b/doc/src/sgml/.gitignore @@ -20,3 +20,5 @@ /postgres.xml /INSTALL.html /INSTALL.xml +/postgres-US.fo +/postgres-A4.fo -- cgit v1.2.3 From 6accefd46639db9f20bcc4c2e15c9844bae0d184 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 15 May 2017 22:06:27 -0400 Subject: Update CREATE SUBSCRIPTION docs for recent syntax change. Masahiko Sawada --- doc/src/sgml/ref/create_subscription.sgml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml index f2da662499..2c91eb6f50 100644 --- a/doc/src/sgml/ref/create_subscription.sgml +++ b/doc/src/sgml/ref/create_subscription.sgml @@ -202,9 +202,10 @@ CREATE SUBSCRIPTION subscription_name - Since no connection is made when this option is specified, the - tables are not subscribed, and so after you enable the subscription - nothing will be replicated. It is required to run + Since no connection is made when this option is set + to false, the tables are not subscribed, and so + after you enable the subscription nothing will be replicated. + It is required to run ALTER SUBSCRIPTION ... REFRESH PUBLICATION in order for tables to be subscribed. -- cgit v1.2.3 From 8e709a612f4c10cdc4b19a734cd67ac019d0a2ec Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Tue, 16 May 2017 11:35:23 -0400 Subject: doc: Remove unnecessary RETURN statements from example. Paul Jungwirth, reviewed by Ashutosh Bapat. Discussion: http://postgr.es/m/e24a6a6d-5670-739b-00f3-41a226a80f25@illuminatedcomputing.com --- doc/src/sgml/plpgsql.sgml | 3 --- 1 file changed, 3 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/plpgsql.sgml b/doc/src/sgml/plpgsql.sgml index a6088e9c07..dc29e7cd0f 100644 --- a/doc/src/sgml/plpgsql.sgml +++ b/doc/src/sgml/plpgsql.sgml @@ -3999,13 +3999,10 @@ CREATE OR REPLACE FUNCTION process_emp_audit() RETURNS TRIGGER AS $emp_audit$ -- IF (TG_OP = 'DELETE') THEN INSERT INTO emp_audit SELECT 'D', now(), user, OLD.*; - RETURN OLD; ELSIF (TG_OP = 'UPDATE') THEN INSERT INTO emp_audit SELECT 'U', now(), user, NEW.*; - RETURN NEW; ELSIF (TG_OP = 'INSERT') THEN INSERT INTO emp_audit SELECT 'I', now(), user, NEW.*; - RETURN NEW; END IF; RETURN NULL; -- result is ignored since this is an AFTER trigger END; -- cgit v1.2.3 From 3ec76ff1f2cf52e9b900349957b42d28128b7bc7 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Thu, 18 May 2017 13:48:10 -0400 Subject: Don't explicitly mark range partitioning columns NOT NULL. This seemed like a good idea originally because there's no way to mark a range partition as accepting NULL, but that now seems more like a current limitation than something we want to lock down for all time. For example, there's a proposal to add the notion of a default partition which accepts all rows not otherwise routed, which directly conflicts with the idea that a range-partitioned table should never allow nulls anywhere. So let's change this while we still can, by putting the NOT NULL test into the partition constraint instead of changing the column properties. Amit Langote and Robert Haas, reviewed by Amit Kapila Discussion: http://postgr.es/m/8e2dd63d-c6fb-bb74-3c2b-ed6d63629c9d@lab.ntt.co.jp --- contrib/postgres_fdw/expected/postgres_fdw.out | 4 +- doc/src/sgml/ref/create_table.sgml | 5 -- src/backend/catalog/partition.c | 71 ++++++++++++++++---------- src/backend/commands/tablecmds.c | 57 +-------------------- src/test/regress/expected/alter_table.out | 7 --- src/test/regress/expected/create_table.out | 40 +++++---------- src/test/regress/expected/insert.out | 5 +- src/test/regress/sql/alter_table.sql | 8 --- src/test/regress/sql/create_table.sql | 5 -- 9 files changed, 64 insertions(+), 138 deletions(-) (limited to 'doc/src') diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index d1bc5b0660..4d86ab54dd 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -6987,7 +6987,7 @@ FDW Options: (schema_name 'import_source', table_name 't3') Foreign table "import_dest1.t4" Column | Type | Collation | Nullable | Default | FDW Options --------+---------+-----------+----------+---------+-------------------- - c1 | integer | | not null | | (column_name 'c1') + c1 | integer | | | | (column_name 'c1') Server: loopback FDW Options: (schema_name 'import_source', table_name 't4') @@ -7051,7 +7051,7 @@ FDW Options: (schema_name 'import_source', table_name 't3') Foreign table "import_dest2.t4" Column | Type | Collation | Nullable | Default | FDW Options --------+---------+-----------+----------+---------+-------------------- - c1 | integer | | not null | | (column_name 'c1') + c1 | integer | | | | (column_name 'c1') Server: loopback FDW Options: (schema_name 'import_source', table_name 't4') diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml index 484f81898b..0478e40447 100644 --- a/doc/src/sgml/ref/create_table.sgml +++ b/doc/src/sgml/ref/create_table.sgml @@ -454,11 +454,6 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI these constraints on individual partitions. - - When using range partitioning, a NOT NULL constraint - is added to each non-expression column in the partition key. - - diff --git a/src/backend/catalog/partition.c b/src/backend/catalog/partition.c index acd9f2d273..7304f6c29a 100644 --- a/src/backend/catalog/partition.c +++ b/src/backend/catalog/partition.c @@ -1449,17 +1449,18 @@ get_range_key_properties(PartitionKey key, int keynum, * as the lower bound tuple and (au, bu, cu) as the upper bound tuple, we * generate an expression tree of the following form: * + * (a IS NOT NULL) and (b IS NOT NULL) and (c IS NOT NULL) + * AND * (a > al OR (a = al AND b > bl) OR (a = al AND b = bl AND c >= cl)) * AND * (a < au OR (a = au AND b < bu) OR (a = au AND b = bu AND c < cu)) * - * If, say, b were an expression key instead of a simple column, we also - * append (b IS NOT NULL) to the AND's argument list. - * * It is often the case that a prefix of lower and upper bound tuples contains * the same values, for example, (al = au), in which case, we will emit an * expression tree of the following form: * + * (a IS NOT NULL) and (b IS NOT NULL) and (c IS NOT NULL) + * AND * (a = al) * AND * (b > bl OR (b = bl AND c >= cl)) @@ -1472,11 +1473,11 @@ get_range_key_properties(PartitionKey key, int keynum, * (b < bu) OR (b = bu), which is simplified to (b <= bu) * * In most common cases with only one partition column, say a, the following - * expression tree will be generated: a >= al AND a < au + * expression tree will be generated: a IS NOT NULL AND a >= al AND a < au * * If all values of both lower and upper bounds are UNBOUNDED, the partition * does not really have a constraint, except the IS NOT NULL constraint for - * any expression keys. + * partition keys. * * If we end up with an empty result list, we append return a single-member * list containing a constant-true expression in that case, because callers @@ -1512,32 +1513,37 @@ get_qual_for_range(PartitionKey key, PartitionBoundSpec *spec) num_or_arms = key->partnatts; /* - * A range-partitioned table does not allow partition keys to be null. For - * simple columns, their NOT NULL constraint suffices for the enforcement - * of non-nullability. But for the expression keys, which are still - * nullable, we must emit a IS NOT NULL expression. Collect them in - * result first. + * A range-partitioned table does not currently allow partition keys to + * be null, so emit an IS NOT NULL expression for each key column. */ partexprs_item = list_head(key->partexprs); for (i = 0; i < key->partnatts; i++) { - if (key->partattrs[i] == 0) - { - Expr *keyCol; + Expr *keyCol; + if (key->partattrs[i] != 0) + { + keyCol = (Expr *) makeVar(1, + key->partattrs[i], + key->parttypid[i], + key->parttypmod[i], + key->parttypcoll[i], + 0); + } + else + { if (partexprs_item == NULL) elog(ERROR, "wrong number of partition key expressions"); - keyCol = lfirst(partexprs_item); + keyCol = copyObject(lfirst(partexprs_item)); partexprs_item = lnext(partexprs_item); - Assert(!IsA(keyCol, Var)); - - nulltest = makeNode(NullTest); - nulltest->arg = keyCol; - nulltest->nulltesttype = IS_NOT_NULL; - nulltest->argisrow = false; - nulltest->location = -1; - result = lappend(result, nulltest); } + + nulltest = makeNode(NullTest); + nulltest->arg = keyCol; + nulltest->nulltesttype = IS_NOT_NULL; + nulltest->argisrow = false; + nulltest->location = -1; + result = lappend(result, nulltest); } /* @@ -1948,7 +1954,8 @@ get_partition_for_tuple(PartitionDispatch *pd, { *failed_at = parent; *failed_slot = slot; - return -1; + result = -1; + goto error_exit; } /* @@ -1964,12 +1971,21 @@ get_partition_for_tuple(PartitionDispatch *pd, if (key->strategy == PARTITION_STRATEGY_RANGE) { - /* Disallow nulls in the range partition key of the tuple */ + /* + * Since we cannot route tuples with NULL partition keys through + * a range-partitioned table, simply return that no partition + * exists + */ for (i = 0; i < key->partnatts; i++) + { if (isnull[i]) - ereport(ERROR, - (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("range partition key of row contains null"))); + { + *failed_at = parent; + *failed_slot = slot; + result = -1; + goto error_exit; + } + } } /* @@ -2032,6 +2048,7 @@ get_partition_for_tuple(PartitionDispatch *pd, parent = pd[-parent->indexes[cur_index]]; } +error_exit: ecxt->ecxt_scantuple = ecxt_scantuple_old; return result; } diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 7319aa597e..99c51b812d 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -805,13 +805,11 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, if (stmt->partspec) { char strategy; - int partnatts, - i; + int partnatts; AttrNumber partattrs[PARTITION_MAX_KEYS]; Oid partopclass[PARTITION_MAX_KEYS]; Oid partcollation[PARTITION_MAX_KEYS]; List *partexprs = NIL; - List *cmds = NIL; /* * We need to transform the raw parsetrees corresponding to partition @@ -828,33 +826,6 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, partnatts = list_length(stmt->partspec->partParams); StorePartitionKey(rel, strategy, partnatts, partattrs, partexprs, partopclass, partcollation); - - /* Force key columns to be NOT NULL when using range partitioning */ - if (strategy == PARTITION_STRATEGY_RANGE) - { - for (i = 0; i < partnatts; i++) - { - AttrNumber partattno = partattrs[i]; - Form_pg_attribute attform = descriptor->attrs[partattno - 1]; - - if (partattno != 0 && !attform->attnotnull) - { - /* Add a subcommand to make this one NOT NULL */ - AlterTableCmd *cmd = makeNode(AlterTableCmd); - - cmd->subtype = AT_SetNotNull; - cmd->name = pstrdup(NameStr(attform->attname)); - cmds = lappend(cmds, cmd); - } - } - - /* - * Although, there cannot be any partitions yet, we still need to - * pass true for recurse; ATPrepSetNotNull() complains if we don't - */ - if (cmds != NIL) - AlterTableInternal(RelationGetRelid(rel), cmds, true); - } } /* @@ -5702,32 +5673,6 @@ ATExecDropNotNull(Relation rel, const char *colName, LOCKMODE lockmode) heap_close(parent, AccessShareLock); } - /* - * If the table is a range partitioned table, check that the column is not - * in the partition key. - */ - if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) - { - PartitionKey key = RelationGetPartitionKey(rel); - - if (get_partition_strategy(key) == PARTITION_STRATEGY_RANGE) - { - int partnatts = get_partition_natts(key), - i; - - for (i = 0; i < partnatts; i++) - { - AttrNumber partattnum = get_partition_col_attnum(key, i); - - if (partattnum == attnum) - ereport(ERROR, - (errcode(ERRCODE_INVALID_TABLE_DEFINITION), - errmsg("column \"%s\" is in range partition key", - colName))); - } - } - } - /* * Okay, actually perform the catalog change ... if needed */ diff --git a/src/test/regress/expected/alter_table.out b/src/test/regress/expected/alter_table.out index 6ebebb4a03..c88fd76848 100644 --- a/src/test/regress/expected/alter_table.out +++ b/src/test/regress/expected/alter_table.out @@ -3079,13 +3079,6 @@ ALTER TABLE partitioned DROP COLUMN b; ERROR: cannot drop column referenced in partition key expression ALTER TABLE partitioned ALTER COLUMN b TYPE char(5); ERROR: cannot alter type of column referenced in partition key expression --- cannot drop NOT NULL on columns in the range partition key -ALTER TABLE partitioned ALTER COLUMN a DROP NOT NULL; -ERROR: column "a" is in range partition key --- it's fine however to drop one on the list partition key column -CREATE TABLE list_partitioned (a int not null) partition by list (a); -ALTER TABLE list_partitioned ALTER a DROP NOT NULL; -DROP TABLE list_partitioned; -- partitioned table cannot participate in regular inheritance CREATE TABLE nonpartitioned ( a int, diff --git a/src/test/regress/expected/create_table.out b/src/test/regress/expected/create_table.out index bbf039ccad..39edf04cb4 100644 --- a/src/test/regress/expected/create_table.out +++ b/src/test/regress/expected/create_table.out @@ -407,18 +407,6 @@ SELECT relkind FROM pg_class WHERE relname = 'partitioned'; p (1 row) --- check that range partition key columns are marked NOT NULL -SELECT attname, attnotnull FROM pg_attribute - WHERE attrelid = 'partitioned'::regclass AND attnum > 0 - ORDER BY attnum; - attname | attnotnull ----------+------------ - a | t - b | f - c | t - d | t -(4 rows) - -- prevent a function referenced in partition key from being dropped DROP FUNCTION plusone(int); ERROR: cannot drop function plusone(integer) because other objects depend on it @@ -435,10 +423,10 @@ ERROR: cannot inherit from partitioned table "partitioned2" Table "public.partitioned" Column | Type | Collation | Nullable | Default --------+---------+-----------+----------+--------- - a | integer | | not null | + a | integer | | | b | integer | | | - c | text | | not null | - d | text | | not null | + c | text | | | + d | text | | | Partition key: RANGE (a oid_ops, plusone(b), c, d COLLATE "C") \d partitioned2 @@ -544,9 +532,9 @@ CREATE TABLE part_forced_oids PARTITION OF oids_parted FOR VALUES FROM (1) TO (1 Table "public.part_forced_oids" Column | Type | Collation | Nullable | Default | Storage | Stats target | Description --------+---------+-----------+----------+---------+---------+--------------+------------- - a | integer | | not null | | plain | | + a | integer | | | | plain | | Partition of: oids_parted FOR VALUES FROM (1) TO (10) -Partition constraint: ((a >= 1) AND (a < 10)) +Partition constraint: ((a IS NOT NULL) AND (a >= 1) AND (a < 10)) Has OIDs: yes DROP TABLE oids_parted, part_forced_oids; @@ -678,7 +666,7 @@ Partitions: part_c_1_10 FOR VALUES FROM (1) TO (10) a | text | | | | extended | | b | integer | | not null | 0 | plain | | Partition of: part_c FOR VALUES FROM (1) TO (10) -Partition constraint: ((a IS NOT NULL) AND (a = ANY (ARRAY['c'::text])) AND (b >= 1) AND (b < 10)) +Partition constraint: ((a IS NOT NULL) AND (a = ANY (ARRAY['c'::text])) AND (b IS NOT NULL) AND (b >= 1) AND (b < 10)) Check constraints: "check_a" CHECK (length(a) > 0) @@ -706,9 +694,9 @@ CREATE TABLE unbounded_range_part PARTITION OF range_parted4 FOR VALUES FROM (UN --------+---------+-----------+----------+---------+---------+--------------+------------- a | integer | | | | plain | | b | integer | | | | plain | | - c | integer | | not null | | plain | | + c | integer | | | | plain | | Partition of: range_parted4 FOR VALUES FROM (UNBOUNDED, UNBOUNDED, UNBOUNDED) TO (UNBOUNDED, UNBOUNDED, UNBOUNDED) -Partition constraint: ((abs(a) IS NOT NULL) AND (abs(b) IS NOT NULL)) +Partition constraint: ((abs(a) IS NOT NULL) AND (abs(b) IS NOT NULL) AND (c IS NOT NULL)) DROP TABLE unbounded_range_part; CREATE TABLE range_parted4_1 PARTITION OF range_parted4 FOR VALUES FROM (UNBOUNDED, UNBOUNDED, UNBOUNDED) TO (1, UNBOUNDED, UNBOUNDED); @@ -718,9 +706,9 @@ CREATE TABLE range_parted4_1 PARTITION OF range_parted4 FOR VALUES FROM (UNBOUND --------+---------+-----------+----------+---------+---------+--------------+------------- a | integer | | | | plain | | b | integer | | | | plain | | - c | integer | | not null | | plain | | + c | integer | | | | plain | | Partition of: range_parted4 FOR VALUES FROM (UNBOUNDED, UNBOUNDED, UNBOUNDED) TO (1, UNBOUNDED, UNBOUNDED) -Partition constraint: ((abs(a) IS NOT NULL) AND (abs(b) IS NOT NULL) AND (abs(a) <= 1)) +Partition constraint: ((abs(a) IS NOT NULL) AND (abs(b) IS NOT NULL) AND (c IS NOT NULL) AND (abs(a) <= 1)) CREATE TABLE range_parted4_2 PARTITION OF range_parted4 FOR VALUES FROM (3, 4, 5) TO (6, 7, UNBOUNDED); \d+ range_parted4_2 @@ -729,9 +717,9 @@ CREATE TABLE range_parted4_2 PARTITION OF range_parted4 FOR VALUES FROM (3, 4, 5 --------+---------+-----------+----------+---------+---------+--------------+------------- a | integer | | | | plain | | b | integer | | | | plain | | - c | integer | | not null | | plain | | + c | integer | | | | plain | | Partition of: range_parted4 FOR VALUES FROM (3, 4, 5) TO (6, 7, UNBOUNDED) -Partition constraint: ((abs(a) IS NOT NULL) AND (abs(b) IS NOT NULL) AND ((abs(a) > 3) OR ((abs(a) = 3) AND (abs(b) > 4)) OR ((abs(a) = 3) AND (abs(b) = 4) AND (c >= 5))) AND ((abs(a) < 6) OR ((abs(a) = 6) AND (abs(b) <= 7)))) +Partition constraint: ((abs(a) IS NOT NULL) AND (abs(b) IS NOT NULL) AND (c IS NOT NULL) AND ((abs(a) > 3) OR ((abs(a) = 3) AND (abs(b) > 4)) OR ((abs(a) = 3) AND (abs(b) = 4) AND (c >= 5))) AND ((abs(a) < 6) OR ((abs(a) = 6) AND (abs(b) <= 7)))) CREATE TABLE range_parted4_3 PARTITION OF range_parted4 FOR VALUES FROM (6, 8, UNBOUNDED) TO (9, UNBOUNDED, UNBOUNDED); \d+ range_parted4_3 @@ -740,9 +728,9 @@ CREATE TABLE range_parted4_3 PARTITION OF range_parted4 FOR VALUES FROM (6, 8, U --------+---------+-----------+----------+---------+---------+--------------+------------- a | integer | | | | plain | | b | integer | | | | plain | | - c | integer | | not null | | plain | | + c | integer | | | | plain | | Partition of: range_parted4 FOR VALUES FROM (6, 8, UNBOUNDED) TO (9, UNBOUNDED, UNBOUNDED) -Partition constraint: ((abs(a) IS NOT NULL) AND (abs(b) IS NOT NULL) AND ((abs(a) > 6) OR ((abs(a) = 6) AND (abs(b) >= 8))) AND (abs(a) <= 9)) +Partition constraint: ((abs(a) IS NOT NULL) AND (abs(b) IS NOT NULL) AND (c IS NOT NULL) AND ((abs(a) > 6) OR ((abs(a) = 6) AND (abs(b) >= 8))) AND (abs(a) <= 9)) DROP TABLE range_parted4; -- cleanup diff --git a/src/test/regress/expected/insert.out b/src/test/regress/expected/insert.out index 02429a37e3..8b0752a0d2 100644 --- a/src/test/regress/expected/insert.out +++ b/src/test/regress/expected/insert.out @@ -189,7 +189,7 @@ DETAIL: Failing row contains (a, 10). insert into part4 values ('b', 10); -- fail (partition key a has a NOT NULL constraint) insert into part1 values (null); -ERROR: null value in column "a" violates not-null constraint +ERROR: new row for relation "part1" violates partition constraint DETAIL: Failing row contains (null, null). -- fail (expression key (b+0) cannot be null either) insert into part1 values (1); @@ -247,7 +247,8 @@ insert into range_parted values ('b', 1); insert into range_parted values ('b', 10); -- fail (partition key (b+0) is null) insert into range_parted values ('a'); -ERROR: range partition key of row contains null +ERROR: no partition of relation "range_parted" found for row +DETAIL: Partition key of the failing row contains (a, (b + 0)) = (a, null). select tableoid::regclass, * from range_parted; tableoid | a | b ----------+---+---- diff --git a/src/test/regress/sql/alter_table.sql b/src/test/regress/sql/alter_table.sql index f014675628..c0e29720dc 100644 --- a/src/test/regress/sql/alter_table.sql +++ b/src/test/regress/sql/alter_table.sql @@ -1947,14 +1947,6 @@ ALTER TABLE partitioned ALTER COLUMN a TYPE char(5); ALTER TABLE partitioned DROP COLUMN b; ALTER TABLE partitioned ALTER COLUMN b TYPE char(5); --- cannot drop NOT NULL on columns in the range partition key -ALTER TABLE partitioned ALTER COLUMN a DROP NOT NULL; - --- it's fine however to drop one on the list partition key column -CREATE TABLE list_partitioned (a int not null) partition by list (a); -ALTER TABLE list_partitioned ALTER a DROP NOT NULL; -DROP TABLE list_partitioned; - -- partitioned table cannot participate in regular inheritance CREATE TABLE nonpartitioned ( a int, diff --git a/src/test/regress/sql/create_table.sql b/src/test/regress/sql/create_table.sql index 766f35a3ed..5a2774395e 100644 --- a/src/test/regress/sql/create_table.sql +++ b/src/test/regress/sql/create_table.sql @@ -410,11 +410,6 @@ CREATE TABLE partitioned ( -- check relkind SELECT relkind FROM pg_class WHERE relname = 'partitioned'; --- check that range partition key columns are marked NOT NULL -SELECT attname, attnotnull FROM pg_attribute - WHERE attrelid = 'partitioned'::regclass AND attnum > 0 - ORDER BY attnum; - -- prevent a function referenced in partition key from being dropped DROP FUNCTION plusone(int); -- cgit v1.2.3 From f4205c4c1fcfc88d0c3885c160f24873bcbcc04d Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Thu, 18 May 2017 21:37:57 -0400 Subject: doc: Fix ALTER SUBSCRIPTION option syntax synopsis Author: Masahiko Sawada --- doc/src/sgml/ref/alter_subscription.sgml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/alter_subscription.sgml b/doc/src/sgml/ref/alter_subscription.sgml index 6320de06ed..113e32bfd0 100644 --- a/doc/src/sgml/ref/alter_subscription.sgml +++ b/doc/src/sgml/ref/alter_subscription.sgml @@ -22,8 +22,8 @@ PostgreSQL documentation ALTER SUBSCRIPTION name CONNECTION 'conninfo' -ALTER SUBSCRIPTION name SET PUBLICATION publication_name [, ...] { REFRESH [ WITH ( refresh_option value [, ... ] ) ] | SKIP REFRESH } -ALTER SUBSCRIPTION name REFRESH PUBLICATION [ WITH ( refresh_option value [, ... ] ) ] +ALTER SUBSCRIPTION name SET PUBLICATION publication_name [, ...] { REFRESH [ WITH ( refresh_option [= value] [, ... ] ) ] | SKIP REFRESH } +ALTER SUBSCRIPTION name REFRESH PUBLICATION [ WITH ( refresh_option [= value] [, ... ] ) ] ALTER SUBSCRIPTION name ENABLE ALTER SUBSCRIPTION name DISABLE ALTER SUBSCRIPTION name SET ( subscription_parameter [= value] [, ... ] ) -- cgit v1.2.3 From f45d86d762b8bcdac42d5448734016242a1d738b Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Fri, 19 May 2017 12:10:10 -0400 Subject: doc: fix PG 10 release notes with proper attribution and commit Fix for hot_standby=on change. Reported-by: Huong Dangminh Author: Huong Dangminh --- doc/src/sgml/release-10.sgml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/release-10.sgml b/doc/src/sgml/release-10.sgml index eff1fbc719..cb6440be69 100644 --- a/doc/src/sgml/release-10.sgml +++ b/doc/src/sgml/release-10.sgml @@ -1452,11 +1452,12 @@ Reduce configuration necessary to perform streaming backup and - replication (Magnus Hagander) + replication (Magnus Hagander, Dang Minh Huong) -- cgit v1.2.3 From f6d5d22ea9497aecdfdfa92717a11f0bcdf17223 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Fri, 19 May 2017 12:16:53 -0400 Subject: doc: remove duplicate PG 10 release notes entry Reported-by: Daniel Gustafsson --- doc/src/sgml/release-10.sgml | 11 ----------- 1 file changed, 11 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/release-10.sgml b/doc/src/sgml/release-10.sgml index cb6440be69..441e6c8d8a 100644 --- a/doc/src/sgml/release-10.sgml +++ b/doc/src/sgml/release-10.sgml @@ -137,17 +137,6 @@ - - - - RenameWAL-related functions and views to use lsn - instead of location (David Rowley) - - - + One user reports: @@ -2264,7 +2264,7 @@ hosts=local4,bind4 Memory Management - + AIX can be somewhat peculiar with regards to the way it does diff --git a/doc/src/sgml/problems.sgml b/doc/src/sgml/problems.sgml index 3f79c6ef90..6bf74bb399 100644 --- a/doc/src/sgml/problems.sgml +++ b/doc/src/sgml/problems.sgml @@ -304,7 +304,7 @@ Another method is to fill in the bug report web-form available at the project's - web site. + web site. Entering a bug report this way causes it to be mailed to the pgsql-bugs@postgresql.org mailing list. diff --git a/doc/src/sgml/release.sgml b/doc/src/sgml/release.sgml index c8776c4f97..f1f4e91252 100644 --- a/doc/src/sgml/release.sgml +++ b/doc/src/sgml/release.sgml @@ -56,9 +56,9 @@ For new features, add links to the documentation sections. A complete list of changes for each release can be obtained by viewing the Git logs for each release. The pgsql-committers + url="https://archives.postgresql.org/pgsql-committers/">pgsql-committers email list records all source code changes as well. There is also - a web + a web interface that shows changes to specific files. diff --git a/doc/src/sgml/sepgsql.sgml b/doc/src/sgml/sepgsql.sgml index 6f80933abd..0b611eeeca 100644 --- a/doc/src/sgml/sepgsql.sgml +++ b/doc/src/sgml/sepgsql.sgml @@ -753,7 +753,7 @@ ERROR: SELinux: security policy violation External Resources - SE-PostgreSQL Introduction + SE-PostgreSQL Introduction This wiki page provides a brief overview, security design, architecture, diff --git a/doc/src/sgml/sourcerepo.sgml b/doc/src/sgml/sourcerepo.sgml index d82706b40b..f8f6bf2de1 100644 --- a/doc/src/sgml/sourcerepo.sgml +++ b/doc/src/sgml/sourcerepo.sgml @@ -12,7 +12,7 @@ Our wiki, , + url="https://wiki.postgresql.org/wiki/Working_with_Git">, has some discussion on working with Git. @@ -64,10 +64,10 @@ git clone git://git.postgresql.org/git/postgresql.git The Git mirror can also be reached via the HTTP protocol, if for example a firewall is blocking access to the Git protocol. Just change the URL - prefix to http, as in: + prefix to https, as in: -git clone http://git.postgresql.org/git/postgresql.git +git clone https://git.postgresql.org/git/postgresql.git The HTTP protocol is less efficient than the Git protocol, so it will be diff --git a/doc/src/sgml/stylesheet.xsl b/doc/src/sgml/stylesheet.xsl index c948c612ac..22dd3b93c6 100644 --- a/doc/src/sgml/stylesheet.xsl +++ b/doc/src/sgml/stylesheet.xsl @@ -23,7 +23,7 @@ stylesheet.css - http://www.postgresql.org/media/css/docs.css + https://www.postgresql.org/media/css/docs.css -- cgit v1.2.3 From d761fe2182d9e26f9483e4b7ac303e38bfbd7a24 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 21 May 2017 13:05:16 -0400 Subject: Fix precision and rounding issues in money multiplication and division. The cash_div_intX functions applied rint() to the result of the division. That's not merely useless (because the result is already an integer) but it causes precision loss for values larger than 2^52 or so, because of the forced conversion to float8. On the other hand, the cash_mul_fltX functions neglected to apply rint() to their multiplication results, thus possibly causing off-by-one outputs. Per C standard, arithmetic between any integral value and a float value is performed in float format. Thus, cash_mul_flt4 and cash_div_flt4 produced answers good to only about six digits, even when the float value is exact. We can improve matters noticeably by widening the float inputs to double. (It's tempting to consider using "long double" arithmetic if available, but that's probably too much of a stretch for a back-patched fix.) Also, document that cash_div_intX operators truncate rather than round. Per bug #14663 from Richard Pistole. Back-patch to all supported branches. Discussion: https://postgr.es/m/22403.1495223615@sss.pgh.pa.us --- doc/src/sgml/datatype.sgml | 5 ++++ src/backend/utils/adt/cash.c | 16 ++++++------ src/test/regress/expected/money.out | 50 +++++++++++++++++++++++++++++++++++++ src/test/regress/sql/money.sql | 12 +++++++++ 4 files changed, 75 insertions(+), 8 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/datatype.sgml b/doc/src/sgml/datatype.sgml index 42b2bb71bb..a322049005 100644 --- a/doc/src/sgml/datatype.sgml +++ b/doc/src/sgml/datatype.sgml @@ -983,6 +983,11 @@ SELECT '52093.89'::money::numeric::float8; + Division of a money value by an integer value is performed + with truncation of the fractional part towards zero. To get a rounded + result, divide by a floating-point value, or cast the money + value to numeric before dividing and back to money + afterwards. (The latter is preferable to avoid risking precision loss.) When a money value is divided by another money value, the result is double precision (i.e., a pure number, not money); the currency units cancel each other out in the division. diff --git a/src/backend/utils/adt/cash.c b/src/backend/utils/adt/cash.c index 5cb086e50e..a170294b94 100644 --- a/src/backend/utils/adt/cash.c +++ b/src/backend/utils/adt/cash.c @@ -667,7 +667,7 @@ cash_mul_flt8(PG_FUNCTION_ARGS) float8 f = PG_GETARG_FLOAT8(1); Cash result; - result = c * f; + result = rint(c * f); PG_RETURN_CASH(result); } @@ -682,7 +682,7 @@ flt8_mul_cash(PG_FUNCTION_ARGS) Cash c = PG_GETARG_CASH(1); Cash result; - result = f * c; + result = rint(f * c); PG_RETURN_CASH(result); } @@ -717,7 +717,7 @@ cash_mul_flt4(PG_FUNCTION_ARGS) float4 f = PG_GETARG_FLOAT4(1); Cash result; - result = c * f; + result = rint(c * (float8) f); PG_RETURN_CASH(result); } @@ -732,7 +732,7 @@ flt4_mul_cash(PG_FUNCTION_ARGS) Cash c = PG_GETARG_CASH(1); Cash result; - result = f * c; + result = rint((float8) f * c); PG_RETURN_CASH(result); } @@ -753,7 +753,7 @@ cash_div_flt4(PG_FUNCTION_ARGS) (errcode(ERRCODE_DIVISION_BY_ZERO), errmsg("division by zero"))); - result = rint(c / f); + result = rint(c / (float8) f); PG_RETURN_CASH(result); } @@ -802,7 +802,7 @@ cash_div_int8(PG_FUNCTION_ARGS) (errcode(ERRCODE_DIVISION_BY_ZERO), errmsg("division by zero"))); - result = rint(c / i); + result = c / i; PG_RETURN_CASH(result); } @@ -854,7 +854,7 @@ cash_div_int4(PG_FUNCTION_ARGS) (errcode(ERRCODE_DIVISION_BY_ZERO), errmsg("division by zero"))); - result = rint(c / i); + result = c / i; PG_RETURN_CASH(result); } @@ -904,7 +904,7 @@ cash_div_int2(PG_FUNCTION_ARGS) (errcode(ERRCODE_DIVISION_BY_ZERO), errmsg("division by zero"))); - result = rint(c / s); + result = c / s; PG_RETURN_CASH(result); } diff --git a/src/test/regress/expected/money.out b/src/test/regress/expected/money.out index 0cc69f925f..ab86595fc0 100644 --- a/src/test/regress/expected/money.out +++ b/src/test/regress/expected/money.out @@ -359,6 +359,56 @@ SELECT '92233720368547758.075'::money; ERROR: value "92233720368547758.075" is out of range for type money LINE 1: SELECT '92233720368547758.075'::money; ^ +-- rounding vs. truncation in division +SELECT '878.08'::money / 11::float8; + ?column? +---------- + $79.83 +(1 row) + +SELECT '878.08'::money / 11::float4; + ?column? +---------- + $79.83 +(1 row) + +SELECT '878.08'::money / 11::bigint; + ?column? +---------- + $79.82 +(1 row) + +SELECT '878.08'::money / 11::int; + ?column? +---------- + $79.82 +(1 row) + +SELECT '878.08'::money / 11::smallint; + ?column? +---------- + $79.82 +(1 row) + +-- check for precision loss in division +SELECT '90000000000000099.00'::money / 10::bigint; + ?column? +--------------------------- + $9,000,000,000,000,009.90 +(1 row) + +SELECT '90000000000000099.00'::money / 10::int; + ?column? +--------------------------- + $9,000,000,000,000,009.90 +(1 row) + +SELECT '90000000000000099.00'::money / 10::smallint; + ?column? +--------------------------- + $9,000,000,000,000,009.90 +(1 row) + -- Cast int4/int8/numeric to money SELECT 1234567890::money; money diff --git a/src/test/regress/sql/money.sql b/src/test/regress/sql/money.sql index f5a92f2a69..37b9ecce1f 100644 --- a/src/test/regress/sql/money.sql +++ b/src/test/regress/sql/money.sql @@ -97,6 +97,18 @@ SELECT '92233720368547758.08'::money; SELECT '-92233720368547758.085'::money; SELECT '92233720368547758.075'::money; +-- rounding vs. truncation in division +SELECT '878.08'::money / 11::float8; +SELECT '878.08'::money / 11::float4; +SELECT '878.08'::money / 11::bigint; +SELECT '878.08'::money / 11::int; +SELECT '878.08'::money / 11::smallint; + +-- check for precision loss in division +SELECT '90000000000000099.00'::money / 10::bigint; +SELECT '90000000000000099.00'::money / 10::int; +SELECT '90000000000000099.00'::money / 10::smallint; + -- Cast int4/int8/numeric to money SELECT 1234567890::money; SELECT 12345678901234567::money; -- cgit v1.2.3 From eb67e2e35a046f700172fbce52ad2331fe6c57ac Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Thu, 25 May 2017 13:17:57 -0400 Subject: extended stats: Clarify behavior of omitting stat type clause Pointed out by Jeff Janes Discussion: https://postgr.es/m/CAMkU=1zGhK-nW10RAXhokcT3MM=YBg=j5LkG9RMDwmu3i0H0Og@mail.gmail.com --- doc/src/sgml/ref/create_statistics.sgml | 2 ++ 1 file changed, 2 insertions(+) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/create_statistics.sgml b/doc/src/sgml/ref/create_statistics.sgml index 854746de24..f319a6ea9c 100644 --- a/doc/src/sgml/ref/create_statistics.sgml +++ b/doc/src/sgml/ref/create_statistics.sgml @@ -84,6 +84,8 @@ CREATE STATISTICS [ IF NOT EXISTS ] statistics_na ndistinct, which enables n-distinct statistics, and dependencies, which enables functional dependency statistics. + If this clause is omitted, all supported statistic types are + included in the statistics object. For more information, see and . -- cgit v1.2.3 From 9c34a05b7d2304eac3fdc5e9db9fcd0d7c72883f Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Fri, 26 May 2017 10:58:15 -0400 Subject: Remove docs mention of PGREALM variable This variable was only used with Kerberos v4. That support was removed in 2005, but we forgot to remove the documentation. Noted by Shinichi Matsuda --- doc/src/sgml/libpq.sgml | 16 ---------------- 1 file changed, 16 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/libpq.sgml b/doc/src/sgml/libpq.sgml index 493cc12d61..23e6f1c3dc 100644 --- a/doc/src/sgml/libpq.sgml +++ b/doc/src/sgml/libpq.sgml @@ -7009,22 +7009,6 @@ myEventProc(PGEventId evtId, void *evtInfo, void *passThrough) - - - - PGREALM - - PGREALM sets the Kerberos realm to use with - PostgreSQL, if it is different from the - local realm. If PGREALM is set, - libpq applications will attempt - authentication with servers for this realm and use separate ticket - files to avoid conflicts with local ticket files. This - environment variable is only used if GSSAPI authentication is - selected by the server. - - - -- cgit v1.2.3 From 1c8b88ab9b72fe4c7c4193170d0810aa42b889cf Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 28 May 2017 15:49:44 -0400 Subject: Improve v10 release notes' discussion of money operator changes. Mention the rounding behavioral change for money/int8. Discussion: https://postgr.es/m/20170519164653.29941.19098@wrigleys.postgresql.org --- doc/src/sgml/release-10.sgml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/release-10.sgml b/doc/src/sgml/release-10.sgml index 441e6c8d8a..013632ed7a 100644 --- a/doc/src/sgml/release-10.sgml +++ b/doc/src/sgml/release-10.sgml @@ -1929,9 +1929,19 @@ 2017-01-17 [323b96aa3] Register missing money operators in system catalogs --> - Add MONEY operators - for multiplication and division with INT8 values (Peter - Eisentraut) + Add operators for multiplication and division + of money values + with int8 values (Peter Eisentraut) + + + + Previously such cases would result in converting the int8 + values to float8 and then using + the money-and-float8 operators. The new behavior + avoids possible precision loss. But note that division + of money by int8 now truncates the quotient, like + other integer-division cases, while the previous behavior would have + rounded. @@ -1941,7 +1951,7 @@ 2016-09-14 [656df624c] Add overflow checks to money type input function --> - More strictly check the MONEY type for overflow operations + More strictly check the money type for overflow operations (Peter Eisentraut) -- cgit v1.2.3 From 54bb322ec7e1f7c3a674b4ea02f7010a51d51f99 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 28 May 2017 16:42:22 -0400 Subject: Format v10 release notes' commit references more like previous releases. Left-justify these comments, remove committer names, remove SGML markup that was randomly added to some of them. Aside from being more consistent with previous practice, this keeps the lines shorter than 80 characters, improving readability in standard terminal windows. --- doc/src/sgml/release-10.sgml | 1432 ++++++++++++++++++------------------------ 1 file changed, 600 insertions(+), 832 deletions(-) (limited to 'doc/src') diff --git a/doc/src/sgml/release-10.sgml b/doc/src/sgml/release-10.sgml index 013632ed7a..3ccac676ad 100644 --- a/doc/src/sgml/release-10.sgml +++ b/doc/src/sgml/release-10.sgml @@ -47,11 +47,9 @@ - + pg_upgrade-ed hash indexes from previous major Postgres versions must be rebuilt. @@ -63,10 +61,9 @@ - + Change the default log directory from pg_log to log (Andreas @@ -75,10 +72,9 @@ - + Rename pg_xlog to pg_wal (Michael Paquier) @@ -91,16 +87,12 @@ - + Rename SQL functions, tools, and options that reference xlog to wal (Robert Haas) @@ -116,10 +108,9 @@ - + Rename transaction status directory pg_clog directory to pg_xact (Michael Paquier) @@ -127,10 +118,9 @@ - + Rename WAL-related functions and views to use lsn instead of location (David Rowley) @@ -138,12 +128,10 @@ - + Allow COALESCE and CASE to return multiple rows when evaluating set-returning functions (Andres Freund). @@ -157,10 +145,9 @@ - + Have stream the WAL needed to restore the backup by default (Magnus @@ -177,10 +164,9 @@ - + Make all actions wait by default for completion (Peter Eisentraut) @@ -193,10 +179,9 @@ - + Remove the ability to store unencrypted passwords on the server (Heikki Linnakangas) @@ -215,12 +200,10 @@ - + Allow multi-dimensional arrays to be passed into PL/Python functions, and returned as nested Python lists (Alexey Grishchenko, Dave Cramer, @@ -239,10 +222,9 @@ - + Remove PL/Tcl's "module" auto-loading facility (Tom Lane) @@ -253,10 +235,9 @@ - + Remove sql_inheritance GUC (Robert Haas) @@ -270,10 +251,9 @@ - + Add GUCs and to control @@ -287,10 +267,9 @@ - + Remove pg_dump/pg_dumpall support for dumping from pre-8.0 servers (Tom Lane) @@ -303,10 +282,9 @@ - + Remove support for floating-point datetimes/timestamps (Tom Lane) @@ -319,10 +297,9 @@ - + Remove support for client/server protocol version 1.0 (Tom Lane) @@ -333,10 +310,9 @@ - + Remove contrib/tsearch2 (Robert Haas) @@ -348,10 +324,9 @@ - + Remove createlang and droplang command-line applications (Peter Eisentraut) @@ -359,10 +334,9 @@ - + Remove support for version-0 function calling conventions (Andres Freund) @@ -370,10 +344,9 @@ - + Remove SCO and Unixware ports (Tom Lane) @@ -402,14 +375,11 @@ - + Support parallel btree index scans (Rahila Syed, Amit Kapila, Robert Haas, Rafia Sabih) @@ -422,12 +392,10 @@ - + Support parallel bitmap heap scans (Dilip Kumar) @@ -439,30 +407,27 @@ - + Allow merge joins to be performed in parallel (Dilip Kumar) - + Allow non-correlated subqueries to be run in parallel (Amit Kapila) - + Improve ability of parallel workers to return pre-sorted data (Rushabh Lathia) @@ -470,10 +435,9 @@ - + Increase parallel query usage in procedural language functions (Robert Haas, Rafia Sabih) @@ -481,12 +445,10 @@ - + Add GUC to limit the number of worker processes that can be used for @@ -510,10 +472,9 @@ - + Add SP-GiST index support for INET and CIDR data types (Emre Hasegeli) @@ -525,10 +486,9 @@ - + Reduce page locking during vacuuming of GIN indexes (Andrey Borodin) @@ -536,10 +496,9 @@ - + Add option to allow BRIN index summarization to happen more aggressively (Álvaro Herrera) @@ -554,10 +513,9 @@ - + Add functions to remove and re-add BRIN summarization for BRIN index ranges (Álvaro @@ -575,10 +533,9 @@ - + Improve accuracy in determining if a BRIN index scan is beneficial (David Rowley, Emre Hasegeli) @@ -586,10 +543,9 @@ - + Allow faster GiST inserts and updates by reusing index space more efficiently (Andrey Borodin) @@ -597,21 +553,18 @@ - + <link linkend="indexes-types">Hash Indexes</link> - + Add write-ahead logging support to hash indexes (Amit Kapila) @@ -623,13 +576,10 @@ - + Improve hash bucket split performance by reducing locking requirements (Amit Kapila, Mithun Cy) @@ -641,20 +591,18 @@ - + Improve efficiency of hash index growth (Amit Kapila, Mithun Cy) - + Allow single-page hash pruning (Ashutosh Sharma) @@ -673,10 +621,9 @@ - + Only check for REFERENCES permission on referenced tables (Tom Lane) @@ -689,10 +636,9 @@ - + Reduce locking required for adding values to enum types (Andrew Dunstan, Tom Lane) @@ -707,10 +653,9 @@ - + Allow tuning of predicate lock promotion thresholds (Dagfinn Ilmari Mannsåker) @@ -733,12 +678,10 @@ - + Add multi-column optimizer statistics to compute the correlation ratio and number of distinct values (Tomas Vondra, David Rowley, @@ -757,20 +700,18 @@ - + Improve planner matching of boolean indexes (Tom Lane) - + Improve performance of queries referencing row-level security restrictions (Tom Lane) @@ -792,10 +733,9 @@ - + Speed up SUM() calculations (Heikki Linnakangas) @@ -806,10 +746,9 @@ - + Improve the performance of character encoding conversions by using radix trees (Kyotaro Horiguchi, Heikki Linnakangas) @@ -817,10 +756,9 @@ - + Reduce the function call overhead during query execution (Andres Freund) @@ -832,20 +770,18 @@ - + Improve the performance of grouping sets (Andrew Gierth) - + Use uniqueness guarantees to optimize certain join types (David Rowley) @@ -853,20 +789,18 @@ - + Improve sort performance of the macaddr data type (Brandur Leach) - + Reduce statistics tracking overhead in sessions that reference many thousands of relations (Aleksander Alekseev) @@ -883,12 +817,10 @@ - + Create a pg_sequence @@ -905,10 +837,9 @@ - + Allow explicit control over EXPLAIN's display of planning and @@ -924,10 +855,9 @@ - + Properly update the statistics collector during REFRESH MATERIALIZED @@ -936,10 +866,9 @@ - + Add default monitoring roles (Dave Page) @@ -952,17 +881,16 @@ - + Logging - + Change default to include current timestamp with milliseconds and the process id (Christoph @@ -975,10 +903,9 @@ - + Add functions to return the log and WAL directory names (Dave Page) @@ -996,10 +923,9 @@ - + Add function pg_current_logfile() @@ -1009,12 +935,10 @@ - + Report the address and port number of successful startup socket binding in the server logs (Tom Lane) @@ -1026,10 +950,9 @@ - + Reduce log chatter about the starting and stopping of launcher subprocesses (Tom Lane) @@ -1041,10 +964,9 @@ - + Reduce message verbosity of lower-numbered debug levels controlled by @@ -1060,17 +982,16 @@ - + <link linkend="pg-stat-activity-view"><structname>pg_stat_activity</></link> - + Add pg_stat_activity reporting of latch wait states (Michael Paquier, Robert Haas) @@ -1083,10 +1004,9 @@ - + Add pg_stat_activity reporting of waits on reads, writes, and fsyncs (Rushabh Lathia) @@ -1094,10 +1014,9 @@ - + Show auxiliary processes and background workers in pg_stat_activity (Kuntal Ghosh) @@ -1110,10 +1029,9 @@ - + Display walsender processes in pg_stat_activity (Michael Paquier) @@ -1125,10 +1043,9 @@ - + Allow pg_stat_activity to show the source query being executed by parallel workers (Rafia Sabih) @@ -1136,10 +1053,9 @@ - + Rename pg_stat_activity.wait_event_type @@ -1155,7 +1071,6 @@ - @@ -1164,16 +1079,12 @@ - + Add SCRAM-SHA-256 support for password negotiation and storage (Michael Paquier, @@ -1187,10 +1098,9 @@ - + Change GUC from boolean to enum (Michael Paquier) @@ -1202,10 +1112,9 @@ - + Add view pg_hba_file_rules @@ -1219,10 +1128,9 @@ - + Support multiple RADIUS servers (Magnus Hagander) @@ -1243,14 +1151,11 @@ - + Allow SSL configuration to be updated during configuration reload (Andreas Karlsson, Tom Lane) @@ -1266,10 +1171,9 @@ - + Remove documented restriction about using large shared buffers on Windows (Takayuki Tsunakawa) @@ -1277,12 +1181,10 @@ - + Reduce locking required to change table params (Simon Riggs, Fabrízio Mello) @@ -1296,10 +1198,9 @@ - + Make the maximum value of effectively unlimited @@ -1317,10 +1218,9 @@ - + Perform an fsync on the directory after creating or unlinking files (Michael Paquier) @@ -1339,10 +1239,9 @@ - + Prevent checkpoints and WAL archiving on otherwise-idle systems (Michael Paquier) @@ -1350,12 +1249,10 @@ - + Add GUC to add details to WAL that can be sanity-checked on @@ -1368,10 +1265,9 @@ - + Increase the maximum configurable WAL segment size to one gigabyte (Beena Emerson) @@ -1385,7 +1281,7 @@ - + @@ -1398,14 +1294,11 @@ - + Add the ability to logically replicate tables to standby servers (Petr Jelinek) @@ -1419,10 +1312,9 @@ - + Allow waiting for commit acknowledgement from standby servers irrespective of the order they appear in - + Reduce configuration necessary to perform streaming backup and replication (Magnus Hagander, Dang Minh Huong) @@ -1458,10 +1349,9 @@ - + Enable replication from localhost connections by default in pg_hba.conf @@ -1476,10 +1366,9 @@ - + Add columns to pg_stat_replication @@ -1493,10 +1382,9 @@ - + Add specification of a Log Sequence Number (LSN) stopping point in @@ -1511,10 +1399,9 @@ - + Allow users to disable pg_stop_backup()'s @@ -1528,10 +1415,9 @@ - + Allow creation of temporary replication slots @@ -1544,10 +1430,9 @@ - + Improve performance of hot standby replay with better tracking of Access Exclusive locks (Simon Riggs, David Rowley) @@ -1555,10 +1440,9 @@ - + Speed up two-phase commit recovery performance (Stas Kelvich, Nikhil Sontakke, Michael Paquier) @@ -1575,10 +1459,9 @@ - + Add XMLTABLE @@ -1588,10 +1471,9 @@ - + Allow ROW to supply values to UPDATE ... SET (column_list) (Tom Lane) @@ -1603,10 +1485,9 @@ - + Fix regular expression locale class handling for bytes greater than U+7FF (Tom Lane) @@ -1627,10 +1508,9 @@ - + Add table partitioning syntax that automatically creates partition constraints and @@ -1643,14 +1523,11 @@ - + Add AFTER trigger transition table to record changed rows (Kevin Grittner) @@ -1662,10 +1539,9 @@ - + Allow restrictive row-level security policies (Stephen Frost) @@ -1679,10 +1555,9 @@ - + Allow default permissions on schemas (Matheus Oliveira) @@ -1694,10 +1569,9 @@ - + Add CREATE SEQUENCE AS command to create a sequence matching an integer data type @@ -1711,10 +1585,9 @@ - + Allow COPY view FROM on views with INSTEAD INSERT triggers (Haribabu Kommi) @@ -1726,10 +1599,9 @@ - + Allow the specification of a function name without arguments in DDL commands, when unique (Peter Eisentraut) @@ -1744,11 +1616,9 @@ - + Allow multiple functions, operators, and aggregates to be dropped with a single DROP command (Peter Eisentraut) @@ -1756,10 +1626,9 @@ - + Add IF NOT EXISTS for CREATE SERVER and @@ -1770,10 +1639,9 @@ - + Add IF NOT EXISTS clause to CREATE COLLATION @@ -1782,12 +1650,10 @@ - + Have VACUUM VERBOSE report the number of skipped frozen pages and oldest xmin (Masahiko @@ -1801,10 +1667,9 @@ - + Improve speed of VACUUM's removal of trailing empty heap pages (Claudio Freire, Álvaro Herrera) @@ -1817,10 +1682,9 @@ - + Fix check_srf_call_placement() to handle VALUES cases correctly (Tom Lane) @@ -1841,10 +1705,9 @@ - + Add full text search support for JSON and JSONB (Dmitry Dolgov) @@ -1857,10 +1720,9 @@ - + Add support for EUI-64 MAC addresses as MACADDR8 (Haribabu @@ -1874,10 +1736,9 @@ - + Add identity columns for assigning a numeric value to columns on insert (Peter Eisentraut) @@ -1890,10 +1751,9 @@ - + Allow ENUM values to be renamed (Dagfinn Ilmari Mannsåker) @@ -1906,10 +1766,9 @@ - + Properly treat array pseudotypes (anyarray) as arrays in - + Add operators for multiplication and division of money values @@ -1946,10 +1804,9 @@ - + More strictly check the money type for overflow operations (Peter Eisentraut) @@ -1966,10 +1823,9 @@ - + Add simplified regexp_match() @@ -1983,10 +1839,9 @@ - + Add version of jsonb's delete operator that takes @@ -1999,10 +1854,9 @@ - + Improve json_populate_record @@ -2015,10 +1869,9 @@ - + Add function txid_current_ifassigned() @@ -2035,10 +1888,9 @@ - + Add function txid_status() @@ -2053,10 +1905,9 @@ - + Allow make_date() @@ -2066,10 +1917,9 @@ - + Have to_timestamp() and to_date() check input values for validity (Artur Zakirov) @@ -2091,10 +1941,9 @@ - + Allow the PL/Python plan object to call cursor and execute methods (Peter Eisentraut) @@ -2106,10 +1955,9 @@ - + Allow PL/pgSQL's GET DIAGNOSTICS to retrieve values into array elements (Tom Lane) @@ -2117,10 +1965,9 @@ - + Remove SPI functions SPI_push(), SPI_pop(), SPI_restore_connection() @@ -2141,10 +1988,9 @@ - + Allow PL/Tcl functions to return composite types and sets (Jim Nasby) @@ -2152,10 +1998,9 @@ - + Add a subtransaction command to PL/Tcl (Victor Wagner) @@ -2167,10 +2012,9 @@ - + Add GUCs to allow initialization routines to be called on PL/Tcl startup (Tom Lane) @@ -2183,9 +2027,8 @@ - + - @@ -2194,10 +2037,9 @@ - + Allow libpq to connect to multiple specified host names @@ -2210,10 +2052,9 @@ - + Allow the libpq connection string to request a read/write host @@ -2228,10 +2069,9 @@ - + Allow password file name to be specified as a libpq connection parameter (Julian Markwort) @@ -2243,10 +2083,9 @@ - + Add function PQencryptPasswordConn() @@ -2263,10 +2102,9 @@ - + ecpg preprocessor version changed from 4.12 to 10 (Tom Lane) @@ -2280,7 +2118,7 @@ - + Client Applications @@ -2290,14 +2128,11 @@ - + Add conditional branch support to psql (Corey Huinker) @@ -2310,10 +2145,9 @@ - + Add psql \gx command to perform \g(execute) in expanded mode (\x) (Christoph Berg) @@ -2321,10 +2155,9 @@ - + Improve psql's \d (relation) and \dD (domain) commands to specify collation, nullable, and default in separate @@ -2337,10 +2170,9 @@ - + Expand psql colon variables when used in backtick-executed contexts (Tom Lane) @@ -2353,14 +2185,11 @@ - + Prevent psql special variables from being set to invalid values (Daniel Vérité, Tom Lane) @@ -2377,28 +2206,18 @@ - + Improve psql's tab completion (Jeff Janes, Ian Barwick, Andreas Karlsson, Sehrope Sarkuni, Thomas Munro, @@ -2416,10 +2235,9 @@ - + Add pgbench option - + Allow pgbench's meta commands to span multiple lines via a line-terminating backslash (Fabien Coelho) @@ -2449,10 +2266,9 @@ - + Add pg_receivewal @@ -2462,10 +2278,9 @@ - + Add pg_recvlogical option @@ -2478,10 +2293,9 @@ - + Rename initdb options - + Improve pg_basebackup's handling of which directories to skip (David Steele) @@ -2642,17 +2447,16 @@ - + <application><xref linkend="app-pg-ctl"></> - + Add wait option for 's promote operation (Peter Eisentraut) @@ -2660,10 +2464,9 @@ - + Add long options for pg_ctl wait ( - + Add long options flag for pg_ctl options (Peter Eisentraut) @@ -2686,9 +2488,8 @@ - + - @@ -2697,10 +2498,9 @@ - + New major version numbering (Peter Eisentraut, Tom Lane) @@ -2713,10 +2513,9 @@ - + Allow the ICU library to optionally be used for collation support (Peter Eisentraut) @@ -2731,10 +2530,9 @@ - + Automatically mark all PG_FUNCTION_INFO_V1 functions @@ -2750,14 +2548,11 @@ - + Allow shared memory to be dynamically allocated (Thomas Munro, Robert Haas) @@ -2765,10 +2560,9 @@ - + Add slab-like memory allocator for efficient fixed-size allocations (Tomas Vondra) @@ -2776,10 +2570,9 @@ - + Use POSIX semaphores rather than SysV semaphores on Linux and - + Improve support for 64-bit atomics (Andres Freund) - + Enable 64-bit atomic operations on ARM64 (Roman Shaposhnik) @@ -2813,10 +2604,9 @@ - + Switch to using clock_gettime(), if available, for duration measurements (Tom Lane) @@ -2829,10 +2619,9 @@ - + Add more robust random number generators to be used for cryptographic secure uses (Magnus Hagander, Michael Paquier, @@ -2849,12 +2638,10 @@ - + Overhaul documentation build process (Alexander Lakhin, Alexander Law) @@ -2862,10 +2649,9 @@ - + Use XSLT to build the Postgres documentation (Peter Eisentraut) @@ -2878,10 +2664,9 @@ - + Build HTML documentation using XSLT stylesheets by default (Peter Eisentraut) @@ -2898,10 +2683,9 @@ - + Allow file_fdw to read from program output as well as files (Corey Huinker, Adam Gomaa) @@ -2909,12 +2693,10 @@ - + Push aggregates to foreign data wrapper servers, where possible (Jeevan Chalke, Ashutosh Bapat) @@ -2931,10 +2713,9 @@ - + Allow push down of FULL JOIN queries containing subqueries in the @@ -2943,10 +2724,9 @@ - + Properly support OID columns in postgres_fdw tables (Etsuro Fujita) @@ -2958,10 +2738,9 @@ - + Allow btree_gist and btree_gin to @@ -2974,10 +2753,9 @@ - + Add indexing support to btree_gist for the UUID data type (Paul Jungwirth) @@ -2985,10 +2763,9 @@ - + Add amcheck which can check the validity of btree indexes (Peter Geoghegan) @@ -2996,10 +2773,9 @@ - + Show ignored constants as $N rather than ? in @@ -3010,10 +2786,9 @@ - + Improve cube's handling of zero-dimensional cubes (Tom Lane) @@ -3026,10 +2801,9 @@ - + Allow pg_buffercache to run @@ -3042,10 +2816,9 @@ - + Add pgstathashindex() function to pgstattuple to view hash @@ -3054,10 +2827,9 @@ - + Allow pgstattuple to use GRANT permissions (Stephen Frost) @@ -3070,10 +2842,9 @@ - + Reduce locking when pgstattuple examines hash @@ -3082,10 +2853,9 @@ - + Add page_checksum() function to pageinspect (Tomas Vondra) @@ -3093,10 +2863,9 @@ - + Add hash index support to pageinspect (Jesper @@ -3105,10 +2874,9 @@ - + Add pageinspect function bt_page_items() to print page items from a -- cgit v1.2.3 From 6fd65f6b877512eb1c35897d4c4c6779d100e459 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 29 May 2017 09:31:33 +0300 Subject: Fix reference to RFC specifying SCRAM. Noted by Peter Eisentraut --- doc/src/sgml/protocol.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index d23df0261c..4837be5016 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -1401,7 +1401,7 @@ ErrorMessage. SCRAM-SHA-256 (called just SCRAM from now on) is the only implemented SASL mechanism, at the moment. It is described in detail - in RFC 7677 and RFC 5741. + in RFC 7677 and RFC 5802. -- cgit v1.2.3 From f3db7f164a29c5cbdc1d6d5d0d23854df58783c1 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 29 May 2017 17:08:16 -0400 Subject: Prevent running pg_resetwal/pg_resetxlog against wrong-version data dirs. pg_resetwal (formerly pg_resetxlog) doesn't insist on finding a matching version number in pg_control, and that seems like an important thing to preserve since recovering from corrupt pg_control is a prime reason to need to run it. However, that means you can try to run it against a data directory of a different major version, which is at best useless and at worst disastrous. So as to provide some protection against that type of pilot error, inspect PG_VERSION at startup and refuse to do anything if it doesn't match. PG_VERSION is read-only after initdb, so it's unlikely to get corrupted, and even if it were corrupted it would be easy to fix by hand. This hazard has been there all along, so back-patch to all supported branches. Michael Paquier, with some kibitzing by me Discussion: https://postgr.es/m/f4b8eb91-b934-8a0d-b3cc-68f06e2279d1@enterprisedb.com --- doc/src/sgml/ref/pg_resetwal.sgml | 5 +++ src/bin/pg_resetwal/pg_resetwal.c | 70 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 74 insertions(+), 1 deletion(-) (limited to 'doc/src') diff --git a/doc/src/sgml/ref/pg_resetwal.sgml b/doc/src/sgml/ref/pg_resetwal.sgml index 0d93b56ddd..defaf170dc 100644 --- a/doc/src/sgml/ref/pg_resetwal.sgml +++ b/doc/src/sgml/ref/pg_resetwal.sgml @@ -281,6 +281,11 @@ PostgreSQL documentation pg_resetwal to run. But before you do so, make doubly certain that there is no server process still alive. + + + pg_resetwal works only with servers of the same + major version. + diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c index a3ecccb035..7f01067581 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c @@ -71,6 +71,7 @@ static MultiXactOffset set_mxoff = (MultiXactOffset) -1; static uint32 minXlogTli = 0; static XLogSegNo minXlogSegNo = 0; +static void CheckDataVersion(void); static bool ReadControlFile(void); static void GuessControlValues(void); static void PrintControlValues(bool guessed); @@ -319,6 +320,9 @@ main(int argc, char *argv[]) exit(1); } + /* Check that data directory matches our server version */ + CheckDataVersion(); + /* * Check for a postmaster lock file --- if there is one, refuse to * proceed, on grounds we might be interfering with a live installation. @@ -452,6 +456,70 @@ main(int argc, char *argv[]) } +/* + * Look at the version string stored in PG_VERSION and decide if this utility + * can be run safely or not. + * + * We don't want to inject pg_control and WAL files that are for a different + * major version; that can't do anything good. Note that we don't treat + * mismatching version info in pg_control as a reason to bail out, because + * recovering from a corrupted pg_control is one of the main reasons for this + * program to exist at all. However, PG_VERSION is unlikely to get corrupted, + * and if it were it would be easy to fix by hand. So let's make this check + * to prevent simple user errors. + */ +static void +CheckDataVersion(void) +{ + const char *ver_file = "PG_VERSION"; + FILE *ver_fd; + char rawline[64]; + int len; + + if ((ver_fd = fopen(ver_file, "r")) == NULL) + { + fprintf(stderr, _("%s: could not open file \"%s\" for reading: %s\n"), + progname, ver_file, strerror(errno)); + exit(1); + } + + /* version number has to be the first line read */ + if (!fgets(rawline, sizeof(rawline), ver_fd)) + { + if (!ferror(ver_fd)) + { + fprintf(stderr, _("%s: unexpected empty file \"%s\"\n"), + progname, ver_file); + } + else + { + fprintf(stderr, _("%s: could not read file \"%s\": %s\n"), + progname, ver_file, strerror(errno)); + } + exit(1); + } + + /* remove trailing newline, handling Windows newlines as well */ + len = strlen(rawline); + if (len > 0 && rawline[len - 1] == '\n') + { + rawline[--len] = '\0'; + if (len > 0 && rawline[len - 1] == '\r') + rawline[--len] = '\0'; + } + + if (strcmp(rawline, PG_MAJORVERSION) != 0) + { + fprintf(stderr, _("%s: data directory is of wrong version\n" + "File \"%s\" contains \"%s\", which is not compatible with this program's version \"%s\".\n"), + progname, ver_file, rawline, PG_MAJORVERSION); + exit(1); + } + + fclose(ver_fd); +} + + /* * Try to read the existing pg_control file. * @@ -521,7 +589,7 @@ ReadControlFile(void) } /* Looks like it's a mess. */ - fprintf(stderr, _("%s: pg_control exists but is broken or unknown version; ignoring it\n"), + fprintf(stderr, _("%s: pg_control exists but is broken or wrong version; ignoring it\n"), progname); return false; } -- cgit v1.2.3