From 72d6b83ff52350cfae6fd71274c5fdcdd398debe Mon Sep 17 00:00:00 2001 From: Vignesh C Date: Fri, 20 Sep 2024 08:45:21 +0530 Subject: [PATCH 1/5] Introduce pg_sequence_state function for enhanced sequence management This patch introduces a new function, 'pg_sequence_state', which allows retrieval of sequence values, including the associated LSN. --- doc/src/sgml/func.sgml | 26 ++++++++++ src/backend/commands/sequence.c | 70 ++++++++++++++++++++++++++ src/include/catalog/pg_proc.dat | 8 +++ src/test/regress/expected/sequence.out | 6 +++ src/test/regress/sql/sequence.sql | 1 + 5 files changed, 111 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 574a544d9fa4..be01319caf97 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -19935,6 +19935,32 @@ SELECT setval('myseq', 42, false); Next nextvalSELECT privilege on the last used sequence. + + + + + pg_sequence_state + + pg_sequence_state ( regclass ) + record + ( page_lsn pg_lsn, + last_value bigint, + log_cnt bigint, + is_called bool ) + + + Returns information about the sequence. page_lsn is + the page LSN of the sequence, last_value is the + current value of the sequence, log_cnt shows how + many fetches remain before a new WAL record must be written, and + is_called indicates whether the sequence has been + used. + + + This function requires USAGE + or SELECT privilege on the sequence. + + diff --git a/src/backend/commands/sequence.c b/src/backend/commands/sequence.c index 451ae6f7f694..2e5b6cbecd1a 100644 --- a/src/backend/commands/sequence.c +++ b/src/backend/commands/sequence.c @@ -45,6 +45,7 @@ #include "utils/acl.h" #include "utils/builtins.h" #include "utils/lsyscache.h" +#include "utils/pg_lsn.h" #include "utils/resowner.h" #include "utils/syscache.h" #include "utils/varlena.h" @@ -1885,6 +1886,75 @@ pg_sequence_last_value(PG_FUNCTION_ARGS) PG_RETURN_NULL(); } +/* + * Return the current on-disk state of the sequence. + * + * Note: This is roughly equivalent to selecting the data from the sequence, + * except that it also returns the page LSN. + */ +Datum +pg_sequence_state(PG_FUNCTION_ARGS) +{ + Oid seq_relid = PG_GETARG_OID(0); + SeqTable elm; + Relation seqrel; + Buffer buf; + Page page; + HeapTupleData seqtuple; + Form_pg_sequence_data seq; + Datum result; + + XLogRecPtr lsn; + int64 last_value; + int64 log_cnt; + bool is_called; + + TupleDesc tupdesc; + HeapTuple tuple; + Datum values[4]; + bool nulls[4] = {0}; + + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + + /* open and lock sequence */ + init_sequence(seq_relid, &elm, &seqrel); + + if (pg_class_aclcheck(elm->relid, GetUserId(), + ACL_SELECT | ACL_USAGE) != ACLCHECK_OK) + ereport(ERROR, + errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("permission denied for sequence %s", + RelationGetRelationName(seqrel))); + + seq = read_seq_tuple(seqrel, &buf, &seqtuple); + page = BufferGetPage(buf); + + lsn = PageGetLSN(page); + last_value = seq->last_value; + log_cnt = seq->log_cnt; + is_called = seq->is_called; + + UnlockReleaseBuffer(buf); + relation_close(seqrel, NoLock); + + /* Page LSN for the sequence */ + values[0] = LSNGetDatum(lsn); + + /* The value most recently returned by nextval in the current session */ + values[1] = Int64GetDatum(last_value); + + /* How many fetches remain before a new WAL record must be written */ + values[2] = Int64GetDatum(log_cnt); + + /* Indicates whether the sequence has been used */ + values[3] = BoolGetDatum(is_called); + + tuple = heap_form_tuple(tupdesc, values, nulls); + result = HeapTupleGetDatum(tuple); + + PG_RETURN_DATUM(result); +} void seq_redo(XLogReaderState *record) diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 62beb71da288..8071134643c3 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -3433,6 +3433,14 @@ proname => 'pg_sequence_last_value', provolatile => 'v', proparallel => 'u', prorettype => 'int8', proargtypes => 'regclass', prosrc => 'pg_sequence_last_value' }, +{ oid => '8051', + descr => 'current on-disk sequence state', + proname => 'pg_sequence_state', provolatile => 'v', + prorettype => 'record', proargtypes => 'regclass', + proallargtypes => '{regclass,pg_lsn,int8,int8,bool}', + proargmodes => '{i,o,o,o,o}', + proargnames => '{seq_oid,page_lsn,last_value,log_cnt,is_called}', + prosrc => 'pg_sequence_state' }, { oid => '9876', descr => 'return sequence tuple, for use by pg_dump', proname => 'pg_get_sequence_data', provolatile => 'v', proparallel => 'u', prorettype => 'record', proargtypes => 'regclass', diff --git a/src/test/regress/expected/sequence.out b/src/test/regress/expected/sequence.out index 15925d99c8a3..4bc21c7af957 100644 --- a/src/test/regress/expected/sequence.out +++ b/src/test/regress/expected/sequence.out @@ -161,6 +161,12 @@ SELECT nextval('serialTest2_f6_seq'); CREATE SEQUENCE sequence_test; CREATE SEQUENCE IF NOT EXISTS sequence_test; NOTICE: relation "sequence_test" already exists, skipping +SELECT last_value, log_cnt, is_called FROM pg_sequence_state('sequence_test'); + last_value | log_cnt | is_called +------------+---------+----------- + 1 | 0 | f +(1 row) + SELECT nextval('sequence_test'::text); nextval --------- diff --git a/src/test/regress/sql/sequence.sql b/src/test/regress/sql/sequence.sql index 2c220b60749e..23341a36caab 100644 --- a/src/test/regress/sql/sequence.sql +++ b/src/test/regress/sql/sequence.sql @@ -112,6 +112,7 @@ SELECT nextval('serialTest2_f6_seq'); CREATE SEQUENCE sequence_test; CREATE SEQUENCE IF NOT EXISTS sequence_test; +SELECT last_value, log_cnt, is_called FROM pg_sequence_state('sequence_test'); SELECT nextval('sequence_test'::text); SELECT nextval('sequence_test'::regclass); SELECT currval('sequence_test'::text); From 5b0ca81ed00e76a89277e3ce0d1b44c3ee29a73d Mon Sep 17 00:00:00 2001 From: Vignesh Date: Mon, 3 Feb 2025 09:53:31 +0530 Subject: [PATCH 2/5] Introduce "ALL SEQUENCES" support for PostgreSQL logical replication This commit enhances logical replication by enabling the inclusion of all sequences in publications. Furthermore, enhancements to psql commands now display which publications contain the specified sequence (\d command), and if a specified publication includes all sequences (\dRp command). Note: This patch currently supports only the "ALL SEQUENCES" clause. Handling of clauses such as "FOR SEQUENCE" and "FOR SEQUENCES IN SCHEMA" will be addressed in a subsequent patch. --- doc/src/sgml/ref/create_publication.sgml | 63 ++- src/backend/catalog/pg_publication.c | 40 +- src/backend/commands/publicationcmds.c | 52 +- src/backend/parser/gram.y | 84 +++- src/bin/pg_dump/pg_dump.c | 14 +- src/bin/pg_dump/pg_dump.h | 1 + src/bin/pg_dump/t/002_pg_dump.pl | 22 + src/bin/psql/describe.c | 202 +++++--- src/bin/psql/tab-complete.in.c | 8 +- src/include/catalog/pg_publication.h | 8 + src/include/nodes/parsenodes.h | 18 + src/test/regress/expected/psql.out | 6 +- src/test/regress/expected/publication.out | 556 ++++++++++++---------- src/test/regress/sql/publication.sql | 37 ++ src/tools/pgindent/typedefs.list | 2 + 15 files changed, 760 insertions(+), 353 deletions(-) diff --git a/doc/src/sgml/ref/create_publication.sgml b/doc/src/sgml/ref/create_publication.sgml index 73f0c8d89fb0..dcf1a68308f4 100644 --- a/doc/src/sgml/ref/create_publication.sgml +++ b/doc/src/sgml/ref/create_publication.sgml @@ -22,14 +22,15 @@ PostgreSQL documentation CREATE PUBLICATION name - [ FOR ALL TABLES - | FOR publication_object [, ... ] ] + [ FOR publication_object [, ... ] ] [ WITH ( publication_parameter [= value] [, ... ] ) ] where publication_object is one of: TABLE [ ONLY ] table_name [ * ] [ ( column_name [, ... ] ) ] [ WHERE ( expression ) ] [, ... ] TABLES IN SCHEMA { schema_name | CURRENT_SCHEMA } [, ... ] + ALL TABLES + ALL SEQUENCES @@ -118,16 +119,6 @@ CREATE PUBLICATION name - - FOR ALL TABLES - - - Marks the publication as one that replicates changes for all tables in - the database, including tables created in the future. - - - - FOR TABLES IN SCHEMA @@ -159,6 +150,26 @@ CREATE PUBLICATION name + + FOR ALL TABLES + + + Marks the publication as one that replicates changes for all tables in + the database, including tables created in the future. + + + + + + FOR ALL SEQUENCES + + + Marks the publication as one that synchronizes changes for all sequences + in the database, including sequences created in the future. + + + + WITH ( publication_parameter [= value] [, ... ] ) @@ -277,10 +288,10 @@ CREATE PUBLICATION name Notes - If FOR TABLE, FOR ALL TABLES or - FOR TABLES IN SCHEMA are not specified, then the - publication starts out with an empty set of tables. That is useful if - tables or schemas are to be added later. + If FOR TABLE, FOR TABLES IN SCHEMA, + FOR ALL TABLES or FOR ALL SEQUENCES + are not specified, then the publication starts out with an empty set of + tables. That is useful if tables or schemas are to be added later. @@ -296,8 +307,9 @@ CREATE PUBLICATION name To add a table to a publication, the invoking user must have ownership - rights on the table. The FOR ALL TABLES and - FOR TABLES IN SCHEMA clauses require the invoking + rights on the table. The FOR TABLES IN SCHEMA, + FOR ALL TABLES and + FOR ALL SEQUENCES clauses require the invoking user to be a superuser. @@ -447,6 +459,21 @@ CREATE PUBLICATION sales_publication FOR TABLES IN SCHEMA marketing, sales; CREATE PUBLICATION users_filtered FOR TABLE users (user_id, firstname); + + + Create a publication that publishes all sequences for synchronization: + +CREATE PUBLICATION all_sequences FOR ALL SEQUENCES; + + + + + Create a publication that publishes all changes in all tables, and + all sequences for synchronization: + +CREATE PUBLICATION all_tables_sequences FOR ALL TABLES, ALL SEQUENCES; + + diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c index d6f94db5d999..617ed0b82c9b 100644 --- a/src/backend/catalog/pg_publication.c +++ b/src/backend/catalog/pg_publication.c @@ -134,7 +134,8 @@ static bool is_publishable_class(Oid relid, Form_pg_class reltuple) { return (reltuple->relkind == RELKIND_RELATION || - reltuple->relkind == RELKIND_PARTITIONED_TABLE) && + reltuple->relkind == RELKIND_PARTITIONED_TABLE || + reltuple->relkind == RELKIND_SEQUENCE) && !IsCatalogRelationOid(relid) && reltuple->relpersistence == RELPERSISTENCE_PERMANENT && relid >= FirstNormalObjectId; @@ -1061,6 +1062,42 @@ GetAllSchemaPublicationRelations(Oid pubid, PublicationPartOpt pub_partopt) return result; } +/* + * Gets list of all relations published by FOR ALL SEQUENCES publication(s). + */ +List * +GetAllSequencesPublicationRelations(void) +{ + Relation classRel; + ScanKeyData key[1]; + TableScanDesc scan; + HeapTuple tuple; + List *result = NIL; + + classRel = table_open(RelationRelationId, AccessShareLock); + + ScanKeyInit(&key[0], + Anum_pg_class_relkind, + BTEqualStrategyNumber, F_CHAREQ, + CharGetDatum(RELKIND_SEQUENCE)); + + scan = table_beginscan_catalog(classRel, 1, key); + + while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) + { + Form_pg_class relForm = (Form_pg_class) GETSTRUCT(tuple); + Oid relid = relForm->oid; + + if (is_publishable_class(relid, relForm)) + result = lappend_oid(result, relid); + } + + table_endscan(scan); + + table_close(classRel, AccessShareLock); + return result; +} + /* * Get publication using oid * @@ -1083,6 +1120,7 @@ GetPublication(Oid pubid) pub->oid = pubid; pub->name = pstrdup(NameStr(pubform->pubname)); pub->alltables = pubform->puballtables; + pub->allsequences = pubform->puballsequences; pub->pubactions.pubinsert = pubform->pubinsert; pub->pubactions.pubupdate = pubform->pubupdate; pub->pubactions.pubdelete = pubform->pubdelete; diff --git a/src/backend/commands/publicationcmds.c b/src/backend/commands/publicationcmds.c index 0b23d94c38e2..ef13cf618d3e 100644 --- a/src/backend/commands/publicationcmds.c +++ b/src/backend/commands/publicationcmds.c @@ -848,11 +848,17 @@ CreatePublication(ParseState *pstate, CreatePublicationStmt *stmt) aclcheck_error(aclresult, OBJECT_DATABASE, get_database_name(MyDatabaseId)); - /* FOR ALL TABLES requires superuser */ - if (stmt->for_all_tables && !superuser()) - ereport(ERROR, - (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("must be superuser to create FOR ALL TABLES publication"))); + if (!superuser()) + { + if (stmt->for_all_tables) + ereport(ERROR, + errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("must be superuser to create a FOR ALL TABLES publication")); + if (stmt->for_all_sequences) + ereport(ERROR, + errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("must be superuser to create a FOR ALL SEQUENCES publication")); + } rel = table_open(PublicationRelationId, RowExclusiveLock); @@ -886,6 +892,8 @@ CreatePublication(ParseState *pstate, CreatePublicationStmt *stmt) values[Anum_pg_publication_oid - 1] = ObjectIdGetDatum(puboid); values[Anum_pg_publication_puballtables - 1] = BoolGetDatum(stmt->for_all_tables); + values[Anum_pg_publication_puballsequences - 1] = + BoolGetDatum(stmt->for_all_sequences); values[Anum_pg_publication_pubinsert - 1] = BoolGetDatum(pubactions.pubinsert); values[Anum_pg_publication_pubupdate - 1] = @@ -2019,19 +2027,27 @@ AlterPublicationOwner_internal(Relation rel, HeapTuple tup, Oid newOwnerId) aclcheck_error(aclresult, OBJECT_DATABASE, get_database_name(MyDatabaseId)); - if (form->puballtables && !superuser_arg(newOwnerId)) - ereport(ERROR, - (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("permission denied to change owner of publication \"%s\"", - NameStr(form->pubname)), - errhint("The owner of a FOR ALL TABLES publication must be a superuser."))); - - if (!superuser_arg(newOwnerId) && is_schema_publication(form->oid)) - ereport(ERROR, - (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("permission denied to change owner of publication \"%s\"", - NameStr(form->pubname)), - errhint("The owner of a FOR TABLES IN SCHEMA publication must be a superuser."))); + if (!superuser_arg(newOwnerId)) + { + if (form->puballtables) + ereport(ERROR, + errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("permission denied to change owner of publication \"%s\"", + NameStr(form->pubname)), + errhint("The owner of a FOR ALL TABLES publication must be a superuser.")); + if (form->puballsequences) + ereport(ERROR, + errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("permission denied to change owner of publication \"%s\"", + NameStr(form->pubname)), + errhint("The owner of a FOR ALL SEQUENCES publication must be a superuser.")); + if (is_schema_publication(form->oid)) + ereport(ERROR, + errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("permission denied to change owner of publication \"%s\"", + NameStr(form->pubname)), + errhint("The owner of a FOR TABLES IN SCHEMA publication must be a superuser.")); + } } form->pubowner = newOwnerId; diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 3c4268b271a4..1c094d7d6053 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -204,6 +204,10 @@ static PartitionStrategy parsePartitionStrategy(char *strategy, int location, core_yyscan_t yyscanner); static void preprocess_pubobj_list(List *pubobjspec_list, core_yyscan_t yyscanner); +static void preprocess_pub_all_objtype_list(List *all_objects_list, + bool *all_tables, + bool *all_sequences, + core_yyscan_t yyscanner); static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); %} @@ -260,6 +264,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); PartitionBoundSpec *partboundspec; RoleSpec *rolespec; PublicationObjSpec *publicationobjectspec; + PublicationAllObjSpec *publicationallobjectspec; struct SelectLimit *selectlimit; SetQuantifier setquantifier; struct GroupClause *groupclause; @@ -446,7 +451,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); transform_element_list transform_type_list TriggerTransitions TriggerReferencing vacuum_relation_list opt_vacuum_relation_list - drop_option_list pub_obj_list + drop_option_list pub_obj_list pub_obj_type_list %type returning_clause %type returning_option @@ -585,6 +590,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); %type var_value zone_value %type auth_ident RoleSpec opt_granted_by %type PublicationObjSpec +%type PublicationAllObjSpec %type unreserved_keyword type_func_name_keyword %type col_name_keyword reserved_keyword @@ -10614,7 +10620,12 @@ AlterOwnerStmt: ALTER AGGREGATE aggregate_with_argtypes OWNER TO RoleSpec * * CREATE PUBLICATION name [WITH options] * - * CREATE PUBLICATION FOR ALL TABLES [WITH options] + * CREATE PUBLICATION FOR ALL pub_obj_type [, ...] [WITH options] + * + * pub_obj_type is one of: + * + * TABLES + * SEQUENCES * * CREATE PUBLICATION FOR pub_obj [, ...] [WITH options] * @@ -10634,13 +10645,13 @@ CreatePublicationStmt: n->options = $4; $$ = (Node *) n; } - | CREATE PUBLICATION name FOR ALL TABLES opt_definition + | CREATE PUBLICATION name FOR pub_obj_type_list opt_definition { CreatePublicationStmt *n = makeNode(CreatePublicationStmt); n->pubname = $3; - n->options = $7; - n->for_all_tables = true; + preprocess_pub_all_objtype_list($5, &n->for_all_tables, &n->for_all_sequences, yyscanner); + n->options = $6; $$ = (Node *) n; } | CREATE PUBLICATION name FOR pub_obj_list opt_definition @@ -10752,6 +10763,28 @@ pub_obj_list: PublicationObjSpec { $$ = lappend($1, $3); } ; +PublicationAllObjSpec: + ALL TABLES + { + $$ = makeNode(PublicationAllObjSpec); + $$->pubobjtype = PUBLICATION_ALL_TABLES; + $$->location = @1; + } + | ALL SEQUENCES + { + $$ = makeNode(PublicationAllObjSpec); + $$->pubobjtype = PUBLICATION_ALL_SEQUENCES; + $$->location = @1; + } + ; + +pub_obj_type_list: PublicationAllObjSpec + { $$ = list_make1($1); } + | pub_obj_type_list ',' PublicationAllObjSpec + { $$ = lappend($1, $3); } + ; + + /***************************************************************************** * * ALTER PUBLICATION name SET ( options ) @@ -19631,6 +19664,47 @@ parsePartitionStrategy(char *strategy, int location, core_yyscan_t yyscanner) } +/* + * Process all_objects_list to set all_tables/all_sequences. + * Also, checks if the pub_object_type has been specified more than once. + */ +static void +preprocess_pub_all_objtype_list(List *all_objects_list, bool *all_tables, + bool *all_sequences, core_yyscan_t yyscanner) +{ + if (!all_objects_list) + return; + + *all_tables = false; + *all_sequences = false; + + foreach_ptr(PublicationAllObjSpec, obj, all_objects_list) + { + if (obj->pubobjtype == PUBLICATION_ALL_TABLES) + { + if (*all_tables) + ereport(ERROR, + errcode(ERRCODE_SYNTAX_ERROR), + errmsg("invalid publication object list"), + errdetail("ALL TABLES can be specified only once."), + parser_errposition(obj->location)); + + *all_tables = true; + } + else if (obj->pubobjtype == PUBLICATION_ALL_SEQUENCES) + { + if (*all_sequences) + ereport(ERROR, + errcode(ERRCODE_SYNTAX_ERROR), + errmsg("invalid publication object list"), + errdetail("ALL SEQUENCES can be specified only once."), + parser_errposition(obj->location)); + + *all_sequences = true; + } + } +} + /* * Process pubobjspec_list to check for errors in any of the objects and * convert PUBLICATIONOBJ_CONTINUATION into appropriate PublicationObjSpecType. diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 105e917aa7b9..bd41c0092152 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -4390,6 +4390,7 @@ getPublications(Archive *fout) int i_pubname; int i_pubowner; int i_puballtables; + int i_puballsequences; int i_pubinsert; int i_pubupdate; int i_pubdelete; @@ -4420,9 +4421,9 @@ getPublications(Archive *fout) appendPQExpBufferStr(query, "false AS pubviaroot, "); if (fout->remoteVersion >= 180000) - appendPQExpBufferStr(query, "p.pubgencols "); + appendPQExpBufferStr(query, "p.pubgencols, p.puballsequences "); else - appendPQExpBuffer(query, "'%c' AS pubgencols ", PUBLISH_GENCOLS_NONE); + appendPQExpBuffer(query, "'%c' AS pubgencols, false AS puballsequences ", PUBLISH_GENCOLS_NONE); appendPQExpBufferStr(query, "FROM pg_publication p"); @@ -4438,6 +4439,7 @@ getPublications(Archive *fout) i_pubname = PQfnumber(res, "pubname"); i_pubowner = PQfnumber(res, "pubowner"); i_puballtables = PQfnumber(res, "puballtables"); + i_puballsequences = PQfnumber(res, "puballsequences"); i_pubinsert = PQfnumber(res, "pubinsert"); i_pubupdate = PQfnumber(res, "pubupdate"); i_pubdelete = PQfnumber(res, "pubdelete"); @@ -4458,6 +4460,8 @@ getPublications(Archive *fout) pubinfo[i].rolname = getRoleName(PQgetvalue(res, i, i_pubowner)); pubinfo[i].puballtables = (strcmp(PQgetvalue(res, i, i_puballtables), "t") == 0); + pubinfo[i].puballsequences = + (strcmp(PQgetvalue(res, i, i_puballsequences), "t") == 0); pubinfo[i].pubinsert = (strcmp(PQgetvalue(res, i, i_pubinsert), "t") == 0); pubinfo[i].pubupdate = @@ -4509,8 +4513,12 @@ dumpPublication(Archive *fout, const PublicationInfo *pubinfo) appendPQExpBuffer(query, "CREATE PUBLICATION %s", qpubname); - if (pubinfo->puballtables) + if (pubinfo->puballtables && pubinfo->puballsequences) + appendPQExpBufferStr(query, " FOR ALL TABLES, ALL SEQUENCES"); + else if (pubinfo->puballtables) appendPQExpBufferStr(query, " FOR ALL TABLES"); + else if (pubinfo->puballsequences) + appendPQExpBufferStr(query, " FOR ALL SEQUENCES"); appendPQExpBufferStr(query, " WITH (publish = '"); if (pubinfo->pubinsert) diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h index b426b5e47361..76aa26fa7143 100644 --- a/src/bin/pg_dump/pg_dump.h +++ b/src/bin/pg_dump/pg_dump.h @@ -660,6 +660,7 @@ typedef struct _PublicationInfo DumpableObject dobj; const char *rolname; bool puballtables; + bool puballsequences; bool pubinsert; bool pubupdate; bool pubdelete; diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl index 6c03eca8e501..f953cad69efb 100644 --- a/src/bin/pg_dump/t/002_pg_dump.pl +++ b/src/bin/pg_dump/t/002_pg_dump.pl @@ -3159,6 +3159,28 @@ like => { %full_runs, section_post_data => 1, }, }, + 'CREATE PUBLICATION pub5' => { + create_order => 50, + create_sql => 'CREATE PUBLICATION pub5 + FOR ALL SEQUENCES + WITH (publish = \'\');', + regexp => qr/^ + \QCREATE PUBLICATION pub5 FOR ALL SEQUENCES WITH (publish = '');\E + /xm, + like => { %full_runs, section_post_data => 1, }, + }, + + 'CREATE PUBLICATION pub6' => { + create_order => 50, + create_sql => 'CREATE PUBLICATION pub6 + FOR ALL SEQUENCES, ALL TABLES + WITH (publish = \'\');', + regexp => qr/^ + \QCREATE PUBLICATION pub6 FOR ALL TABLES, ALL SEQUENCES WITH (publish = '');\E + /xm, + like => { %full_runs, section_post_data => 1, }, + }, + 'CREATE SUBSCRIPTION sub1' => { create_order => 50, create_sql => 'CREATE SUBSCRIPTION sub1 diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index 1d08268393e3..3d38f32f6ab1 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -1757,28 +1757,19 @@ describeOneTableDetails(const char *schemaname, if (tableinfo.relkind == RELKIND_SEQUENCE) { PGresult *result = NULL; - printQueryOpt myopt = pset.popt; - char *footers[2] = {NULL, NULL}; if (pset.sversion >= 100000) { printfPQExpBuffer(&buf, - "SELECT pg_catalog.format_type(seqtypid, NULL) AS \"%s\",\n" - " seqstart AS \"%s\",\n" - " seqmin AS \"%s\",\n" - " seqmax AS \"%s\",\n" - " seqincrement AS \"%s\",\n" - " CASE WHEN seqcycle THEN '%s' ELSE '%s' END AS \"%s\",\n" - " seqcache AS \"%s\"\n", - gettext_noop("Type"), - gettext_noop("Start"), - gettext_noop("Minimum"), - gettext_noop("Maximum"), - gettext_noop("Increment"), + "SELECT pg_catalog.format_type(seqtypid, NULL),\n" + " seqstart,\n" + " seqmin,\n" + " seqmax,\n" + " seqincrement,\n" + " CASE WHEN seqcycle THEN '%s' ELSE '%s' END,\n" + " seqcache\n", gettext_noop("yes"), - gettext_noop("no"), - gettext_noop("Cycles?"), - gettext_noop("Cache")); + gettext_noop("no")); appendPQExpBuffer(&buf, "FROM pg_catalog.pg_sequence\n" "WHERE seqrelid = '%s';", @@ -1787,22 +1778,15 @@ describeOneTableDetails(const char *schemaname, else { printfPQExpBuffer(&buf, - "SELECT 'bigint' AS \"%s\",\n" - " start_value AS \"%s\",\n" - " min_value AS \"%s\",\n" - " max_value AS \"%s\",\n" - " increment_by AS \"%s\",\n" - " CASE WHEN is_cycled THEN '%s' ELSE '%s' END AS \"%s\",\n" - " cache_value AS \"%s\"\n", - gettext_noop("Type"), - gettext_noop("Start"), - gettext_noop("Minimum"), - gettext_noop("Maximum"), - gettext_noop("Increment"), + "SELECT 'bigint',\n" + " start_value,\n" + " min_value,\n" + " max_value,\n" + " increment_by,\n" + " CASE WHEN is_cycled THEN '%s' ELSE '%s' END,\n" + " cache_value\n", gettext_noop("yes"), - gettext_noop("no"), - gettext_noop("Cycles?"), - gettext_noop("Cache")); + gettext_noop("no")); appendPQExpBuffer(&buf, "FROM %s", fmtId(schemaname)); /* must be separate because fmtId isn't reentrant */ appendPQExpBuffer(&buf, ".%s;", fmtId(relationname)); @@ -1812,6 +1796,59 @@ describeOneTableDetails(const char *schemaname, if (!res) goto error_return; + numrows = PQntuples(res); + + /* + * XXX reset to use expanded output for sequences (maybe we should + * keep this disabled, just like for tables?) + */ + myopt.expanded = pset.popt.topt.expanded; + + printTableInit(&cont, &myopt, title.data, 7, numrows); + printTableInitialized = true; + + if (tableinfo.relpersistence == RELPERSISTENCE_UNLOGGED) + printfPQExpBuffer(&title, _("Unlogged sequence \"%s.%s\""), + schemaname, relationname); + else + printfPQExpBuffer(&title, _("Sequence \"%s.%s\""), + schemaname, relationname); + + printTableAddHeader(&cont, gettext_noop("Type"), true, 'l'); + printTableAddHeader(&cont, gettext_noop("Start"), true, 'r'); + printTableAddHeader(&cont, gettext_noop("Minimum"), true, 'r'); + printTableAddHeader(&cont, gettext_noop("Maximum"), true, 'r'); + printTableAddHeader(&cont, gettext_noop("Increment"), true, 'r'); + printTableAddHeader(&cont, gettext_noop("Cycles?"), true, 'l'); + printTableAddHeader(&cont, gettext_noop("Cache"), true, 'r'); + + /* Generate table cells to be printed */ + for (i = 0; i < numrows; i++) + { + /* Type */ + printTableAddCell(&cont, PQgetvalue(res, i, 0), false, false); + + /* Start */ + printTableAddCell(&cont, PQgetvalue(res, i, 1), false, false); + + /* Minimum */ + printTableAddCell(&cont, PQgetvalue(res, i, 2), false, false); + + /* Maximum */ + printTableAddCell(&cont, PQgetvalue(res, i, 3), false, false); + + /* Increment */ + printTableAddCell(&cont, PQgetvalue(res, i, 4), false, false); + + /* Cycles? */ + printTableAddCell(&cont, PQgetvalue(res, i, 5), false, false); + + /* Cache */ + printTableAddCell(&cont, PQgetvalue(res, i, 6), false, false); + } + + /* Footer information about a sequence */ + /* Get the column that owns this sequence */ printfPQExpBuffer(&buf, "SELECT pg_catalog.quote_ident(nspname) || '.' ||" "\n pg_catalog.quote_ident(relname) || '.' ||" @@ -1843,32 +1880,53 @@ describeOneTableDetails(const char *schemaname, switch (PQgetvalue(result, 0, 1)[0]) { case 'a': - footers[0] = psprintf(_("Owned by: %s"), - PQgetvalue(result, 0, 0)); + printTableAddFooter(&cont, + psprintf(_("Owned by: %s"), + PQgetvalue(result, 0, 0))); break; case 'i': - footers[0] = psprintf(_("Sequence for identity column: %s"), - PQgetvalue(result, 0, 0)); + printTableAddFooter(&cont, + psprintf(_("Sequence for identity column: %s"), + PQgetvalue(result, 0, 0))); break; } } PQclear(result); - if (tableinfo.relpersistence == RELPERSISTENCE_UNLOGGED) - printfPQExpBuffer(&title, _("Unlogged sequence \"%s.%s\""), - schemaname, relationname); - else - printfPQExpBuffer(&title, _("Sequence \"%s.%s\""), - schemaname, relationname); + /* Print any publications */ + if (pset.sversion >= 180000) + { + int tuples; - myopt.footers = footers; - myopt.topt.default_footer = false; - myopt.title = title.data; - myopt.translate_header = true; + printfPQExpBuffer(&buf, + "SELECT pubname\n" + "FROM pg_catalog.pg_publication p\n" + "WHERE p.puballsequences AND pg_catalog.pg_relation_is_publishable('%s')\n" + "ORDER BY 1;", + oid); - printQuery(res, &myopt, pset.queryFout, false, pset.logfile); + result = PSQLexec(buf.data); + if (!result) + goto error_return; - free(footers[0]); + /* Might be an empty set - that's ok */ + tuples = PQntuples(result); + if (tuples > 0) + { + printTableAddFooter(&cont, _("Publications:")); + + for (i = 0; i < tuples; i++) + { + printfPQExpBuffer(&buf, " \"%s\"", + PQgetvalue(result, i, 0)); + + printTableAddFooter(&cont, buf.data); + } + } + PQclear(result); + } + + printTable(&cont, pset.queryFout, false, pset.logfile); retval = true; goto error_return; /* not an error, just return early */ @@ -6397,7 +6455,7 @@ listPublications(const char *pattern) PQExpBufferData buf; PGresult *res; printQueryOpt myopt = pset.popt; - static const bool translate_columns[] = {false, false, false, false, false, false, false, false, false}; + static const bool translate_columns[] = {false, false, false, false, false, false, false, false, false, false}; if (pset.sversion < 100000) { @@ -6414,13 +6472,20 @@ listPublications(const char *pattern) printfPQExpBuffer(&buf, "SELECT pubname AS \"%s\",\n" " pg_catalog.pg_get_userbyid(pubowner) AS \"%s\",\n" - " puballtables AS \"%s\",\n" - " pubinsert AS \"%s\",\n" - " pubupdate AS \"%s\",\n" - " pubdelete AS \"%s\"", + " puballtables AS \"%s\"", gettext_noop("Name"), gettext_noop("Owner"), - gettext_noop("All tables"), + gettext_noop("All tables")); + + if (pset.sversion >= 180000) + appendPQExpBuffer(&buf, + ",\n puballsequences AS \"%s\"", + gettext_noop("All sequences")); + + appendPQExpBuffer(&buf, + ",\n pubinsert AS \"%s\",\n" + " pubupdate AS \"%s\",\n" + " pubdelete AS \"%s\"", gettext_noop("Inserts"), gettext_noop("Updates"), gettext_noop("Deletes")); @@ -6531,6 +6596,7 @@ describePublications(const char *pattern) bool has_pubtruncate; bool has_pubgencols; bool has_pubviaroot; + bool has_pubsequence; PQExpBufferData title; printTableContent cont; @@ -6545,6 +6611,7 @@ describePublications(const char *pattern) return true; } + has_pubsequence = (pset.sversion >= 180000); has_pubtruncate = (pset.sversion >= 110000); has_pubgencols = (pset.sversion >= 180000); has_pubviaroot = (pset.sversion >= 130000); @@ -6554,7 +6621,18 @@ describePublications(const char *pattern) printfPQExpBuffer(&buf, "SELECT oid, pubname,\n" " pg_catalog.pg_get_userbyid(pubowner) AS owner,\n" - " puballtables, pubinsert, pubupdate, pubdelete"); + " puballtables"); + + if (has_pubsequence) + appendPQExpBufferStr(&buf, + ", puballsequences"); + else + appendPQExpBufferStr(&buf, + ", false AS puballsequences"); + + appendPQExpBufferStr(&buf, + ", pubinsert, pubupdate, pubdelete"); + if (has_pubtruncate) appendPQExpBufferStr(&buf, ", pubtruncate"); @@ -6629,6 +6707,8 @@ describePublications(const char *pattern) bool puballtables = strcmp(PQgetvalue(res, i, 3), "t") == 0; printTableOpt myopt = pset.popt.topt; + if (has_pubsequence) + ncols++; if (has_pubtruncate) ncols++; if (has_pubgencols) @@ -6642,6 +6722,8 @@ describePublications(const char *pattern) printTableAddHeader(&cont, gettext_noop("Owner"), true, align); printTableAddHeader(&cont, gettext_noop("All tables"), true, align); + if (has_pubsequence) + printTableAddHeader(&cont, gettext_noop("All sequences"), true, align); printTableAddHeader(&cont, gettext_noop("Inserts"), true, align); printTableAddHeader(&cont, gettext_noop("Updates"), true, align); printTableAddHeader(&cont, gettext_noop("Deletes"), true, align); @@ -6654,15 +6736,17 @@ describePublications(const char *pattern) printTableAddCell(&cont, PQgetvalue(res, i, 2), false, false); printTableAddCell(&cont, PQgetvalue(res, i, 3), false, false); - printTableAddCell(&cont, PQgetvalue(res, i, 4), false, false); + if (has_pubsequence) + printTableAddCell(&cont, PQgetvalue(res, i, 4), false, false); printTableAddCell(&cont, PQgetvalue(res, i, 5), false, false); printTableAddCell(&cont, PQgetvalue(res, i, 6), false, false); + printTableAddCell(&cont, PQgetvalue(res, i, 7), false, false); if (has_pubtruncate) - printTableAddCell(&cont, PQgetvalue(res, i, 7), false, false); - if (has_pubgencols) printTableAddCell(&cont, PQgetvalue(res, i, 8), false, false); - if (has_pubviaroot) + if (has_pubgencols) printTableAddCell(&cont, PQgetvalue(res, i, 9), false, false); + if (has_pubviaroot) + printTableAddCell(&cont, PQgetvalue(res, i, 10), false, false); if (!puballtables) { diff --git a/src/bin/psql/tab-complete.in.c b/src/bin/psql/tab-complete.in.c index c916b9299a80..10dc03cd7cb9 100644 --- a/src/bin/psql/tab-complete.in.c +++ b/src/bin/psql/tab-complete.in.c @@ -3524,12 +3524,12 @@ match_previous_words(int pattern_id, /* CREATE PUBLICATION */ else if (Matches("CREATE", "PUBLICATION", MatchAny)) - COMPLETE_WITH("FOR TABLE", "FOR ALL TABLES", "FOR TABLES IN SCHEMA", "WITH ("); + COMPLETE_WITH("FOR TABLE", "FOR TABLES IN SCHEMA", "FOR ALL TABLES", "FOR ALL SEQUENCES", "WITH ("); else if (Matches("CREATE", "PUBLICATION", MatchAny, "FOR")) - COMPLETE_WITH("TABLE", "ALL TABLES", "TABLES IN SCHEMA"); + COMPLETE_WITH("TABLE", "TABLES IN SCHEMA", "ALL TABLES", "ALL SEQUENCES"); else if (Matches("CREATE", "PUBLICATION", MatchAny, "FOR", "ALL")) - COMPLETE_WITH("TABLES"); - else if (Matches("CREATE", "PUBLICATION", MatchAny, "FOR", "ALL", "TABLES")) + COMPLETE_WITH("TABLES", "SEQUENCES"); + else if (Matches("CREATE", "PUBLICATION", MatchAny, "FOR", "ALL", "TABLES|SEQUENCES")) COMPLETE_WITH("WITH ("); else if (Matches("CREATE", "PUBLICATION", MatchAny, "FOR", "TABLES")) COMPLETE_WITH("IN SCHEMA"); diff --git a/src/include/catalog/pg_publication.h b/src/include/catalog/pg_publication.h index 48c7d1a86152..283c0b111956 100644 --- a/src/include/catalog/pg_publication.h +++ b/src/include/catalog/pg_publication.h @@ -40,6 +40,12 @@ CATALOG(pg_publication,6104,PublicationRelationId) */ bool puballtables; + /* + * indicates that this is special publication which should encompass all + * sequences in the database (except for the unlogged and temp ones) + */ + bool puballsequences; + /* true if inserts are published */ bool pubinsert; @@ -129,6 +135,7 @@ typedef struct Publication Oid oid; char *name; bool alltables; + bool allsequences; bool pubviaroot; PublishGencolsType pubgencols_type; PublicationActions pubactions; @@ -164,6 +171,7 @@ typedef enum PublicationPartOpt extern List *GetPublicationRelations(Oid pubid, PublicationPartOpt pub_partopt); extern List *GetAllTablesPublications(void); extern List *GetAllTablesPublicationRelations(bool pubviaroot); +extern List *GetAllSequencesPublicationRelations(void); extern List *GetPublicationSchemas(Oid pubid); extern List *GetSchemaPublications(Oid schemaid); extern List *GetSchemaPublicationRelations(Oid schemaid, diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 4610fc61293b..9b9656dd6e3c 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -4253,6 +4253,22 @@ typedef struct PublicationObjSpec ParseLoc location; /* token location, or -1 if unknown */ } PublicationObjSpec; +/* + * Publication types supported by FOR ALL ... + */ +typedef enum PublicationAllObjType +{ + PUBLICATION_ALL_TABLES, + PUBLICATION_ALL_SEQUENCES, +} PublicationAllObjType; + +typedef struct PublicationAllObjSpec +{ + NodeTag type; + PublicationAllObjType pubobjtype; /* type of this publication object */ + ParseLoc location; /* token location, or -1 if unknown */ +} PublicationAllObjSpec; + typedef struct CreatePublicationStmt { NodeTag type; @@ -4260,6 +4276,8 @@ typedef struct CreatePublicationStmt List *options; /* List of DefElem nodes */ List *pubobjects; /* Optional list of publication objects */ bool for_all_tables; /* Special publication for all tables in db */ + bool for_all_sequences; /* Special publication for all sequences + * in db */ } CreatePublicationStmt; typedef enum AlterPublicationAction diff --git a/src/test/regress/expected/psql.out b/src/test/regress/expected/psql.out index cf48ae6d0c2e..fb05755449d5 100644 --- a/src/test/regress/expected/psql.out +++ b/src/test/regress/expected/psql.out @@ -6443,9 +6443,9 @@ List of schemas (0 rows) \dRp "no.such.publication" - List of publications - Name | Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root -------+-------+------------+---------+---------+---------+-----------+-------------------+---------- + List of publications + Name | Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +------+-------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- (0 rows) \dRs "no.such.subscription" diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out index 4de96c04f9de..c128322be05e 100644 --- a/src/test/regress/expected/publication.out +++ b/src/test/regress/expected/publication.out @@ -36,20 +36,20 @@ LINE 1: ...pub_xxx WITH (publish_generated_columns = stored, publish_ge... CREATE PUBLICATION testpub_xxx WITH (publish_generated_columns = foo); ERROR: publish_generated_columns requires a "none" or "stored" value \dRp - List of publications - Name | Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------+--------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - testpub_default | regress_publication_user | f | f | t | f | f | none | f - testpub_ins_trunct | regress_publication_user | f | t | f | f | f | none | f + List of publications + Name | Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------+--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + testpub_default | regress_publication_user | f | f | f | t | f | f | none | f + testpub_ins_trunct | regress_publication_user | f | f | t | f | f | f | none | f (2 rows) ALTER PUBLICATION testpub_default SET (publish = 'insert, update, delete'); \dRp - List of publications - Name | Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------+--------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - testpub_default | regress_publication_user | f | t | t | t | f | none | f - testpub_ins_trunct | regress_publication_user | f | t | f | f | f | none | f + List of publications + Name | Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------+--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + testpub_default | regress_publication_user | f | f | t | t | t | f | none | f + testpub_ins_trunct | regress_publication_user | f | f | t | f | f | f | none | f (2 rows) --- adding tables @@ -93,10 +93,10 @@ RESET client_min_messages; -- should be able to add schema to 'FOR TABLE' publication ALTER PUBLICATION testpub_fortable ADD TABLES IN SCHEMA pub_test; \dRp+ testpub_fortable - Publication testpub_fortable - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub_fortable + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables: "public.testpub_tbl1" Tables from schemas: @@ -105,20 +105,20 @@ Tables from schemas: -- should be able to drop schema from 'FOR TABLE' publication ALTER PUBLICATION testpub_fortable DROP TABLES IN SCHEMA pub_test; \dRp+ testpub_fortable - Publication testpub_fortable - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub_fortable + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables: "public.testpub_tbl1" -- should be able to set schema to 'FOR TABLE' publication ALTER PUBLICATION testpub_fortable SET TABLES IN SCHEMA pub_test; \dRp+ testpub_fortable - Publication testpub_fortable - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub_fortable + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables from schemas: "pub_test" @@ -129,10 +129,10 @@ CREATE PUBLICATION testpub_forschema FOR TABLES IN SCHEMA pub_test; CREATE PUBLICATION testpub_for_tbl_schema FOR TABLES IN SCHEMA pub_test, TABLE pub_test.testpub_nopk; RESET client_min_messages; \dRp+ testpub_for_tbl_schema - Publication testpub_for_tbl_schema - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub_for_tbl_schema + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables: "pub_test.testpub_nopk" Tables from schemas: @@ -150,10 +150,10 @@ LINE 1: ...CATION testpub_parsertst FOR TABLES IN SCHEMA foo, test.foo; -- should be able to add a table of the same schema to the schema publication ALTER PUBLICATION testpub_forschema ADD TABLE pub_test.testpub_nopk; \dRp+ testpub_forschema - Publication testpub_forschema - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub_forschema + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables: "pub_test.testpub_nopk" Tables from schemas: @@ -162,10 +162,10 @@ Tables from schemas: -- should be able to drop the table ALTER PUBLICATION testpub_forschema DROP TABLE pub_test.testpub_nopk; \dRp+ testpub_forschema - Publication testpub_forschema - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub_forschema + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables from schemas: "pub_test" @@ -176,10 +176,10 @@ ERROR: relation "testpub_nopk" is not part of the publication -- should be able to set table to schema publication ALTER PUBLICATION testpub_forschema SET TABLE pub_test.testpub_nopk; \dRp+ testpub_forschema - Publication testpub_forschema - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub_forschema + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables: "pub_test.testpub_nopk" @@ -203,10 +203,10 @@ Not-null constraints: "testpub_tbl2_id_not_null" NOT NULL "id" \dRp+ testpub_foralltables - Publication testpub_foralltables - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | t | t | t | f | f | none | f + Publication testpub_foralltables + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | t | f | t | t | f | f | none | f (1 row) DROP TABLE testpub_tbl2; @@ -218,24 +218,96 @@ CREATE PUBLICATION testpub3 FOR TABLE testpub_tbl3; CREATE PUBLICATION testpub4 FOR TABLE ONLY testpub_tbl3; RESET client_min_messages; \dRp+ testpub3 - Publication testpub3 - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub3 + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables: "public.testpub_tbl3" "public.testpub_tbl3a" \dRp+ testpub4 - Publication testpub4 - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub4 + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables: "public.testpub_tbl3" DROP TABLE testpub_tbl3, testpub_tbl3a; DROP PUBLICATION testpub3, testpub4; +--- Tests for publications with SEQUENCES +CREATE SEQUENCE regress_pub_seq0; +CREATE SEQUENCE pub_test.regress_pub_seq1; +-- FOR ALL SEQUENCES +SET client_min_messages = 'ERROR'; +CREATE PUBLICATION regress_pub_forallsequences1 FOR ALL SEQUENCES; +RESET client_min_messages; +SELECT pubname, puballtables, puballsequences FROM pg_publication WHERE pubname = 'regress_pub_forallsequences1'; + pubname | puballtables | puballsequences +------------------------------+--------------+----------------- + regress_pub_forallsequences1 | f | t +(1 row) + +\d+ regress_pub_seq0 + Sequence "public.regress_pub_seq0" + Type | Start | Minimum | Maximum | Increment | Cycles? | Cache +--------+-------+---------+---------------------+-----------+---------+------- + bigint | 1 | 1 | 9223372036854775807 | 1 | no | 1 +Publications: + "regress_pub_forallsequences1" + +\dRp+ regress_pub_forallsequences1 + Publication regress_pub_forallsequences1 + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | t | t | t | t | t | none | f +(1 row) + +SET client_min_messages = 'ERROR'; +CREATE PUBLICATION regress_pub_forallsequences2 FOR ALL SEQUENCES; +RESET client_min_messages; +-- check that describe sequence lists both publications the sequence belongs to +\d+ pub_test.regress_pub_seq1 + Sequence "pub_test.regress_pub_seq1" + Type | Start | Minimum | Maximum | Increment | Cycles? | Cache +--------+-------+---------+---------------------+-----------+---------+------- + bigint | 1 | 1 | 9223372036854775807 | 1 | no | 1 +Publications: + "regress_pub_forallsequences1" + "regress_pub_forallsequences2" + +--- Specifying both ALL TABLES and ALL SEQUENCES +SET client_min_messages = 'ERROR'; +CREATE PUBLICATION regress_pub_for_allsequences_alltables FOR ALL SEQUENCES, ALL TABLES; +RESET client_min_messages; +SELECT pubname, puballtables, puballsequences FROM pg_publication WHERE pubname = 'regress_pub_for_allsequences_alltables'; + pubname | puballtables | puballsequences +----------------------------------------+--------------+----------------- + regress_pub_for_allsequences_alltables | t | t +(1 row) + +\dRp+ regress_pub_for_allsequences_alltables + Publication regress_pub_for_allsequences_alltables + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | t | t | t | t | t | t | none | f +(1 row) + +DROP SEQUENCE regress_pub_seq0, pub_test.regress_pub_seq1; +DROP PUBLICATION regress_pub_forallsequences1, regress_pub_forallsequences2, regress_pub_for_allsequences_alltables; +-- fail - Specifying ALL TABLES more than once +CREATE PUBLICATION regress_pub_for_allsequences_alltables FOR ALL SEQUENCES, ALL TABLES, ALL TABLES; +ERROR: invalid publication object list +LINE 1: ...equences_alltables FOR ALL SEQUENCES, ALL TABLES, ALL TABLES... + ^ +DETAIL: ALL TABLES can be specified only once. +-- fail - Specifying ALL SEQUENCES more than once +CREATE PUBLICATION regress_pub_for_allsequences_alltables FOR ALL SEQUENCES, ALL TABLES, ALL SEQUENCES; +ERROR: invalid publication object list +LINE 1: ...equences_alltables FOR ALL SEQUENCES, ALL TABLES, ALL SEQUEN... + ^ +DETAIL: ALL SEQUENCES can be specified only once. -- Tests for partitioned tables SET client_min_messages = 'ERROR'; CREATE PUBLICATION testpub_forparted; @@ -251,10 +323,10 @@ UPDATE testpub_parted1 SET a = 1; -- only parent is listed as being in publication, not the partition ALTER PUBLICATION testpub_forparted ADD TABLE testpub_parted; \dRp+ testpub_forparted - Publication testpub_forparted - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub_forparted + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables: "public.testpub_parted" @@ -269,10 +341,10 @@ ALTER TABLE testpub_parted DETACH PARTITION testpub_parted1; UPDATE testpub_parted1 SET a = 1; ALTER PUBLICATION testpub_forparted SET (publish_via_partition_root = true); \dRp+ testpub_forparted - Publication testpub_forparted - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | t + Publication testpub_forparted + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | t Tables: "public.testpub_parted" @@ -301,10 +373,10 @@ SET client_min_messages = 'ERROR'; CREATE PUBLICATION testpub5 FOR TABLE testpub_rf_tbl1, testpub_rf_tbl2 WHERE (c <> 'test' AND d < 5) WITH (publish = 'insert'); RESET client_min_messages; \dRp+ testpub5 - Publication testpub5 - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | f | f | f | none | f + Publication testpub5 + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | f | f | f | none | f Tables: "public.testpub_rf_tbl1" "public.testpub_rf_tbl2" WHERE ((c <> 'test'::text) AND (d < 5)) @@ -317,10 +389,10 @@ Tables: ALTER PUBLICATION testpub5 ADD TABLE testpub_rf_tbl3 WHERE (e > 1000 AND e < 2000); \dRp+ testpub5 - Publication testpub5 - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | f | f | f | none | f + Publication testpub5 + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | f | f | f | none | f Tables: "public.testpub_rf_tbl1" "public.testpub_rf_tbl2" WHERE ((c <> 'test'::text) AND (d < 5)) @@ -336,10 +408,10 @@ Publications: ALTER PUBLICATION testpub5 DROP TABLE testpub_rf_tbl2; \dRp+ testpub5 - Publication testpub5 - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | f | f | f | none | f + Publication testpub5 + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | f | f | f | none | f Tables: "public.testpub_rf_tbl1" "public.testpub_rf_tbl3" WHERE ((e > 1000) AND (e < 2000)) @@ -347,10 +419,10 @@ Tables: -- remove testpub_rf_tbl1 and add testpub_rf_tbl3 again (another WHERE expression) ALTER PUBLICATION testpub5 SET TABLE testpub_rf_tbl3 WHERE (e > 300 AND e < 500); \dRp+ testpub5 - Publication testpub5 - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | f | f | f | none | f + Publication testpub5 + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | f | f | f | none | f Tables: "public.testpub_rf_tbl3" WHERE ((e > 300) AND (e < 500)) @@ -383,10 +455,10 @@ SET client_min_messages = 'ERROR'; CREATE PUBLICATION testpub_syntax1 FOR TABLE testpub_rf_tbl1, ONLY testpub_rf_tbl3 WHERE (e < 999) WITH (publish = 'insert'); RESET client_min_messages; \dRp+ testpub_syntax1 - Publication testpub_syntax1 - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | f | f | f | none | f + Publication testpub_syntax1 + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | f | f | f | none | f Tables: "public.testpub_rf_tbl1" "public.testpub_rf_tbl3" WHERE (e < 999) @@ -396,10 +468,10 @@ SET client_min_messages = 'ERROR'; CREATE PUBLICATION testpub_syntax2 FOR TABLE testpub_rf_tbl1, testpub_rf_schema1.testpub_rf_tbl5 WHERE (h < 999) WITH (publish = 'insert'); RESET client_min_messages; \dRp+ testpub_syntax2 - Publication testpub_syntax2 - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | f | f | f | none | f + Publication testpub_syntax2 + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | f | f | f | none | f Tables: "public.testpub_rf_tbl1" "testpub_rf_schema1.testpub_rf_tbl5" WHERE (h < 999) @@ -514,10 +586,10 @@ CREATE PUBLICATION testpub6 FOR TABLES IN SCHEMA testpub_rf_schema2; ALTER PUBLICATION testpub6 SET TABLES IN SCHEMA testpub_rf_schema2, TABLE testpub_rf_schema2.testpub_rf_tbl6 WHERE (i < 99); RESET client_min_messages; \dRp+ testpub6 - Publication testpub6 - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub6 + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables: "testpub_rf_schema2.testpub_rf_tbl6" WHERE (i < 99) Tables from schemas: @@ -803,10 +875,10 @@ CREATE PUBLICATION testpub_table_ins WITH (publish = 'insert, truncate'); RESET client_min_messages; ALTER PUBLICATION testpub_table_ins ADD TABLE testpub_tbl5 (a); -- ok \dRp+ testpub_table_ins - Publication testpub_table_ins - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | f | f | t | none | f + Publication testpub_table_ins + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | f | f | t | none | f Tables: "public.testpub_tbl5" (a) @@ -996,10 +1068,10 @@ CREATE TABLE testpub_tbl_both_filters (a int, b int, c int, PRIMARY KEY (a,c)); ALTER TABLE testpub_tbl_both_filters REPLICA IDENTITY USING INDEX testpub_tbl_both_filters_pkey; ALTER PUBLICATION testpub_both_filters ADD TABLE testpub_tbl_both_filters (a,c) WHERE (c != 1); \dRp+ testpub_both_filters - Publication testpub_both_filters - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub_both_filters + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables: "public.testpub_tbl_both_filters" (a, c) WHERE (c <> 1) @@ -1207,10 +1279,10 @@ ERROR: relation "testpub_tbl1" is already member of publication "testpub_fortbl CREATE PUBLICATION testpub_fortbl FOR TABLE testpub_tbl1; ERROR: publication "testpub_fortbl" already exists \dRp+ testpub_fortbl - Publication testpub_fortbl - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub_fortbl + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables: "pub_test.testpub_nopk" "public.testpub_tbl1" @@ -1250,10 +1322,10 @@ Not-null constraints: "testpub_tbl1_id_not_null" NOT NULL "id" \dRp+ testpub_default - Publication testpub_default - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | f | none | f + Publication testpub_default + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | f | none | f Tables: "pub_test.testpub_nopk" "public.testpub_tbl1" @@ -1333,10 +1405,10 @@ REVOKE CREATE ON DATABASE regression FROM regress_publication_user2; DROP TABLE testpub_parted; DROP TABLE testpub_tbl1; \dRp+ testpub_default - Publication testpub_default - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | f | none | f + Publication testpub_default + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | f | none | f (1 row) -- fail - must be owner of publication @@ -1346,20 +1418,20 @@ ERROR: must be owner of publication testpub_default RESET ROLE; ALTER PUBLICATION testpub_default RENAME TO testpub_foo; \dRp testpub_foo - List of publications - Name | Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root --------------+--------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - testpub_foo | regress_publication_user | f | t | t | t | f | none | f + List of publications + Name | Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +-------------+--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + testpub_foo | regress_publication_user | f | f | t | t | t | f | none | f (1 row) -- rename back to keep the rest simple ALTER PUBLICATION testpub_foo RENAME TO testpub_default; ALTER PUBLICATION testpub_default OWNER TO regress_publication_user2; \dRp testpub_default - List of publications - Name | Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ------------------+---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - testpub_default | regress_publication_user2 | f | t | t | t | f | none | f + List of publications + Name | Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +-----------------+---------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + testpub_default | regress_publication_user2 | f | f | t | t | t | f | none | f (1 row) -- adding schemas and tables @@ -1375,19 +1447,19 @@ CREATE TABLE "CURRENT_SCHEMA"."CURRENT_SCHEMA"(id int); SET client_min_messages = 'ERROR'; CREATE PUBLICATION testpub1_forschema FOR TABLES IN SCHEMA pub_test1; \dRp+ testpub1_forschema - Publication testpub1_forschema - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub1_forschema + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables from schemas: "pub_test1" CREATE PUBLICATION testpub2_forschema FOR TABLES IN SCHEMA pub_test1, pub_test2, pub_test3; \dRp+ testpub2_forschema - Publication testpub2_forschema - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub2_forschema + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables from schemas: "pub_test1" "pub_test2" @@ -1401,44 +1473,44 @@ CREATE PUBLICATION testpub6_forschema FOR TABLES IN SCHEMA "CURRENT_SCHEMA", CUR CREATE PUBLICATION testpub_fortable FOR TABLE "CURRENT_SCHEMA"."CURRENT_SCHEMA"; RESET client_min_messages; \dRp+ testpub3_forschema - Publication testpub3_forschema - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub3_forschema + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables from schemas: "public" \dRp+ testpub4_forschema - Publication testpub4_forschema - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub4_forschema + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables from schemas: "CURRENT_SCHEMA" \dRp+ testpub5_forschema - Publication testpub5_forschema - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub5_forschema + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables from schemas: "CURRENT_SCHEMA" "public" \dRp+ testpub6_forschema - Publication testpub6_forschema - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub6_forschema + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables from schemas: "CURRENT_SCHEMA" "public" \dRp+ testpub_fortable - Publication testpub_fortable - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub_fortable + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables: "CURRENT_SCHEMA.CURRENT_SCHEMA" @@ -1472,10 +1544,10 @@ ERROR: schema "testpub_view" does not exist -- dropping the schema should reflect the change in publication DROP SCHEMA pub_test3; \dRp+ testpub2_forschema - Publication testpub2_forschema - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub2_forschema + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables from schemas: "pub_test1" "pub_test2" @@ -1483,20 +1555,20 @@ Tables from schemas: -- renaming the schema should reflect the change in publication ALTER SCHEMA pub_test1 RENAME to pub_test1_renamed; \dRp+ testpub2_forschema - Publication testpub2_forschema - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub2_forschema + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables from schemas: "pub_test1_renamed" "pub_test2" ALTER SCHEMA pub_test1_renamed RENAME to pub_test1; \dRp+ testpub2_forschema - Publication testpub2_forschema - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub2_forschema + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables from schemas: "pub_test1" "pub_test2" @@ -1504,10 +1576,10 @@ Tables from schemas: -- alter publication add schema ALTER PUBLICATION testpub1_forschema ADD TABLES IN SCHEMA pub_test2; \dRp+ testpub1_forschema - Publication testpub1_forschema - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub1_forschema + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables from schemas: "pub_test1" "pub_test2" @@ -1516,10 +1588,10 @@ Tables from schemas: ALTER PUBLICATION testpub1_forschema ADD TABLES IN SCHEMA non_existent_schema; ERROR: schema "non_existent_schema" does not exist \dRp+ testpub1_forschema - Publication testpub1_forschema - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub1_forschema + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables from schemas: "pub_test1" "pub_test2" @@ -1528,10 +1600,10 @@ Tables from schemas: ALTER PUBLICATION testpub1_forschema ADD TABLES IN SCHEMA pub_test1; ERROR: schema "pub_test1" is already member of publication "testpub1_forschema" \dRp+ testpub1_forschema - Publication testpub1_forschema - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub1_forschema + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables from schemas: "pub_test1" "pub_test2" @@ -1539,10 +1611,10 @@ Tables from schemas: -- alter publication drop schema ALTER PUBLICATION testpub1_forschema DROP TABLES IN SCHEMA pub_test2; \dRp+ testpub1_forschema - Publication testpub1_forschema - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub1_forschema + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables from schemas: "pub_test1" @@ -1550,10 +1622,10 @@ Tables from schemas: ALTER PUBLICATION testpub1_forschema DROP TABLES IN SCHEMA pub_test2; ERROR: tables from schema "pub_test2" are not part of the publication \dRp+ testpub1_forschema - Publication testpub1_forschema - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub1_forschema + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables from schemas: "pub_test1" @@ -1561,29 +1633,29 @@ Tables from schemas: ALTER PUBLICATION testpub1_forschema DROP TABLES IN SCHEMA non_existent_schema; ERROR: schema "non_existent_schema" does not exist \dRp+ testpub1_forschema - Publication testpub1_forschema - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub1_forschema + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables from schemas: "pub_test1" -- drop all schemas ALTER PUBLICATION testpub1_forschema DROP TABLES IN SCHEMA pub_test1; \dRp+ testpub1_forschema - Publication testpub1_forschema - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub1_forschema + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f (1 row) -- alter publication set multiple schema ALTER PUBLICATION testpub1_forschema SET TABLES IN SCHEMA pub_test1, pub_test2; \dRp+ testpub1_forschema - Publication testpub1_forschema - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub1_forschema + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables from schemas: "pub_test1" "pub_test2" @@ -1592,10 +1664,10 @@ Tables from schemas: ALTER PUBLICATION testpub1_forschema SET TABLES IN SCHEMA non_existent_schema; ERROR: schema "non_existent_schema" does not exist \dRp+ testpub1_forschema - Publication testpub1_forschema - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub1_forschema + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables from schemas: "pub_test1" "pub_test2" @@ -1604,10 +1676,10 @@ Tables from schemas: -- removing the duplicate schemas ALTER PUBLICATION testpub1_forschema SET TABLES IN SCHEMA pub_test1, pub_test1; \dRp+ testpub1_forschema - Publication testpub1_forschema - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub1_forschema + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables from schemas: "pub_test1" @@ -1686,18 +1758,18 @@ SET client_min_messages = 'ERROR'; CREATE PUBLICATION testpub3_forschema; RESET client_min_messages; \dRp+ testpub3_forschema - Publication testpub3_forschema - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub3_forschema + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f (1 row) ALTER PUBLICATION testpub3_forschema SET TABLES IN SCHEMA pub_test1; \dRp+ testpub3_forschema - Publication testpub3_forschema - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub3_forschema + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables from schemas: "pub_test1" @@ -1707,20 +1779,20 @@ CREATE PUBLICATION testpub_forschema_fortable FOR TABLES IN SCHEMA pub_test1, TA CREATE PUBLICATION testpub_fortable_forschema FOR TABLE pub_test2.tbl1, TABLES IN SCHEMA pub_test1; RESET client_min_messages; \dRp+ testpub_forschema_fortable - Publication testpub_forschema_fortable - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub_forschema_fortable + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables: "pub_test2.tbl1" Tables from schemas: "pub_test1" \dRp+ testpub_fortable_forschema - Publication testpub_fortable_forschema - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication testpub_fortable_forschema + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables: "pub_test2.tbl1" Tables from schemas: @@ -1842,26 +1914,26 @@ DROP SCHEMA sch2 cascade; SET client_min_messages = 'ERROR'; CREATE PUBLICATION pub1 FOR ALL TABLES WITH (publish_generated_columns = stored); \dRp+ pub1 - Publication pub1 - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | t | t | t | t | t | stored | f + Publication pub1 + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | t | f | t | t | t | t | stored | f (1 row) CREATE PUBLICATION pub2 FOR ALL TABLES WITH (publish_generated_columns = none); \dRp+ pub2 - Publication pub2 - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | t | t | t | t | t | none | f + Publication pub2 + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | t | f | t | t | t | t | none | f (1 row) CREATE PUBLICATION pub3 FOR ALL TABLES WITH (publish_generated_columns); \dRp+ pub3 - Publication pub3 - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | t | t | t | t | t | stored | f + Publication pub3 + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | t | f | t | t | t | t | stored | f (1 row) DROP PUBLICATION pub1; @@ -1873,50 +1945,50 @@ CREATE TABLE gencols (a int, gen1 int GENERATED ALWAYS AS (a * 2) STORED); -- Generated columns in column list, when 'publish_generated_columns'='none' CREATE PUBLICATION pub1 FOR table gencols(a, gen1) WITH (publish_generated_columns = none); \dRp+ pub1 - Publication pub1 - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication pub1 + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables: "public.gencols" (a, gen1) -- Generated columns in column list, when 'publish_generated_columns'='stored' CREATE PUBLICATION pub2 FOR table gencols(a, gen1) WITH (publish_generated_columns = stored); \dRp+ pub2 - Publication pub2 - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | stored | f + Publication pub2 + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | stored | f Tables: "public.gencols" (a, gen1) -- Generated columns in column list, then set 'publish_generated_columns'='none' ALTER PUBLICATION pub2 SET (publish_generated_columns = none); \dRp+ pub2 - Publication pub2 - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication pub2 + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables: "public.gencols" (a, gen1) -- Remove generated columns from column list, when 'publish_generated_columns'='none' ALTER PUBLICATION pub2 SET TABLE gencols(a); \dRp+ pub2 - Publication pub2 - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication pub2 + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables: "public.gencols" (a) -- Add generated columns in column list, when 'publish_generated_columns'='none' ALTER PUBLICATION pub2 SET TABLE gencols(a, gen1); \dRp+ pub2 - Publication pub2 - Owner | All tables | Inserts | Updates | Deletes | Truncates | Generated columns | Via root ---------------------------+------------+---------+---------+---------+-----------+-------------------+---------- - regress_publication_user | f | t | t | t | t | none | f + Publication pub2 + Owner | All tables | All sequences | Inserts | Updates | Deletes | Truncates | Generated columns | Via root +--------------------------+------------+---------------+---------+---------+---------+-----------+-------------------+---------- + regress_publication_user | f | f | t | t | t | t | none | f Tables: "public.gencols" (a, gen1) diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql index 68001de4000f..97ea0f593b9e 100644 --- a/src/test/regress/sql/publication.sql +++ b/src/test/regress/sql/publication.sql @@ -119,6 +119,43 @@ RESET client_min_messages; DROP TABLE testpub_tbl3, testpub_tbl3a; DROP PUBLICATION testpub3, testpub4; +--- Tests for publications with SEQUENCES +CREATE SEQUENCE regress_pub_seq0; +CREATE SEQUENCE pub_test.regress_pub_seq1; + +-- FOR ALL SEQUENCES +SET client_min_messages = 'ERROR'; +CREATE PUBLICATION regress_pub_forallsequences1 FOR ALL SEQUENCES; +RESET client_min_messages; + +SELECT pubname, puballtables, puballsequences FROM pg_publication WHERE pubname = 'regress_pub_forallsequences1'; +\d+ regress_pub_seq0 +\dRp+ regress_pub_forallsequences1 + +SET client_min_messages = 'ERROR'; +CREATE PUBLICATION regress_pub_forallsequences2 FOR ALL SEQUENCES; +RESET client_min_messages; + +-- check that describe sequence lists both publications the sequence belongs to +\d+ pub_test.regress_pub_seq1 + +--- Specifying both ALL TABLES and ALL SEQUENCES +SET client_min_messages = 'ERROR'; +CREATE PUBLICATION regress_pub_for_allsequences_alltables FOR ALL SEQUENCES, ALL TABLES; +RESET client_min_messages; + +SELECT pubname, puballtables, puballsequences FROM pg_publication WHERE pubname = 'regress_pub_for_allsequences_alltables'; +\dRp+ regress_pub_for_allsequences_alltables + +DROP SEQUENCE regress_pub_seq0, pub_test.regress_pub_seq1; +DROP PUBLICATION regress_pub_forallsequences1, regress_pub_forallsequences2, regress_pub_for_allsequences_alltables; + +-- fail - Specifying ALL TABLES more than once +CREATE PUBLICATION regress_pub_for_allsequences_alltables FOR ALL SEQUENCES, ALL TABLES, ALL TABLES; + +-- fail - Specifying ALL SEQUENCES more than once +CREATE PUBLICATION regress_pub_for_allsequences_alltables FOR ALL SEQUENCES, ALL TABLES, ALL SEQUENCES; + -- Tests for partitioned tables SET client_min_messages = 'ERROR'; CREATE PUBLICATION testpub_forparted; diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index e5879e00dffe..74dad46568ac 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2342,6 +2342,8 @@ PsqlScanStateData PsqlSettings Publication PublicationActions +PublicationAllObjSpec +PublicationAllObjType PublicationDesc PublicationInfo PublicationObjSpec From 19ac78c20986a833b226877e6b411a7943093658 Mon Sep 17 00:00:00 2001 From: Vignesh Date: Tue, 25 Mar 2025 09:23:48 +0530 Subject: [PATCH 3/5] Reorganize tablesync Code and Introduce syncutils Reorganized the tablesync code by creating a new syncutils file. This refactoring will facilitate the development of sequence synchronization worker code. This commit separates code reorganization from functional changes, making it clearer to reviewers that only existing code has been moved. The changes in this patch can be merged with subsequent patches during the commit process. --- src/backend/catalog/pg_subscription.c | 4 +- src/backend/replication/logical/Makefile | 1 + .../replication/logical/applyparallelworker.c | 2 +- src/backend/replication/logical/meson.build | 1 + src/backend/replication/logical/syncutils.c | 190 ++++++++++++++++++ src/backend/replication/logical/tablesync.c | 186 ++--------------- src/backend/replication/logical/worker.c | 18 +- src/include/catalog/pg_subscription_rel.h | 2 +- src/include/replication/worker_internal.h | 13 +- src/tools/pgindent/typedefs.list | 2 +- 10 files changed, 232 insertions(+), 187 deletions(-) create mode 100644 src/backend/replication/logical/syncutils.c diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c index 1395032413e3..1c71161e7237 100644 --- a/src/backend/catalog/pg_subscription.c +++ b/src/backend/catalog/pg_subscription.c @@ -488,13 +488,13 @@ RemoveSubscriptionRel(Oid subid, Oid relid) } /* - * Does the subscription have any relations? + * Does the subscription have any tables? * * Use this function only to know true/false, and when you have no need for the * List returned by GetSubscriptionRelations. */ bool -HasSubscriptionRelations(Oid subid) +HasSubscriptionTables(Oid subid) { Relation rel; ScanKeyData skey[1]; diff --git a/src/backend/replication/logical/Makefile b/src/backend/replication/logical/Makefile index 1e08bbbd4eb1..c62c8c67521c 100644 --- a/src/backend/replication/logical/Makefile +++ b/src/backend/replication/logical/Makefile @@ -28,6 +28,7 @@ OBJS = \ reorderbuffer.o \ slotsync.o \ snapbuild.o \ + syncutils.o \ tablesync.o \ worker.o diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c index d25085d35153..d2b663267adc 100644 --- a/src/backend/replication/logical/applyparallelworker.c +++ b/src/backend/replication/logical/applyparallelworker.c @@ -962,7 +962,7 @@ ParallelApplyWorkerMain(Datum main_arg) * the subscription relation state. */ CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP, - invalidate_syncing_table_states, + SyncInvalidateRelationStates, (Datum) 0); set_apply_error_context_origin(originname); diff --git a/src/backend/replication/logical/meson.build b/src/backend/replication/logical/meson.build index 6f19614c79d8..9283e996ef4a 100644 --- a/src/backend/replication/logical/meson.build +++ b/src/backend/replication/logical/meson.build @@ -14,6 +14,7 @@ backend_sources += files( 'reorderbuffer.c', 'slotsync.c', 'snapbuild.c', + 'syncutils.c', 'tablesync.c', 'worker.c', ) diff --git a/src/backend/replication/logical/syncutils.c b/src/backend/replication/logical/syncutils.c new file mode 100644 index 000000000000..3d405ff2dc67 --- /dev/null +++ b/src/backend/replication/logical/syncutils.c @@ -0,0 +1,190 @@ +/*------------------------------------------------------------------------- + * syncutils.c + * PostgreSQL logical replication: common synchronization code + * + * Copyright (c) 2025, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/replication/logical/syncutils.c + * + * NOTES + * This file contains code common to table synchronization workers, and + * the sequence synchronization worker. + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "catalog/pg_subscription_rel.h" +#include "pgstat.h" +#include "replication/logicallauncher.h" +#include "replication/origin.h" +#include "replication/slot.h" +#include "replication/worker_internal.h" +#include "storage/ipc.h" +#include "utils/lsyscache.h" +#include "utils/memutils.h" + +/* + * Enum for phases of the subscription relations state. + * + * SYNC_RELATIONS_STATE_NEEDS_REBUILD indicates that the subscription relations + * state is no longer valid, and the subscription relations should be rebuilt. + * + * SYNC_RELATIONS_STATE_REBUILD_STARTED indicates that the subscription + * relations state is being rebuilt. + * + * SYNC_RELATIONS_STATE_VALID indicates that the subscription relation state is + * up-to-date and valid. + */ +typedef enum +{ + SYNC_RELATIONS_STATE_NEEDS_REBUILD, + SYNC_RELATIONS_STATE_REBUILD_STARTED, + SYNC_RELATIONS_STATE_VALID, +} SyncingRelationsState; + +static SyncingRelationsState relation_states_validity = SYNC_RELATIONS_STATE_NEEDS_REBUILD; + +/* + * Exit routine for synchronization worker. + */ +pg_noreturn void +SyncFinishWorker(void) +{ + /* + * Commit any outstanding transaction. This is the usual case, unless + * there was nothing to do for the table. + */ + if (IsTransactionState()) + { + CommitTransactionCommand(); + pgstat_report_stat(true); + } + + /* And flush all writes. */ + XLogFlush(GetXLogWriteRecPtr()); + + StartTransactionCommand(); + ereport(LOG, + (errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished", + MySubscription->name, + get_rel_name(MyLogicalRepWorker->relid)))); + CommitTransactionCommand(); + + /* Find the leader apply worker and signal it. */ + logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid); + + /* Stop gracefully */ + proc_exit(0); +} + +/* + * Callback from syscache invalidation. + */ +void +SyncInvalidateRelationStates(Datum arg, int cacheid, uint32 hashvalue) +{ + relation_states_validity = SYNC_RELATIONS_STATE_NEEDS_REBUILD; +} + +/* + * Process possible state change(s) of relations that are being synchronized. + */ +void +SyncProcessRelations(XLogRecPtr current_lsn) +{ + switch (MyLogicalRepWorker->type) + { + case WORKERTYPE_PARALLEL_APPLY: + /* + * Skip for parallel apply workers because they only operate on + * tables that are in a READY state. See pa_can_start() and + * should_apply_changes_for_rel(). + */ + break; + + case WORKERTYPE_TABLESYNC: + ProcessSyncingTablesForSync(current_lsn); + break; + + case WORKERTYPE_APPLY: + ProcessSyncingTablesForApply(current_lsn); + break; + + case WORKERTYPE_UNKNOWN: + /* Should never happen. */ + elog(ERROR, "Unknown worker type"); + } +} + +/* + * Common code to fetch the up-to-date sync state info into the static lists. + * + * Returns true if subscription has 1 or more tables, else false. + * + * Note: If this function started the transaction (indicated by the parameter) + * then it is the caller's responsibility to commit it. + */ +bool +SyncFetchRelationStates(bool *started_tx) +{ + static bool has_subtables = false; + + *started_tx = false; + + if (relation_states_validity != SYNC_RELATIONS_STATE_VALID) + { + MemoryContext oldctx; + List *rstates; + ListCell *lc; + SubscriptionRelState *rstate; + + relation_states_validity = SYNC_RELATIONS_STATE_REBUILD_STARTED; + + /* Clean the old lists. */ + list_free_deep(table_states_not_ready); + table_states_not_ready = NIL; + + if (!IsTransactionState()) + { + StartTransactionCommand(); + *started_tx = true; + } + + /* Fetch tables that are in non-ready state. */ + rstates = GetSubscriptionRelations(MySubscription->oid, true); + + /* Allocate the tracking info in a permanent memory context. */ + oldctx = MemoryContextSwitchTo(CacheMemoryContext); + foreach(lc, rstates) + { + rstate = palloc(sizeof(SubscriptionRelState)); + memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState)); + table_states_not_ready = lappend(table_states_not_ready, rstate); + } + MemoryContextSwitchTo(oldctx); + + /* + * Does the subscription have tables? + * + * If there were not-READY tables found then we know it does. But if + * table_states_not_ready was empty we still need to check again to + * see if there are 0 tables. + */ + has_subtables = (table_states_not_ready != NIL) || + HasSubscriptionTables(MySubscription->oid); + + /* + * If the subscription relation cache has been invalidated since we + * entered this routine, we still use and return the relations we just + * finished constructing, to avoid infinite loops, but we leave the + * table states marked as stale so that we'll rebuild it again on next + * access. Otherwise, we mark the table states as valid. + */ + if (relation_states_validity == SYNC_RELATIONS_STATE_REBUILD_STARTED) + relation_states_validity = SYNC_RELATIONS_STATE_VALID; + } + + return has_subtables; +} diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index 8e1e8762f625..9bd51ceef481 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -117,58 +117,15 @@ #include "utils/array.h" #include "utils/builtins.h" #include "utils/lsyscache.h" -#include "utils/memutils.h" #include "utils/rls.h" #include "utils/snapmgr.h" #include "utils/syscache.h" #include "utils/usercontext.h" -typedef enum -{ - SYNC_TABLE_STATE_NEEDS_REBUILD, - SYNC_TABLE_STATE_REBUILD_STARTED, - SYNC_TABLE_STATE_VALID, -} SyncingTablesState; - -static SyncingTablesState table_states_validity = SYNC_TABLE_STATE_NEEDS_REBUILD; -static List *table_states_not_ready = NIL; -static bool FetchTableStates(bool *started_tx); +List *table_states_not_ready = NIL; static StringInfo copybuf = NULL; -/* - * Exit routine for synchronization worker. - */ -pg_noreturn static void -finish_sync_worker(void) -{ - /* - * Commit any outstanding transaction. This is the usual case, unless - * there was nothing to do for the table. - */ - if (IsTransactionState()) - { - CommitTransactionCommand(); - pgstat_report_stat(true); - } - - /* And flush all writes. */ - XLogFlush(GetXLogWriteRecPtr()); - - StartTransactionCommand(); - ereport(LOG, - (errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished", - MySubscription->name, - get_rel_name(MyLogicalRepWorker->relid)))); - CommitTransactionCommand(); - - /* Find the leader apply worker and signal it. */ - logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid); - - /* Stop gracefully */ - proc_exit(0); -} - /* * Wait until the relation sync state is set in the catalog to the expected * one; return true when it happens. @@ -180,7 +137,7 @@ finish_sync_worker(void) * CATCHUP state to SYNCDONE. */ static bool -wait_for_relation_state_change(Oid relid, char expected_state) +wait_for_table_state_change(Oid relid, char expected_state) { char state; @@ -273,15 +230,6 @@ wait_for_worker_state_change(char expected_state) return false; } -/* - * Callback from syscache invalidation. - */ -void -invalidate_syncing_table_states(Datum arg, int cacheid, uint32 hashvalue) -{ - table_states_validity = SYNC_TABLE_STATE_NEEDS_REBUILD; -} - /* * Handle table synchronization cooperation from the synchronization * worker. @@ -290,8 +238,8 @@ invalidate_syncing_table_states(Datum arg, int cacheid, uint32 hashvalue) * predetermined synchronization point in the WAL stream, mark the table as * SYNCDONE and finish. */ -static void -process_syncing_tables_for_sync(XLogRecPtr current_lsn) +void +ProcessSyncingTablesForSync(XLogRecPtr current_lsn) { SpinLockAcquire(&MyLogicalRepWorker->relmutex); @@ -348,9 +296,9 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn) /* * Start a new transaction to clean up the tablesync origin tracking. - * This transaction will be ended within the finish_sync_worker(). - * Now, even, if we fail to remove this here, the apply worker will - * ensure to clean it up afterward. + * This transaction will be ended within the SyncFinishWorker(). Now, + * even, if we fail to remove this here, the apply worker will ensure + * to clean it up afterward. * * We need to do this after the table state is set to SYNCDONE. * Otherwise, if an error occurs while performing the database @@ -386,7 +334,7 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn) */ replorigin_drop_by_name(originname, true, false); - finish_sync_worker(); + SyncFinishWorker(); } else SpinLockRelease(&MyLogicalRepWorker->relmutex); @@ -413,8 +361,8 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn) * If the synchronization position is reached (SYNCDONE), then the table can * be marked as READY and is no longer tracked. */ -static void -process_syncing_tables_for_apply(XLogRecPtr current_lsn) +void +ProcessSyncingTablesForApply(XLogRecPtr current_lsn) { struct tablesync_start_time_mapping { @@ -429,7 +377,7 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn) Assert(!IsTransactionState()); /* We need up-to-date sync state info for subscription tables here. */ - FetchTableStates(&started_tx); + SyncFetchRelationStates(&started_tx); /* * Prepare a hash table for tracking last start times of workers, to avoid @@ -567,8 +515,8 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn) StartTransactionCommand(); started_tx = true; - wait_for_relation_state_change(rstate->relid, - SUBREL_STATE_SYNCDONE); + wait_for_table_state_change(rstate->relid, + SUBREL_STATE_SYNCDONE); } else LWLockRelease(LogicalRepWorkerLock); @@ -659,37 +607,6 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn) } } -/* - * Process possible state change(s) of tables that are being synchronized. - */ -void -process_syncing_tables(XLogRecPtr current_lsn) -{ - switch (MyLogicalRepWorker->type) - { - case WORKERTYPE_PARALLEL_APPLY: - - /* - * Skip for parallel apply workers because they only operate on - * tables that are in a READY state. See pa_can_start() and - * should_apply_changes_for_rel(). - */ - break; - - case WORKERTYPE_TABLESYNC: - process_syncing_tables_for_sync(current_lsn); - break; - - case WORKERTYPE_APPLY: - process_syncing_tables_for_apply(current_lsn); - break; - - case WORKERTYPE_UNKNOWN: - /* Should never happen. */ - elog(ERROR, "Unknown worker type"); - } -} - /* * Create list of columns for COPY based on logical relation mapping. */ @@ -1326,7 +1243,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos) case SUBREL_STATE_SYNCDONE: case SUBREL_STATE_READY: case SUBREL_STATE_UNKNOWN: - finish_sync_worker(); /* doesn't return */ + SyncFinishWorker(); /* doesn't return */ } /* Calculate the name of the tablesync slot. */ @@ -1567,77 +1484,6 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos) return slotname; } -/* - * Common code to fetch the up-to-date sync state info into the static lists. - * - * Returns true if subscription has 1 or more tables, else false. - * - * Note: If this function started the transaction (indicated by the parameter) - * then it is the caller's responsibility to commit it. - */ -static bool -FetchTableStates(bool *started_tx) -{ - static bool has_subrels = false; - - *started_tx = false; - - if (table_states_validity != SYNC_TABLE_STATE_VALID) - { - MemoryContext oldctx; - List *rstates; - ListCell *lc; - SubscriptionRelState *rstate; - - table_states_validity = SYNC_TABLE_STATE_REBUILD_STARTED; - - /* Clean the old lists. */ - list_free_deep(table_states_not_ready); - table_states_not_ready = NIL; - - if (!IsTransactionState()) - { - StartTransactionCommand(); - *started_tx = true; - } - - /* Fetch all non-ready tables. */ - rstates = GetSubscriptionRelations(MySubscription->oid, true); - - /* Allocate the tracking info in a permanent memory context. */ - oldctx = MemoryContextSwitchTo(CacheMemoryContext); - foreach(lc, rstates) - { - rstate = palloc(sizeof(SubscriptionRelState)); - memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState)); - table_states_not_ready = lappend(table_states_not_ready, rstate); - } - MemoryContextSwitchTo(oldctx); - - /* - * Does the subscription have tables? - * - * If there were not-READY relations found then we know it does. But - * if table_states_not_ready was empty we still need to check again to - * see if there are 0 tables. - */ - has_subrels = (table_states_not_ready != NIL) || - HasSubscriptionRelations(MySubscription->oid); - - /* - * If the subscription relation cache has been invalidated since we - * entered this routine, we still use and return the relations we just - * finished constructing, to avoid infinite loops, but we leave the - * table states marked as stale so that we'll rebuild it again on next - * access. Otherwise, we mark the table states as valid. - */ - if (table_states_validity == SYNC_TABLE_STATE_REBUILD_STARTED) - table_states_validity = SYNC_TABLE_STATE_VALID; - } - - return has_subrels; -} - /* * Execute the initial sync with error handling. Disable the subscription, * if it's required. @@ -1723,7 +1569,7 @@ TablesyncWorkerMain(Datum main_arg) run_tablesync_worker(); - finish_sync_worker(); + SyncFinishWorker(); } /* @@ -1741,7 +1587,7 @@ AllTablesyncsReady(void) bool has_subrels = false; /* We need up-to-date sync state info for subscription tables here. */ - has_subrels = FetchTableStates(&started_tx); + has_subrels = SyncFetchRelationStates(&started_tx); if (started_tx) { diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 4151a4b2a96b..765754bfc3c6 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -91,7 +91,7 @@ * behave as if two_phase = off. When the apply worker detects that all * tablesyncs have become READY (while the tri-state was PENDING) it will * restart the apply worker process. This happens in - * process_syncing_tables_for_apply. + * ProcessSyncingTablesForApply. * * When the (re-started) apply worker finds that all tablesyncs are READY for a * two_phase tri-state of PENDING it start streaming messages with the @@ -1030,7 +1030,7 @@ apply_handle_commit(StringInfo s) apply_handle_commit_internal(&commit_data); /* Process any tables that are being synchronized in parallel. */ - process_syncing_tables(commit_data.end_lsn); + SyncProcessRelations(commit_data.end_lsn); pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); @@ -1152,7 +1152,7 @@ apply_handle_prepare(StringInfo s) in_remote_transaction = false; /* Process any tables that are being synchronized in parallel. */ - process_syncing_tables(prepare_data.end_lsn); + SyncProcessRelations(prepare_data.end_lsn); /* * Since we have already prepared the transaction, in a case where the @@ -1208,7 +1208,7 @@ apply_handle_commit_prepared(StringInfo s) in_remote_transaction = false; /* Process any tables that are being synchronized in parallel. */ - process_syncing_tables(prepare_data.end_lsn); + SyncProcessRelations(prepare_data.end_lsn); clear_subscription_skip_lsn(prepare_data.end_lsn); @@ -1274,7 +1274,7 @@ apply_handle_rollback_prepared(StringInfo s) in_remote_transaction = false; /* Process any tables that are being synchronized in parallel. */ - process_syncing_tables(rollback_data.rollback_end_lsn); + SyncProcessRelations(rollback_data.rollback_end_lsn); pgstat_report_activity(STATE_IDLE, NULL); reset_apply_error_context_info(); @@ -1409,7 +1409,7 @@ apply_handle_stream_prepare(StringInfo s) pgstat_report_stat(false); /* Process any tables that are being synchronized in parallel. */ - process_syncing_tables(prepare_data.end_lsn); + SyncProcessRelations(prepare_data.end_lsn); /* * Similar to prepare case, the subskiplsn could be left in a case of @@ -2251,7 +2251,7 @@ apply_handle_stream_commit(StringInfo s) } /* Process any tables that are being synchronized in parallel. */ - process_syncing_tables(commit_data.end_lsn); + SyncProcessRelations(commit_data.end_lsn); pgstat_report_activity(STATE_IDLE, NULL); @@ -3728,7 +3728,7 @@ LogicalRepApplyLoop(XLogRecPtr last_received) maybe_reread_subscription(); /* Process any table synchronization changes. */ - process_syncing_tables(last_received); + SyncProcessRelations(last_received); } /* Cleanup the memory. */ @@ -4797,7 +4797,7 @@ SetupApplyOrSyncWorker(int worker_slot) * the subscription relation state. */ CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP, - invalidate_syncing_table_states, + SyncInvalidateRelationStates, (Datum) 0); } diff --git a/src/include/catalog/pg_subscription_rel.h b/src/include/catalog/pg_subscription_rel.h index c91797c869c2..ea869588d842 100644 --- a/src/include/catalog/pg_subscription_rel.h +++ b/src/include/catalog/pg_subscription_rel.h @@ -89,7 +89,7 @@ extern void UpdateSubscriptionRelState(Oid subid, Oid relid, char state, extern char GetSubscriptionRelState(Oid subid, Oid relid, XLogRecPtr *sublsn); extern void RemoveSubscriptionRel(Oid subid, Oid relid); -extern bool HasSubscriptionRelations(Oid subid); +extern bool HasSubscriptionTables(Oid subid); extern List *GetSubscriptionRelations(Oid subid, bool not_ready); #endif /* PG_SUBSCRIPTION_REL_H */ diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h index 30b2775952c3..082e2b3d86c9 100644 --- a/src/include/replication/worker_internal.h +++ b/src/include/replication/worker_internal.h @@ -237,6 +237,8 @@ extern PGDLLIMPORT bool in_remote_transaction; extern PGDLLIMPORT bool InitializingApplyWorker; +extern PGDLLIMPORT List *table_states_not_ready; + extern void logicalrep_worker_attach(int slot); extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid, bool only_running); @@ -259,9 +261,14 @@ extern void ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid, extern bool AllTablesyncsReady(void); extern void UpdateTwoPhaseState(Oid suboid, char new_state); -extern void process_syncing_tables(XLogRecPtr current_lsn); -extern void invalidate_syncing_table_states(Datum arg, int cacheid, - uint32 hashvalue); +extern void ProcessSyncingTablesForSync(XLogRecPtr current_lsn); +extern void ProcessSyncingTablesForApply(XLogRecPtr current_lsn); + +pg_noreturn extern void SyncFinishWorker(void); +extern void SyncInvalidateRelationStates(Datum arg, int cacheid, + uint32 hashvalue); +extern void SyncProcessRelations(XLogRecPtr current_lsn); +extern bool SyncFetchRelationStates(bool *started_tx); extern void stream_start_internal(TransactionId xid, bool first_segment); extern void stream_stop_internal(TransactionId xid); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 74dad46568ac..82af9d8a7411 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2902,7 +2902,7 @@ SyncRepStandbyData SyncRequestHandler SyncRequestType SyncStandbySlotsConfigData -SyncingTablesState +SyncingRelationsState SysFKRelationship SysScanDesc SyscacheCallbackFunction From 1aabaf5ff82b1c4684c772680a80d04ad42cd614 Mon Sep 17 00:00:00 2001 From: Vignesh Date: Mon, 28 Apr 2025 11:41:50 +0530 Subject: [PATCH 4/5] Enhance sequence synchronization during subscription management This patch introduces sequence synchronization: Sequences have 2 states: - INIT (needs synchronizing) - READY (is already synchronized) A new sequencesync worker is launched as needed to synchronize sequences. It does the following: a) Retrieves remote values of sequences with pg_sequence_state() INIT. b) Logs a warning if the sequence parameters differ between the publisher and subscriber. c) Sets the local sequence values accordingly. d) Updates the local sequence state to READY. e) Repeats until all done; Commits synchronized sequences in batches of 100 Sequence synchronization occurs in 3 places: 1) CREATE SUBSCRIPTION - (PG18 command syntax is unchanged) - The subscriber retrieves sequences associated with publications. - Published sequences are added to pg_subscription_rel with INIT state. - Initiate the sequencesync worker (see above) to synchronize all sequences. 2) ALTER SUBSCRIPTION ... REFRESH PUBLICATION - (PG18 command syntax is unchanged) - Drop published sequences are removed from pg_subscription_rel. - Newly published sequences are added to pg_subscription_rel with INIT state. - Initiate the sequencesync worker (see above) to synchronize only newly added sequences. 3) ALTER SUBSCRIPTION ... REFRESH PUBLICATION SEQUENCES - The patch introduces this new command to refresh all sequences - Drop published sequences are removed from pg_subscription_rel. - Newly published sequences are added to pg_subscription_rel - All sequences in pg_subscription_rel are reset to INIT state. - Initiate the sequencesync worker (see above) to synchronize all sequences. --- src/backend/catalog/pg_publication.c | 46 ++ src/backend/catalog/pg_subscription.c | 63 +- src/backend/catalog/system_views.sql | 10 + src/backend/commands/sequence.c | 26 +- src/backend/commands/subscriptioncmds.c | 322 +++++++-- src/backend/executor/execReplication.c | 4 +- src/backend/parser/gram.y | 11 +- src/backend/postmaster/bgworker.c | 5 +- src/backend/replication/logical/Makefile | 1 + src/backend/replication/logical/launcher.c | 71 +- src/backend/replication/logical/meson.build | 1 + .../replication/logical/sequencesync.c | 658 ++++++++++++++++++ src/backend/replication/logical/syncutils.c | 75 +- src/backend/replication/logical/tablesync.c | 45 +- src/backend/replication/logical/worker.c | 58 +- src/backend/utils/misc/guc_tables.c | 2 +- src/bin/pg_dump/common.c | 4 +- src/bin/pg_dump/pg_dump.c | 8 +- src/bin/pg_dump/pg_dump.h | 2 +- src/bin/psql/tab-complete.in.c | 2 +- src/include/catalog/pg_proc.dat | 5 + src/include/catalog/pg_subscription_rel.h | 4 +- src/include/commands/sequence.h | 3 + src/include/nodes/parsenodes.h | 3 +- src/include/replication/logicalworker.h | 3 +- src/include/replication/worker_internal.h | 30 +- src/test/regress/expected/rules.out | 8 + src/test/regress/expected/subscription.out | 4 +- src/test/subscription/meson.build | 1 + src/test/subscription/t/036_sequences.pl | 227 ++++++ 30 files changed, 1523 insertions(+), 179 deletions(-) create mode 100644 src/backend/replication/logical/sequencesync.c create mode 100644 src/test/subscription/t/036_sequences.pl diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c index 617ed0b82c9b..ec46b126304c 100644 --- a/src/backend/catalog/pg_publication.c +++ b/src/backend/catalog/pg_publication.c @@ -1370,3 +1370,49 @@ pg_get_publication_tables(PG_FUNCTION_ARGS) SRF_RETURN_DONE(funcctx); } + +/* + * Returns Oids of sequences in a publication. + */ +Datum +pg_get_publication_sequences(PG_FUNCTION_ARGS) +{ + FuncCallContext *funcctx; + List *sequences = NIL; + + /* stuff done only on the first call of the function */ + if (SRF_IS_FIRSTCALL()) + { + char *pubname = text_to_cstring(PG_GETARG_TEXT_PP(0)); + Publication *publication; + MemoryContext oldcontext; + + /* create a function context for cross-call persistence */ + funcctx = SRF_FIRSTCALL_INIT(); + + /* switch to memory context appropriate for multiple function calls */ + oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + + publication = GetPublicationByName(pubname, false); + + if (publication->allsequences) + sequences = GetAllSequencesPublicationRelations(); + + funcctx->user_fctx = (void *) sequences; + + MemoryContextSwitchTo(oldcontext); + } + + /* stuff done on every call of the function */ + funcctx = SRF_PERCALL_SETUP(); + sequences = (List *) funcctx->user_fctx; + + if (funcctx->call_cntr < list_length(sequences)) + { + Oid relid = list_nth_oid(sequences, funcctx->call_cntr); + + SRF_RETURN_NEXT(funcctx, ObjectIdGetDatum(relid)); + } + + SRF_RETURN_DONE(funcctx); +} diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c index 1c71161e7237..68b55bb5ea5c 100644 --- a/src/backend/catalog/pg_subscription.c +++ b/src/backend/catalog/pg_subscription.c @@ -27,6 +27,7 @@ #include "utils/array.h" #include "utils/builtins.h" #include "utils/fmgroids.h" +#include "utils/memutils.h" #include "utils/lsyscache.h" #include "utils/pg_lsn.h" #include "utils/rel.h" @@ -462,7 +463,9 @@ RemoveSubscriptionRel(Oid subid, Oid relid) * leave tablesync slots or origins in the system when the * corresponding table is dropped. */ - if (!OidIsValid(subid) && subrel->srsubstate != SUBREL_STATE_READY) + if (!OidIsValid(subid) && + get_rel_relkind(subrel->srrelid) != RELKIND_SEQUENCE && + subrel->srsubstate != SUBREL_STATE_READY) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), @@ -499,7 +502,8 @@ HasSubscriptionTables(Oid subid) Relation rel; ScanKeyData skey[1]; SysScanDesc scan; - bool has_subrels; + HeapTuple tup; + bool has_subrels = false; rel = table_open(SubscriptionRelRelationId, AccessShareLock); @@ -511,8 +515,22 @@ HasSubscriptionTables(Oid subid) scan = systable_beginscan(rel, InvalidOid, false, NULL, 1, skey); - /* If even a single tuple exists then the subscription has tables. */ - has_subrels = HeapTupleIsValid(systable_getnext(scan)); + while (HeapTupleIsValid(tup = systable_getnext(scan))) + { + Form_pg_subscription_rel subrel; + + subrel = (Form_pg_subscription_rel) GETSTRUCT(tup); + + /* + * Skip sequence tuples. If even a single table tuple exists then the + * subscription has tables. + */ + if (get_rel_relkind(subrel->srrelid) != RELKIND_SEQUENCE) + { + has_subrels = true; + break; + } + } /* Cleanup */ systable_endscan(scan); @@ -524,12 +542,22 @@ HasSubscriptionTables(Oid subid) /* * Get the relations for the subscription. * - * If not_ready is true, return only the relations that are not in a ready - * state, otherwise return all the relations of the subscription. The - * returned list is palloc'ed in the current memory context. + * get_tables: get relations for tables of the subscription. + * + * get_sequences: get relations for sequences of the subscription. + * + * all_states: + * If getting tables, if all_states is true get all tables, otherwise + * only get tables that have not reached READY state. + * If getting sequences, if all_states is true get all sequences, + * otherwise only get sequences that have not reached READY state (i.e. are + * still in INIT state). + * + * The returned list is palloc'ed in the current memory context. */ List * -GetSubscriptionRelations(Oid subid, bool not_ready) +GetSubscriptionRelations(Oid subid, bool get_tables, bool get_sequences, + bool all_states) { List *res = NIL; Relation rel; @@ -538,6 +566,9 @@ GetSubscriptionRelations(Oid subid, bool not_ready) ScanKeyData skey[2]; SysScanDesc scan; + /* One or both of 'get_tables' and 'get_sequences' must be true. */ + Assert(get_tables || get_sequences); + rel = table_open(SubscriptionRelRelationId, AccessShareLock); ScanKeyInit(&skey[nkeys++], @@ -545,7 +576,7 @@ GetSubscriptionRelations(Oid subid, bool not_ready) BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(subid)); - if (not_ready) + if (!all_states) ScanKeyInit(&skey[nkeys++], Anum_pg_subscription_rel_srsubstate, BTEqualStrategyNumber, F_CHARNE, @@ -560,9 +591,23 @@ GetSubscriptionRelations(Oid subid, bool not_ready) SubscriptionRelState *relstate; Datum d; bool isnull; + bool issequence; + bool istable; subrel = (Form_pg_subscription_rel) GETSTRUCT(tup); + /* Relation is either a sequence or a table */ + issequence = get_rel_relkind(subrel->srrelid) == RELKIND_SEQUENCE; + istable = !issequence; + + /* Skip sequences if they were not requested */ + if (!get_sequences && issequence) + continue; + + /* Skip tables if they were not requested */ + if (!get_tables && istable) + continue; + relstate = (SubscriptionRelState *) palloc(sizeof(SubscriptionRelState)); relstate->relid = subrel->srrelid; relstate->state = subrel->srsubstate; diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 15efb02badb8..998fc05d7c24 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -394,6 +394,16 @@ CREATE VIEW pg_publication_tables AS pg_class C JOIN pg_namespace N ON (N.oid = C.relnamespace) WHERE C.oid = GPT.relid; +CREATE VIEW pg_publication_sequences AS + SELECT + P.pubname AS pubname, + N.nspname AS schemaname, + C.relname AS sequencename + FROM pg_publication P, + LATERAL pg_get_publication_sequences(P.pubname) GPS, + pg_class C JOIN pg_namespace N ON (N.oid = C.relnamespace) + WHERE C.oid = GPS.relid; + CREATE VIEW pg_locks AS SELECT * FROM pg_lock_status() AS L; diff --git a/src/backend/commands/sequence.c b/src/backend/commands/sequence.c index 2e5b6cbecd1a..8c5c81818ca1 100644 --- a/src/backend/commands/sequence.c +++ b/src/backend/commands/sequence.c @@ -110,7 +110,6 @@ static void init_params(ParseState *pstate, List *options, bool for_identity, Form_pg_sequence_data seqdataform, bool *need_seq_rewrite, List **owned_by); -static void do_setval(Oid relid, int64 next, bool iscalled); static void process_owned_by(Relation seqrel, List *owned_by, bool for_identity); @@ -941,9 +940,12 @@ lastval(PG_FUNCTION_ARGS) * restore the state of a sequence exactly during data-only restores - * it is the only way to clear the is_called flag in an existing * sequence. + * + * log_cnt is currently used only by the sequence syncworker to set the + * log_cnt for sequences while synchronizing values from the publisher. */ -static void -do_setval(Oid relid, int64 next, bool iscalled) +void +SetSequence(Oid relid, int64 next, bool is_called, int64 log_cnt) { SeqTable elm; Relation seqrel; @@ -994,7 +996,7 @@ do_setval(Oid relid, int64 next, bool iscalled) minv, maxv))); /* Set the currval() state only if iscalled = true */ - if (iscalled) + if (is_called) { elm->last = next; /* last returned number */ elm->last_valid = true; @@ -1011,8 +1013,8 @@ do_setval(Oid relid, int64 next, bool iscalled) START_CRIT_SECTION(); seq->last_value = next; /* last fetched number */ - seq->is_called = iscalled; - seq->log_cnt = 0; + seq->is_called = is_called; + seq->log_cnt = log_cnt; MarkBufferDirty(buf); @@ -1044,7 +1046,7 @@ do_setval(Oid relid, int64 next, bool iscalled) /* * Implement the 2 arg setval procedure. - * See do_setval for discussion. + * See SetSequence for discussion. */ Datum setval_oid(PG_FUNCTION_ARGS) @@ -1052,14 +1054,14 @@ setval_oid(PG_FUNCTION_ARGS) Oid relid = PG_GETARG_OID(0); int64 next = PG_GETARG_INT64(1); - do_setval(relid, next, true); + SetSequence(relid, next, true, SEQ_LOG_CNT_INVALID); PG_RETURN_INT64(next); } /* * Implement the 3 arg setval procedure. - * See do_setval for discussion. + * See SetSequence for discussion. */ Datum setval3_oid(PG_FUNCTION_ARGS) @@ -1068,7 +1070,7 @@ setval3_oid(PG_FUNCTION_ARGS) int64 next = PG_GETARG_INT64(1); bool iscalled = PG_GETARG_BOOL(2); - do_setval(relid, next, iscalled); + SetSequence(relid, next, iscalled, SEQ_LOG_CNT_INVALID); PG_RETURN_INT64(next); } @@ -1889,6 +1891,10 @@ pg_sequence_last_value(PG_FUNCTION_ARGS) /* * Return the current on-disk state of the sequence. * + * The page LSN will be used in logical replication of sequences to record the + * LSN of the sequence page in the pg_subscription_rel system catalog. It + * reflects the LSN of the remote sequence at the time it was synchronized. + * * Note: This is roughly equivalent to selecting the data from the sequence, * except that it also returns the page LSN. */ diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c index 4aec73bcc6bb..83be0bae0622 100644 --- a/src/backend/commands/subscriptioncmds.c +++ b/src/backend/commands/subscriptioncmds.c @@ -26,6 +26,7 @@ #include "catalog/objectaddress.h" #include "catalog/pg_authid_d.h" #include "catalog/pg_database_d.h" +#include "catalog/pg_sequence.h" #include "catalog/pg_subscription.h" #include "catalog/pg_subscription_rel.h" #include "catalog/pg_type.h" @@ -103,6 +104,7 @@ typedef struct SubOpts } SubOpts; static List *fetch_table_list(WalReceiverConn *wrconn, List *publications); +static List *fetch_sequence_list(WalReceiverConn *wrconn, List *publications); static void check_publications_origin(WalReceiverConn *wrconn, List *publications, bool copydata, char *origin, Oid *subrel_local_oids, @@ -692,6 +694,12 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, recordDependencyOnOwner(SubscriptionRelationId, subid, owner); + /* + * XXX: If the subscription is for a sequence-only publication, creating + * this origin is unnecessary. It can be created later during the ALTER + * SUBSCRIPTION ... REFRESH command, if the publication is updated to + * include tables. + */ ReplicationOriginNameForLogicalRep(subid, InvalidOid, originname, sizeof(originname)); replorigin_create(originname); @@ -703,9 +711,6 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, { char *err; WalReceiverConn *wrconn; - List *tables; - ListCell *lc; - char table_state; bool must_use_password; /* Try to connect to the publisher. */ @@ -720,6 +725,10 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, PG_TRY(); { + bool has_tables; + List *relations; + char table_state; + check_publications(wrconn, publications); check_publications_origin(wrconn, publications, opts.copy_data, opts.origin, NULL, 0, stmt->subname); @@ -731,13 +740,16 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, table_state = opts.copy_data ? SUBREL_STATE_INIT : SUBREL_STATE_READY; /* - * Get the table list from publisher and build local table status - * info. + * Build local relation status info. Relations are for both tables + * and sequences from the publisher. */ - tables = fetch_table_list(wrconn, publications); - foreach(lc, tables) + relations = fetch_table_list(wrconn, publications); + has_tables = relations != NIL; + relations = list_concat(relations, + fetch_sequence_list(wrconn, publications)); + + foreach_ptr(RangeVar, rv, relations) { - RangeVar *rv = (RangeVar *) lfirst(lc); Oid relid; relid = RangeVarGetRelid(rv, AccessShareLock, false); @@ -754,6 +766,12 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, * If requested, create permanent slot for the subscription. We * won't use the initial snapshot for anything, so no need to * export it. + * + * XXX: If the subscription is for a sequence-only publication, + * creating this slot is unnecessary. It can be created later + * during the ALTER SUBSCRIPTION ... REFRESH PUBLICATION or ALTER + * SUBSCRIPTION ... REFRESH PUBLICATION SEQUENCES command, if the + * publication is updated to include tables. */ if (opts.create_slot) { @@ -777,7 +795,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, * PENDING, to allow ALTER SUBSCRIPTION ... REFRESH * PUBLICATION to work. */ - if (opts.twophase && !opts.copy_data && tables != NIL) + if (opts.twophase && !opts.copy_data && has_tables) twophase_enabled = true; walrcv_create_slot(wrconn, opts.slot_name, false, twophase_enabled, @@ -816,12 +834,50 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt, return myself; } +/* + * Update the subscription to refresh both the publication and the publication + * objects associated with the subscription. + * + * Parameters: + * + * If 'copy_data' is true, the function will set the state to INIT; otherwise, + * it will set the state to READY. + * + * If 'validate_publications' is provided with a publication list, the + * function checks that the specified publications exist on the publisher. + * + * If 'refresh_tables' is true, update the subscription by adding or removing + * tables that have been added or removed since the last subscription creation + * or refresh publication. + * + * If 'refresh_sequences' is true, update the subscription by adding or removing + * sequences that have been added or removed since the last subscription + * creation or refresh publication. + * + * Note, this is a common function for handling different REFRESH commands + * according to the parameter 'resync_all_sequences' + * + * 1. ALTER SUBSCRIPTION ... REFRESH PUBLICATION SEQUENCES + * (when parameter resync_all_sequences is true) + * + * The function will mark all sequences with INIT state. + * Assert copy_data is true. + * Assert refresh_tables is false. + * Assert refresh_sequences is true. + * + * 2. ALTER SUBSCRIPTION ... REFRESH PUBLICATION [WITH (copy_data=true|false)] + * (when parameter resync_all_sequences is false) + * + * The function will update only the newly added tables and/or sequences + * based on the copy_data parameter. + */ static void AlterSubscription_refresh(Subscription *sub, bool copy_data, - List *validate_publications) + List *validate_publications, bool refresh_tables, + bool refresh_sequences, bool resync_all_sequences) { char *err; - List *pubrel_names; + List *pubrel_names = NIL; List *subrel_states; Oid *subrel_local_oids; Oid *pubrel_local_oids; @@ -839,6 +895,12 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data, WalReceiverConn *wrconn; bool must_use_password; +#ifdef USE_ASSERT_CHECKING + /* Sanity checks for parameter values */ + if (resync_all_sequences) + Assert(copy_data && !refresh_tables && refresh_sequences); +#endif + /* Load the library providing us libpq calls. */ load_file("libpqwalreceiver", false); @@ -858,10 +920,17 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data, check_publications(wrconn, validate_publications); /* Get the table list from publisher. */ - pubrel_names = fetch_table_list(wrconn, sub->publications); + if (refresh_tables) + pubrel_names = fetch_table_list(wrconn, sub->publications); + + /* Get the sequence list from publisher. */ + if (refresh_sequences) + pubrel_names = list_concat(pubrel_names, + fetch_sequence_list(wrconn, + sub->publications)); /* Get local table list. */ - subrel_states = GetSubscriptionRelations(sub->oid, false); + subrel_states = GetSubscriptionRelations(sub->oid, refresh_tables, refresh_sequences, true); subrel_count = list_length(subrel_states); /* @@ -880,9 +949,10 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data, qsort(subrel_local_oids, subrel_count, sizeof(Oid), oid_cmp); - check_publications_origin(wrconn, sub->publications, copy_data, - sub->origin, subrel_local_oids, - subrel_count, sub->name); + if (refresh_tables) + check_publications_origin(wrconn, sub->publications, copy_data, + sub->origin, subrel_local_oids, + subrel_count, sub->name); /* * Rels that we want to remove from subscription and drop any slots @@ -904,12 +974,13 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data, { RangeVar *rv = (RangeVar *) lfirst(lc); Oid relid; + char relkind; relid = RangeVarGetRelid(rv, AccessShareLock, false); /* Check for supported relkind. */ - CheckSubscriptionRelkind(get_rel_relkind(relid), - rv->schemaname, rv->relname); + relkind = get_rel_relkind(relid); + CheckSubscriptionRelkind(relkind, rv->schemaname, rv->relname); pubrel_local_oids[off++] = relid; @@ -920,8 +991,9 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data, copy_data ? SUBREL_STATE_INIT : SUBREL_STATE_READY, InvalidXLogRecPtr, true); ereport(DEBUG1, - (errmsg_internal("table \"%s.%s\" added to subscription \"%s\"", - rv->schemaname, rv->relname, sub->name))); + errmsg_internal("%s \"%s.%s\" added to subscription \"%s\"", + relkind == RELKIND_SEQUENCE ? "sequence" : "table", + rv->schemaname, rv->relname, sub->name)); } } @@ -937,11 +1009,31 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data, { Oid relid = subrel_local_oids[off]; - if (!bsearch(&relid, pubrel_local_oids, - list_length(pubrel_names), sizeof(Oid), oid_cmp)) + if (bsearch(&relid, pubrel_local_oids, + list_length(pubrel_names), sizeof(Oid), oid_cmp)) + { + /* + * The resync_all_sequences flag will only be set to true for + * the REFRESH PUBLICATION SEQUENCES command, indicating that + * the existing sequences need to be re-synchronized by + * resetting the relation to its initial state. + */ + if (resync_all_sequences) + { + UpdateSubscriptionRelState(sub->oid, relid, SUBREL_STATE_INIT, + InvalidXLogRecPtr); + ereport(DEBUG1, + errmsg_internal("sequence \"%s.%s\" of subscription \"%s\" set to INIT state", + get_namespace_name(get_rel_namespace(relid)), + get_rel_name(relid), + sub->name)); + } + } + else { char state; XLogRecPtr statelsn; + char relkind = get_rel_relkind(relid); /* * Lock pg_subscription_rel with AccessExclusiveLock to @@ -963,41 +1055,51 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data, /* Last known rel state. */ state = GetSubscriptionRelState(sub->oid, relid, &statelsn); - sub_remove_rels[remove_rel_len].relid = relid; - sub_remove_rels[remove_rel_len++].state = state; - RemoveSubscriptionRel(sub->oid, relid); - logicalrep_worker_stop(sub->oid, relid); - /* - * For READY state, we would have already dropped the - * tablesync origin. + * A single sequencesync worker synchronizes all sequences, so + * only stop workers when relation kind is not sequence. */ - if (state != SUBREL_STATE_READY) + if (relkind != RELKIND_SEQUENCE) { - char originname[NAMEDATALEN]; + sub_remove_rels[remove_rel_len].relid = relid; + sub_remove_rels[remove_rel_len++].state = state; + + logicalrep_worker_stop(sub->oid, relid, WORKERTYPE_TABLESYNC); /* - * Drop the tablesync's origin tracking if exists. - * - * It is possible that the origin is not yet created for - * tablesync worker, this can happen for the states before - * SUBREL_STATE_FINISHEDCOPY. The tablesync worker or - * apply worker can also concurrently try to drop the - * origin and by this time the origin might be already - * removed. For these reasons, passing missing_ok = true. + * For READY state, we would have already dropped the + * tablesync origin. */ - ReplicationOriginNameForLogicalRep(sub->oid, relid, originname, - sizeof(originname)); - replorigin_drop_by_name(originname, true, false); + if (state != SUBREL_STATE_READY) + { + char originname[NAMEDATALEN]; + + /* + * Drop the tablesync's origin tracking if exists. + * + * It is possible that the origin is not yet created + * for tablesync worker, this can happen for the + * states before SUBREL_STATE_FINISHEDCOPY. The + * tablesync worker or apply worker can also + * concurrently try to drop the origin and by this + * time the origin might be already removed. For these + * reasons, passing missing_ok = true. + */ + ReplicationOriginNameForLogicalRep(sub->oid, relid, + originname, + sizeof(originname)); + replorigin_drop_by_name(originname, true, false); + } } ereport(DEBUG1, - (errmsg_internal("table \"%s.%s\" removed from subscription \"%s\"", - get_namespace_name(get_rel_namespace(relid)), - get_rel_name(relid), - sub->name))); + errmsg_internal("%s \"%s.%s\" removed from subscription \"%s\"", + relkind == RELKIND_SEQUENCE ? "sequence" : "table", + get_namespace_name(get_rel_namespace(relid)), + get_rel_name(relid), + sub->name)); } } @@ -1393,8 +1495,8 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, errhint("Use ALTER SUBSCRIPTION ... SET PUBLICATION ... WITH (refresh = false)."))); /* - * See ALTER_SUBSCRIPTION_REFRESH for details why this is - * not allowed. + * See ALTER_SUBSCRIPTION_REFRESH_PUBLICATION for details + * why this is not allowed. */ if (sub->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED && opts.copy_data) ereport(ERROR, @@ -1408,7 +1510,8 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, sub->publications = stmt->publication; AlterSubscription_refresh(sub, opts.copy_data, - stmt->publication); + stmt->publication, true, true, + false); } break; @@ -1448,8 +1551,8 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, "ALTER SUBSCRIPTION ... DROP PUBLICATION ... WITH (refresh = false)"))); /* - * See ALTER_SUBSCRIPTION_REFRESH for details why this is - * not allowed. + * See ALTER_SUBSCRIPTION_REFRESH_PUBLICATION for details + * why this is not allowed. */ if (sub->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED && opts.copy_data) ereport(ERROR, @@ -1467,18 +1570,19 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, sub->publications = publist; AlterSubscription_refresh(sub, opts.copy_data, - validate_publications); + validate_publications, true, true, + false); } break; } - case ALTER_SUBSCRIPTION_REFRESH: + case ALTER_SUBSCRIPTION_REFRESH_PUBLICATION: { if (!sub->enabled) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), - errmsg("ALTER SUBSCRIPTION ... REFRESH is not allowed for disabled subscriptions"))); + errmsg("ALTER SUBSCRIPTION ... REFRESH PUBLICATION is not allowed for disabled subscriptions"))); parse_subscription_options(pstate, stmt->options, SUBOPT_COPY_DATA, &opts); @@ -1490,8 +1594,8 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, * * But, having reached this two-phase commit "enabled" state * we must not allow any subsequent table initialization to - * occur. So the ALTER SUBSCRIPTION ... REFRESH is disallowed - * when the user had requested two_phase = on mode. + * occur. So the ALTER SUBSCRIPTION ... REFRESH PUBLICATION is + * disallowed when the user had requested two_phase = on mode. * * The exception to this restriction is when copy_data = * false, because when copy_data is false the tablesync will @@ -1503,12 +1607,26 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt, if (sub->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED && opts.copy_data) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("ALTER SUBSCRIPTION ... REFRESH with copy_data is not allowed when two_phase is enabled"), - errhint("Use ALTER SUBSCRIPTION ... REFRESH with copy_data = false, or use DROP/CREATE SUBSCRIPTION."))); + errmsg("ALTER SUBSCRIPTION ... REFRESH PUBLICATION with copy_data is not allowed when two_phase is enabled"), + errhint("Use ALTER SUBSCRIPTION ... REFRESH PUBLICATION with copy_data = false, or use DROP/CREATE SUBSCRIPTION."))); + + PreventInTransactionBlock(isTopLevel, "ALTER SUBSCRIPTION ... REFRESH PUBLICATION"); - PreventInTransactionBlock(isTopLevel, "ALTER SUBSCRIPTION ... REFRESH"); + AlterSubscription_refresh(sub, opts.copy_data, NULL, true, true, false); - AlterSubscription_refresh(sub, opts.copy_data, NULL); + break; + } + + case ALTER_SUBSCRIPTION_REFRESH_PUBLICATION_SEQUENCES: + { + if (!sub->enabled) + ereport(ERROR, + errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("ALTER SUBSCRIPTION ... REFRESH PUBLICATION SEQUENCES is not allowed for disabled subscriptions")); + + PreventInTransactionBlock(isTopLevel, "ALTER SUBSCRIPTION ... REFRESH PUBLICATION SEQUENCES"); + + AlterSubscription_refresh(sub, true, NULL, false, true, true); break; } @@ -1750,7 +1868,7 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel) { LogicalRepWorker *w = (LogicalRepWorker *) lfirst(lc); - logicalrep_worker_stop(w->subid, w->relid); + logicalrep_worker_stop(w->subid, w->relid, w->type); } list_free(subworkers); @@ -1773,7 +1891,7 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel) * the apply and tablesync workers and they can't restart because of * exclusive lock on the subscription. */ - rstates = GetSubscriptionRelations(subid, true); + rstates = GetSubscriptionRelations(subid, true, false, false); foreach(lc, rstates) { SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc); @@ -2087,8 +2205,8 @@ AlterSubscriptionOwner_oid(Oid subid, Oid newOwnerId) * its partition ancestors (if it's a partition), or its partition children (if * it's a partitioned table), from some other publishers. This check is * required only if "copy_data = true" and "origin = none" for CREATE - * SUBSCRIPTION and ALTER SUBSCRIPTION ... REFRESH statements to notify the - * user that data having origin might have been copied. + * SUBSCRIPTION and ALTER SUBSCRIPTION ... REFRESH PUBLICATION statements to + * notify the user that data having origin might have been copied. * * This check need not be performed on the tables that are already added * because incremental sync for those tables will happen through WAL and the @@ -2127,18 +2245,23 @@ check_publications_origin(WalReceiverConn *wrconn, List *publications, appendStringInfoString(&cmd, ")\n"); /* - * In case of ALTER SUBSCRIPTION ... REFRESH, subrel_local_oids contains - * the list of relation oids that are already present on the subscriber. - * This check should be skipped for these tables. + * In case of ALTER SUBSCRIPTION ... REFRESH PUBLICATION, + * subrel_local_oids contains the list of relation oids that are already + * present on the subscriber. This check should be skipped for these + * tables. */ for (i = 0; i < subrel_count; i++) { Oid relid = subrel_local_oids[i]; - char *schemaname = get_namespace_name(get_rel_namespace(relid)); - char *tablename = get_rel_name(relid); - appendStringInfo(&cmd, "AND NOT (N.nspname = '%s' AND C.relname = '%s')\n", - schemaname, tablename); + if (get_rel_relkind(relid) != RELKIND_SEQUENCE) + { + char *schemaname = get_namespace_name(get_rel_namespace(relid)); + char *tablename = get_rel_name(relid); + + appendStringInfo(&cmd, "AND NOT (N.nspname = '%s' AND C.relname = '%s')\n", + schemaname, tablename); + } } res = walrcv_exec(wrconn, cmd.data, 1, tableRow); @@ -2307,6 +2430,63 @@ fetch_table_list(WalReceiverConn *wrconn, List *publications) return tablelist; } +/* + * Get the list of sequences which belong to specified publications on the + * publisher connection. + */ +static List * +fetch_sequence_list(WalReceiverConn *wrconn, List *publications) +{ + WalRcvExecResult *res; + StringInfoData cmd; + TupleTableSlot *slot; + Oid tableRow[2] = {TEXTOID, TEXTOID}; + List *seqlist = NIL; + + Assert(list_length(publications) > 0); + + initStringInfo(&cmd); + + appendStringInfoString(&cmd, + "SELECT DISTINCT s.schemaname, s.sequencename\n" + "FROM pg_catalog.pg_publication_sequences s\n" + "WHERE s.pubname IN ("); + GetPublicationsStr(publications, &cmd, true); + appendStringInfoChar(&cmd, ')'); + + res = walrcv_exec(wrconn, cmd.data, 2, tableRow); + pfree(cmd.data); + + if (res->status != WALRCV_OK_TUPLES) + ereport(ERROR, + errmsg("could not receive list of sequences from the publisher: %s", + res->err)); + + /* Process sequences. */ + slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple); + while (tuplestore_gettupleslot(res->tuplestore, true, false, slot)) + { + char *nspname; + char *relname; + bool isnull; + RangeVar *rv; + + nspname = TextDatumGetCString(slot_getattr(slot, 1, &isnull)); + Assert(!isnull); + relname = TextDatumGetCString(slot_getattr(slot, 2, &isnull)); + Assert(!isnull); + + rv = makeRangeVar(nspname, relname, -1); + seqlist = lappend(seqlist, rv); + ExecClearTuple(slot); + } + + ExecDropSingleTupleTableSlot(slot); + walrcv_clear_result(res); + + return seqlist; +} + /* * This is to report the connection failure while dropping replication slots. * Here, we report the WARNING for all tablesync slots so that user can drop diff --git a/src/backend/executor/execReplication.c b/src/backend/executor/execReplication.c index 53ddd25c42db..3dfa086faa87 100644 --- a/src/backend/executor/execReplication.c +++ b/src/backend/executor/execReplication.c @@ -877,7 +877,9 @@ void CheckSubscriptionRelkind(char relkind, const char *nspname, const char *relname) { - if (relkind != RELKIND_RELATION && relkind != RELKIND_PARTITIONED_TABLE) + if (relkind != RELKIND_RELATION && + relkind != RELKIND_PARTITIONED_TABLE && + relkind != RELKIND_SEQUENCE) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot use relation \"%s.%s\" as logical replication target", diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 1c094d7d6053..d470c1cd2fa2 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -10894,11 +10894,20 @@ AlterSubscriptionStmt: AlterSubscriptionStmt *n = makeNode(AlterSubscriptionStmt); - n->kind = ALTER_SUBSCRIPTION_REFRESH; + n->kind = ALTER_SUBSCRIPTION_REFRESH_PUBLICATION; n->subname = $3; n->options = $6; $$ = (Node *) n; } + | ALTER SUBSCRIPTION name REFRESH PUBLICATION SEQUENCES + { + AlterSubscriptionStmt *n = + makeNode(AlterSubscriptionStmt); + + n->kind = ALTER_SUBSCRIPTION_REFRESH_PUBLICATION_SEQUENCES; + n->subname = $3; + $$ = (Node *) n; + } | ALTER SUBSCRIPTION name ADD_P PUBLICATION name_list opt_definition { AlterSubscriptionStmt *n = diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c index 116ddf7b835f..81e0e369fb03 100644 --- a/src/backend/postmaster/bgworker.c +++ b/src/backend/postmaster/bgworker.c @@ -131,7 +131,10 @@ static const struct "ParallelApplyWorkerMain", ParallelApplyWorkerMain }, { - "TablesyncWorkerMain", TablesyncWorkerMain + "TableSyncWorkerMain", TableSyncWorkerMain + }, + { + "SequenceSyncWorkerMain", SequenceSyncWorkerMain } }; diff --git a/src/backend/replication/logical/Makefile b/src/backend/replication/logical/Makefile index c62c8c67521c..c719af1f8a94 100644 --- a/src/backend/replication/logical/Makefile +++ b/src/backend/replication/logical/Makefile @@ -26,6 +26,7 @@ OBJS = \ proto.o \ relation.o \ reorderbuffer.o \ + sequencesync.o \ slotsync.o \ snapbuild.o \ syncutils.o \ diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c index 10677da56b2b..fb3be0236dee 100644 --- a/src/backend/replication/logical/launcher.c +++ b/src/backend/replication/logical/launcher.c @@ -226,19 +226,18 @@ WaitForReplicationWorkerAttach(LogicalRepWorker *worker, /* * Walks the workers array and searches for one that matches given - * subscription id and relid. - * - * We are only interested in the leader apply worker or table sync worker. + * subscription id, relid and type. */ LogicalRepWorker * -logicalrep_worker_find(Oid subid, Oid relid, bool only_running) +logicalrep_worker_find(Oid subid, Oid relid, LogicalRepWorkerType wtype, + bool only_running) { int i; LogicalRepWorker *res = NULL; Assert(LWLockHeldByMe(LogicalRepWorkerLock)); - /* Search for attached worker for a given subscription id. */ + /* Search for the attached worker matching the specified criteria. */ for (i = 0; i < max_logical_replication_workers; i++) { LogicalRepWorker *w = &LogicalRepCtx->workers[i]; @@ -248,7 +247,7 @@ logicalrep_worker_find(Oid subid, Oid relid, bool only_running) continue; if (w->in_use && w->subid == subid && w->relid == relid && - (!only_running || w->proc)) + w->type == wtype && (!only_running || w->proc)) { res = w; break; @@ -308,6 +307,7 @@ logicalrep_worker_launch(LogicalRepWorkerType wtype, int nparallelapplyworkers; TimestampTz now; bool is_tablesync_worker = (wtype == WORKERTYPE_TABLESYNC); + bool is_sequencesync_worker = (wtype == WORKERTYPE_SEQUENCESYNC); bool is_parallel_apply_worker = (wtype == WORKERTYPE_PARALLEL_APPLY); /*---------- @@ -393,7 +393,8 @@ logicalrep_worker_launch(LogicalRepWorkerType wtype, * sync worker limit per subscription. So, just return silently as we * might get here because of an otherwise harmless race condition. */ - if (is_tablesync_worker && nsyncworkers >= max_sync_workers_per_subscription) + if ((is_tablesync_worker || is_sequencesync_worker) && + nsyncworkers >= max_sync_workers_per_subscription) { LWLockRelease(LogicalRepWorkerLock); return false; @@ -479,8 +480,16 @@ logicalrep_worker_launch(LogicalRepWorkerType wtype, memcpy(bgw.bgw_extra, &subworker_dsm, sizeof(dsm_handle)); break; + case WORKERTYPE_SEQUENCESYNC: + snprintf(bgw.bgw_function_name, BGW_MAXLEN, "SequenceSyncWorkerMain"); + snprintf(bgw.bgw_name, BGW_MAXLEN, + "logical replication sequencesync worker for subscription %u", + subid); + snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication sequencesync worker"); + break; + case WORKERTYPE_TABLESYNC: - snprintf(bgw.bgw_function_name, BGW_MAXLEN, "TablesyncWorkerMain"); + snprintf(bgw.bgw_function_name, BGW_MAXLEN, "TableSyncWorkerMain"); snprintf(bgw.bgw_name, BGW_MAXLEN, "logical replication tablesync worker for subscription %u sync %u", subid, @@ -603,13 +612,13 @@ logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo) * Stop the logical replication worker for subid/relid, if any. */ void -logicalrep_worker_stop(Oid subid, Oid relid) +logicalrep_worker_stop(Oid subid, Oid relid, LogicalRepWorkerType wtype) { LogicalRepWorker *worker; LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); - worker = logicalrep_worker_find(subid, relid, false); + worker = logicalrep_worker_find(subid, relid, wtype, false); if (worker) { @@ -676,7 +685,7 @@ logicalrep_worker_wakeup(Oid subid, Oid relid) LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); - worker = logicalrep_worker_find(subid, relid, true); + worker = logicalrep_worker_find(subid, relid, WORKERTYPE_APPLY, true); if (worker) logicalrep_worker_wakeup_ptr(worker); @@ -806,6 +815,37 @@ logicalrep_launcher_onexit(int code, Datum arg) LogicalRepCtx->launcher_pid = 0; } +/* + * Set the sequencesync worker failure time. + */ +void +logicalrep_seqsyncworker_set_failuretime() +{ + LogicalRepWorker *worker; + + LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + + worker = logicalrep_worker_find(MyLogicalRepWorker->subid, InvalidOid, + WORKERTYPE_APPLY, true); + if (worker) + worker->sequencesync_failure_time = GetCurrentTimestamp(); + + LWLockRelease(LogicalRepWorkerLock); +} + +/* + * Update the failure time of the sequencesync worker in the subscription's + * apply worker. + * + * This function is invoked when the sequencesync worker exits due to a + * failure. + */ +void +logicalrep_seqsyncworker_failure(int code, Datum arg) +{ + logicalrep_seqsyncworker_set_failuretime(); +} + /* * Cleanup function. * @@ -854,7 +894,7 @@ logicalrep_sync_worker_count(Oid subid) { LogicalRepWorker *w = &LogicalRepCtx->workers[i]; - if (isTablesyncWorker(w) && w->subid == subid) + if (w->subid == subid && (isTableSyncWorker(w) || isSequenceSyncWorker(w))) res++; } @@ -1169,7 +1209,7 @@ ApplyLauncherMain(Datum main_arg) continue; LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); - w = logicalrep_worker_find(sub->oid, InvalidOid, false); + w = logicalrep_worker_find(sub->oid, InvalidOid, WORKERTYPE_APPLY, false); LWLockRelease(LogicalRepWorkerLock); if (w != NULL) @@ -1305,7 +1345,7 @@ pg_stat_get_subscription(PG_FUNCTION_ARGS) worker_pid = worker.proc->pid; values[0] = ObjectIdGetDatum(worker.subid); - if (isTablesyncWorker(&worker)) + if (isTableSyncWorker(&worker)) values[1] = ObjectIdGetDatum(worker.relid); else nulls[1] = true; @@ -1345,6 +1385,9 @@ pg_stat_get_subscription(PG_FUNCTION_ARGS) case WORKERTYPE_PARALLEL_APPLY: values[9] = CStringGetTextDatum("parallel apply"); break; + case WORKERTYPE_SEQUENCESYNC: + values[9] = CStringGetTextDatum("sequence synchronization"); + break; case WORKERTYPE_TABLESYNC: values[9] = CStringGetTextDatum("table synchronization"); break; diff --git a/src/backend/replication/logical/meson.build b/src/backend/replication/logical/meson.build index 9283e996ef4a..a2268d8361ee 100644 --- a/src/backend/replication/logical/meson.build +++ b/src/backend/replication/logical/meson.build @@ -12,6 +12,7 @@ backend_sources += files( 'proto.c', 'relation.c', 'reorderbuffer.c', + 'sequencesync.c', 'slotsync.c', 'snapbuild.c', 'syncutils.c', diff --git a/src/backend/replication/logical/sequencesync.c b/src/backend/replication/logical/sequencesync.c new file mode 100644 index 000000000000..e6a36b0bfca9 --- /dev/null +++ b/src/backend/replication/logical/sequencesync.c @@ -0,0 +1,658 @@ +/*------------------------------------------------------------------------- + * sequencesync.c + * PostgreSQL logical replication: sequence synchronization + * + * Copyright (c) 2025, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/replication/logical/sequencesync.c + * + * NOTES + * This file contains code for sequence synchronization for + * logical replication. + * + * Sequences to be synchronized by the sequencesync worker will + * be added to pg_subscription_rel in INIT state when one of the following + * commands is executed: + * CREATE SUBSCRIPTION + * ALTER SUBSCRIPTION ... REFRESH PUBLICATION + * ALTER SUBSCRIPTION ... REFRESH PUBLICATION SEQUENCES + * + * The apply worker will periodically check if there are any sequences in INIT + * state and will start a sequencesync worker if needed. + * + * The sequencesync worker retrieves the sequences to be synchronized from the + * pg_subscription_rel catalog table. It synchronizes multiple sequences per + * single transaction by fetching the sequence value and page LSN from the + * remote publisher and updating them in the local subscriber sequence. After + * synchronization, it sets the sequence state to READY. + * + * So the state progression is always just: INIT -> READY. + * + * To avoid creating too many transactions, up to MAX_SEQUENCES_SYNC_PER_BATCH + * (100) sequences are synchronized per transaction. The locks on the sequence + * relation will be periodically released at each transaction commit. + * + * XXX: An alternative design was considered where the launcher process would + * periodically check for sequences that need syncing and then start the + * sequencesync worker. However, the approach of having the apply worker + * manage the sequencesync worker was chosen for the following reasons: + * a) It avoids overloading the launcher, which handles various other + * subscription requests. + * b) It offers a more straightforward path for extending support for + * incremental sequence synchronization. + * c) It utilizes the existing tablesync worker code to start the sequencesync + * process, thus preventing code duplication in the launcher. + * d) It simplifies code maintenance by consolidating changes to a single + * location rather than multiple components. + * e) The apply worker can access the sequences that need to be synchronized + * from the pg_subscription_rel system catalog. Whereas the launcher process + * operates without direct database access so would need a framework to + * establish connections with the databases to retrieve the sequences for + * synchronization. + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "access/table.h" +#include "catalog/pg_sequence.h" +#include "catalog/pg_subscription_rel.h" +#include "commands/sequence.h" +#include "pgstat.h" +#include "replication/logicallauncher.h" +#include "replication/logicalworker.h" +#include "replication/worker_internal.h" +#include "utils/acl.h" +#include "utils/builtins.h" +#include "utils/catcache.h" +#include "utils/lsyscache.h" +#include "utils/pg_lsn.h" +#include "utils/rls.h" +#include "utils/syscache.h" +#include "utils/usercontext.h" + +List *sequence_states_not_ready = NIL; + +/* + * Handle sequence synchronization cooperation from the apply worker. + * + * Walk over all subscription sequences that are individually tracked by the + * apply process (currently, all that have state SUBREL_STATE_INIT) and manage + * synchronization for them. + * + * If a sequencesync worker is running already, there is no need to start a new + * one; the existing sequencesync worker will synchronize all the sequences. If + * there are still any sequences to be synced after the sequencesync worker + * exited, then a new sequencesync worker can be started in the next iteration. + */ +void +ProcessSyncingSequencesForApply(void) +{ + bool started_tx = false; + + Assert(!IsTransactionState()); + + /* Start the sequencesync worker if needed, and there is not one already. */ + foreach_ptr(SubscriptionRelState, rstate, sequence_states_not_ready) + { + LogicalRepWorker *sequencesync_worker; + int nsyncworkers; + + if (!started_tx) + { + StartTransactionCommand(); + started_tx = true; + } + + Assert(get_rel_relkind(rstate->relid) == RELKIND_SEQUENCE); + + if (rstate->state != SUBREL_STATE_INIT) + continue; + + /* + * Check if there is a sequencesync worker already running? + */ + LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); + + sequencesync_worker = logicalrep_worker_find(MyLogicalRepWorker->subid, + InvalidOid, + WORKERTYPE_SEQUENCESYNC, + true); + if (sequencesync_worker) + { + /* Now safe to release the LWLock */ + LWLockRelease(LogicalRepWorkerLock); + break; + } + + /* + * Count running sync workers for this subscription, while we have the + * lock. + */ + nsyncworkers = logicalrep_sync_worker_count(MyLogicalRepWorker->subid); + + /* Now safe to release the LWLock */ + LWLockRelease(LogicalRepWorkerLock); + + /* + * If there is a free sync worker slot, start a new sequencesync worker, + * and break from the loop. + */ + if (nsyncworkers < max_sync_workers_per_subscription) + { + TimestampTz now = GetCurrentTimestamp(); + + /* + * To prevent starting the sequencesync worker at a high frequency + * after a failure, we store its last failure time. We start the + * sequencesync worker again after waiting at least + * wal_retrieve_retry_interval. + */ + if (!MyLogicalRepWorker->sequencesync_failure_time || + TimestampDifferenceExceeds(MyLogicalRepWorker->sequencesync_failure_time, + now, wal_retrieve_retry_interval)) + { + MyLogicalRepWorker->sequencesync_failure_time = 0; + + logicalrep_worker_launch(WORKERTYPE_SEQUENCESYNC, + MyLogicalRepWorker->dbid, + MySubscription->oid, + MySubscription->name, + MyLogicalRepWorker->userid, + InvalidOid, + DSM_HANDLE_INVALID); + break; + } + } + } + + if (started_tx) + { + CommitTransactionCommand(); + pgstat_report_stat(true); + } +} + +/* + * fetch_remote_sequence_data + * + * Retrieves sequence data (last_value, log_cnt, page_lsn, and is_called) and + * parameters (seqtypid, seqstart, seqincrement, seqmin, seqmax and seqcycle) + * from a remote node. + * + * Output Parameters: + * - log_cnt: The log count of the sequence. + * - is_called: Indicates if the sequence has been called. + * - page_lsn: The log sequence number of the sequence page. + * - last_value: The last value of the sequence. + * + * Returns: + * - TRUE if parameters match for the local and remote sequences. + * - FALSE if parameters differ for the local and remote sequences. + */ +static bool +fetch_remote_sequence_data(WalReceiverConn *conn, Oid relid, Oid remoteid, + char *nspname, char *relname, int64 *log_cnt, + bool *is_called, XLogRecPtr *page_lsn, + int64 *last_value) +{ +#define REMOTE_SEQ_COL_COUNT 10 + Oid seqRow[REMOTE_SEQ_COL_COUNT] = {INT8OID, INT8OID, BOOLOID, + LSNOID, OIDOID, INT8OID, INT8OID, INT8OID, INT8OID, BOOLOID}; + + WalRcvExecResult *res; + StringInfoData cmd; + TupleTableSlot *slot; + bool isnull; + Oid seqtypid; + int64 seqstart; + int64 seqincrement; + int64 seqmin; + int64 seqmax; + bool seqcycle; + bool seq_params_match; + HeapTuple tup; + Form_pg_sequence seqform; + int col = 0; + + initStringInfo(&cmd); + appendStringInfo(&cmd, + "SELECT last_value, log_cnt, is_called, page_lsn,\n" + "seqtypid, seqstart, seqincrement, seqmin, seqmax, seqcycle\n" + "FROM pg_sequence_state(%d), pg_sequence WHERE seqrelid = %d", + remoteid, remoteid); + + res = walrcv_exec(conn, cmd.data, REMOTE_SEQ_COL_COUNT, seqRow); + pfree(cmd.data); + + if (res->status != WALRCV_OK_TUPLES) + ereport(ERROR, + (errcode(ERRCODE_CONNECTION_FAILURE), + errmsg("could not fetch sequence info for sequence \"%s.%s\" from publisher: %s", + nspname, relname, res->err))); + + /* Process the sequence. */ + slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple); + if (!tuplestore_gettupleslot(res->tuplestore, true, false, slot)) + ereport(ERROR, + errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("sequence \"%s.%s\" not found on publisher", + nspname, relname)); + + *last_value = DatumGetInt64(slot_getattr(slot, ++col, &isnull)); + Assert(!isnull); + + *log_cnt = DatumGetInt64(slot_getattr(slot, ++col, &isnull)); + Assert(!isnull); + + *is_called = DatumGetBool(slot_getattr(slot, ++col, &isnull)); + Assert(!isnull); + + *page_lsn = DatumGetLSN(slot_getattr(slot, ++col, &isnull)); + Assert(!isnull); + + seqtypid = DatumGetObjectId(slot_getattr(slot, ++col, &isnull)); + Assert(!isnull); + + seqstart = DatumGetInt64(slot_getattr(slot, ++col, &isnull)); + Assert(!isnull); + + seqincrement = DatumGetInt64(slot_getattr(slot, ++col, &isnull)); + Assert(!isnull); + + seqmin = DatumGetInt64(slot_getattr(slot, ++col, &isnull)); + Assert(!isnull); + + seqmax = DatumGetInt64(slot_getattr(slot, ++col, &isnull)); + Assert(!isnull); + + seqcycle = DatumGetBool(slot_getattr(slot, ++col, &isnull)); + Assert(!isnull); + + /* Sanity check */ + Assert(col == REMOTE_SEQ_COL_COUNT); + + /* Get the local sequence */ + tup = SearchSysCache1(SEQRELID, ObjectIdGetDatum(relid)); + if (!HeapTupleIsValid(tup)) + elog(ERROR, "cache lookup failed for sequence \"%s.%s\"", + nspname, relname); + + seqform = (Form_pg_sequence) GETSTRUCT(tup); + + seq_params_match = seqform->seqtypid == seqtypid && + seqform->seqmin == seqmin && seqform->seqmax == seqmax && + seqform->seqcycle == seqcycle && + seqform->seqstart == seqstart && + seqform->seqincrement == seqincrement; + + ReleaseSysCache(tup); + ExecDropSingleTupleTableSlot(slot); + walrcv_clear_result(res); + + return seq_params_match; +} + +/* + * Copy existing data of a sequence from the publisher. + * + * Fetch the sequence value from the publisher and set the subscriber sequence + * with the same value. Caller is responsible for locking the local + * relation. + * + * The output parameter 'sequence_mismatch' indicates if a local/remote + * sequence parameter mismatch was detected. + */ +static XLogRecPtr +copy_sequence(WalReceiverConn *conn, Relation rel, bool *sequence_mismatch) +{ + StringInfoData cmd; + int64 seq_last_value; + int64 seq_log_cnt; + bool seq_is_called; + XLogRecPtr seq_page_lsn = InvalidXLogRecPtr; + WalRcvExecResult *res; + Oid seqRow[] = {OIDOID, CHAROID}; + TupleTableSlot *slot; + LogicalRepRelId remoteid; /* unique id of the relation */ + char relkind PG_USED_FOR_ASSERTS_ONLY; + bool isnull; + char *nspname = get_namespace_name(RelationGetNamespace(rel)); + char *relname = RelationGetRelationName(rel); + Oid relid = RelationGetRelid(rel); + + Assert(!*sequence_mismatch); + + /* Fetch Oid. */ + initStringInfo(&cmd); + appendStringInfo(&cmd, "SELECT c.oid, c.relkind\n" + "FROM pg_catalog.pg_class c\n" + "INNER JOIN pg_catalog.pg_namespace n\n" + " ON (c.relnamespace = n.oid)\n" + "WHERE n.nspname = %s AND c.relname = %s", + quote_literal_cstr(nspname), + quote_literal_cstr(relname)); + + res = walrcv_exec(conn, cmd.data, lengthof(seqRow), seqRow); + if (res->status != WALRCV_OK_TUPLES) + ereport(ERROR, + errcode(ERRCODE_CONNECTION_FAILURE), + errmsg("sequence \"%s.%s\" info could not be fetched from publisher: %s", + nspname, relname, res->err)); + + slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple); + if (!tuplestore_gettupleslot(res->tuplestore, true, false, slot)) + ereport(ERROR, + errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("sequence \"%s.%s\" not found on publisher", + nspname, relname)); + + remoteid = DatumGetObjectId(slot_getattr(slot, 1, &isnull)); + Assert(!isnull); + relkind = DatumGetChar(slot_getattr(slot, 2, &isnull)); + Assert(!isnull); + Assert(relkind == RELKIND_SEQUENCE); + + ExecDropSingleTupleTableSlot(slot); + walrcv_clear_result(res); + + *sequence_mismatch = !fetch_remote_sequence_data(conn, relid, remoteid, + nspname, relname, + &seq_log_cnt, &seq_is_called, + &seq_page_lsn, &seq_last_value); + + /* Update the sequence only if the parameters are identical. */ + if (*sequence_mismatch) + return InvalidXLogRecPtr; + else + { + SetSequence(RelationGetRelid(rel), seq_last_value, seq_is_called, + seq_log_cnt); + + /* Return the LSN when the sequence state was set. */ + return seq_page_lsn; + } +} + +/* + * report_mismatched_sequences + * + * Report any sequence mismatches as a single warning log. + */ +static void +report_mismatched_sequences(StringInfo mismatched_seqs) +{ + if (mismatched_seqs->len) + { + ereport(WARNING, + errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("parameters differ for the remote and local sequences (%s) for subscription \"%s\"", + mismatched_seqs->data, MySubscription->name), + errhint("Alter/Re-create local sequences to have the same parameters as the remote sequences.")); + + resetStringInfo(mismatched_seqs); + } +} + +/* + * append_mismatched_sequences + * + * Appends schema name and sequence name of sequences that have discrepancies + * between the publisher and subscriber to the mismatched_seqs string. + */ +static void +append_mismatched_sequences(StringInfo mismatched_seqs, Relation seqrel) +{ + if (mismatched_seqs->len) + appendStringInfoString(mismatched_seqs, ", "); + + appendStringInfo(mismatched_seqs, "\"%s.%s\"", + get_namespace_name(RelationGetNamespace(seqrel)), + RelationGetRelationName(seqrel)); +} + +/* + * Start syncing the sequences in the sequencesync worker. + */ +static void +LogicalRepSyncSequences(void) +{ + char *err; + bool must_use_password; + List *sequences; + List *sequences_not_synced = NIL; + AclResult aclresult; + UserContext ucxt; + bool run_as_owner = false; + int curr_seq = 0; + int seq_count; + int curr_batch_seq = 0; + bool start_txn = true; + bool sequence_sync_error = false; + Oid subid = MyLogicalRepWorker->subid; + MemoryContext oldctx; + StringInfo mismatched_seqs = makeStringInfo(); + StringInfoData app_name; + +/* + * We batch synchronize multiple sequences per transaction, because the + * alternative of synchronizing each sequence individually incurs overhead of + * starting and committing transactions repeatedly. On the other hand, we want + * to avoid keeping this batch transaction open for extended periods so it is + * currently limited to 100 sequences per batch. + */ +#define MAX_SEQUENCES_SYNC_PER_BATCH 100 + + StartTransactionCommand(); + + /* Get the sequences that should be synchronized. */ + sequences = GetSubscriptionRelations(subid, false, true, false); + + /* Allocate the tracking info in a permanent memory context. */ + oldctx = MemoryContextSwitchTo(CacheMemoryContext); + foreach_ptr(SubscriptionRelState, seq_state, sequences) + { + SubscriptionRelState *rstate = palloc(sizeof(SubscriptionRelState)); + + memcpy(rstate, seq_state, sizeof(SubscriptionRelState)); + sequences_not_synced = lappend(sequences_not_synced, rstate); + } + MemoryContextSwitchTo(oldctx); + + CommitTransactionCommand(); + + /* Is the use of a password mandatory? */ + must_use_password = MySubscription->passwordrequired && + !MySubscription->ownersuperuser; + + initStringInfo(&app_name); + appendStringInfo(&app_name, "%s_%s", MySubscription->name, "sequencesync worker"); + + /* + * Establish the connection to the publisher for sequence synchronization. + */ + LogRepWorkerWalRcvConn = + walrcv_connect(MySubscription->conninfo, true, true, + must_use_password, + app_name.data, &err); + if (LogRepWorkerWalRcvConn == NULL) + ereport(ERROR, + errcode(ERRCODE_CONNECTION_FAILURE), + errmsg("could not connect to the publisher: %s", err)); + + pfree(app_name.data); + + seq_count = list_length(sequences_not_synced); + foreach_ptr(SubscriptionRelState, seqinfo, sequences_not_synced) + { + Relation sequence_rel; + XLogRecPtr sequence_lsn; + bool sequence_mismatch = false; + + CHECK_FOR_INTERRUPTS(); + + if (start_txn) + { + StartTransactionCommand(); + start_txn = false; + } + + sequence_rel = table_open(seqinfo->relid, RowExclusiveLock); + + /* + * Make sure that the copy command runs as the sequence owner, unless + * the user has opted out of that behaviour. + */ + run_as_owner = MySubscription->runasowner; + if (!run_as_owner) + SwitchToUntrustedUser(sequence_rel->rd_rel->relowner, &ucxt); + + /* + * Check that our sequencesync worker has permission to insert into + * the target sequence. + */ + aclresult = pg_class_aclcheck(RelationGetRelid(sequence_rel), GetUserId(), + ACL_INSERT); + if (aclresult != ACLCHECK_OK) + aclcheck_error(aclresult, + get_relkind_objtype(sequence_rel->rd_rel->relkind), + RelationGetRelationName(sequence_rel)); + + /* + * In case sequence copy fails, throw a warning for the sequences that + * did not match before exiting. + */ + PG_TRY(); + { + sequence_lsn = copy_sequence(LogRepWorkerWalRcvConn, sequence_rel, + &sequence_mismatch); + } + PG_CATCH(); + { + report_mismatched_sequences(mismatched_seqs); + PG_RE_THROW(); + } + PG_END_TRY(); + + if (sequence_mismatch) + append_mismatched_sequences(mismatched_seqs, sequence_rel); + else + UpdateSubscriptionRelState(subid, seqinfo->relid, + SUBREL_STATE_READY, sequence_lsn); + + table_close(sequence_rel, NoLock); + + curr_seq++; + curr_batch_seq++; + + /* + * Have we reached the end of the current batch of sequences, or last + * remaining sequences to synchronize? + */ + if (curr_batch_seq == MAX_SEQUENCES_SYNC_PER_BATCH || + curr_seq == seq_count) + { + if (message_level_is_interesting(DEBUG1)) + { + /* LOG all the sequences synchronized during current batch. */ + for (int i = 0; i < curr_batch_seq; i++) + { + SubscriptionRelState *done_seq; + + done_seq = (SubscriptionRelState *) lfirst(list_nth_cell(sequences_not_synced, + (curr_seq - curr_batch_seq) + i)); + + ereport(DEBUG1, + errmsg_internal("logical replication synchronization for subscription \"%s\", sequence \"%s\" has finished", + get_subscription_name(subid, false), + get_rel_name(done_seq->relid))); + } + } + + if (mismatched_seqs->len) + { + sequence_sync_error = true; + report_mismatched_sequences(mismatched_seqs); + } + + ereport(LOG, + errmsg("logical replication synchronized %d of %d sequences for subscription \"%s\" ", + curr_seq, seq_count, get_subscription_name(subid, false))); + + /* Commit this batch, and prepare for next batch. */ + CommitTransactionCommand(); + start_txn = true; + + /* Prepare for next batch */ + curr_batch_seq = 0; + } + } + + /* + * Sequence synchronization failed due to a parameter mismatch. Set the + * failure time to prevent immediate initiation of the sequencesync worker. + */ + if (sequence_sync_error) + { + logicalrep_seqsyncworker_set_failuretime(); + ereport(LOG, + errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("sequence synchronization failed because the parameters between the publisher and subscriber do not match for all sequences")); + } + + list_free_deep(sequences_not_synced); + if (!run_as_owner && seq_count) + RestoreUserContext(&ucxt); +} + +/* + * Execute the initial sync with error handling. Disable the subscription, + * if required. + * + * Allocate the slot name in long-lived context on return. Note that we don't + * handle FATAL errors which are probably because of system resource error and + * are not repeatable. + */ +static void +start_sequence_sync() +{ + Assert(am_sequencesync_worker()); + + PG_TRY(); + { + /* Call initial sync. */ + LogicalRepSyncSequences(); + } + PG_CATCH(); + { + if (MySubscription->disableonerr) + DisableSubscriptionAndExit(); + else + { + /* + * Report the worker failed during sequence synchronization. Abort + * the current transaction so that the stats message is sent in an + * idle state. + */ + AbortOutOfAnyTransaction(); + pgstat_report_subscription_error(MySubscription->oid, false); + + PG_RE_THROW(); + } + } + PG_END_TRY(); +} + +/* Logical Replication sequencesync worker entry point */ +void +SequenceSyncWorkerMain(Datum main_arg) +{ + int worker_slot = DatumGetInt32(main_arg); + + SetupApplyOrSyncWorker(worker_slot); + + start_sequence_sync(); + + SyncFinishWorker(WORKERTYPE_SEQUENCESYNC); +} diff --git a/src/backend/replication/logical/syncutils.c b/src/backend/replication/logical/syncutils.c index 3d405ff2dc67..1d7d7543af59 100644 --- a/src/backend/replication/logical/syncutils.c +++ b/src/backend/replication/logical/syncutils.c @@ -50,8 +50,10 @@ static SyncingRelationsState relation_states_validity = SYNC_RELATIONS_STATE_NEE * Exit routine for synchronization worker. */ pg_noreturn void -SyncFinishWorker(void) +SyncFinishWorker(LogicalRepWorkerType wtype) { + Assert(wtype == WORKERTYPE_TABLESYNC || wtype == WORKERTYPE_SEQUENCESYNC); + /* * Commit any outstanding transaction. This is the usual case, unless * there was nothing to do for the table. @@ -66,15 +68,24 @@ SyncFinishWorker(void) XLogFlush(GetXLogWriteRecPtr()); StartTransactionCommand(); - ereport(LOG, - (errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished", - MySubscription->name, - get_rel_name(MyLogicalRepWorker->relid)))); + if (wtype == WORKERTYPE_TABLESYNC) + ereport(LOG, + errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished", + MySubscription->name, + get_rel_name(MyLogicalRepWorker->relid))); + else + ereport(LOG, + errmsg("logical replication sequence synchronization worker for subscription \"%s\" has finished", + MySubscription->name)); CommitTransactionCommand(); /* Find the leader apply worker and signal it. */ logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid); + /* This is a clean exit, so no need for any sequence failure logic. */ + if (wtype == WORKERTYPE_SEQUENCESYNC) + cancel_before_shmem_exit(logicalrep_seqsyncworker_failure, 0); + /* Stop gracefully */ proc_exit(0); } @@ -89,7 +100,9 @@ SyncInvalidateRelationStates(Datum arg, int cacheid, uint32 hashvalue) } /* - * Process possible state change(s) of relations that are being synchronized. + * Process possible state change(s) of relations that are being synchronized + * and start new tablesync workers for the newly added tables. Also, start a + * new sequencesync worker for the newly added sequences. */ void SyncProcessRelations(XLogRecPtr current_lsn) @@ -109,7 +122,19 @@ SyncProcessRelations(XLogRecPtr current_lsn) break; case WORKERTYPE_APPLY: + /* + * We need up-to-date sync state info for subscription tables and + * sequences here. + */ + SyncFetchRelationStates(); + ProcessSyncingTablesForApply(current_lsn); + ProcessSyncingSequencesForApply(); + break; + + case WORKERTYPE_SEQUENCESYNC: + /* Should never happen. */ + Assert(0); break; case WORKERTYPE_UNKNOWN: @@ -121,17 +146,22 @@ SyncProcessRelations(XLogRecPtr current_lsn) /* * Common code to fetch the up-to-date sync state info into the static lists. * - * Returns true if subscription has 1 or more tables, else false. + * The pg_subscription_rel catalog is shared by tables and sequences. Changes + * to either sequences or tables can affect the validity of relation states, so + * we update both table_states_not_ready and sequence_states_not_ready + * simultaneously to ensure consistency. * - * Note: If this function started the transaction (indicated by the parameter) - * then it is the caller's responsibility to commit it. + * Returns true if subscription has 1 or more tables, else false. */ bool -SyncFetchRelationStates(bool *started_tx) +SyncFetchRelationStates() { + /* + * has_subtables is declared as static, since the same value can be used + * until the system table is invalidated. + */ static bool has_subtables = false; - - *started_tx = false; + bool started_tx = false; if (relation_states_validity != SYNC_RELATIONS_STATE_VALID) { @@ -144,16 +174,19 @@ SyncFetchRelationStates(bool *started_tx) /* Clean the old lists. */ list_free_deep(table_states_not_ready); + list_free_deep(sequence_states_not_ready); table_states_not_ready = NIL; + sequence_states_not_ready = NIL; if (!IsTransactionState()) { StartTransactionCommand(); - *started_tx = true; + started_tx = true; } - /* Fetch tables that are in non-ready state. */ - rstates = GetSubscriptionRelations(MySubscription->oid, true); + /* Fetch tables and sequences that are in non-ready state. */ + rstates = GetSubscriptionRelations(MySubscription->oid, true, true, + false); /* Allocate the tracking info in a permanent memory context. */ oldctx = MemoryContextSwitchTo(CacheMemoryContext); @@ -161,7 +194,11 @@ SyncFetchRelationStates(bool *started_tx) { rstate = palloc(sizeof(SubscriptionRelState)); memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState)); - table_states_not_ready = lappend(table_states_not_ready, rstate); + + if (get_rel_relkind(rstate->relid) == RELKIND_SEQUENCE) + sequence_states_not_ready = lappend(sequence_states_not_ready, rstate); + else + table_states_not_ready = lappend(table_states_not_ready, rstate); } MemoryContextSwitchTo(oldctx); @@ -186,5 +223,11 @@ SyncFetchRelationStates(bool *started_tx) relation_states_validity = SYNC_RELATIONS_STATE_VALID; } + if (started_tx) + { + CommitTransactionCommand(); + pgstat_report_stat(true); + } + return has_subtables; } diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index 9bd51ceef481..688e5c85c474 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -161,7 +161,7 @@ wait_for_table_state_change(Oid relid, char expected_state) /* Check if the sync worker is still running and bail if not. */ LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); worker = logicalrep_worker_find(MyLogicalRepWorker->subid, relid, - false); + WORKERTYPE_TABLESYNC, false); LWLockRelease(LogicalRepWorkerLock); if (!worker) break; @@ -208,7 +208,7 @@ wait_for_worker_state_change(char expected_state) */ LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); worker = logicalrep_worker_find(MyLogicalRepWorker->subid, - InvalidOid, false); + InvalidOid, WORKERTYPE_APPLY, false); if (worker && worker->proc) logicalrep_worker_wakeup_ptr(worker); LWLockRelease(LogicalRepWorkerLock); @@ -334,7 +334,7 @@ ProcessSyncingTablesForSync(XLogRecPtr current_lsn) */ replorigin_drop_by_name(originname, true, false); - SyncFinishWorker(); + SyncFinishWorker(WORKERTYPE_TABLESYNC); } else SpinLockRelease(&MyLogicalRepWorker->relmutex); @@ -376,9 +376,6 @@ ProcessSyncingTablesForApply(XLogRecPtr current_lsn) Assert(!IsTransactionState()); - /* We need up-to-date sync state info for subscription tables here. */ - SyncFetchRelationStates(&started_tx); - /* * Prepare a hash table for tracking last start times of workers, to avoid * immediate restarts. We don't need it if there are no tables that need @@ -411,6 +408,14 @@ ProcessSyncingTablesForApply(XLogRecPtr current_lsn) { SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc); + if (!started_tx) + { + StartTransactionCommand(); + started_tx = true; + } + + Assert(get_rel_relkind(rstate->relid) != RELKIND_SEQUENCE); + if (rstate->state == SUBREL_STATE_SYNCDONE) { /* @@ -424,11 +429,6 @@ ProcessSyncingTablesForApply(XLogRecPtr current_lsn) rstate->state = SUBREL_STATE_READY; rstate->lsn = current_lsn; - if (!started_tx) - { - StartTransactionCommand(); - started_tx = true; - } /* * Remove the tablesync origin tracking if exists. @@ -465,8 +465,8 @@ ProcessSyncingTablesForApply(XLogRecPtr current_lsn) LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); syncworker = logicalrep_worker_find(MyLogicalRepWorker->subid, - rstate->relid, false); - + rstate->relid, + WORKERTYPE_TABLESYNC, true); if (syncworker) { /* Found one, update our copy of its state */ @@ -1243,7 +1243,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos) case SUBREL_STATE_SYNCDONE: case SUBREL_STATE_READY: case SUBREL_STATE_UNKNOWN: - SyncFinishWorker(); /* doesn't return */ + SyncFinishWorker(WORKERTYPE_TABLESYNC); /* doesn't return */ } /* Calculate the name of the tablesync slot. */ @@ -1561,7 +1561,7 @@ run_tablesync_worker() /* Logical Replication Tablesync worker entry point */ void -TablesyncWorkerMain(Datum main_arg) +TableSyncWorkerMain(Datum main_arg) { int worker_slot = DatumGetInt32(main_arg); @@ -1569,7 +1569,7 @@ TablesyncWorkerMain(Datum main_arg) run_tablesync_worker(); - SyncFinishWorker(); + SyncFinishWorker(WORKERTYPE_TABLESYNC); } /* @@ -1583,23 +1583,16 @@ TablesyncWorkerMain(Datum main_arg) bool AllTablesyncsReady(void) { - bool started_tx = false; - bool has_subrels = false; + bool has_tables; /* We need up-to-date sync state info for subscription tables here. */ - has_subrels = SyncFetchRelationStates(&started_tx); - - if (started_tx) - { - CommitTransactionCommand(); - pgstat_report_stat(true); - } + has_tables = SyncFetchRelationStates(); /* * Return false when there are no tables in subscription or not all tables * are in ready state; true otherwise. */ - return has_subrels && (table_states_not_ready == NIL); + return has_tables && (table_states_not_ready == NIL); } /* diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 765754bfc3c6..1742968427a8 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -489,6 +489,11 @@ should_apply_changes_for_rel(LogicalRepRelMapEntry *rel) (rel->state == SUBREL_STATE_SYNCDONE && rel->statelsn <= remote_final_lsn)); + case WORKERTYPE_SEQUENCESYNC: + /* Should never happen. */ + Assert(0); + break; + case WORKERTYPE_UNKNOWN: /* Should never happen. */ elog(ERROR, "Unknown worker type"); @@ -1029,7 +1034,10 @@ apply_handle_commit(StringInfo s) apply_handle_commit_internal(&commit_data); - /* Process any tables that are being synchronized in parallel. */ + /* + * Process any tables that are being synchronized in parallel and any + * newly added relations. + */ SyncProcessRelations(commit_data.end_lsn); pgstat_report_activity(STATE_IDLE, NULL); @@ -1151,7 +1159,10 @@ apply_handle_prepare(StringInfo s) in_remote_transaction = false; - /* Process any tables that are being synchronized in parallel. */ + /* + * Process any tables that are being synchronized in parallel and any + * newly added relations. + */ SyncProcessRelations(prepare_data.end_lsn); /* @@ -1207,7 +1218,10 @@ apply_handle_commit_prepared(StringInfo s) store_flush_position(prepare_data.end_lsn, XactLastCommitEnd); in_remote_transaction = false; - /* Process any tables that are being synchronized in parallel. */ + /* + * Process any tables that are being synchronized in parallel and any + * newly added relations. + */ SyncProcessRelations(prepare_data.end_lsn); clear_subscription_skip_lsn(prepare_data.end_lsn); @@ -1273,7 +1287,10 @@ apply_handle_rollback_prepared(StringInfo s) store_flush_position(rollback_data.rollback_end_lsn, InvalidXLogRecPtr); in_remote_transaction = false; - /* Process any tables that are being synchronized in parallel. */ + /* + * Process any tables that are being synchronized in parallel and any + * newly added relations. + */ SyncProcessRelations(rollback_data.rollback_end_lsn); pgstat_report_activity(STATE_IDLE, NULL); @@ -1408,7 +1425,10 @@ apply_handle_stream_prepare(StringInfo s) pgstat_report_stat(false); - /* Process any tables that are being synchronized in parallel. */ + /* + * Process any tables that are being synchronized in parallel and any + * newly added relations. + */ SyncProcessRelations(prepare_data.end_lsn); /* @@ -2250,7 +2270,10 @@ apply_handle_stream_commit(StringInfo s) break; } - /* Process any tables that are being synchronized in parallel. */ + /* + * Process any tables that are being synchronized in parallel and any + * newly added relations. + */ SyncProcessRelations(commit_data.end_lsn); pgstat_report_activity(STATE_IDLE, NULL); @@ -3727,7 +3750,10 @@ LogicalRepApplyLoop(XLogRecPtr last_received) AcceptInvalidationMessages(); maybe_reread_subscription(); - /* Process any table synchronization changes. */ + /* + * Process any tables that are being synchronized in parallel and + * any newly added relations. + */ SyncProcessRelations(last_received); } @@ -4648,8 +4674,8 @@ run_apply_worker() } /* - * Common initialization for leader apply worker, parallel apply worker and - * tablesync worker. + * Common initialization for leader apply worker, parallel apply worker, + * tablesync worker and sequencesync worker. * * Initialize the database connection, in-memory subscription and necessary * config options. @@ -4728,6 +4754,10 @@ InitializeLogRepWorker(void) (errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has started", MySubscription->name, get_rel_name(MyLogicalRepWorker->relid)))); + else if (am_sequencesync_worker()) + ereport(LOG, + (errmsg("logical replication sequence synchronization worker for subscription \"%s\" has started", + MySubscription->name))); else ereport(LOG, (errmsg("logical replication apply worker for subscription \"%s\" has started", @@ -4747,14 +4777,17 @@ replorigin_reset(int code, Datum arg) replorigin_session_origin_timestamp = 0; } -/* Common function to setup the leader apply or tablesync worker. */ +/* + * Common function to setup the leader apply, tablesync worker and sequencesync + * worker. + */ void SetupApplyOrSyncWorker(int worker_slot) { /* Attach to slot */ logicalrep_worker_attach(worker_slot); - Assert(am_tablesync_worker() || am_leader_apply_worker()); + Assert(am_tablesync_worker() || am_sequencesync_worker() || am_leader_apply_worker()); /* Setup signal handling */ pqsignal(SIGHUP, SignalHandlerForConfigReload); @@ -4799,6 +4832,9 @@ SetupApplyOrSyncWorker(int worker_slot) CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP, SyncInvalidateRelationStates, (Datum) 0); + + if (am_sequencesync_worker()) + before_shmem_exit(logicalrep_seqsyncworker_failure, (Datum) 0); } /* Logical Replication Apply worker entry point */ diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 2f8cbd867599..c8779efe1836 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3356,7 +3356,7 @@ struct config_int ConfigureNamesInt[] = {"max_sync_workers_per_subscription", PGC_SIGHUP, REPLICATION_SUBSCRIBERS, - gettext_noop("Maximum number of table synchronization workers per subscription."), + gettext_noop("Maximum number of workers per subscription for synchronizing tables and sequences."), NULL, }, &max_sync_workers_per_subscription, diff --git a/src/bin/pg_dump/common.c b/src/bin/pg_dump/common.c index 56b6c368acf8..5c5a775d40d7 100644 --- a/src/bin/pg_dump/common.c +++ b/src/bin/pg_dump/common.c @@ -243,8 +243,8 @@ getSchemaData(Archive *fout, int *numTablesPtr) pg_log_info("reading subscriptions"); getSubscriptions(fout); - pg_log_info("reading subscription membership of tables"); - getSubscriptionTables(fout); + pg_log_info("reading subscription membership of relations"); + getSubscriptionRelations(fout); free(inhinfo); /* not needed any longer */ diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index bd41c0092152..95ce19a88435 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -5137,12 +5137,12 @@ getSubscriptions(Archive *fout) } /* - * getSubscriptionTables - * Get information about subscription membership for dumpable tables. This + * getSubscriptionRelations + * Get information about subscription membership for dumpable relations. This * will be used only in binary-upgrade mode for PG17 or later versions. */ void -getSubscriptionTables(Archive *fout) +getSubscriptionRelations(Archive *fout) { DumpOptions *dopt = fout->dopt; SubscriptionInfo *subinfo = NULL; @@ -5196,7 +5196,7 @@ getSubscriptionTables(Archive *fout) tblinfo = findTableByOid(relid); if (tblinfo == NULL) - pg_fatal("failed sanity check, table with OID %u not found", + pg_fatal("failed sanity check, relation with OID %u not found", relid); /* OK, make a DumpableObject for this relationship */ diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h index 76aa26fa7143..b43c44e4b052 100644 --- a/src/bin/pg_dump/pg_dump.h +++ b/src/bin/pg_dump/pg_dump.h @@ -817,6 +817,6 @@ extern void getPublicationNamespaces(Archive *fout); extern void getPublicationTables(Archive *fout, TableInfo tblinfo[], int numTables); extern void getSubscriptions(Archive *fout); -extern void getSubscriptionTables(Archive *fout); +extern void getSubscriptionRelations(Archive *fout); #endif /* PG_DUMP_H */ diff --git a/src/bin/psql/tab-complete.in.c b/src/bin/psql/tab-complete.in.c index 10dc03cd7cb9..6fddb5ea6356 100644 --- a/src/bin/psql/tab-complete.in.c +++ b/src/bin/psql/tab-complete.in.c @@ -2288,7 +2288,7 @@ match_previous_words(int pattern_id, "ADD PUBLICATION", "DROP PUBLICATION"); /* ALTER SUBSCRIPTION REFRESH PUBLICATION */ else if (Matches("ALTER", "SUBSCRIPTION", MatchAny, MatchAnyN, "REFRESH", "PUBLICATION")) - COMPLETE_WITH("WITH ("); + COMPLETE_WITH("SEQUENCES", "WITH ("); /* ALTER SUBSCRIPTION REFRESH PUBLICATION WITH ( */ else if (Matches("ALTER", "SUBSCRIPTION", MatchAny, MatchAnyN, "REFRESH", "PUBLICATION", "WITH", "(")) COMPLETE_WITH("copy_data"); diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 8071134643c3..a84fb5065718 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -12268,6 +12268,11 @@ proargmodes => '{v,o,o,o,o}', proargnames => '{pubname,pubid,relid,attrs,qual}', prosrc => 'pg_get_publication_tables' }, +{ oid => '8052', descr => 'get OIDs of sequences in a publication', + proname => 'pg_get_publication_sequences', prorows => '1000', proretset => 't', + provolatile => 's', prorettype => 'oid', proargtypes => 'text', + proallargtypes => '{text,oid}', proargmodes => '{i,o}', + proargnames => '{pubname,relid}', prosrc => 'pg_get_publication_sequences' }, { oid => '6121', descr => 'returns whether a relation can be part of a publication', proname => 'pg_relation_is_publishable', provolatile => 's', diff --git a/src/include/catalog/pg_subscription_rel.h b/src/include/catalog/pg_subscription_rel.h index ea869588d842..0c706bd9cd57 100644 --- a/src/include/catalog/pg_subscription_rel.h +++ b/src/include/catalog/pg_subscription_rel.h @@ -90,6 +90,8 @@ extern char GetSubscriptionRelState(Oid subid, Oid relid, XLogRecPtr *sublsn); extern void RemoveSubscriptionRel(Oid subid, Oid relid); extern bool HasSubscriptionTables(Oid subid); -extern List *GetSubscriptionRelations(Oid subid, bool not_ready); +extern List *GetSubscriptionRelations(Oid subid, bool get_tables, + bool get_sequences, + bool all_states); #endif /* PG_SUBSCRIPTION_REL_H */ diff --git a/src/include/commands/sequence.h b/src/include/commands/sequence.h index 9ac0b67683d3..26e3c9096ae8 100644 --- a/src/include/commands/sequence.h +++ b/src/include/commands/sequence.h @@ -45,6 +45,8 @@ typedef FormData_pg_sequence_data *Form_pg_sequence_data; /* XLOG stuff */ #define XLOG_SEQ_LOG 0x00 +#define SEQ_LOG_CNT_INVALID 0 + typedef struct xl_seq_rec { RelFileLocator locator; @@ -60,6 +62,7 @@ extern ObjectAddress AlterSequence(ParseState *pstate, AlterSeqStmt *stmt); extern void SequenceChangePersistence(Oid relid, char newrelpersistence); extern void DeleteSequenceTuple(Oid relid); extern void ResetSequence(Oid seq_relid); +extern void SetSequence(Oid relid, int64 next, bool is_called, int64 log_cnt); extern void ResetSequenceCaches(void); extern void seq_redo(XLogReaderState *record); diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 9b9656dd6e3c..e3db33e85fb2 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -4321,7 +4321,8 @@ typedef enum AlterSubscriptionType ALTER_SUBSCRIPTION_SET_PUBLICATION, ALTER_SUBSCRIPTION_ADD_PUBLICATION, ALTER_SUBSCRIPTION_DROP_PUBLICATION, - ALTER_SUBSCRIPTION_REFRESH, + ALTER_SUBSCRIPTION_REFRESH_PUBLICATION, + ALTER_SUBSCRIPTION_REFRESH_PUBLICATION_SEQUENCES, ALTER_SUBSCRIPTION_ENABLED, ALTER_SUBSCRIPTION_SKIP, } AlterSubscriptionType; diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h index 88912606e4d5..56fa79b648e7 100644 --- a/src/include/replication/logicalworker.h +++ b/src/include/replication/logicalworker.h @@ -18,7 +18,8 @@ extern PGDLLIMPORT volatile sig_atomic_t ParallelApplyMessagePending; extern void ApplyWorkerMain(Datum main_arg); extern void ParallelApplyWorkerMain(Datum main_arg); -extern void TablesyncWorkerMain(Datum main_arg); +extern void TableSyncWorkerMain(Datum main_arg); +extern void SequenceSyncWorkerMain(Datum main_arg); extern bool IsLogicalWorker(void); extern bool IsLogicalParallelApplyWorker(void); diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h index 082e2b3d86c9..7b6fe125b995 100644 --- a/src/include/replication/worker_internal.h +++ b/src/include/replication/worker_internal.h @@ -30,6 +30,7 @@ typedef enum LogicalRepWorkerType { WORKERTYPE_UNKNOWN = 0, WORKERTYPE_TABLESYNC, + WORKERTYPE_SEQUENCESYNC, WORKERTYPE_APPLY, WORKERTYPE_PARALLEL_APPLY, } LogicalRepWorkerType; @@ -92,6 +93,8 @@ typedef struct LogicalRepWorker TimestampTz last_recv_time; XLogRecPtr reply_lsn; TimestampTz reply_time; + + TimestampTz sequencesync_failure_time; } LogicalRepWorker; /* @@ -238,9 +241,11 @@ extern PGDLLIMPORT bool in_remote_transaction; extern PGDLLIMPORT bool InitializingApplyWorker; extern PGDLLIMPORT List *table_states_not_ready; +extern PGDLLIMPORT List *sequence_states_not_ready; extern void logicalrep_worker_attach(int slot); extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid, + LogicalRepWorkerType wtype, bool only_running); extern List *logicalrep_workers_find(Oid subid, bool only_running, bool acquire_lock); @@ -248,13 +253,17 @@ extern bool logicalrep_worker_launch(LogicalRepWorkerType wtype, Oid dbid, Oid subid, const char *subname, Oid userid, Oid relid, dsm_handle subworker_dsm); -extern void logicalrep_worker_stop(Oid subid, Oid relid); +extern void logicalrep_worker_stop(Oid subid, Oid relid, + LogicalRepWorkerType wtype); extern void logicalrep_pa_worker_stop(ParallelApplyWorkerInfo *winfo); extern void logicalrep_worker_wakeup(Oid subid, Oid relid); extern void logicalrep_worker_wakeup_ptr(LogicalRepWorker *worker); extern int logicalrep_sync_worker_count(Oid subid); +extern void logicalrep_seqsyncworker_set_failuretime(void); +extern void logicalrep_seqsyncworker_failure(int code, Datum arg); + extern void ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid, char *originname, Size szoriginname); @@ -263,12 +272,13 @@ extern void UpdateTwoPhaseState(Oid suboid, char new_state); extern void ProcessSyncingTablesForSync(XLogRecPtr current_lsn); extern void ProcessSyncingTablesForApply(XLogRecPtr current_lsn); +extern void ProcessSyncingSequencesForApply(void); -pg_noreturn extern void SyncFinishWorker(void); +pg_noreturn extern void SyncFinishWorker(LogicalRepWorkerType wtype); extern void SyncInvalidateRelationStates(Datum arg, int cacheid, uint32 hashvalue); extern void SyncProcessRelations(XLogRecPtr current_lsn); -extern bool SyncFetchRelationStates(bool *started_tx); +extern bool SyncFetchRelationStates(void); extern void stream_start_internal(TransactionId xid, bool first_segment); extern void stream_stop_internal(TransactionId xid); @@ -333,15 +343,25 @@ extern void pa_decr_and_wait_stream_block(void); extern void pa_xact_finish(ParallelApplyWorkerInfo *winfo, XLogRecPtr remote_lsn); +#define isApplyWorker(worker) ((worker)->in_use && \ + (worker)->type == WORKERTYPE_APPLY) #define isParallelApplyWorker(worker) ((worker)->in_use && \ (worker)->type == WORKERTYPE_PARALLEL_APPLY) -#define isTablesyncWorker(worker) ((worker)->in_use && \ +#define isTableSyncWorker(worker) ((worker)->in_use && \ (worker)->type == WORKERTYPE_TABLESYNC) +#define isSequenceSyncWorker(worker) ((worker)->in_use && \ + (worker)->type == WORKERTYPE_SEQUENCESYNC) static inline bool am_tablesync_worker(void) { - return isTablesyncWorker(MyLogicalRepWorker); + return isTableSyncWorker(MyLogicalRepWorker); +} + +static inline bool +am_sequencesync_worker(void) +{ + return isSequenceSyncWorker(MyLogicalRepWorker); } static inline bool diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 6cf828ca8d0d..2c4d1b78649c 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1458,6 +1458,14 @@ pg_prepared_xacts| SELECT p.transaction, FROM ((pg_prepared_xact() p(transaction, gid, prepared, ownerid, dbid) LEFT JOIN pg_authid u ON ((p.ownerid = u.oid))) LEFT JOIN pg_database d ON ((p.dbid = d.oid))); +pg_publication_sequences| SELECT p.pubname, + n.nspname AS schemaname, + c.relname AS sequencename + FROM pg_publication p, + LATERAL pg_get_publication_sequences((p.pubname)::text) gps(relid), + (pg_class c + JOIN pg_namespace n ON ((n.oid = c.relnamespace))) + WHERE (c.oid = gps.relid); pg_publication_tables| SELECT p.pubname, n.nspname AS schemaname, c.relname AS tablename, diff --git a/src/test/regress/expected/subscription.out b/src/test/regress/expected/subscription.out index 1443e1d92929..66dcd71eefac 100644 --- a/src/test/regress/expected/subscription.out +++ b/src/test/regress/expected/subscription.out @@ -107,7 +107,7 @@ HINT: To initiate replication, you must manually create the replication slot, e ALTER SUBSCRIPTION regress_testsub3 ENABLE; ERROR: cannot enable subscription that does not have a slot name ALTER SUBSCRIPTION regress_testsub3 REFRESH PUBLICATION; -ERROR: ALTER SUBSCRIPTION ... REFRESH is not allowed for disabled subscriptions +ERROR: ALTER SUBSCRIPTION ... REFRESH PUBLICATION is not allowed for disabled subscriptions -- fail - origin must be either none or any CREATE SUBSCRIPTION regress_testsub4 CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (slot_name = NONE, connect = false, origin = foo); ERROR: unrecognized origin value: "foo" @@ -352,7 +352,7 @@ ERROR: ALTER SUBSCRIPTION with refresh cannot run inside a transaction block END; BEGIN; ALTER SUBSCRIPTION regress_testsub REFRESH PUBLICATION; -ERROR: ALTER SUBSCRIPTION ... REFRESH cannot run inside a transaction block +ERROR: ALTER SUBSCRIPTION ... REFRESH PUBLICATION cannot run inside a transaction block END; CREATE FUNCTION func() RETURNS VOID AS $$ ALTER SUBSCRIPTION regress_testsub SET PUBLICATION mypub WITH (refresh = true) $$ LANGUAGE SQL; diff --git a/src/test/subscription/meson.build b/src/test/subscription/meson.build index 586ffba434e1..a6c267a8a2ce 100644 --- a/src/test/subscription/meson.build +++ b/src/test/subscription/meson.build @@ -42,6 +42,7 @@ tests += { 't/033_run_as_table_owner.pl', 't/034_temporal.pl', 't/035_conflicts.pl', + 't/036_sequences.pl', 't/100_bugs.pl', ], }, diff --git a/src/test/subscription/t/036_sequences.pl b/src/test/subscription/t/036_sequences.pl new file mode 100644 index 000000000000..cf5904f3e061 --- /dev/null +++ b/src/test/subscription/t/036_sequences.pl @@ -0,0 +1,227 @@ + +# Copyright (c) 2025, PostgreSQL Global Development Group + +# This tests that sequences are synced correctly to the subscriber +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Initialize publisher node +my $node_publisher = PostgreSQL::Test::Cluster->new('publisher'); + +# Avoid checkpoint during the test, otherwise, extra values will be fetched for +# the sequences which will cause the test to fail randomly. +$node_publisher->init(allows_streaming => 'logical'); +$node_publisher->append_conf('postgresql.conf', 'checkpoint_timeout = 1h'); +$node_publisher->start; + +# Initialize subscriber node +my $node_subscriber = PostgreSQL::Test::Cluster->new('subscriber'); +$node_subscriber->init(allows_streaming => 'logical'); +$node_subscriber->start; + +# Setup structure on the publisher +my $ddl = qq( + CREATE TABLE regress_seq_test (v BIGINT); + CREATE SEQUENCE regress_s1; +); +$node_publisher->safe_psql('postgres', $ddl); + +# Setup the same structure on the subscriber, plus some extra sequences that +# we'll create on the publisher later +$ddl = qq( + CREATE TABLE regress_seq_test (v BIGINT); + CREATE SEQUENCE regress_s1; + CREATE SEQUENCE regress_s2; + CREATE SEQUENCE regress_s3; + CREATE SEQUENCE regress_s4 +); +$node_subscriber->safe_psql('postgres', $ddl); + +# Insert initial test data +$node_publisher->safe_psql( + 'postgres', qq( + -- generate a number of values using the sequence + INSERT INTO regress_seq_test SELECT nextval('regress_s1') FROM generate_series(1,100); +)); + +# Setup logical replication pub/sub +my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; +$node_publisher->safe_psql('postgres', + "CREATE PUBLICATION regress_seq_pub FOR ALL SEQUENCES"); +$node_subscriber->safe_psql('postgres', + "CREATE SUBSCRIPTION regress_seq_sub CONNECTION '$publisher_connstr' PUBLICATION regress_seq_pub" +); + +# Wait for initial sync to finish +my $synced_query = + "SELECT count(1) = 0 FROM pg_subscription_rel WHERE srsubstate NOT IN ('r');"; +$node_subscriber->poll_query_until('postgres', $synced_query) + or die "Timed out while waiting for subscriber to synchronize data"; + +# Check the initial data on subscriber +my $result = $node_subscriber->safe_psql( + 'postgres', qq( + SELECT last_value, log_cnt, is_called FROM regress_s1; +)); +is($result, '100|32|t', 'initial test data replicated'); + +########## +## ALTER SUBSCRIPTION ... REFRESH PUBLICATION should cause sync of new +# sequences of the publisher, but changes to existing sequences should +# not be synced. +########## + +# Create a new sequence 'regress_s2', and update existing sequence 'regress_s1' +$node_publisher->safe_psql( + 'postgres', qq( + CREATE SEQUENCE regress_s2; + INSERT INTO regress_seq_test SELECT nextval('regress_s2') FROM generate_series(1,100); + + -- Existing sequence + INSERT INTO regress_seq_test SELECT nextval('regress_s1') FROM generate_series(1,100); +)); + +# Do ALTER SUBSCRIPTION ... REFRESH PUBLICATION +$result = $node_subscriber->safe_psql( + 'postgres', qq( + ALTER SUBSCRIPTION regress_seq_sub REFRESH PUBLICATION +)); +$node_subscriber->poll_query_until('postgres', $synced_query) + or die "Timed out while waiting for subscriber to synchronize data"; + +$result = $node_publisher->safe_psql( + 'postgres', qq( + SELECT last_value, log_cnt, is_called FROM regress_s1; +)); +is($result, '200|31|t', 'Check sequence value in the publisher'); + +# Check - existing sequence is not synced +$result = $node_subscriber->safe_psql( + 'postgres', qq( + SELECT last_value, log_cnt, is_called FROM regress_s1; +)); +is($result, '100|32|t', + 'REFRESH PUBLICATION does not sync existing sequence'); + +# Check - newly published sequence is synced +$result = $node_subscriber->safe_psql( + 'postgres', qq( + SELECT last_value, log_cnt, is_called FROM regress_s2; +)); +is($result, '100|32|t', + 'REFRESH PUBLICATION will sync newly published sequence'); + +########## +## ALTER SUBSCRIPTION ... REFRESH PUBLICATION SEQUENCES should cause sync of +# new sequences of the publisher, and changes to existing sequences should +# also be synced. +########## + +# Create a new sequence 'regress_s3', and update the existing sequence +# 'regress_s2'. +$node_publisher->safe_psql( + 'postgres', qq( + CREATE SEQUENCE regress_s3; + INSERT INTO regress_seq_test SELECT nextval('regress_s3') FROM generate_series(1,100); + + -- Existing sequence + INSERT INTO regress_seq_test SELECT nextval('regress_s2') FROM generate_series(1,100); +)); + +# Do ALTER SUBSCRIPTION ... REFRESH PUBLICATION SEQUENCES +$result = $node_subscriber->safe_psql( + 'postgres', qq( + ALTER SUBSCRIPTION regress_seq_sub REFRESH PUBLICATION SEQUENCES +)); +$node_subscriber->poll_query_until('postgres', $synced_query) + or die "Timed out while waiting for subscriber to synchronize data"; + +# Check - existing sequences are synced +$result = $node_subscriber->safe_psql( + 'postgres', qq( + SELECT last_value, log_cnt, is_called FROM regress_s1; +)); +is($result, '200|31|t', + 'REFRESH PUBLICATION SEQUENCES will sync existing sequences'); +$result = $node_subscriber->safe_psql( + 'postgres', qq( + SELECT last_value, log_cnt, is_called FROM regress_s2; +)); +is($result, '200|31|t', + 'REFRESH PUBLICATION SEQUENCES will sync existing sequences'); + +# Check - newly published sequence is synced +$result = $node_subscriber->safe_psql( + 'postgres', qq( + SELECT last_value, log_cnt, is_called FROM regress_s3; +)); +is($result, '100|32|t', + 'REFRESH PUBLICATION SEQUENCES will sync newly published sequence'); + +########## +## ALTER SUBSCRIPTION ... REFRESH PUBLICATION with (copy_data = off) should +# not update the sequence values for the new sequence. +########## + +# Create a new sequence 'regress_s4' +$node_publisher->safe_psql( + 'postgres', qq( + CREATE SEQUENCE regress_s4; + INSERT INTO regress_seq_test SELECT nextval('regress_s4') FROM generate_series(1,100); +)); + +# Do ALTER SUBSCRIPTION ... REFRESH PUBLICATION +$result = $node_subscriber->safe_psql( + 'postgres', qq( + ALTER SUBSCRIPTION regress_seq_sub REFRESH PUBLICATION with (copy_data = false); +)); +$node_subscriber->poll_query_until('postgres', $synced_query) + or die "Timed out while waiting for subscriber to synchronize data"; + +$result = $node_publisher->safe_psql( + 'postgres', qq( + SELECT last_value, log_cnt, is_called FROM regress_s4; +)); +is($result, '100|32|t', 'Check sequence value in the publisher'); + +# Check - newly published sequence values are not updated +$result = $node_subscriber->safe_psql( + 'postgres', qq( + SELECT last_value, log_cnt, is_called FROM regress_s4; +)); +is($result, '1|0|f', + 'REFRESH PUBLICATION will not sync newly published sequence with copy_data as off'); + +########## +# ALTER SUBSCRIPTION ... REFRESH PUBLICATION SEQUENCES should throw a warning +# for sequence definition not matching between the publisher and the subscriber. +########## + +# Create a new sequence 'regress_s5' whose START value is not the same in the +# publisher and subscriber. +$node_publisher->safe_psql( + 'postgres', qq( + CREATE SEQUENCE regress_s5 START 1 INCREMENT 2; +)); + +$node_subscriber->safe_psql( + 'postgres', qq( + CREATE SEQUENCE regress_s5 START 10 INCREMENT 2; +)); + +my $log_offset = -s $node_subscriber->logfile; + +# Do ALTER SUBSCRIPTION ... REFRESH PUBLICATION SEQUENCES +$node_subscriber->safe_psql( + 'postgres', " + ALTER SUBSCRIPTION regress_seq_sub REFRESH PUBLICATION SEQUENCES" +); + +# Confirm that the warning for parameters differing is logged. +$node_subscriber->wait_for_log( + qr/WARNING: ( [A-Z0-9]+:)? parameters differ for the remote and local sequences \("public.regress_s5"\) for subscription "regress_seq_sub"/, + $log_offset); +done_testing(); From fd43dc890bc7304ba1cfe5a80c84b8b2a215e5cd Mon Sep 17 00:00:00 2001 From: Vignesh Date: Mon, 3 Feb 2025 10:30:51 +0530 Subject: [PATCH 5/5] Documentation for sequence synchronization feature. Documentation for sequence synchronization feature. --- doc/src/sgml/catalogs.sgml | 29 ++- doc/src/sgml/config.sgml | 16 +- doc/src/sgml/logical-replication.sgml | 241 ++++++++++++++++++++-- doc/src/sgml/monitoring.sgml | 5 +- doc/src/sgml/ref/alter_subscription.sgml | 55 ++++- doc/src/sgml/ref/create_subscription.sgml | 6 + doc/src/sgml/system-views.sgml | 67 ++++++ 7 files changed, 377 insertions(+), 42 deletions(-) diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index cbd4e40a320b..31bbfe08d005 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -8155,16 +8155,19 @@ SCRAM-SHA-256$<iteration count>:&l - The catalog pg_subscription_rel contains the - state for each replicated relation in each subscription. This is a - many-to-many mapping. + The catalog pg_subscription_rel stores the + state of each replicated table and sequence for each subscription. This + is a many-to-many mapping. - This catalog only contains tables known to the subscription after running - either CREATE SUBSCRIPTION or - ALTER SUBSCRIPTION ... REFRESH - PUBLICATION. + This catalog only contains tables and sequences known to the subscription + after running + CREATE SUBSCRIPTION or + + ALTER SUBSCRIPTION ... REFRESH PUBLICATION or + + ALTER SUBSCRIPTION ... REFRESH PUBLICATION SEQUENCES. @@ -8198,7 +8201,7 @@ SCRAM-SHA-256$<iteration count>:&l (references pg_class.oid) - Reference to relation + Reference to table or sequence @@ -8207,12 +8210,20 @@ SCRAM-SHA-256$<iteration count>:&l srsubstate char - State code: + State code for the table or sequence. + + + State codes for tables: i = initialize, d = data is being copied, f = finished table copy, s = synchronized, r = ready (normal replication) + + + State codes for sequences: + i = initialize, + r = ready diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 14661ac2cc63..a4b77ea76ae5 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -5167,9 +5167,9 @@ ANY num_sync ( num_sync ( num_sync ( . @@ -1786,6 +1790,201 @@ test_sub=# SELECT * from tab_gen_to_gen; + + Replicating Sequences + + + To replicate sequences from a publisher to a subscriber, first publish them + using + CREATE PUBLICATION ... FOR ALL SEQUENCES. + + + + At the subscriber side: + + + + use CREATE SUBSCRIPTION + to initially synchronize the published sequences. + + + + + use + ALTER SUBSCRIPTION ... REFRESH PUBLICATION + to synchronize only newly added sequences. + + + + + use + ALTER SUBSCRIPTION ... REFRESH PUBLICATION SEQUENCES + to re-synchronize all sequences. + + + + + + + A new sequence synchronization worker will be started + after executing any of the above subscriber commands, and will exit once the + sequences are synchronized. + + + The ability to launch a sequence synchronization worker is limited by the + + max_sync_workers_per_subscription + configuration. + + + + Sequence Definition Mismatches + + + During sequence synchronization, the sequence definitions of the publisher + and the subscriber are compared. A WARNING is logged if any differences + are detected. + + + + To resolve this, use + ALTER SEQUENCE + to align the subscriber's sequence parameters with those of the publisher. + Then, execute + ALTER SUBSCRIPTION ... REFRESH PUBLICATION SEQUENCES. + + + + + Refreshing Stale Sequences + + Subscriber side sequence values may frequently become out of sync due to + updates on the publisher. + + + To verify, compare the sequence values between the publisher and + subscriber, and if necessary, execute + + ALTER SUBSCRIPTION ... REFRESH PUBLICATION SEQUENCES. + + + + + Examples + + + Create some sequences on the publisher. + +test_pub=# CREATE SEQUENCE s1 START WITH 10 INCREMENT BY 1; +CREATE SEQUENCE +test_pub=# CREATE SEQUENCE s2 START WITH 100 INCREMENT BY 10; +CREATE SEQUENCE + + + + Create the same sequences on the subscriber. + +test_sub=# CREATE SEQUENCE s1 START WITH 10 INCREMENT BY 1 +CREATE SEQUENCE +test_sub=# CREATE SEQUENCE s2 START WITH 100 INCREMENT BY 10; +CREATE SEQUENCE + + + + Update the sequences at the publisher side few times. + +test_pub=# SELECT nextval('s1'); + nextval +--------- + 10 +(1 row) +test_pub=# SELECT NEXTVAL('s1'); + nextval +--------- + 11 +(1 row) +test_pub=# SELECT nextval('s2'); + nextval +--------- + 100 +(1 row) +test_pub=# SELECT nextval('s2'); + nextval +--------- + 110 +(1 row) + + + + Create a publication for the sequences. + +test_pub=# CREATE PUBLICATION pub1 FOR ALL SEQUENCES; +CREATE PUBLICATION + + + + Subscribe to the publication. + +test_sub=# CREATE SUBSCRIPTION sub1 +test_sub-# CONNECTION 'host=localhost dbname=test_pub application_name=sub1' +test_sub-# PUBLICATION pub1; +CREATE SUBSCRIPTION + + + + Observe that initial sequence values are synchronized. + +test_sub=# SELECT * FROM s1; + last_value | log_cnt | is_called +------------+---------+----------- + 11 | 31 | t +(1 row) + +test_sub=# SELECT * FROM s2; + last_value | log_cnt | is_called +------------+---------+----------- + 110 | 31 | t +(1 row) + + + + Update the sequences at the publisher side. + +test_pub=# SELECT nextval('s1'); + nextval +--------- + 12 +(1 row) +test_pub=# SELECT nextval('s2'); + nextval +--------- + 120 +(1 row) + + + + Re-synchronize all the sequences at the subscriber side using + + ALTER SUBSCRIPTION ... REFRESH PUBLICATION SEQUENCES. + +test_sub=# ALTER SUBSCRIPTION sub1 REFRESH PUBLICATION SEQUENCES; +ALTER SUBSCRIPTION + +test_sub=# SELECT * FROM s1; + last_value | log_cnt | is_called +------------+---------+----------- + 12 | 30 | t +(1 row) + +test_sub=# SELECT * FROM s2 + last_value | log_cnt | is_called +------------+---------+----------- + 120 | 30 | t +(1 row) + + + + Conflicts @@ -2115,16 +2314,19 @@ CONTEXT: processing remote data for replication origin "pg_16395" during "INSER - Sequence data is not replicated. The data in serial or identity columns - backed by sequences will of course be replicated as part of the table, - but the sequence itself would still show the start value on the - subscriber. If the subscriber is used as a read-only database, then this - should typically not be a problem. If, however, some kind of switchover - or failover to the subscriber database is intended, then the sequences - would need to be updated to the latest values, either by copying the - current data from the publisher (perhaps - using pg_dump) or by determining a sufficiently high - value from the tables themselves. + Incremental sequence changes are not replicated. Although the data in + serial or identity columns backed by sequences will of course be + replicated as part of the table, the sequences themselves do not replicate + ongoing changes. On the subscriber, a sequence will retain the last value + it synchronized from the publisher. If the subscriber is used as a + read-only database, then this should typically not be a problem. If, + however, some kind of switchover or failover to the subscriber database is + intended, then the sequences would need to be updated to the latest + values, either by executing + ALTER SUBSCRIPTION ... REFRESH PUBLICATION SEQUENCES + or by copying the current data from the publisher (perhaps using + pg_dump) or by determining a sufficiently high value + from the tables themselves. @@ -2442,8 +2644,8 @@ CONTEXT: processing remote data for replication origin "pg_16395" during "INSER max_logical_replication_workers must be set to at least the number of subscriptions (for leader apply - workers), plus some reserve for the table synchronization workers and - parallel apply workers. + workers), plus some reserve for the parallel apply workers, table synchronization workers, and a sequence + synchronization worker. @@ -2456,8 +2658,9 @@ CONTEXT: processing remote data for replication origin "pg_16395" during "INSER max_sync_workers_per_subscription - controls the amount of parallelism of the initial data copy during the - subscription initialization or when new tables are added. + controls how many tables can be synchronized in parallel during + subscription initialization or when new tables are added. One additional + worker is also needed for sequence synchronization. diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index d768ea065c55..08d88c796879 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -2025,8 +2025,9 @@ description | Waiting for a newly initialized WAL file to reach durable storage Type of the subscription worker process. Possible types are - apply, parallel apply, and - table synchronization. + apply, parallel apply, + table synchronization, and + sequence synchronization. diff --git a/doc/src/sgml/ref/alter_subscription.sgml b/doc/src/sgml/ref/alter_subscription.sgml index fdc648d007f1..0ecc91b6fc17 100644 --- a/doc/src/sgml/ref/alter_subscription.sgml +++ b/doc/src/sgml/ref/alter_subscription.sgml @@ -26,6 +26,7 @@ ALTER SUBSCRIPTION name SET PUBLICA ALTER SUBSCRIPTION name ADD PUBLICATION publication_name [, ...] [ WITH ( publication_option [= value] [, ... ] ) ] ALTER SUBSCRIPTION name DROP PUBLICATION publication_name [, ...] [ WITH ( publication_option [= value] [, ... ] ) ] ALTER SUBSCRIPTION name REFRESH PUBLICATION [ WITH ( refresh_option [= value] [, ... ] ) ] +ALTER SUBSCRIPTION name REFRESH PUBLICATION SEQUENCES ALTER SUBSCRIPTION name ENABLE ALTER SUBSCRIPTION name DISABLE ALTER SUBSCRIPTION name SET ( subscription_parameter [= value] [, ... ] ) @@ -67,6 +68,7 @@ ALTER SUBSCRIPTION name RENAME TO < Commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION, + ALTER SUBSCRIPTION ... REFRESH PUBLICATION SEQUENCES, ALTER SUBSCRIPTION ... {SET|ADD|DROP} PUBLICATION ... with refresh option as true, ALTER SUBSCRIPTION ... SET (failover = true|false) and @@ -158,30 +160,51 @@ ALTER SUBSCRIPTION name RENAME TO < REFRESH PUBLICATION - Fetch missing table information from publisher. This will start + Fetch missing table information from the publisher. This will start replication of tables that were added to the subscribed-to publications since CREATE SUBSCRIPTION or the last invocation of REFRESH PUBLICATION. + + Also, fetch missing sequence information from the publisher. + + + + The system catalog pg_subscription_rel + is updated to record all tables and sequences known to the subscription, + that are still part of the publication. + + refresh_option specifies additional options for the - refresh operation. The supported options are: + refresh operation. The only supported option is: copy_data (boolean) - Specifies whether to copy pre-existing data in the publications - that are being subscribed to when the replication starts. - The default is true. + Specifies whether to copy pre-existing data for tables and synchronize + sequences in the publications that are being subscribed to when the replication + starts. The default is true. Previously subscribed tables are not copied, even if a table's row filter WHERE clause has since been modified. + + Previously subscribed sequences are not re-synchronized. To do that, + see + ALTER SUBSCRIPTION ... REFRESH PUBLICATION SEQUENCES. + + + See for recommendations on how + to handle any warnings about sequence definition differences between + the publisher and the subscriber, which might occur when + copy_data = true. + See for details of how copy_data = true can interact with the @@ -200,6 +223,28 @@ ALTER SUBSCRIPTION name RENAME TO < + + REFRESH PUBLICATION SEQUENCES + + + Fetch missing sequence information from the publisher, then re-synchronize + sequence data with the publisher. Unlike + ALTER SUBSCRIPTION ... REFRESH PUBLICATION which + only synchronizes newly added sequences, REFRESH PUBLICATION SEQUENCES + will re-synchronize the sequence data for all subscribed sequences. + + + See for + recommendations on how to handle any warnings about sequence definition + differences between the publisher and the subscriber. + + + See for recommendations on how to + identify and handle out-of-sync sequences. + + + + ENABLE diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml index 57dec28a5df6..44308515bbbc 100644 --- a/doc/src/sgml/ref/create_subscription.sgml +++ b/doc/src/sgml/ref/create_subscription.sgml @@ -263,6 +263,12 @@ CREATE SUBSCRIPTION subscription_namecopy_data = true can interact with the origin parameter. + + See + for recommendations on how to handle any warnings about sequence + definition differences between the publisher and the subscriber, + which might occur when copy_data = true. + diff --git a/doc/src/sgml/system-views.sgml b/doc/src/sgml/system-views.sgml index b58c52ea50f5..066a8c526db0 100644 --- a/doc/src/sgml/system-views.sgml +++ b/doc/src/sgml/system-views.sgml @@ -131,6 +131,11 @@ prepared transactions + + pg_publication_sequences + publications and information of their associated sequences + + pg_publication_tables publications and information of their associated tables @@ -2475,6 +2480,68 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx + + <structname>pg_publication_sequences</structname> + + + pg_publication_sequences + + + + The view pg_publication_sequences provides + information about the mapping between publications and information of + sequences they contain. + + +
+ <structname>pg_publication_sequences</structname> Columns + + + + + Column Type + + + Description + + + + + + + + pubname name + (references pg_publication.pubname) + + + Name of publication + + + + + + schemaname name + (references pg_namespace.nspname) + + + Name of schema containing sequence + + + + + + sequencename name + (references pg_class.relname) + + + Name of sequence + + + + +
+ + <structname>pg_publication_tables</structname>