Send shared invalidation messages to other backends upon completion of a

command.

Since multiple backends may cooperate in a distributed transaction, it's
important that when one of the backends make changes to catalogs, which
requires cache invalidation, the other cooperating backends see those changes
immediately instead of end-of-transaction.

Also ensure that invalidation messages are accepted even when a relation lock
was already held by the backend, if it's running in a distributed transaction.

6 files changed, 173 insertions, 2 deletions

diff --git a/src/backend/storage/lmgr/lmgr.c b/src/backend/storage/lmgr/lmgr.c
index 6757def565..501bcb0c20 100644
--- a/src/backend/storage/lmgr/lmgr.c
+++ b/src/backend/storage/lmgr/lmgr.c
@@ -21,6 +21,7 @@
 #include "catalog/catalog.h"
 #include "miscadmin.h"
 #include "storage/lmgr.h"
+#include "storage/proc.h"
 #include "storage/procarray.h"
 #include "utils/inval.h"
 
@@ -120,8 +121,18 @@ LockRelationOid(Oid relid, LOCKMODE lockmode)
 	 * relcache entry in an undesirable way.  (In the case where our own xact
 	 * modifies the rel, the relcache update happens via
 	 * CommandCounterIncrement, not here.)
+	 *
+	 * In Postgres-XL, multiple backends may run concurrently to serve a
+	 * distributed transaction. In that case, a conflicting lock may be granted
+	 * to another backend running the same distributed transaction. But it's
+	 * important that such backends process invalidation messages to ensure
+	 * that relcache entry modified by the other cooperating backend is truly
+	 * reflected. For example, the other backend may TRUNCATE the table and
+	 * change the relfilenode. So we must see that change and work with the new
+	 * relfilenode.
 	 */
-	if (res != LOCKACQUIRE_ALREADY_HELD)
+	if ((res != LOCKACQUIRE_ALREADY_HELD) ||
+		(MyProc->coordPid && MyProc->coordId))
 		AcceptInvalidationMessages();
 }
 
diff --git a/src/backend/utils/cache/inval.c b/src/backend/utils/cache/inval.c
index 3cbf8c73fd..99b6deb9cf 100644
--- a/src/backend/utils/cache/inval.c
+++ b/src/backend/utils/cache/inval.c
@@ -1071,6 +1071,20 @@ CommandEndInvalidationMessages(void)
 
 	ProcessInvalidationMessages(&transInvalInfo->CurrentCmdInvalidMsgs,
 								LocalExecuteInvalidationMessage);
+
+	/*
+	 * In Postgres-XL, multiple backends can work for a distributed
+	 * transaction. When catalog changes are made by one backend, the other
+	 * backend participating in the distributed transaction must also see the
+	 * changes immediately. So send the messages using shared invalidation
+	 *
+	 * XXX We could investigate if this can cause performance problems and
+	 * check if the messages can be sent only to participating backends
+	 */
+	if (IS_PGXC_DATANODE)
+		ProcessInvalidationMessagesMulti(&transInvalInfo->CurrentCmdInvalidMsgs,
+			SendSharedInvalidMessages);
+
 	AppendInvalidationMessages(&transInvalInfo->PriorCmdInvalidMsgs,
 							   &transInvalInfo->CurrentCmdInvalidMsgs);
 }
diff --git a/src/test/regress/expected/xl_distributed_xact.out b/src/test/regress/expected/xl_distributed_xact.out
new file mode 100644
index 0000000000..0087690348
--- /dev/null
+++ b/src/test/regress/expected/xl_distributed_xact.out
@@ -0,0 +1,87 @@
+CREATE TABLE testr_x14 (a int, b int);
+CREATE TABLE testl_x14 (a int, b int);
+INSERT INTO testl_x14 SELECT generate_series(1,10000), generate_series(1,10000);
+CREATE TABLE testlr_x14 (al int, bl int, ar int, br int);
+-- Create a scenario where datanode-datanode connections are established and
+-- they access a relation, which is subsequently altered or modified by some
+-- other connections. As seen from a bug report, if cache invalidation is not
+-- done correctly, the datanode-datanode connections may use stale cached
+-- information, leading to many problems
+BEGIN;
+INSERT INTO testlr_x14 SELECT * FROM testr_x14 RIGHT OUTER JOIN testl_x14 ON testr_x14.a = testl_x14.b;
+TRUNCATE testr_x14;
+INSERT INTO testr_x14 SELECT * FROM testl_x14;
+SELECT count(*) FROM testlr_x14 WHERE al IS NOT NULL;
+ count 
+-------
+     0
+(1 row)
+
+TRUNCATE testlr_x14;
+INSERT INTO testlr_x14 SELECT * FROM testr_x14 RIGHT OUTER JOIN testl_x14 ON testr_x14.a = testl_x14.b;
+SELECT count(*) FROM testlr_x14 WHERE al IS NOT NULL;
+ count 
+-------
+ 10000
+(1 row)
+
+ROLLBACK;
+INSERT INTO testlr_x14 SELECT * FROM testr_x14 RIGHT OUTER JOIN testl_x14 ON testr_x14.a = testl_x14.b;
+TRUNCATE testr_x14;
+INSERT INTO testr_x14 SELECT * FROM testl_x14;
+SELECT count(*) FROM testlr_x14 WHERE al IS NOT NULL;
+ count 
+-------
+     0
+(1 row)
+
+TRUNCATE testlr_x14;
+INSERT INTO testlr_x14 SELECT * FROM testr_x14 RIGHT OUTER JOIN testl_x14 ON testr_x14.a = testl_x14.b;
+SELECT count(*) FROM testlr_x14 WHERE al IS NOT NULL;
+ count 
+-------
+ 10000
+(1 row)
+
+DROP TABLE testlr_x14, testr_x14, testl_x14;
+CREATE TABLE testr_x14 (a int, b int);
+CREATE TABLE testl_x14 (a int, b int);
+INSERT INTO testl_x14 SELECT generate_series(1,10000), generate_series(1,10000);
+CREATE TABLE testlr_x14 (al int, bl int, ar int, br int);
+BEGIN;
+INSERT INTO testlr_x14 SELECT * FROM testr_x14 RIGHT OUTER JOIN testl_x14 ON testr_x14.a = testl_x14.b;
+ALTER TABLE testr_x14 ALTER COLUMN b SET DATA TYPE text;
+INSERT INTO testr_x14 SELECT * FROM testl_x14;
+SELECT count(*) FROM testlr_x14 WHERE al IS NOT NULL;
+ count 
+-------
+     0
+(1 row)
+
+TRUNCATE testlr_x14;
+INSERT INTO testlr_x14 SELECT testr_x14.a, testr_x14.b::integer, testl_x14.* FROM testr_x14 RIGHT OUTER JOIN testl_x14 ON testr_x14.a = testl_x14.b;
+SELECT count(*) FROM testlr_x14 WHERE al IS NOT NULL;
+ count 
+-------
+ 10000
+(1 row)
+
+ROLLBACK;
+INSERT INTO testlr_x14 SELECT * FROM testr_x14 RIGHT OUTER JOIN testl_x14 ON testr_x14.a = testl_x14.b;
+ALTER TABLE testr_x14 ALTER COLUMN b SET DATA TYPE text;
+INSERT INTO testr_x14 SELECT * FROM testl_x14;
+SELECT count(*) FROM testlr_x14 WHERE al IS NOT NULL;
+ count 
+-------
+     0
+(1 row)
+
+TRUNCATE testlr_x14;
+INSERT INTO testlr_x14 SELECT testr_x14.a, testr_x14.b::integer, testl_x14.* FROM testr_x14 RIGHT OUTER JOIN testl_x14 ON testr_x14.a = testl_x14.b;
+SELECT count(*) FROM testlr_x14 WHERE al IS NOT NULL;
+ count 
+-------
+ 10000
+(1 row)
+
+DROP TABLE testlr_x14, testr_x14, testl_x14;
diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule
index ed93ed9104..ff7c8cac2e 100644
--- a/src/test/regress/parallel_schedule
+++ b/src/test/regress/parallel_schedule
@@ -139,4 +139,4 @@ test: xc_prepared_xacts
 test: xc_notrans_block
 
 # This runs XL specific tests
-test: xl_primary_key xl_foreign_key xl_distribution_column_types xl_alter_table xl_distribution_column_types_modulo xl_plan_pushdown xl_functions xl_limitations xl_user_defined_functions xl_join
+test: xl_primary_key xl_foreign_key xl_distribution_column_types xl_alter_table xl_distribution_column_types_modulo xl_plan_pushdown xl_functions xl_limitations xl_user_defined_functions xl_join xl_distributed_xact
diff --git a/src/test/regress/serial_schedule b/src/test/regress/serial_schedule
index de2d0a5a59..e4ced6c15f 100644
--- a/src/test/regress/serial_schedule
+++ b/src/test/regress/serial_schedule
@@ -195,3 +195,4 @@ test: xl_functions
 test: xl_limitations
 test: xl_user_defined_functions
 test: xl_join
+test: xl_distributed_xact
diff --git a/src/test/regress/sql/xl_distributed_xact.sql b/src/test/regress/sql/xl_distributed_xact.sql
new file mode 100644
index 0000000000..02ba65fe8e
--- /dev/null
+++ b/src/test/regress/sql/xl_distributed_xact.sql
@@ -0,0 +1,58 @@
+
+CREATE TABLE testr_x14 (a int, b int);
+CREATE TABLE testl_x14 (a int, b int);
+INSERT INTO testl_x14 SELECT generate_series(1,10000), generate_series(1,10000);
+CREATE TABLE testlr_x14 (al int, bl int, ar int, br int);
+
+-- Create a scenario where datanode-datanode connections are established and
+-- they access a relation, which is subsequently altered or modified by some
+-- other connections. As seen from a bug report, if cache invalidation is not
+-- done correctly, the datanode-datanode connections may use stale cached
+-- information, leading to many problems
+
+BEGIN;
+INSERT INTO testlr_x14 SELECT * FROM testr_x14 RIGHT OUTER JOIN testl_x14 ON testr_x14.a = testl_x14.b;
+TRUNCATE testr_x14;
+INSERT INTO testr_x14 SELECT * FROM testl_x14;
+SELECT count(*) FROM testlr_x14 WHERE al IS NOT NULL;
+TRUNCATE testlr_x14;
+INSERT INTO testlr_x14 SELECT * FROM testr_x14 RIGHT OUTER JOIN testl_x14 ON testr_x14.a = testl_x14.b;
+SELECT count(*) FROM testlr_x14 WHERE al IS NOT NULL;
+ROLLBACK;
+
+INSERT INTO testlr_x14 SELECT * FROM testr_x14 RIGHT OUTER JOIN testl_x14 ON testr_x14.a = testl_x14.b;
+TRUNCATE testr_x14;
+INSERT INTO testr_x14 SELECT * FROM testl_x14;
+SELECT count(*) FROM testlr_x14 WHERE al IS NOT NULL;
+TRUNCATE testlr_x14;
+INSERT INTO testlr_x14 SELECT * FROM testr_x14 RIGHT OUTER JOIN testl_x14 ON testr_x14.a = testl_x14.b;
+SELECT count(*) FROM testlr_x14 WHERE al IS NOT NULL;
+
+DROP TABLE testlr_x14, testr_x14, testl_x14;
+
+
+CREATE TABLE testr_x14 (a int, b int);
+CREATE TABLE testl_x14 (a int, b int);
+INSERT INTO testl_x14 SELECT generate_series(1,10000), generate_series(1,10000);
+CREATE TABLE testlr_x14 (al int, bl int, ar int, br int);
+
+BEGIN;
+INSERT INTO testlr_x14 SELECT * FROM testr_x14 RIGHT OUTER JOIN testl_x14 ON testr_x14.a = testl_x14.b;
+ALTER TABLE testr_x14 ALTER COLUMN b SET DATA TYPE text;
+INSERT INTO testr_x14 SELECT * FROM testl_x14;
+SELECT count(*) FROM testlr_x14 WHERE al IS NOT NULL;
+TRUNCATE testlr_x14;
+INSERT INTO testlr_x14 SELECT testr_x14.a, testr_x14.b::integer, testl_x14.* FROM testr_x14 RIGHT OUTER JOIN testl_x14 ON testr_x14.a = testl_x14.b;
+SELECT count(*) FROM testlr_x14 WHERE al IS NOT NULL;
+ROLLBACK;
+
+INSERT INTO testlr_x14 SELECT * FROM testr_x14 RIGHT OUTER JOIN testl_x14 ON testr_x14.a = testl_x14.b;
+ALTER TABLE testr_x14 ALTER COLUMN b SET DATA TYPE text;
+INSERT INTO testr_x14 SELECT * FROM testl_x14;
+SELECT count(*) FROM testlr_x14 WHERE al IS NOT NULL;
+TRUNCATE testlr_x14;
+INSERT INTO testlr_x14 SELECT testr_x14.a, testr_x14.b::integer, testl_x14.* FROM testr_x14 RIGHT OUTER JOIN testl_x14 ON testr_x14.a = testl_x14.b;
+SELECT count(*) FROM testlr_x14 WHERE al IS NOT NULL;
+
+DROP TABLE testlr_x14, testr_x14, testl_x14;
+