Partial fix for bug 3310399: Autovacuum workers using same connections to GTM

This fixes a problem with autovacuum worker/launchers that tended to use
the connection allocated for postmaster to connect to GTM.
In the case of multiple vacuums running at the same time, this tended to mess the
way autovacuum was receiving GXID and snapshots from GTM.
This commit also adds some debug messages to look at the connection activity to GTM
and more strict connection control of autovacuum backends to GTM.

4 files changed, 55 insertions, 7 deletions

diff --git a/src/backend/access/transam/gtm.c b/src/backend/access/transam/gtm.c
index e9f02298c9..f05e38de86 100644
--- a/src/backend/access/transam/gtm.c
+++ b/src/backend/access/transam/gtm.c
@@ -28,17 +28,28 @@ extern bool FirstSnapshotSet;
 
 static GTM_Conn *conn;
 
-#define CheckConnection() \
-	if (GTMPQstatus(conn) != CONNECTION_OK) InitGTM()
-
 bool
 IsGTMConnected()
 {
 	return conn != NULL;
 }
 
+static void
+CheckConnection(void)
+{
+	/* Be sure that a backend does not use a postmaster connection */
+	if (IsUnderPostmaster && GTMPQispostmaster(conn) == 1)
+	{
+		InitGTM();
+		return;
+	}
+
+	if (GTMPQstatus(conn) != CONNECTION_OK)
+		InitGTM();
+}
+
 void
-InitGTM()
+InitGTM(void)
 {
 	/* 256 bytes should be enough */
 	char conn_str[256];
@@ -55,10 +66,23 @@ InitGTM()
 
 		sprintf(conn_str, "host=%s port=%d pgxc_node_id=%d remote_type=%d postmaster=1",
 								GtmHost, GtmPort, PGXCNodeId, remote_type);
+
+		/* Log activity of GTM connections */
+		elog(DEBUG1, "Postmaster: connection established to GTM with string %s", conn_str);
 	}
 	else
+	{
 		sprintf(conn_str, "host=%s port=%d pgxc_node_id=%d", GtmHost, GtmPort, PGXCNodeId);
 
+		/* Log activity of GTM connections */
+		if (IsAutoVacuumWorkerProcess())
+			elog(DEBUG1, "Autovacuum worker: connection established to GTM with string %s", conn_str);
+		else if (IsAutoVacuumLauncherProcess())
+			elog(DEBUG1, "Autovacuum launcher: connection established to GTM with string %s", conn_str);
+		else
+			elog(DEBUG1, "Postmaster child: connection established to GTM with string %s", conn_str);
+	}
+
 	conn = PQconnectGTM(conn_str);
 	if (GTMPQstatus(conn) != CONNECTION_OK)
 	{
@@ -79,6 +103,16 @@ CloseGTM(void)
 {
 	GTMPQfinish(conn);
 	conn = NULL;
+
+	/* Log activity of GTM connections */
+	if (!IsUnderPostmaster)
+		elog(DEBUG1, "Postmaster: connection to GTM closed");
+	else if (IsAutoVacuumWorkerProcess())
+		elog(DEBUG1, "Autovacuum worker: connection to GTM closed");
+	else if (IsAutoVacuumLauncherProcess())
+		elog(DEBUG1, "Autovacuum launcher: connection to GTM closed");
+	else
+		elog(DEBUG1, "Postmaster child: connection to GTM closed");
 }
 
 GlobalTransactionId
@@ -114,7 +148,8 @@ BeginTranAutovacuumGTM(void)
 	if (conn)
 		xid =  begin_transaction_autovacuum(conn, GTM_ISOLATION_RC);
 
-	/* If something went wrong (timeout), try and reset GTM connection and retry.
+	/*
+	 * If something went wrong (timeout), try and reset GTM connection and retry.
 	 * This is safe at the beginning of a transaction.
 	 */
 	if (!TransactionIdIsValid(xid))
@@ -147,6 +182,11 @@ CommitTranGTM(GlobalTransactionId gxid)
 		CloseGTM();
 		InitGTM();
 	}
+
+	/* Close connection in case commit is done by autovacuum worker or launcher */
+	if (IsAutoVacuumWorkerProcess() || IsAutoVacuumLauncherProcess())
+		CloseGTM();
+
 	return ret;
 }
 
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index 9dc1617e72..71b139895a 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -226,8 +226,7 @@ static void
 DataNodeShutdown (int code, Datum arg)
 {
 	/* Close connection with GTM, if active */
-	if (IsAutoVacuumWorkerProcess())
-		CloseGTM();
+	CloseGTM();
 }
 #endif
 
diff --git a/src/gtm/client/fe-connect.c b/src/gtm/client/fe-connect.c
index 61697cbdf2..7cd3440e4e 100644
--- a/src/gtm/client/fe-connect.c
+++ b/src/gtm/client/fe-connect.c
@@ -1254,6 +1254,14 @@ GTMPQstatus(const GTM_Conn *conn)
 	return conn->status;
 }
 
+int
+GTMPQispostmaster(const GTM_Conn *conn)
+{
+	if (!conn)
+		return 0;
+	return conn->is_postmaster;
+}
+
 char *
 GTMPQerrorMessage(const GTM_Conn *conn)
 {
diff --git a/src/include/gtm/libpq-fe.h b/src/include/gtm/libpq-fe.h
index 1ae16c1dbd..31274698c8 100644
--- a/src/include/gtm/libpq-fe.h
+++ b/src/include/gtm/libpq-fe.h
@@ -119,6 +119,7 @@ extern void GTMPQconninfoFree(GTMPQconninfoOption *connOptions);
 extern char *GTMPQhost(const GTM_Conn *conn);
 extern char *GTMPQport(const GTM_Conn *conn);
 extern ConnStatusType GTMPQstatus(const GTM_Conn *conn);
+extern int GTMPQispostmaster(const GTM_Conn *conn);
 extern char *GTMPQerrorMessage(const GTM_Conn *conn);
 extern int	GTMPQsocket(const GTM_Conn *conn);