Some improvements to debug/error logging.

We now log name of the remote node and remote PID if an error is received on a
connection. Some more debugging messages added to the pooler code.

3 files changed, 66 insertions, 2 deletions

diff --git a/src/backend/pgxc/pool/pgxcnode.c b/src/backend/pgxc/pool/pgxcnode.c
index bb8e94c829..7ca3691dcd 100644
--- a/src/backend/pgxc/pool/pgxcnode.c
+++ b/src/backend/pgxc/pool/pgxcnode.c
@@ -213,12 +213,16 @@ InitMultinodeExecutor(bool is_force)
 		init_pgxc_handle(&dn_handles[count]);
 		dn_handles[count].nodeoid = dnOids[count];
 		dn_handles[count].nodeid = get_pgxc_node_id(dnOids[count]);
+		strncpy(dn_handles[count].nodename, get_pgxc_nodename(dnOids[count]),
+				NAMEDATALEN);
 	}
 	for (count = 0; count < NumCoords; count++)
 	{
 		init_pgxc_handle(&co_handles[count]);
 		co_handles[count].nodeoid = coOids[count];
 		co_handles[count].nodeid = get_pgxc_node_id(coOids[count]);
+		strncpy(co_handles[count].nodename, get_pgxc_nodename(coOids[count]),
+				NAMEDATALEN);
 	}
 
 	datanode_count = 0;
@@ -1885,7 +1889,8 @@ pgxc_node_send_timestamp(PGXCNodeHandle *handle, TimestampTz timestamp)
 void
 add_error_message(PGXCNodeHandle *handle, const char *message)
 {
-	elog(LOG, "Connection error %s", message);
+	elog(LOG, "Remote node \"%s\", running with pid %d returned an error: %s",
+			handle->nodename, handle->backend_pid, message);
 	handle->transaction_status = 'E';
 	if (handle->error)
 	{
@@ -1962,6 +1967,7 @@ get_any_handle(List *datanodelist)
 					int	   *pids;
 					int    *fds = PoolManagerGetConnections(allocate, NIL,
 							&pids);
+					PGXCNodeHandle		*node_handle;
 
 					if (!fds)
 					{
@@ -1978,9 +1984,14 @@ get_any_handle(List *datanodelist)
 									 "max_connections and max_pool_size configuration "
 									 "parameters")));
 					}
-					pgxc_node_init(&dn_handles[node], fds[0], true, pids[0]);
+					node_handle = &dn_handles[node];
+					pgxc_node_init(node_handle, fds[0], true, pids[0]);
 					datanode_count++;
 
+					elog(DEBUG1, "Established a connection with datanode \"%s\","
+							"remote backend PID %d, socket fd %d, global session %c",
+							node_handle->nodename, (int) pids[0], fds[0], 'T');
+
 					/*
 					 * set load_balancer for next time and return the handle
 					 */
@@ -2227,6 +2238,11 @@ get_handles(List *datanodelist, List *coordlist, bool is_coord_only_query, bool
 				pgxc_node_init(node_handle, fdsock, is_global_session, be_pid);
 				dn_handles[node] = *node_handle;
 				datanode_count++;
+
+				elog(DEBUG1, "Established a connection with datanode \"%s\","
+						"remote backend PID %d, socket fd %d, global session %c",
+						node_handle->nodename, (int) be_pid, fdsock,
+						is_global_session ? 'T' : 'F');
 			}
 		}
 		/* Initialisation for Coordinators */
@@ -2249,6 +2265,11 @@ get_handles(List *datanodelist, List *coordlist, bool is_coord_only_query, bool
 				pgxc_node_init(node_handle, fdsock, is_global_session, be_pid);
 				co_handles[node] = *node_handle;
 				coord_count++;
+
+				elog(DEBUG1, "Established a connection with coordinator \"%s\","
+						"remote backend PID %d, socket fd %d, global session %c",
+						node_handle->nodename, (int) be_pid, fdsock,
+						is_global_session ? 'T' : 'F');
 			}
 		}
 
diff --git a/src/backend/pgxc/pool/poolmgr.c b/src/backend/pgxc/pool/poolmgr.c
index 95514e31a6..27a3f6f99b 100644
--- a/src/backend/pgxc/pool/poolmgr.c
+++ b/src/backend/pgxc/pool/poolmgr.c
@@ -561,6 +561,9 @@ PoolManagerConnect(const char *database, const char *user_name,
 				(errcode(ERRCODE_INTERNAL_ERROR),
 				 errmsg("failed to connect to the pooler process")));
 
+	elog(DEBUG1, "Connecting to PoolManager (user_name %s, database %s, "
+			"pgoptions %s", user_name, database, pgoptions);
+
 	/*
 	 * Special handling for db_user_namespace=on
 	 * We need to handle per-db users and global users. The per-db users will
@@ -632,6 +635,8 @@ PoolManagerConnect(const char *database, const char *user_name,
 void
 PoolManagerReconnect(void)
 {
+	elog(DEBUG1, "Reconnecting to PoolManager");
+
 	/* Connected, disconnect */
 	if (poolHandle)
 		PoolManagerDisconnect();
@@ -655,6 +660,8 @@ PoolManagerLock(bool is_lock)
 		PoolManagerConnect(get_database_name(MyDatabaseId),
 						   GetClusterUserName(), "");
 
+	elog(DEBUG1, "Locking PoolManager");
+
 	/* Message type */
 	pool_putbytes(&poolHandle->port, &msgtype, 1);
 
@@ -1420,7 +1427,14 @@ agent_acquire_connections(PoolAgent *agent, List *datanodelist,
 	/* Check if pooler can accept those requests */
 	if (list_length(datanodelist) > agent->num_dn_connections ||
 			list_length(coordlist) > agent->num_coord_connections)
+	{
+		elog(LOG, "agent_acquire_connections called with invalid arguments -"
+				"list_length(datanodelist) %d, num_dn_connections %d,"
+				"list_length(coordlist) %d, num_coord_connections %d",
+				list_length(datanodelist), agent->num_dn_connections,
+				list_length(coordlist), agent->num_coord_connections);
 		return NULL;
+	}
 
 	/*
 	 * Allocate memory
@@ -1471,6 +1485,8 @@ agent_acquire_connections(PoolAgent *agent, List *datanodelist,
 			{
 				pfree(result);
 				MemoryContextSwitchTo(oldcontext);
+				elog(LOG, "Pooler could not open a connection to node %d",
+						agent->dn_conn_oids[node]);
 				return NULL;
 			}
 
@@ -1503,6 +1519,8 @@ agent_acquire_connections(PoolAgent *agent, List *datanodelist,
 			{
 				pfree(result);
 				MemoryContextSwitchTo(oldcontext);
+				elog(LOG, "Pooler could not open a connection to node %d",
+						agent->coord_conn_oids[node]);
 				return NULL;
 			}
 
@@ -1555,9 +1573,15 @@ cancel_query_on_connections(PoolAgent *agent, List *datanodelist, List *coordlis
 		if (!agent->dn_connections[node])
 			continue;
 
+		elog(DEBUG1, "Canceling query on connection to remote node %d, remote pid %d",
+				agent->dn_conn_oids[node],
+				((PGconn *) agent->dn_connections[node]->conn)->be_pid);
 		bRet = PQcancel((PGcancel *) agent->dn_connections[node]->xc_cancelConn, errbuf, sizeof(errbuf));
 		if (bRet != false)
 		{
+			elog(DEBUG1, "Cancelled query on connection to remote node %d, remote pid %d",
+					agent->dn_conn_oids[node],
+					((PGconn *) agent->dn_connections[node]->conn)->be_pid);
 			nCount++;
 		}
 	}
@@ -1576,9 +1600,15 @@ cancel_query_on_connections(PoolAgent *agent, List *datanodelist, List *coordlis
 		if (!agent->coord_connections[node])
 			continue;
 
+		elog(DEBUG1, "Canceling query on connection to remote node %d, remote pid %d",
+				agent->coord_conn_oids[node],
+				((PGconn *) agent->coord_connections[node]->conn)->be_pid);
 		bRet = PQcancel((PGcancel *) agent->coord_connections[node]->xc_cancelConn, errbuf, sizeof(errbuf));
 		if (bRet != false)
 		{
+			elog(DEBUG1, "Cancelled query on connection to remote node %d, remote pid %d",
+					agent->coord_conn_oids[node],
+					((PGconn *) agent->coord_connections[node]->conn)->be_pid);
 			nCount++;
 		}
 	}
@@ -1600,6 +1630,8 @@ PoolManagerReleaseConnections(bool force)
 	if (!poolHandle)
 		return;
 
+	elog(DEBUG1, "Returning connections back to the pool");
+
 	/* Message type */
 	pool_putbytes(&poolHandle->port, &msgtype, 1);
 
@@ -1710,6 +1742,7 @@ agent_release_connections(PoolAgent *agent, bool force_destroy)
 		if (slot)
 			release_connection(agent->pool, slot, agent->dn_conn_oids[i], force_destroy);
 		agent->dn_connections[i] = NULL;
+		elog(DEBUG1, "Released connection to node %d", agent->dn_conn_oids[i]);
 	}
 	/* Then clean up for Coordinator connections */
 	for (i = 0; i < agent->num_coord_connections; i++)
@@ -1723,6 +1756,7 @@ agent_release_connections(PoolAgent *agent, bool force_destroy)
 		if (slot)
 			release_connection(agent->pool, slot, agent->coord_conn_oids[i], force_destroy);
 		agent->coord_connections[i] = NULL;
+		elog(DEBUG1, "Released connection to node %d", agent->coord_conn_oids[i]);
 	}
 
 	/*
@@ -1753,6 +1787,9 @@ create_database_pool(const char *database, const char *user_name, const char *pg
 	HASHCTL			hinfo;
 	int				hflags;
 
+	elog(DEBUG1, "Creating a connection pool for database %s, user %s,"
+			" with pgoptions %s", database, user_name, pgoptions);
+
 	dbcontext = AllocSetContextCreate(PoolerCoreContext,
 									  "DB Context",
 									  ALLOCSET_DEFAULT_MINSIZE,
@@ -1824,6 +1861,9 @@ destroy_database_pool(const char *database, const char *user_name)
 {
 	DatabasePool *databasePool;
 
+	elog(DEBUG1, "Destroy a connection pool to database %s, user %s",
+			database, user_name);
+
 	/* Delete from the list */
 	databasePool = remove_database_pool(database, user_name);
 	if (databasePool)
@@ -1870,6 +1910,8 @@ reload_database_pools(PoolAgent *agent)
 {
 	DatabasePool *databasePool;
 
+	elog(DEBUG1, "Reloading database pools");
+
 	/*
 	 * Release node connections if any held. It is not guaranteed client session
 	 * does the same so don't ever try to return them to pool and reuse
diff --git a/src/include/pgxc/pgxcnode.h b/src/include/pgxc/pgxcnode.h
index d57aabe18b..e19bbfe2b8 100644
--- a/src/include/pgxc/pgxcnode.h
+++ b/src/include/pgxc/pgxcnode.h
@@ -63,6 +63,7 @@ struct pgxc_node_handle
 {
 	Oid			nodeoid;
 	int			nodeid;
+	char		nodename[NAMEDATALEN];
 
 	/* fd of the connection */
 	int		sock;