postgres-xc-committers Mailing List for Postgres-XC

Project "Postgres-XC".

The branch, master has been updated
       via  bc45a6fdb6afe5dd72526c4919bd8c138f11eb0d (commit)
      from  981e2bbe81c94c0427ed9504d0390119a7770a83 (commit)


- Log -----------------------------------------------------------------
commit bc45a6fdb6afe5dd72526c4919bd8c138f11eb0d
Author: Michael P <mic...@us...>
Date:   Tue Jan 11 11:46:53 2011 +0900

    Improvement of performance with tuple scan
    
    This fixes some performance issues that have been introduced with commit:
    44ca05af2742271abdc5c14f5ca313d5ea307875
    
    Tuple were scanned with the cheapest cost, degrading performance a lot.
    
    Patch written by Benny, with some editorialization by me.

diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c
index 63893b4..ad90109 100644
--- a/src/backend/optimizer/plan/createplan.c
+++ b/src/backend/optimizer/plan/createplan.c
@@ -33,6 +33,7 @@
 #include "parser/parse_clause.h"
 #include "parser/parsetree.h"
 #ifdef PGXC
+#include "pgxc/pgxc.h"
 #include "pgxc/planner.h"
 #include "access/sysattr.h"
 #include "utils/builtins.h"
@@ -573,8 +574,12 @@ create_join_plan(PlannerInfo *root, JoinPath *best_path)
 #endif
 
 #ifdef PGXC
-	/* check if this join can be reduced to an equiv. remote scan node */
-	plan = create_remotejoin_plan(root, best_path, plan, outer_plan, inner_plan);
+	/*
+	 * Check if this join can be reduced to an equiv. remote scan node
+	 * This can only be executed on a remote Coordinator
+	 */
+	if (IS_PGXC_COORDINATOR && IsConnFromCoord())
+		plan = create_remotejoin_plan(root, best_path, plan, outer_plan, inner_plan);
 #endif
 
 	return plan;
diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c
index 04bf594..63b0581 100644
--- a/src/backend/optimizer/util/relnode.c
+++ b/src/backend/optimizer/util/relnode.c
@@ -23,6 +23,9 @@
 #include "parser/parsetree.h"
 #include "utils/hsearch.h"
 
+#ifdef PGXC
+#include "pgxc/pgxc.h"
+#endif
 
 typedef struct JoinHashEntry
 {
@@ -107,10 +110,16 @@ build_simple_rel(PlannerInfo *root, int relid, RelOptKind reloptkind)
 			 *
 			 * These allow for maximum query shipping to the remote
 			 * side later during the planning phase
+			 *
+			 * This has to be set on a remote Coordinator only
+			 * as it hugely penalizes performance on backend Nodes
 			 */
-			rel->pages   = 1;
-			rel->tuples  = 1;
-			rel->rows    = 1;
+			if (IS_PGXC_COORDINATOR && IsConnFromCoord())
+			{
+				rel->pages   = 1;
+				rel->tuples  = 1;
+				rel->rows    = 1;
+			}
 #endif
 			break;
 		case RTE_SUBQUERY:

-----------------------------------------------------------------------

Summary of changes:
 src/backend/optimizer/plan/createplan.c |    9 +++++++--
 src/backend/optimizer/util/relnode.c    |   15 ++++++++++++---
 2 files changed, 19 insertions(+), 5 deletions(-)


hooks/post-receive
-- 
Postgres-XC

Project "Postgres-XC".

The branch, master has been updated
       via  981e2bbe81c94c0427ed9504d0390119a7770a83 (commit)
      from  45e1d4e389e966d072aaf98a49d9702aa253d976 (commit)


- Log -----------------------------------------------------------------
commit 981e2bbe81c94c0427ed9504d0390119a7770a83
Author: Michael P <mic...@us...>
Date:   Tue Jan 11 11:05:18 2011 +0900

    Clean up of execRemote.c
    
    There was some code that was used to clean up connection thread between Coordinator and Datanodes
    that was not really necessary.
    This has been introduced with version 0.9.2 to stabilize the code by consuming messages on connections
    where error happened on backend Noce.
    
    This patch also corrects a bug on Datanode with GXID that was not correctly set
    at initialization.
    This leaded to transactions being committed twice on backend nodes, crashing it with a FATAL error.
    
    Patch written by Andrei Martsinchyk

diff --git a/src/backend/pgxc/pool/execRemote.c b/src/backend/pgxc/pool/execRemote.c
index a387354..855a1b5 100644
--- a/src/backend/pgxc/pool/execRemote.c
+++ b/src/backend/pgxc/pool/execRemote.c
@@ -36,7 +36,6 @@
 #include "pgxc/pgxc.h"
 
 #define END_QUERY_TIMEOUT	20
-#define CLEAR_TIMEOUT		5
 #define DATA_NODE_FETCH_SIZE 1
 
 
@@ -79,8 +78,6 @@ static void clear_write_node_list(void);
 
 static void pfree_pgxc_all_handles(PGXCNodeAllHandles *pgxc_handles);
 
-static int handle_response_clear(PGXCNodeHandle * conn);
-
 static void close_node_cursors(PGXCNodeHandle **connections, int conn_count, char *cursor);
 
 static PGXCNodeAllHandles *pgxc_get_all_transaction_nodes(PGXCNode_HandleRequested status_requested);
@@ -955,14 +952,16 @@ void
 BufferConnection(PGXCNodeHandle *conn)
 {
 	RemoteQueryState *combiner = conn->combiner;
+	MemoryContext oldcontext;
+
+	Assert(conn->state == DN_CONNECTION_STATE_QUERY && combiner);
+
 	/*
 	 * When BufferConnection is invoked CurrentContext is related to other
 	 * portal, which is trying to control the connection.
 	 * TODO See if we can find better context to switch to
 	 */
-	MemoryContext oldcontext = MemoryContextSwitchTo(combiner->ss.ss_ScanTupleSlot->tts_mcxt);
-
-	Assert(conn->state == DN_CONNECTION_STATE_QUERY && combiner);
+	oldcontext = MemoryContextSwitchTo(combiner->ss.ss_ScanTupleSlot->tts_mcxt);
 
 	/* Verify the connection is in use by the combiner */
 	combiner->current_conn = 0;
@@ -1007,10 +1006,11 @@ BufferConnection(PGXCNodeHandle *conn)
 		{
 			/* incomplete message, read more */
 			if (pgxc_node_receive(1, &conn, NULL))
-				ereport(ERROR,
-						(errcode(ERRCODE_INTERNAL_ERROR),
-						 errmsg("Failed to fetch from data node")));
-			continue;
+			{
+				conn->state = DN_CONNECTION_STATE_ERROR_FATAL;
+				add_error_message(conn, "Failed to fetch from data node");
+			}
+			break;
 		}
 		else if (res == RESPONSE_COMPLETE)
 		{
@@ -1261,7 +1261,6 @@ handle_response(PGXCNodeHandle * conn, RemoteQueryState *combiner)
 	for (;;)
 	{
 		Assert(conn->state != DN_CONNECTION_STATE_IDLE);
-		Assert(conn->combiner == combiner || conn->combiner == NULL);
 
 		/*
 		 * If we are in the process of shutting down, we
@@ -1280,6 +1279,8 @@ handle_response(PGXCNodeHandle * conn, RemoteQueryState *combiner)
 		if (!HAS_MESSAGE_BUFFERED(conn))
 			return RESPONSE_EOF;
 
+		Assert(conn->combiner == combiner || conn->combiner == NULL);
+
 		/* TODO handle other possible responses */
 		msg_type = get_message(conn, &msg_len, &msg);
 		switch (msg_type)
@@ -1374,75 +1375,6 @@ handle_response(PGXCNodeHandle * conn, RemoteQueryState *combiner)
 
 
 /*
- * Like handle_response, but for consuming the messages,
- * in case we of an error to clean the data node connection.
- * Return values:
- * RESPONSE_EOF - need to receive more data for the connection
- * RESPONSE_COMPLETE - done with the connection, or done trying (error)
- */
-static int
-handle_response_clear(PGXCNodeHandle * conn)
-{
-	char	   *msg;
-	int			msg_len;
-	char		msg_type;
-
-	for (;;)
-	{
-		/* No data available, exit */
-		if (conn->state == DN_CONNECTION_STATE_QUERY)
-			return RESPONSE_EOF;
-
-		/*
-		 * If we are in the process of shutting down, we
-		 * may be rolling back, and the buffer may contain other messages.
-		 * We want to avoid a procarray exception
-		 * as well as an error stack overflow.
-		 */
-		if (proc_exit_inprogress)
-		{
-			conn->state = DN_CONNECTION_STATE_ERROR_FATAL;
-			return RESPONSE_COMPLETE;
-		}
-
-		msg_type = get_message(conn, &msg_len, &msg);
-		switch (msg_type)
-		{
-			case '\0':			/* Not enough data in the buffer */
-			case 'c':			/* CopyToCommandComplete */
-			case 'C':			/* CommandComplete */
-			case 'T':			/* RowDescription */
-			case 'D':			/* DataRow */
-			case 'H': 			/* CopyOutResponse */
-			case 'd': 			/* CopyOutDataRow */
-			case 'A':			/* NotificationResponse */
-			case 'N':			/* NoticeResponse */
-				break;
-			case 'E':			/* ErrorResponse */
-				/*
-				 * conn->state = DN_CONNECTION_STATE_ERROR_NOT_READY;
-				 * Do not return with an error, we still need to consume Z,
-				 * ready-for-query
-				 */
-				break;
-			case 'Z':			/* ReadyForQuery */
-				conn->transaction_status = msg[0];
-				conn->state = DN_CONNECTION_STATE_IDLE;
-				return RESPONSE_COMPLETE;
-			case 'I':			/* EmptyQuery */
-			default:
-				/* sync lost? */
-				elog(WARNING, "Received unsupported message type: %c", msg_type);
-				conn->state = DN_CONNECTION_STATE_ERROR_FATAL;
-				return RESPONSE_COMPLETE;
-		}
-	}
-
-	return RESPONSE_EOF;
-}
-
-
-/*
  * Send BEGIN command to the Datanodes or Coordinators and receive responses
  */
 static int
@@ -1551,7 +1483,7 @@ finish:
 	if (!autocommit)
 		stat_transaction(pgxc_connections->dn_conn_count);
 	if (!PersistentConnections)
-		release_handles(false);
+		release_handles();
 	autocommit = true;
 	clear_write_node_list();
 
@@ -1650,9 +1582,6 @@ finish:
 		buffer = (char *) repalloc(buffer, 20 + strlen(gid) + 1);
 		sprintf(buffer, "ROLLBACK PREPARED '%s'", gid);
 
-		/* Consume any messages on the Datanodes and Coordinators first if necessary */
-		PGXCNodeConsumeMessages();
-
 		rollback_xid = BeginTranGTM(NULL);
 
 		/*
@@ -1786,7 +1715,7 @@ finish:
 	 * is aborted after the list of nodes in error state has been saved to be sent to GTM
 	 */
 	if (!PersistentConnections && res == 0)
-		release_handles(false);
+		release_handles();
 	autocommit = true;
 	clear_write_node_list();
 
@@ -1918,7 +1847,7 @@ finish:
 	if (!autocommit)
 		stat_transaction(tran_count);
 	if (!PersistentConnections)
-		release_handles(false);
+		release_handles();
 	autocommit = true;
 	clear_write_node_list();
 
@@ -2040,7 +1969,7 @@ finish:
 	if (!autocommit)
 		stat_transaction(tran_count);
 	if (!PersistentConnections)
-		release_handles(true);
+		release_handles();
 	autocommit = true;
 	clear_write_node_list();
 
@@ -2130,7 +2059,7 @@ finish:
 	if (!autocommit)
 		stat_transaction(tran_count);
 	if (!PersistentConnections)
-		release_handles(false);
+		release_handles();
 	autocommit = true;
 	clear_write_node_list();
 
@@ -2184,9 +2113,6 @@ PGXCNodeRollback(void)
 
 	tran_count = pgxc_connections->dn_conn_count + pgxc_connections->co_conn_count;
 
-	/* Consume any messages on the Datanodes and Coordinators first if necessary */
-	PGXCNodeConsumeMessages();
-
 	/*
 	 * If we do not have open transactions we have nothing to rollback just
 	 * report success
@@ -2201,7 +2127,7 @@ finish:
 	if (!autocommit)
 		stat_transaction(tran_count);
 	if (!PersistentConnections)
-		release_handles(true);
+		release_handles();
 	autocommit = true;
 	clear_write_node_list();
 
@@ -2396,7 +2322,7 @@ DataNodeCopyBegin(const char *query, List *nodelist, Snapshot snapshot, bool is_
 			if (need_tran)
 				DataNodeCopyFinish(connections, 0, COMBINE_TYPE_NONE);
 			else if (!PersistentConnections)
-				release_handles(false);
+				release_handles();
 		}
 
 		pfree(connections);
@@ -2620,7 +2546,7 @@ DataNodeCopyOut(ExecNodes *exec_nodes, PGXCNodeHandle** copy_connections, FILE*
 	if (!ValidateAndCloseCombiner(combiner))
 	{
 		if (autocommit && !PersistentConnections)
-			release_handles(false);
+			release_handles();
 		pfree(copy_connections);
 		ereport(ERROR,
 				(errcode(ERRCODE_DATA_CORRUPTED),
@@ -3188,20 +3114,27 @@ do_query(RemoteQueryState *node)
 		primaryconnection->combiner = node;
 		Assert(node->combine_type == COMBINE_TYPE_SAME);
 
-		while (node->command_complete_count < 1)
+		/* Make sure the command is completed on the primary node */
+		while (true)
 		{
-			if (pgxc_node_receive(1, &primaryconnection, NULL))
+			int res;
+			pgxc_node_receive(1, &primaryconnection, NULL);
+			res = handle_response(primaryconnection, node);
+			if (res == RESPONSE_COMPLETE)
+				break;
+			else if (res == RESPONSE_EOF)
+				continue;
+			else
 				ereport(ERROR,
 						(errcode(ERRCODE_INTERNAL_ERROR),
-						 errmsg("Failed to read response from data nodes")));
-			handle_response(primaryconnection, node);
-			if (node->errorMessage)
-			{
-				char *code = node->errorCode;
-				ereport(ERROR,
-						(errcode(MAKE_SQLSTATE(code[0], code[1], code[2], code[3], code[4])),
-						 errmsg("%s", node->errorMessage)));
-			}
+						 errmsg("Unexpected response from data node")));
+		}
+		if (node->errorMessage)
+		{
+			char *code = node->errorCode;
+			ereport(ERROR,
+					(errcode(MAKE_SQLSTATE(code[0], code[1], code[2], code[3], code[4])),
+					 errmsg("%s", node->errorMessage)));
 		}
 	}
 
@@ -3805,65 +3738,6 @@ ParamListToDataRow(ParamListInfo params, char** result)
 }
 
 
-/*
- * Consume any remaining messages on the connections.
- * This is useful for calling after ereport()
- */
-void
-PGXCNodeConsumeMessages(void)
-{
-	int i;
-	int active_count = 0;
-	int res;
-	struct timeval timeout;
-	PGXCNodeHandle *connection = NULL;
-	PGXCNodeHandle **connections = NULL;
-	PGXCNodeHandle *active_connections[NumDataNodes+NumCoords];
-
-	/* Get all active Coordinators and Datanodes */
-	active_count = get_active_nodes(active_connections);
-
-	/* Iterate through handles in use and try and clean */
-	for (i = 0; i < active_count; i++)
-	{
-		elog(WARNING, "Consuming data node messages after error.");
-
-		connection = active_connections[i];
-
-		res = RESPONSE_EOF;
-
-		while (res != RESPONSE_COMPLETE)
-		{
-			int res = handle_response_clear(connection);
-
-			if (res == RESPONSE_EOF)
-			{
-				if (!connections)
-					connections = (PGXCNodeHandle **) palloc(sizeof(PGXCNodeHandle*));
-
-				connections[0] = connection;
-
-				/* Use a timeout so we do not wait forever */
-				timeout.tv_sec = CLEAR_TIMEOUT;
-				timeout.tv_usec = 0;
-				if (pgxc_node_receive(1, connections, &timeout))
-				{
-					/* Mark this as bad, move on to next one */
-					connection->state = DN_CONNECTION_STATE_ERROR_FATAL;
-					break;
-				}
-			}
-			if (connection->state == DN_CONNECTION_STATE_ERROR_FATAL
-					|| connection->state == DN_CONNECTION_STATE_IDLE)
-				break;
-		}
-	}
-
-	if (connections)
-		pfree(connections);
-}
-
-
 /* ----------------------------------------------------------------
  *		ExecRemoteQueryReScan
  *
@@ -3893,9 +3767,6 @@ ExecRemoteQueryReScan(RemoteQueryState *node, ExprContext *exprCtxt)
  *
  * But does not need an Estate instance and does not do some unnecessary work,
  * like allocating tuple slots.
- *
- * Handles are freed when an error occurs during Transaction Abort, it is first necessary
- * to consume all the messages on the connections.
  */
 void
 ExecRemoteUtility(RemoteQuery *node)
@@ -3907,10 +3778,9 @@ ExecRemoteUtility(RemoteQuery *node)
 	GlobalTransactionId gxid = InvalidGlobalTransactionId;
 	Snapshot snapshot = GetActiveSnapshot();
 	PGXCNodeAllHandles *pgxc_connections;
-	PGXCNodeHandle *primaryconnection = NULL;/* For the moment only Datanode has a primary */
-	int			regular_conn_count;
 	int			total_conn_count;
 	int			co_conn_count;
+	int			dn_conn_count;
 	bool		need_tran;
 	int			i;
 
@@ -3920,23 +3790,11 @@ ExecRemoteUtility(RemoteQuery *node)
 
 	pgxc_connections = get_exec_connections(NULL, node->exec_nodes, exec_type);
 
-	primaryconnection = pgxc_connections->primary_handle;
-
-	/* Registering new connections needs the sum of Connections to Datanodes AND to Coordinators */
-	total_conn_count = regular_conn_count = pgxc_connections->dn_conn_count
-												+ pgxc_connections->co_conn_count;
-
-	regular_conn_count = pgxc_connections->dn_conn_count;
+	dn_conn_count = pgxc_connections->dn_conn_count;
 	co_conn_count = pgxc_connections->co_conn_count;
 
-	/*
-	 * Primary connection is counted separately in regular connection count
-	 * but is included in total connection count if used.
-	 */
-	if (primaryconnection)
-	{
-		regular_conn_count--;
-	}
+	/* Registering new connections needs the sum of Connections to Datanodes AND to Coordinators */
+	total_conn_count = dn_conn_count + co_conn_count;
 
 	if (force_autocommit)
 		need_tran = false;
@@ -3949,9 +3807,7 @@ ExecRemoteUtility(RemoteQuery *node)
 
 	if (!is_read_only)
 	{
-		if (primaryconnection)
-			register_write_nodes(1, &primaryconnection);
-		register_write_nodes(regular_conn_count, pgxc_connections->datanode_handles);
+		register_write_nodes(dn_conn_count, pgxc_connections->datanode_handles);
 	}
 
 	gxid = GetCurrentGlobalTransactionId();
@@ -3971,11 +3827,8 @@ ExecRemoteUtility(RemoteQuery *node)
 		PGXCNodeHandle *new_connections[total_conn_count];
 		int 		new_count = 0;
 
-		if (primaryconnection && primaryconnection->transaction_status != 'T')
-			new_connections[new_count++] = primaryconnection;
-
 		/* Check for Datanodes */
-		for (i = 0; i < regular_conn_count; i++)
+		for (i = 0; i < dn_conn_count; i++)
 			if (pgxc_connections->datanode_handles[i]->transaction_status != 'T')
 				new_connections[new_count++] = pgxc_connections->datanode_handles[i];
 
@@ -4005,64 +3858,11 @@ ExecRemoteUtility(RemoteQuery *node)
 		}
 	}
 
-	/* See if we have a primary nodes, execute on it first before the others */
-	if (primaryconnection)
-	{
-		if (primaryconnection->state == DN_CONNECTION_STATE_QUERY)
-			BufferConnection(primaryconnection);
-		/* If explicit transaction is needed gxid is already sent */
-		if (!need_tran && pgxc_node_send_gxid(primaryconnection, gxid))
-		{
-			ereport(ERROR,
-					(errcode(ERRCODE_INTERNAL_ERROR),
-					 errmsg("Failed to send command to data nodes")));
-		}
-		if (snapshot && pgxc_node_send_snapshot(primaryconnection, snapshot))
-		{
-			ereport(ERROR,
-					(errcode(ERRCODE_INTERNAL_ERROR),
-					 errmsg("Failed to send command to data nodes")));
-		}
-		if (pgxc_node_send_query(primaryconnection, node->sql_statement) != 0)
-		{
-			ereport(ERROR,
-					(errcode(ERRCODE_INTERNAL_ERROR),
-					 errmsg("Failed to send command to data nodes")));
-		}
-
-		Assert(remotestate->combine_type == COMBINE_TYPE_SAME);
-
-		while (remotestate->command_complete_count < 1)
-		{
-			PG_TRY();
-			{
-				pgxc_node_receive(1, &primaryconnection, NULL);
-				while (handle_response(primaryconnection, remotestate) == RESPONSE_EOF)
-					pgxc_node_receive(1, &primaryconnection, NULL);
-				if (remotestate->errorMessage)
-				{
-					char *code = remotestate->errorCode;
-					ereport(ERROR,
-							(errcode(MAKE_SQLSTATE(code[0], code[1], code[2], code[3], code[4])),
-							 errmsg("%s", remotestate->errorMessage)));
-				}
-			}
-			/* If we got an error response return immediately */
-			PG_CATCH();
-			{
-				pfree_pgxc_all_handles(pgxc_connections);
-
-				PG_RE_THROW();
-			}
-			PG_END_TRY();
-		}
-	}
-
 	/* Send query down to Datanodes */
 	if (exec_type == EXEC_ON_ALL_NODES ||
 		exec_type == EXEC_ON_DATANODES)
 	{
-		for (i = 0; i < regular_conn_count; i++)
+		for (i = 0; i < dn_conn_count; i++)
 		{
 			PGXCNodeHandle *conn = pgxc_connections->datanode_handles[i];
 
@@ -4126,24 +3926,19 @@ ExecRemoteUtility(RemoteQuery *node)
 	if (exec_type == EXEC_ON_ALL_NODES ||
 		exec_type == EXEC_ON_DATANODES)
 	{
-		while (regular_conn_count > 0)
+		while (dn_conn_count > 0)
 		{
 			int i = 0;
 
-			pgxc_node_receive(regular_conn_count, pgxc_connections->datanode_handles, NULL);
+			pgxc_node_receive(dn_conn_count, pgxc_connections->datanode_handles, NULL);
 			/*
 			 * Handle input from the data nodes.
-			 * If we got a RESPONSE_DATAROW we can break handling to wrap
-			 * it into a tuple and return. Handling will be continued upon
-			 * subsequent invocations.
-			 * If we got 0, we exclude connection from the list. We do not
-			 * expect more input from it. In case of non-SELECT query we quit
-			 * the loop when all nodes finish their work and send ReadyForQuery
-			 * with empty connections array.
+			 * We do not expect data nodes returning tuples when running utility
+			 * command.
 			 * If we got EOF, move to the next connection, will receive more
 			 * data on the next iteration.
 			 */
-			while (i < regular_conn_count)
+			while (i < dn_conn_count)
 			{
 				PGXCNodeHandle *conn = pgxc_connections->datanode_handles[i];
 				int res = handle_response(conn, remotestate);
@@ -4153,9 +3948,9 @@ ExecRemoteUtility(RemoteQuery *node)
 				}
 				else if (res == RESPONSE_COMPLETE)
 				{
-					if (i < --regular_conn_count)
+					if (i < --dn_conn_count)
 						pgxc_connections->datanode_handles[i] =
-							pgxc_connections->datanode_handles[regular_conn_count];
+							pgxc_connections->datanode_handles[dn_conn_count];
 				}
 				else if (res == RESPONSE_TUPDESC)
 				{
@@ -4171,6 +3966,19 @@ ExecRemoteUtility(RemoteQuery *node)
 				}
 			}
 		}
+
+		/*
+		 * We have processed all responses from the data nodes and if we have
+		 * error message pending we can report it. All connections should be in
+		 * consistent state now and can be released to the pool after rollback.
+		 */
+		if (remotestate->errorMessage)
+		{
+			char *code = remotestate->errorCode;
+			ereport(ERROR,
+					(errcode(MAKE_SQLSTATE(code[0], code[1], code[2], code[3], code[4])),
+					 errmsg("%s", remotestate->errorMessage)));
+		}
 	}
 
 	/* Make the same for Coordinators */
@@ -4211,6 +4019,18 @@ ExecRemoteUtility(RemoteQuery *node)
 				}
 			}
 		}
+		/*
+		 * We have processed all responses from the data nodes and if we have
+		 * error message pending we can report it. All connections should be in
+		 * consistent state now and can be released to the pool after rollback.
+		 */
+		if (remotestate->errorMessage)
+		{
+			char *code = remotestate->errorCode;
+			ereport(ERROR,
+					(errcode(MAKE_SQLSTATE(code[0], code[1], code[2], code[3], code[4])),
+					 errmsg("%s", remotestate->errorMessage)));
+		}
 	}
 }
 
@@ -4228,11 +4048,9 @@ PGXCNodeCleanAndRelease(int code, Datum arg)
 
 		/* Rollback on GTM if transaction id opened. */
 		RollbackTranGTM((GlobalTransactionId) GetCurrentTransactionIdIfAny());
-
-		release_handles(true);
-	} else
-		/* Release data node connections */
-		release_handles(false);
+	}
+	/* Release data node connections */
+	release_handles();
 
 	/* Close connection with GTM */
 	CloseGTM();
@@ -4474,5 +4292,5 @@ PGXCNodeGetNodeList(PGXC_NodeId **datanodes,
 	 * and will be sent to GTM.
 	 */
 	if (!PersistentConnections)
-		release_handles(false);
+		release_handles();
 }
diff --git a/src/backend/pgxc/pool/pgxcnode.c b/src/backend/pgxc/pool/pgxcnode.c
index fc63457..dafbec5 100644
--- a/src/backend/pgxc/pool/pgxcnode.c
+++ b/src/backend/pgxc/pool/pgxcnode.c
@@ -327,6 +327,8 @@ retry:
 
 			if (read_status == EOF || read_status < 0)
 			{
+				/* Can not read - no more actions, just discard connection */
+				conn->state = DN_CONNECTION_STATE_ERROR_FATAL;
 				add_error_message(conn, "unexpected EOF on datanode connection");
 				elog(WARNING, "unexpected EOF on datanode connection");
 				/* Should we read from the other connections before returning? */
@@ -577,12 +579,9 @@ get_message(PGXCNodeHandle *conn, int *len, char **msg)
 /*
  * Release all data node connections  and coordinator connections
  * back to pool and release occupied memory
- *
- * If force_drop is true, we force dropping all of the connections, such as after
- * a rollback, which was likely issued due to an error.
  */
 void
-release_handles(bool force_drop)
+release_handles(void)
 {
 	int			i;
 	int 		dn_discard[NumDataNodes];
@@ -604,7 +603,7 @@ release_handles(bool force_drop)
 
 		if (handle->sock != NO_SOCKET)
 		{
-			if (force_drop)
+			if (handle->state == DN_CONNECTION_STATE_ERROR_FATAL)
 				dn_discard[dn_ndisc++] = handle->nodenum;
 			else if (handle->state != DN_CONNECTION_STATE_IDLE)
 			{
@@ -622,7 +621,7 @@ release_handles(bool force_drop)
 
 		if (handle->sock != NO_SOCKET)
 		{
-			if (force_drop)
+			if (handle->state == DN_CONNECTION_STATE_ERROR_FATAL)
 				co_discard[co_ndisc++] = handle->nodenum;
 			else if (handle->state != DN_CONNECTION_STATE_IDLE)
 			{
@@ -899,19 +898,29 @@ pgxc_node_send_bind(PGXCNodeHandle * handle, const char *portal,
 					const char *statement, int paramlen, char *params)
 {
 	uint16		n16;
+	int			pnameLen;
+	int			stmtLen;
+	int 		paramCodeLen;
+	int 		paramValueLen;
+	int 		paramOutLen;
+	int			msgLen;
+
+	/* Invalid connection state, return error */
+	if (handle->state != DN_CONNECTION_STATE_IDLE)
+		return EOF;
+
 	/* portal name size (allow NULL) */
-	int			pnameLen = portal ? strlen(portal) + 1 : 1;
+	pnameLen = portal ? strlen(portal) + 1 : 1;
 	/* statement name size (allow NULL) */
-	int			stmtLen = statement ? strlen(statement) + 1 : 1;
+	stmtLen = statement ? strlen(statement) + 1 : 1;
 	/* size of parameter codes array (always empty for now) */
-	int 		paramCodeLen = 2;
+	paramCodeLen = 2;
 	/* size of parameter values array, 2 if no params */
-	int 		paramValueLen = paramlen ? paramlen : 2;
+	paramValueLen = paramlen ? paramlen : 2;
 	/* size of output parameter codes array (always empty for now) */
-	int 		paramOutLen = 2;
-
+	paramOutLen = 2;
 	/* size + pnameLen + stmtLen + parameters */
-	int			msgLen = 4 + pnameLen + stmtLen + paramCodeLen + paramValueLen + paramOutLen;
+	msgLen = 4 + pnameLen + stmtLen + paramCodeLen + paramValueLen + paramOutLen;
 
 	/* msgType + msgLen */
 	if (ensure_out_buffer_capacity(handle->outEnd + 1 + msgLen, handle) != 0)
@@ -970,11 +979,18 @@ int
 pgxc_node_send_describe(PGXCNodeHandle * handle, bool is_statement,
 						const char *name)
 {
+	int			nameLen;
+	int			msgLen;
+
+	/* Invalid connection state, return error */
+	if (handle->state != DN_CONNECTION_STATE_IDLE)
+		return EOF;
+
 	/* statement or portal name size (allow NULL) */
-	int			nameLen = name ? strlen(name) + 1 : 1;
+	nameLen = name ? strlen(name) + 1 : 1;
 
 	/* size + statement/portal + name */
-	int			msgLen = 4 + 1 + nameLen;
+	msgLen = 4 + 1 + nameLen;
 
 	/* msgType + msgLen */
 	if (ensure_out_buffer_capacity(handle->outEnd + 1 + msgLen, handle) != 0)
@@ -1191,10 +1207,16 @@ pgxc_node_flush(PGXCNodeHandle *handle)
 int
 pgxc_node_send_query(PGXCNodeHandle * handle, const char *query)
 {
-	int			strLen = strlen(query) + 1;
+	int			strLen;
+	int			msgLen;
 
+	/* Invalid connection state, return error */
+	if (handle->state != DN_CONNECTION_STATE_IDLE)
+		return EOF;
+
+	strLen = strlen(query) + 1;
 	/* size + strlen */
-	int			msgLen = 4 + strLen;
+	msgLen = 4 + strLen;
 
 	/* msgType + msgLen */
 	if (ensure_out_buffer_capacity(handle->outEnd + 1 + msgLen, handle) != 0)
@@ -1225,6 +1247,10 @@ pgxc_node_send_gxid(PGXCNodeHandle *handle, GlobalTransactionId gxid)
 	int			msglen = 8;
 	int			i32;
 
+	/* Invalid connection state, return error */
+	if (handle->state != DN_CONNECTION_STATE_IDLE)
+		return EOF;
+
 	/* msgType + msgLen */
 	if (ensure_out_buffer_capacity(handle->outEnd + 1 + msglen, handle) != 0)
 	{
@@ -1254,6 +1280,10 @@ pgxc_node_send_snapshot(PGXCNodeHandle *handle, Snapshot snapshot)
 	int			nval;
 	int			i;
 
+	/* Invalid connection state, return error */
+	if (handle->state != DN_CONNECTION_STATE_IDLE)
+		return EOF;
+
 	/* calculate message length */
 	msglen = 20;
 	if (snapshot->xcnt > 0)
@@ -1307,6 +1337,10 @@ pgxc_node_send_timestamp(PGXCNodeHandle *handle, TimestampTz timestamp)
 	uint32	n32;
 	int64	i = (int64) timestamp;
 
+	/* Invalid connection state, return error */
+	if (handle->state != DN_CONNECTION_STATE_IDLE)
+		return EOF;
+
 	/* msgType + msgLen */
 	if (ensure_out_buffer_capacity(handle->outEnd + 1 + msglen, handle) != 0)
 	{
@@ -1533,7 +1567,9 @@ get_handles(List *datanodelist, List *coordlist, bool is_coord_only_query)
 				list_free(dn_allocate);
 			if (co_allocate)
 				list_free(co_allocate);
-			return NULL;
+			ereport(ERROR,
+					(errcode(ERRCODE_INSUFFICIENT_RESOURCES),
+					 errmsg("Failed to get pooled connections")));
 		}
 		/* Initialisation for Datanodes */
 		if (dn_allocate)
@@ -1607,46 +1643,29 @@ get_transaction_nodes(PGXCNodeHandle **connections, char client_conn_type,
 	{
 		for (i = 0; i < NumDataNodes; i++)
 		{
-			/*
-			 * We may want to consider also not returning connections with a
-			 * state of DN_CONNECTION_STATE_ERROR_NOT_READY or
-			 * DN_CONNECTION_STATE_ERROR_FATAL.
-			 * ERROR_NOT_READY can happen if the data node abruptly disconnects.
-			 */
-			if (status_requested == HANDLE_IDLE)
+			if (dn_handles[i].sock != NO_SOCKET && dn_handles[i].state != DN_CONNECTION_STATE_ERROR_FATAL)
 			{
-				if (dn_handles[i].sock != NO_SOCKET && dn_handles[i].transaction_status == 'I')
+				if (status_requested == HANDLE_IDLE && dn_handles[i].transaction_status == 'I')
 					connections[tran_count++] = &dn_handles[i];
-			}
-			else if (status_requested == HANDLE_ERROR)
-			{
-				if (dn_handles[i].transaction_status == 'E')
+				else if (status_requested == HANDLE_ERROR && dn_handles[i].transaction_status == 'E')
 					connections[tran_count++] = &dn_handles[i];
-			}
-			else
-			{
-				if (dn_handles[i].sock != NO_SOCKET && dn_handles[i].transaction_status != 'I')
+				else if (dn_handles[i].transaction_status != 'I')
 					connections[tran_count++] = &dn_handles[i];
 			}
 		}
 	}
+
 	if (coord_count && client_conn_type == REMOTE_CONN_COORD)
 	{
 		for (i = 0; i < NumCoords; i++)
 		{
-			if (status_requested == HANDLE_IDLE)
+			if (co_handles[i].sock != NO_SOCKET && co_handles[i].state != DN_CONNECTION_STATE_ERROR_FATAL)
 			{
-				if (co_handles[i].sock != NO_SOCKET && co_handles[i].transaction_status == 'I')
+				if (status_requested == HANDLE_IDLE && co_handles[i].transaction_status == 'I')
 					connections[tran_count++] = &co_handles[i];
-			}
-			else if (status_requested == HANDLE_ERROR)
-			{
-				if (co_handles[i].transaction_status == 'E')
-					connections[tran_count++] = &co_handles[i];				
-			}
-			else
-			{
-				if (co_handles[i].sock != NO_SOCKET && co_handles[i].transaction_status != 'I')
+				else if (status_requested == HANDLE_ERROR && co_handles[i].transaction_status == 'E')
+					connections[tran_count++] = &co_handles[i];
+				else if (co_handles[i].transaction_status != 'I')
 					connections[tran_count++] = &co_handles[i];
 			}
 		}
@@ -1789,11 +1808,11 @@ pgxc_all_handles_send_query(PGXCNodeAllHandles *pgxc_handles, const char *buffer
     /* Send to Datanodes */
     for (i = 0; i < dn_conn_count; i++)
 	{
-		/*
-		 * Clean connection if fetch in progress
-		 */
-		if (pgxc_handles->datanode_handles[i]->state == DN_CONNECTION_STATE_QUERY)
-			BufferConnection(pgxc_handles->datanode_handles[i]);
+		if (pgxc_handles->datanode_handles[i]->state != DN_CONNECTION_STATE_IDLE)
+		{
+			pgxc_handles->datanode_handles[i]->state = DN_CONNECTION_STATE_ERROR_FATAL;
+			continue;
+		}
 		if (pgxc_node_send_query(pgxc_handles->datanode_handles[i], buffer))
 		{
 			add_error_message(pgxc_handles->datanode_handles[i], "Can not send request");
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index c83563b..e61d444 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -3615,7 +3615,7 @@ PostgresMain(int argc, char *argv[], const char *username)
 		 * Abort the current transaction in order to recover.
 		 */
 #ifdef PGXC
-		/* 
+		/*
 		 * Temporarily do not abort if we are already in an abort state.
 		 * This change tries to handle the case where the error data stack fills up.
 		*/
@@ -3657,7 +3657,6 @@ PostgresMain(int argc, char *argv[], const char *username)
 	/*
 	 * Non-error queries loop here.
 	 */
-
 	for (;;)
 	{
 		/*
diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c
index 327ba7c..ef5e218 100644
--- a/src/backend/utils/init/postinit.c
+++ b/src/backend/utils/init/postinit.c
@@ -30,6 +30,9 @@
 #include "mb/pg_wchar.h"
 #include "miscadmin.h"
 #include "pgstat.h"
+#ifdef PGXC
+#include "pgxc/pgxc.h"
+#endif
 #include "postmaster/autovacuum.h"
 #include "postmaster/postmaster.h"
 #include "storage/backendid.h"
@@ -491,6 +494,18 @@ InitPostgres(const char *in_dbname, Oid dboid, const char *username,
 	 */
 	on_shmem_exit(ShutdownPostgres, 0);
 
+#ifdef PGXC
+	/*
+	 * The transaction below consumes a xid, and we should let GTM know about
+	 * that. Session being initializing now and value from the coordinator
+	 * is not available, so try and connect to GTM directly
+	 * The check for PostmasterPid is to detect --single mode as it runs
+	 * under initdb. PostmasterPid is not set in this case
+	 */
+	if (!bootstrap && IS_PGXC_DATANODE && PostmasterPid)
+		SetForceXidFromGTM(true);
+#endif
+
 	/*
 	 * Start a new transaction here before first access to db, and get a
 	 * snapshot.  We don't have a use for the snapshot itself, but we're
@@ -655,6 +670,14 @@ InitPostgres(const char *in_dbname, Oid dboid, const char *username,
 	if (!bootstrap)
 		CommitTransactionCommand();
 
+#ifdef PGXC
+	/*
+	 * We changed the flag, set it back to default
+	 */
+	if (!bootstrap && IS_PGXC_DATANODE && PostmasterPid)
+		SetForceXidFromGTM(false);
+#endif
+
 	return am_superuser;
 }
 
diff --git a/src/include/pgxc/execRemote.h b/src/include/pgxc/execRemote.h
index c5c45c0..8d7a348 100644
--- a/src/include/pgxc/execRemote.h
+++ b/src/include/pgxc/execRemote.h
@@ -118,7 +118,6 @@ extern void	PGXCNodeImplicitCommitPrepared(GlobalTransactionId prepare_xid,
 										   GlobalTransactionId commit_xid,
 										   char *gid,
 										   bool is_commit);
-extern void PGXCNodeConsumeMessages(void);
 
 /* Get list of nodes */
 extern void PGXCNodeGetNodeList(PGXC_NodeId **datanodes,
diff --git a/src/include/pgxc/pgxcnode.h b/src/include/pgxc/pgxcnode.h
index 47b0b96..098a7f2 100644
--- a/src/include/pgxc/pgxcnode.h
+++ b/src/include/pgxc/pgxcnode.h
@@ -102,7 +102,7 @@ extern int	PGXCNodeConnClean(NODE_CONNECTION * conn);
 extern void PGXCNodeCleanAndRelease(int code, Datum arg);
 
 extern PGXCNodeAllHandles *get_handles(List *datanodelist, List *coordlist, bool is_query_coord_only);
-extern void release_handles(bool force_drop);
+extern void release_handles(void);
 
 extern int	get_transaction_nodes(PGXCNodeHandle ** connections,
 								  char client_conn_type,

-----------------------------------------------------------------------

Summary of changes:
 src/backend/pgxc/pool/execRemote.c |  344 +++++++++---------------------------
 src/backend/pgxc/pool/pgxcnode.c   |  119 +++++++------
 src/backend/tcop/postgres.c        |    3 +-
 src/backend/utils/init/postinit.c  |   23 +++
 src/include/pgxc/execRemote.h      |    1 -
 src/include/pgxc/pgxcnode.h        |    2 +-
 6 files changed, 175 insertions(+), 317 deletions(-)


hooks/post-receive
-- 
Postgres-XC