Fix bugs around handling of params passed to the datanodes.

This fixes problems reported in plpgsql function calls since
fast-query-shipping was added. But fixing the problem also highlighted other
bugs in the protocol handling. For example, we would set a datanode connection
state to be IDLE upon receiving a CommandComplete. But for simple query
protocol, we should really be waiting for ReadyForQuery before changing the
state. This patch has related fixes.

plpgsql test case's expected output is also fixed now that the underlying bug
has been take care of

10 files changed, 317 insertions, 100 deletions

diff --git a/src/backend/commands/prepare.c b/src/backend/commands/prepare.c
index e66d2bc70b..fa90742b7f 100644
--- a/src/backend/commands/prepare.c
+++ b/src/backend/commands/prepare.c
@@ -490,7 +490,7 @@ SetRemoteStatementName(Plan *plan, const char *stmt_name, int num_params,
 		char name[NAMEDATALEN];
 
 		/* Nothing to do if parameters are already set for this query */
-		if (remotequery->remote_num_params != 0)
+		if (remotequery->rq_num_params != 0)
 			return 0;
 
 		if (stmt_name)
@@ -533,8 +533,8 @@ SetRemoteStatementName(Plan *plan, const char *stmt_name, int num_params,
 					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
 					 errmsg("Passing parameters in PREPARE statement is not supported")));
 
-		remotequery->remote_num_params = num_params;
-		remotequery->remote_param_types = param_types;
+		remotequery->rq_num_params = num_params;
+		remotequery->rq_param_types = param_types;
 	}
 	else if (IsA(plan, ModifyTable))
 	{
diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c
index 261901f6a3..7c56a1c553 100644
--- a/src/backend/nodes/copyfuncs.c
+++ b/src/backend/nodes/copyfuncs.c
@@ -1105,9 +1105,12 @@ _copyRemoteQuery(const RemoteQuery *from)
 	COPY_SCALAR_FIELD(force_autocommit);
 	COPY_STRING_FIELD(statement);
 	COPY_STRING_FIELD(cursor);
-	COPY_SCALAR_FIELD(remote_num_params);
-	COPY_POINTER_FIELD(remote_param_types,
-	   sizeof(from->remote_param_types[0]) * from->remote_num_params);
+	COPY_SCALAR_FIELD(rq_num_params);
+	if (from->rq_param_types)
+		COPY_POINTER_FIELD(rq_param_types,
+			sizeof(from->rq_param_types[0]) * from->rq_num_params);
+	else
+		newnode->rq_param_types = NULL;
 	COPY_SCALAR_FIELD(exec_type);
 
 	COPY_SCALAR_FIELD(reduce_level);
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 99670231fe..9cd2aca7e2 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -758,11 +758,11 @@ _outRemoteQuery(StringInfo str, const RemoteQuery *node)
 	WRITE_BOOL_FIELD(force_autocommit);
 	WRITE_STRING_FIELD(statement);
 	WRITE_STRING_FIELD(cursor);
-	WRITE_INT_FIELD(remote_num_params);
+	WRITE_INT_FIELD(rq_num_params);
 
-	appendStringInfo(str, " :remote_param_types");
-	for (i = 0; i < node->remote_num_params; i++)
-		appendStringInfo(str, " %d", node->remote_param_types[i]);
+	appendStringInfo(str, " :rq_param_types");
+	for (i = 0; i < node->rq_num_params; i++)
+		appendStringInfo(str, " %d", node->rq_param_types[i]);
 
 	WRITE_ENUM_FIELD(exec_type, RemoteQueryExecType);
 	WRITE_BOOL_FIELD(has_row_marks);
diff --git a/src/backend/pgxc/barrier/barrier.c b/src/backend/pgxc/barrier/barrier.c
index e532d6a262..0fe377a4a4 100644
--- a/src/backend/pgxc/barrier/barrier.c
+++ b/src/backend/pgxc/barrier/barrier.c
@@ -200,7 +200,7 @@ SendBarrierPrepareRequest(List *coords, const char *id)
 		memcpy(handle->outBuffer + handle->outEnd, id, barrier_idlen);
 		handle->outEnd += barrier_idlen;
 
-		handle->state = DN_CONNECTION_STATE_QUERY;
+		PGXCNodeSetConnectionState(handle, DN_CONNECTION_STATE_QUERY);
 
 		pgxc_node_flush(handle);
 	}
@@ -286,7 +286,7 @@ SendBarrierEndRequest(PGXCNodeAllHandles *coord_handles, const char *id)
 		memcpy(handle->outBuffer + handle->outEnd, id, barrier_idlen);
 		handle->outEnd += barrier_idlen;
 
-		handle->state = DN_CONNECTION_STATE_QUERY;
+		PGXCNodeSetConnectionState(handle, DN_CONNECTION_STATE_QUERY);
 		pgxc_node_flush(handle);
 	}
 
@@ -398,7 +398,7 @@ ExecuteBarrier(const char *id)
 		memcpy(handle->outBuffer + handle->outEnd, id, barrier_idlen);
 		handle->outEnd += barrier_idlen;
 
-		handle->state = DN_CONNECTION_STATE_QUERY;
+		PGXCNodeSetConnectionState(handle, DN_CONNECTION_STATE_QUERY);
 		pgxc_node_flush(handle);
 	}
 
diff --git a/src/backend/pgxc/pool/execRemote.c b/src/backend/pgxc/pool/execRemote.c
index 7c6204b1b4..eb3807c70e 100644
--- a/src/backend/pgxc/pool/execRemote.c
+++ b/src/backend/pgxc/pool/execRemote.c
@@ -1045,7 +1045,8 @@ BufferConnection(PGXCNodeHandle *conn)
 		{
 			if (pgxc_node_receive(1, &conn, NULL))
 			{
-				conn->state = DN_CONNECTION_STATE_ERROR_FATAL;
+				PGXCNodeSetConnectionState(conn,
+						DN_CONNECTION_STATE_ERROR_FATAL);
 				add_error_message(conn, "Failed to fetch from data node");
 			}
 		}
@@ -1551,6 +1552,8 @@ pgxc_node_receive_responses(const int conn_count, PGXCNodeHandle ** connections,
 		while (i < count)
 		{
 			int result =  handle_response(to_receive[i], combiner);
+			elog(DEBUG5, "Received response %d on connection to node %s",
+					result, to_receive[i]->nodename);
 			switch (result)
 			{
 				case RESPONSE_EOF: /* have something to read, keep receiving */
@@ -1575,13 +1578,12 @@ pgxc_node_receive_responses(const int conn_count, PGXCNodeHandle ** connections,
 					break;
 
 				case RESPONSE_WAITXIDS:
-					break;
-
 				case RESPONSE_ASSIGN_GXID:
+				case RESPONSE_TUPDESC:
 					break;
 
-				case RESPONSE_TUPDESC:
 				case RESPONSE_DATAROW:
+					combiner->currentRow = NULL;
 					break;
 
 				default:
@@ -1633,7 +1635,7 @@ handle_response(PGXCNodeHandle *conn, ResponseCombiner *combiner)
 		 * as well as an error stack overflow.
 		 */
 		if (proc_exit_inprogress)
-			conn->state = DN_CONNECTION_STATE_ERROR_FATAL;
+			PGXCNodeSetConnectionState(conn, DN_CONNECTION_STATE_ERROR_FATAL);
 
 		/*
 		 * Don't read from from the connection if there is a fatal error.
@@ -1653,6 +1655,8 @@ handle_response(PGXCNodeHandle *conn, ResponseCombiner *combiner)
 
 		/* TODO handle other possible responses */
 		msg_type = get_message(conn, &msg_len, &msg);
+		elog(DEBUG5, "handle_response - received message %c, node %s, "
+				"current_state %d", msg_type, conn->nodename, conn->state);
 		switch (msg_type)
 		{
 			case '\0':			/* Not enough data in the buffer */
@@ -1663,8 +1667,13 @@ handle_response(PGXCNodeHandle *conn, ResponseCombiner *combiner)
 			case 'C':			/* CommandComplete */
 				HandleCommandComplete(combiner, msg, msg_len, conn);
 				conn->combiner = NULL;
-				if (conn->state == DN_CONNECTION_STATE_QUERY)
-					conn->state = DN_CONNECTION_STATE_IDLE;
+				/* 
+				 * In case of simple query protocol, wait for the ReadyForQuery
+				 * before marking connection as Idle
+				 */
+				if (combiner->extended_query &&
+					conn->state == DN_CONNECTION_STATE_QUERY)
+					PGXCNodeSetConnectionState(conn, DN_CONNECTION_STATE_IDLE);
 				return RESPONSE_COMPLETE;
 			case 'T':			/* RowDescription */
 #ifdef DN_CONNECTION_DEBUG
@@ -1684,7 +1693,7 @@ handle_response(PGXCNodeHandle *conn, ResponseCombiner *combiner)
 				break;
 			case 's':			/* PortalSuspended */
 				/* No activity is expected on the connection until next query */
-				conn->state = DN_CONNECTION_STATE_IDLE;
+				PGXCNodeSetConnectionState(conn, DN_CONNECTION_STATE_IDLE);
 				conn->combiner = NULL;
 				return RESPONSE_SUSPENDED;
 			case '1': /* ParseComplete */
@@ -1694,16 +1703,16 @@ handle_response(PGXCNodeHandle *conn, ResponseCombiner *combiner)
 				/* simple notifications, continue reading */
 				break;
 			case 'G': /* CopyInResponse */
-				conn->state = DN_CONNECTION_STATE_COPY_IN;
+				PGXCNodeSetConnectionState(conn, DN_CONNECTION_STATE_COPY_IN);
 				HandleCopyIn(combiner);
 				/* Done, return to caller to let it know the data can be passed in */
 				return RESPONSE_COPY;
 			case 'H': /* CopyOutResponse */
-				conn->state = DN_CONNECTION_STATE_COPY_OUT;
+				PGXCNodeSetConnectionState(conn, DN_CONNECTION_STATE_COPY_OUT);
 				HandleCopyOut(combiner);
 				return RESPONSE_COPY;
 			case 'd': /* CopyOutDataRow */
-				conn->state = DN_CONNECTION_STATE_COPY_OUT;
+				PGXCNodeSetConnectionState(conn, DN_CONNECTION_STATE_COPY_OUT);
 				HandleCopyDataRow(combiner, msg, msg_len);
 				break;
 			case 'E':			/* ErrorResponse */
@@ -1727,7 +1736,7 @@ handle_response(PGXCNodeHandle *conn, ResponseCombiner *combiner)
 				 * with the connection
 				 */
 				conn->transaction_status = msg[0];
-				conn->state = DN_CONNECTION_STATE_IDLE;
+				PGXCNodeSetConnectionState(conn, DN_CONNECTION_STATE_IDLE);
 				conn->combiner = NULL;
 #ifdef DN_CONNECTION_DEBUG
 				conn->have_row_desc = false;
@@ -1738,7 +1747,7 @@ handle_response(PGXCNodeHandle *conn, ResponseCombiner *combiner)
 				HandleDatanodeCommandId(combiner, msg, msg_len);
 				break;
 			case 'b':
-				conn->state = DN_CONNECTION_STATE_IDLE;
+				PGXCNodeSetConnectionState(conn, DN_CONNECTION_STATE_IDLE);
 				return RESPONSE_BARRIER_OK;
 			case 'I':			/* EmptyQuery */
 				return RESPONSE_COMPLETE;
@@ -1751,7 +1760,7 @@ handle_response(PGXCNodeHandle *conn, ResponseCombiner *combiner)
 			default:
 				/* sync lost? */
 				elog(WARNING, "Received unsupported message type: %c", msg_type);
-				conn->state = DN_CONNECTION_STATE_ERROR_FATAL;
+				PGXCNodeSetConnectionState(conn, DN_CONNECTION_STATE_ERROR_FATAL);
 				/* stop reading */
 				return RESPONSE_COMPLETE;
 		}
@@ -1780,7 +1789,7 @@ is_data_node_ready(PGXCNodeHandle * conn)
 		 * as well as an error stack overflow.
 		 */
 		if (proc_exit_inprogress)
-			conn->state = DN_CONNECTION_STATE_ERROR_FATAL;
+			PGXCNodeSetConnectionState(conn, DN_CONNECTION_STATE_ERROR_FATAL);
 
 		/* don't read from from the connection if there is a fatal error */
 		if (conn->state == DN_CONNECTION_STATE_ERROR_FATAL)
@@ -1800,7 +1809,7 @@ is_data_node_ready(PGXCNodeHandle * conn)
 			 * with the connection
 			 */
 			conn->transaction_status = msg[0];
-			conn->state = DN_CONNECTION_STATE_IDLE;
+			PGXCNodeSetConnectionState(conn, DN_CONNECTION_STATE_IDLE);
 			conn->combiner = NULL;
 			return true;
 		}
@@ -1859,6 +1868,9 @@ pgxc_node_begin(int conn_count, PGXCNodeHandle **connections,
 			need_tran_block = true;
 		else if (IS_PGXC_REMOTE_COORDINATOR)
 			need_tran_block = false;
+
+		elog(DEBUG5, "need_tran_block %d, connections[%d]->transaction_status %c",
+				need_tran_block, i, connections[i]->transaction_status);
 		/* Send BEGIN if not already in transaction */
 		if (need_tran_block && connections[i]->transaction_status == 'I')
 		{
@@ -1924,6 +1936,9 @@ pgxc_node_remote_cleanup_all(void)
 	char		   *resetcmd = "RESET ALL;RESET SESSION AUTHORIZATION;"
 							   "RESET transaction_isolation;";
 
+	elog(DEBUG5, "pgxc_node_remote_cleanup_all - handles->co_conn_count %d,"
+			"handles->dn_conn_count", handles->co_conn_count,
+			handles->dn_conn_count);
 	/*
 	 * We must handle reader and writer connections both since even a read-only
 	 * needs to be cleaned up.
@@ -1941,7 +1956,7 @@ pgxc_node_remote_cleanup_all(void)
 		/* At this point connection should be in IDLE state */
 		if (handle->state != DN_CONNECTION_STATE_IDLE)
 		{
-			handle->state = DN_CONNECTION_STATE_ERROR_FATAL;
+			PGXCNodeSetConnectionState(handle, DN_CONNECTION_STATE_ERROR_FATAL);
 			continue;
 		}
 
@@ -1954,7 +1969,7 @@ pgxc_node_remote_cleanup_all(void)
 			ereport(WARNING,
 					(errcode(ERRCODE_INTERNAL_ERROR),
 					 errmsg("Failed to clean up data nodes")));
-			handle->state = DN_CONNECTION_STATE_ERROR_FATAL;
+			PGXCNodeSetConnectionState(handle, DN_CONNECTION_STATE_ERROR_FATAL);
 			continue;
 		}
 		new_connections[new_conn_count++] = handle;
@@ -1967,7 +1982,7 @@ pgxc_node_remote_cleanup_all(void)
 		/* At this point connection should be in IDLE state */
 		if (handle->state != DN_CONNECTION_STATE_IDLE)
 		{
-			handle->state = DN_CONNECTION_STATE_ERROR_FATAL;
+			PGXCNodeSetConnectionState(handle, DN_CONNECTION_STATE_ERROR_FATAL);
 			continue;
 		}
 
@@ -1980,7 +1995,7 @@ pgxc_node_remote_cleanup_all(void)
 			ereport(WARNING,
 					(errcode(ERRCODE_INTERNAL_ERROR),
 					 errmsg("Failed to clean up data nodes")));
-			handle->state = DN_CONNECTION_STATE_ERROR_FATAL;
+			PGXCNodeSetConnectionState(handle, DN_CONNECTION_STATE_ERROR_FATAL);
 			continue;
 		}
 		new_connections[new_conn_count++] = handle;
@@ -2601,6 +2616,9 @@ pgxc_node_remote_abort(void)
 
 	SetSendCommandId(false);
 
+	elog(DEBUG5, "pgxc_node_remote_abort - dn_conn_count %d, co_conn_count %d",
+			handles->dn_conn_count, handles->co_conn_count);
+
 	for (i = 0; i < handles->dn_conn_count; i++)
 	{
 		PGXCNodeHandle *conn = handles->datanode_handles[i];
@@ -2609,6 +2627,10 @@ pgxc_node_remote_abort(void)
 		if (conn->sock == NO_SOCKET)
 			continue;
 
+		elog(DEBUG5, "node %s, conn->transaction_status %c",
+				conn->nodename,
+				conn->transaction_status);
+
 		if (conn->transaction_status != 'I')
 		{
 			/* Read in any pending input */
@@ -2619,7 +2641,7 @@ pgxc_node_remote_abort(void)
 			 * Do not matter, is there committed or failed transaction,
 			 * just send down rollback to finish it.
 			 */
-			if (pgxc_node_send_query(conn, rollbackCmd))
+			if (pgxc_node_send_rollback(conn, rollbackCmd))
 			{
 				add_error_message(conn,
 						"failed to send ROLLBACK TRANSACTION command");
@@ -2646,7 +2668,7 @@ pgxc_node_remote_abort(void)
 			 * Do not matter, is there committed or failed transaction,
 			 * just send down rollback to finish it.
 			 */
-			if (pgxc_node_send_query(conn, rollbackCmd))
+			if (pgxc_node_send_rollback(conn, rollbackCmd))
 			{
 				add_error_message(conn,
 						"failed to send ROLLBACK TRANSACTION command");
@@ -3222,6 +3244,9 @@ pgxc_start_command_on_connection(PGXCNodeHandle *connection,
 	RemoteQuery	*step = (RemoteQuery *) combiner->ss.ps.plan;
 	CHECK_OWNERSHIP(connection, combiner);
 
+	elog(DEBUG5, "pgxc_start_command_on_connection - node %s, state %d",
+			connection->nodename, connection->state);
+
 	/*
 	 * Scan descriptor would be valid and would contain a valid snapshot
 	 * in cases when we need to send out of order command id to data node
@@ -3234,7 +3259,7 @@ pgxc_start_command_on_connection(PGXCNodeHandle *connection,
 
 	if (snapshot && pgxc_node_send_snapshot(connection, snapshot))
 		return false;
-	if (step->statement || step->cursor || step->remote_param_types)
+	if (step->statement || step->cursor || remotestate->rqs_num_params)
 	{
 		/* need to use Extended Query Protocol */
 		int	fetch = 0;
@@ -3262,8 +3287,8 @@ pgxc_start_command_on_connection(PGXCNodeHandle *connection,
 							prepared ? NULL : step->sql_statement,
 							step->statement,
 							step->cursor,
-							step->remote_num_params,
-							step->remote_param_types,
+							remotestate->rqs_num_params,
+							remotestate->rqs_param_types,
 							remotestate->paramval_len,
 							remotestate->paramval_data,
 							step->has_row_marks ? true : step->read_only,
@@ -3272,6 +3297,7 @@ pgxc_start_command_on_connection(PGXCNodeHandle *connection,
 	}
 	else
 	{
+		combiner->extended_query = false;
 		if (pgxc_node_send_query(connection, step->sql_statement) != 0)
 			return false;
 	}
@@ -3684,14 +3710,16 @@ ExecCloseRemoteStatement(const char *stmt_name, List *nodelist)
 			 * unclosed statement on the Datanode as a fatal issue and
 			 * force connection is discarded
 			 */
-			connections[i]->state = DN_CONNECTION_STATE_ERROR_FATAL;
+			PGXCNodeSetConnectionState(connections[i],
+					DN_CONNECTION_STATE_ERROR_FATAL);
 			ereport(WARNING,
 					(errcode(ERRCODE_INTERNAL_ERROR),
 					 errmsg("Failed to close Datanode statemrnt")));
 		}
 		if (pgxc_node_send_sync(connections[i]) != 0)
 		{
-			connections[i]->state = DN_CONNECTION_STATE_ERROR_FATAL;
+			PGXCNodeSetConnectionState(connections[i],
+					DN_CONNECTION_STATE_ERROR_FATAL);
 			ereport(WARNING,
 					(errcode(ERRCODE_INTERNAL_ERROR),
 					 errmsg("Failed to close Datanode statement")));
@@ -3709,7 +3737,8 @@ ExecCloseRemoteStatement(const char *stmt_name, List *nodelist)
 		if (pgxc_node_receive(conn_count, connections, NULL))
 		{
 			for (i = 0; i <= conn_count; i++)
-				connections[i]->state = DN_CONNECTION_STATE_ERROR_FATAL;
+				PGXCNodeSetConnectionState(connections[i],
+						DN_CONNECTION_STATE_ERROR_FATAL);
 
 			ereport(ERROR,
 					(errcode(ERRCODE_INTERNAL_ERROR),
@@ -3780,6 +3809,144 @@ DataNodeCopyInBinaryForAll(char *msg_buf, int len, int conn_count,
 }
 
 /*
+ * Encode parameter values to format of DataRow message (the same format is
+ * used in Bind) to prepare for sending down to Datanodes.
+ * The data row is copied to RemoteQueryState.paramval_data.
+ */
+void
+SetDataRowForExtParams(ParamListInfo paraminfo, RemoteQueryState *rq_state)
+{
+	StringInfoData buf;
+	uint16 n16;
+	int i;
+	int real_num_params = 0;
+	RemoteQuery *node = (RemoteQuery*) rq_state->combiner.ss.ps.plan;
+
+	/* If there are no parameters, there is no data to BIND. */
+	if (!paraminfo)
+		return;
+
+	Assert(!rq_state->paramval_data);
+
+	/*
+	 * It is necessary to fetch parameters
+	 * before looking at the output value.
+	 */
+	for (i = 0; i < paraminfo->numParams; i++)
+	{
+		ParamExternData *param;
+
+		param = &paraminfo->params[i];
+
+		if (!OidIsValid(param->ptype) && paraminfo->paramFetch != NULL)
+			(*paraminfo->paramFetch) (paraminfo, i + 1);
+
+		/*
+		 * This is the last parameter found as useful, so we need
+		 * to include all the previous ones to keep silent the remote
+		 * nodes. All the parameters prior to the last usable having no
+		 * type available will be considered as NULL entries.
+		 */
+		if (OidIsValid(param->ptype))
+			real_num_params = i + 1;
+	}
+
+	/*
+	 * If there are no parameters available, simply leave.
+	 * This is possible in the case of a query called through SPI
+	 * and using no parameters.
+	 */
+	if (real_num_params == 0)
+	{
+		rq_state->paramval_data = NULL;
+		rq_state->paramval_len = 0;
+		return;
+	}
+
+	initStringInfo(&buf);
+
+	/* Number of parameter values */
+	n16 = htons(real_num_params);
+	appendBinaryStringInfo(&buf, (char *) &n16, 2);
+
+	/* Parameter values */
+	for (i = 0; i < real_num_params; i++)
+	{
+		ParamExternData *param = &paraminfo->params[i];
+		uint32 n32;
+
+		/*
+		 * Parameters with no types are considered as NULL and treated as integer
+		 * The same trick is used for dropped columns for remote DML generation.
+		 */
+		if (param->isnull || !OidIsValid(param->ptype))
+		{
+			n32 = htonl(-1);
+			appendBinaryStringInfo(&buf, (char *) &n32, 4);
+		}
+		else
+		{
+			Oid		typOutput;
+			bool	typIsVarlena;
+			Datum	pval;
+			char   *pstring;
+			int		len;
+
+			/* Get info needed to output the value */
+			getTypeOutputInfo(param->ptype, &typOutput, &typIsVarlena);
+
+			/*
+			 * If we have a toasted datum, forcibly detoast it here to avoid
+			 * memory leakage inside the type's output routine.
+			 */
+			if (typIsVarlena)
+				pval = PointerGetDatum(PG_DETOAST_DATUM(param->value));
+			else
+				pval = param->value;
+
+			/* Convert Datum to string */
+			pstring = OidOutputFunctionCall(typOutput, pval);
+
+			/* copy data to the buffer */
+			len = strlen(pstring);
+			n32 = htonl(len);
+			appendBinaryStringInfo(&buf, (char *) &n32, 4);
+			appendBinaryStringInfo(&buf, pstring, len);
+		}
+	}
+
+
+	/*
+	 * If parameter types are not already set, infer them from
+	 * the paraminfo.
+	 */
+	if (node->rq_num_params > 0)
+	{
+		/*
+		 * Use the already known param types for BIND. Parameter types
+		 * can be already known when the same plan is executed multiple
+		 * times.
+		 */
+		if (node->rq_num_params != real_num_params)
+			elog(ERROR, "Number of user-supplied parameters do not match "
+						"the number of remote parameters");
+		rq_state->rqs_num_params = node->rq_num_params;
+		rq_state->rqs_param_types = node->rq_param_types;
+	}
+	else
+	{
+		rq_state->rqs_num_params = real_num_params;
+		rq_state->rqs_param_types = (Oid *) palloc(sizeof(Oid) * real_num_params);
+		for (i = 0; i < real_num_params; i++)
+			rq_state->rqs_param_types[i] = paraminfo->params[i].ptype;
+	}
+
+	/* Assign the newly allocated data row to paramval */
+	rq_state->paramval_data = buf.data;
+	rq_state->paramval_len = buf.len;
+}
+
+/*
  * Clear per transaction remote information
  */
 void
@@ -3972,6 +4139,7 @@ PreAbort_Remote(void)
 	 * certain issues for aborted transactions, we drop the connections.
 	 * Revisit and fix the issue
 	 */
+	elog(DEBUG5, "temp_object_included %d", temp_object_included);
 	if (!temp_object_included)
 	{
 		/* Clean up remote sessions */
@@ -4381,14 +4549,9 @@ ExecInitRemoteQuery(RemoteQuery *node, EState *estate, int eflags)
 
 	/*
 	 * If there are parameters supplied, get them into a form to be sent to the
-	 * datanodes with bind message. We should not have had done this before.
+	 * Datanodes with bind message. We should not have had done this before.
 	 */
-	if (estate->es_param_list_info)
-	{
-		Assert(!remotestate->paramval_data);
-		remotestate->paramval_len = ParamListToDataRow(estate->es_param_list_info,
-												&remotestate->paramval_data);
-	}
+	SetDataRowForExtParams(estate->es_param_list_info, remotestate);
 
 	/* We need expression context to evaluate */
 	if (node->exec_nodes && node->exec_nodes->en_expr)
@@ -6111,7 +6274,7 @@ ExecEndRemoteSubplan(RemoteSubplanState *node)
 		 * state will be changed back to IDLE and conn->coordinator will be
 		 * cleared.
 		 */
-		conn->state = DN_CONNECTION_STATE_CLOSE;
+		PGXCNodeSetConnectionState(conn, DN_CONNECTION_STATE_CLOSE);
 	}
 
 	while (combiner->conn_count > 0)
diff --git a/src/backend/pgxc/pool/pgxcnode.c b/src/backend/pgxc/pool/pgxcnode.c
index f55e003e9f..87a2f6167f 100644
--- a/src/backend/pgxc/pool/pgxcnode.c
+++ b/src/backend/pgxc/pool/pgxcnode.c
@@ -403,7 +403,7 @@ pgxc_node_init(PGXCNodeHandle *handle, int sock, bool global_session, int pid)
 	handle->sock = sock;
 	handle->backend_pid = pid;
 	handle->transaction_status = 'I';
-	handle->state = DN_CONNECTION_STATE_IDLE;
+	PGXCNodeSetConnectionState(handle, DN_CONNECTION_STATE_IDLE);
 	handle->read_only = true;
 	handle->ck_resp_rollback = false;
 	handle->combiner = NULL;
@@ -481,7 +481,8 @@ pgxc_node_receive(const int conn_count,
 		else
 		{
 			/* flag as bad, it will be removed from the list */
-			connections[i]->state = DN_CONNECTION_STATE_ERROR_FATAL;
+			PGXCNodeSetConnectionState(connections[i],
+					DN_CONNECTION_STATE_ERROR_FATAL);
 			pool_fd[i].fd = -1;
 			pool_fd[i].events = 0;
 		}
@@ -523,7 +524,8 @@ retry:
 		/* Handle timeout */
 		elog(DEBUG1, "timeout %ld while waiting for any response from %d connections", timeout_ms,conn_count);
 		for (i = 0; i < conn_count; i++)
-			connections[i]->state = DN_CONNECTION_STATE_ERROR_FATAL;
+			PGXCNodeSetConnectionState(connections[i],
+					DN_CONNECTION_STATE_ERROR_FATAL);
 		return NO_ERROR_OCCURED;
 	}
 
@@ -543,7 +545,8 @@ retry:
 				if ( read_status == EOF || read_status < 0 )
 				{
 					/* Can not read - no more actions, just discard connection */
-					conn->state = DN_CONNECTION_STATE_ERROR_FATAL;
+					PGXCNodeSetConnectionState(conn,
+							DN_CONNECTION_STATE_ERROR_FATAL);
 					add_error_message(conn, "unexpected EOF on datanode connection.");
 					elog(WARNING, "unexpected EOF on datanode oid connection: %d", conn->nodeoid);
 
@@ -570,7 +573,8 @@ retry:
 					(pool_fd[i].revents & POLLNVAL)
 					)
 			{
-				connections[i]->state = DN_CONNECTION_STATE_ERROR_FATAL;
+				PGXCNodeSetConnectionState(connections[i],
+						DN_CONNECTION_STATE_ERROR_FATAL);
 				add_error_message(conn, "unexpected network error on datanode connection");
 				elog(WARNING, "unexpected EOF on datanode oid connection: %d with event %d", conn->nodeoid,pool_fd[i].revents);
 				/* Should we check/read from the other connections before returning? */
@@ -690,7 +694,8 @@ retry:
 								"Datanode closed the connection unexpectedly\n"
 					"\tThis probably means the Datanode terminated abnormally\n"
 								"\tbefore or while processing the request.\n");
-				conn->state = DN_CONNECTION_STATE_ERROR_FATAL;	/* No more connection to
+				PGXCNodeSetConnectionState(conn,
+						DN_CONNECTION_STATE_ERROR_FATAL);	/* No more connection to
 															* backend */
 				closesocket(conn->sock);
 				conn->sock = NO_SOCKET;
@@ -1528,7 +1533,7 @@ pgxc_node_send_execute(PGXCNodeHandle * handle, const char *portal, int fetch)
 	memcpy(handle->outBuffer + handle->outEnd, &fetch, 4);
 	handle->outEnd += 4;
 
-	handle->state = DN_CONNECTION_STATE_QUERY;
+	PGXCNodeSetConnectionState(handle, DN_CONNECTION_STATE_QUERY);
 
 	return 0;
 }
@@ -1679,14 +1684,21 @@ pgxc_node_flush_read(PGXCNodeHandle *handle)
 /*
  * Send specified statement down to the PGXC node
  */
-int
-pgxc_node_send_query(PGXCNodeHandle * handle, const char *query)
+static int
+pgxc_node_send_query_internal(PGXCNodeHandle * handle, const char *query,
+		bool rollback)
 {
 	int			strLen;
 	int			msgLen;
 
-	/* Invalid connection state, return error */
-	if (handle->state != DN_CONNECTION_STATE_IDLE)
+	/*
+	 * Its appropriate to send ROLLBACK commands on a failed connection, but
+	 * for everything else we expect the connection to be in a sane state
+	 */
+	elog(DEBUG5, "pgxc_node_send_query - handle->state %d, node %s, query %s",
+			handle->state, handle->nodename, query);
+	if ((handle->state != DN_CONNECTION_STATE_IDLE) &&
+		!(handle->state == DN_CONNECTION_STATE_ERROR_FATAL && rollback))
 		return EOF;
 
 	strLen = strlen(query) + 1;
@@ -1707,11 +1719,23 @@ pgxc_node_send_query(PGXCNodeHandle * handle, const char *query)
 	memcpy(handle->outBuffer + handle->outEnd, query, strLen);
 	handle->outEnd += strLen;
 
-	handle->state = DN_CONNECTION_STATE_QUERY;
+	PGXCNodeSetConnectionState(handle, DN_CONNECTION_STATE_QUERY);
 
  	return pgxc_node_flush(handle);
 }
 
+int
+pgxc_node_send_rollback(PGXCNodeHandle *handle, const char *query)
+{
+	return pgxc_node_send_query_internal(handle, query, true);
+}
+
+int
+pgxc_node_send_query(PGXCNodeHandle *handle, const char *query)
+{
+	return pgxc_node_send_query_internal(handle, query, false);
+}
+
 
 /*
  * Send the GXID down to the PGXC node
@@ -2717,7 +2741,7 @@ pgxc_node_set_query(PGXCNodeHandle *handle, const char *set_query)
 		 * as well as an error stack overflow.
 		 */
 		if (proc_exit_inprogress)
-			handle->state = DN_CONNECTION_STATE_ERROR_FATAL;
+			PGXCNodeSetConnectionState(handle, DN_CONNECTION_STATE_ERROR_FATAL);
 
 		/* don't read from from the connection if there is a fatal error */
 		if (handle->state == DN_CONNECTION_STATE_ERROR_FATAL)
@@ -2738,14 +2762,14 @@ pgxc_node_set_query(PGXCNodeHandle *handle, const char *set_query)
 		if (msgtype == 'E')	/* ErrorResponse */
 		{
 			handle->error = pstrdup(msg);
-			handle->state = DN_CONNECTION_STATE_ERROR_FATAL;
+			PGXCNodeSetConnectionState(handle, DN_CONNECTION_STATE_ERROR_FATAL);
 			break;
 		}
 
 		if (msgtype == 'Z') /* ReadyForQuery */
 		{
 			handle->transaction_status = msg[0];
-			handle->state = DN_CONNECTION_STATE_IDLE;
+			PGXCNodeSetConnectionState(handle, DN_CONNECTION_STATE_IDLE);
 			handle->combiner = NULL;
 			break;
 		}
@@ -2795,3 +2819,11 @@ DoInvalidateRemoteHandles(void)
 
 	return result;
 }
+
+void
+PGXCNodeSetConnectionState(PGXCNodeHandle *handle, DNConnectionState new_state)
+{
+	elog(DEBUG5, "Changing connection state for node %s, old state %d, "
+			"new state %d", handle->nodename, handle->state, new_state);
+	handle->state = new_state;
+}
diff --git a/src/include/pgxc/execRemote.h b/src/include/pgxc/execRemote.h
index eceb7be680..d8c1d6fa1c 100644
--- a/src/include/pgxc/execRemote.h
+++ b/src/include/pgxc/execRemote.h
@@ -151,6 +151,8 @@ typedef struct RemoteQueryState
 	/* Support for parameters */
 	char	   *paramval_data;		/* parameter data, format is like in BIND */
 	int			paramval_len;		/* length of parameter values data */
+	Oid		   *rqs_param_types;	/* Types of the remote params */
+	int			rqs_num_params;
 
 	int			eflags;			/* capability flags to pass to tuplestore */
 	bool		eof_underlying; /* reached end of underlying plan? */
@@ -272,7 +274,7 @@ extern void BufferConnection(PGXCNodeHandle *conn);
 
 extern void ExecRemoteQueryReScan(RemoteQueryState *node, ExprContext *exprCtxt);
 
-extern int ParamListToDataRow(ParamListInfo params, char** result);
+extern void SetDataRowForExtParams(ParamListInfo params, RemoteQueryState *rq_state);
 
 extern void ExecCloseRemoteStatement(const char *stmt_name, List *nodelist);
 extern char *PrePrepare_Remote(char *prepareGID, bool localNode, bool implicit);
diff --git a/src/include/pgxc/pgxcnode.h b/src/include/pgxc/pgxcnode.h
index 163b0e4914..64cf432ee8 100644
--- a/src/include/pgxc/pgxcnode.h
+++ b/src/include/pgxc/pgxcnode.h
@@ -142,6 +142,7 @@ extern int	ensure_in_buffer_capacity(size_t bytes_needed, PGXCNodeHandle * handl
 extern int	ensure_out_buffer_capacity(size_t bytes_needed, PGXCNodeHandle * handle);
 
 extern int	pgxc_node_send_query(PGXCNodeHandle * handle, const char *query);
+extern int	pgxc_node_send_rollback(PGXCNodeHandle * handle, const char *query);
 extern int	pgxc_node_send_describe(PGXCNodeHandle * handle, bool is_statement,
 						const char *name);
 extern int	pgxc_node_send_execute(PGXCNodeHandle * handle, const char *portal, int fetch);
@@ -187,5 +188,7 @@ extern char *PGXCNodeGetSessionParamStr(void);
 extern char *PGXCNodeGetTransactionParamStr(void);
 extern void pgxc_node_set_query(PGXCNodeHandle *handle, const char *set_query);
 extern void RequestInvalidateRemoteHandles(void);
+extern void PGXCNodeSetConnectionState(PGXCNodeHandle *handle,
+		DNConnectionState new_state);
 
 #endif /* PGXCNODE_H */
diff --git a/src/include/pgxc/planner.h b/src/include/pgxc/planner.h
index 54713bcedf..f6357cd711 100644
--- a/src/include/pgxc/planner.h
+++ b/src/include/pgxc/planner.h
@@ -91,11 +91,10 @@ typedef struct
 	bool			force_autocommit;	/* some commands like VACUUM require autocommit mode */
 	char			*statement;		/* if specified use it as a PreparedStatement name on Datanodes */
 	char			*cursor;		/* if specified use it as a Portal name on Datanodes */
-	int			remote_num_params;		/* number of parameters specified for Prepared remote statement */
-	Oid			*remote_param_types;		/* parameter types, this pointer is shared
-							 * across all the RemoteQuery nodes in the
-							 * plan. So, don't change this once set.
-							 */
+	int             rq_num_params;      /* number of parameters present in
+										   remote statement */
+	Oid             *rq_param_types;    /* parameter types for the remote
+										   statement */
 	RemoteQueryExecType	exec_type;
 	int			reduce_level;		/* in case of reduced JOIN, it's level    */
 	char			*outer_alias;
diff --git a/src/test/regress/expected/plpgsql_1.out b/src/test/regress/expected/plpgsql_1.out
index cc4a98231f..4f9dea9c25 100644
--- a/src/test/regress/expected/plpgsql_1.out
+++ b/src/test/regress/expected/plpgsql_1.out
@@ -2053,13 +2053,13 @@ begin
   return x;
 end$$ language plpgsql;
 select subxact_rollback_semantics();
-ERROR:  could not determine data type of parameter $1
-CONTEXT:  SQL statement "insert into foo values(x)"
-PL/pgSQL function subxact_rollback_semantics() line 5 at SQL statement
+ERROR:  Internal subtransactions not supported in Postgres-XL
+CONTEXT:  PL/pgSQL function subxact_rollback_semantics() line 6 during statement block entry
 select * from foo;
  f1 
 ----
-(0 rows)
+  1
+(1 row)
 
 drop table foo;
 create function trap_timeout() returns void as $$
@@ -2292,8 +2292,11 @@ $$ language plpgsql;
 -- PGXCTODO: This is failing due to issue 3522907, complicated SELECT queries in plpgsql functions
 select refcursor_test2(20000, 20000) as "Should be false",
        refcursor_test2(20, 20) as "Should be true";
-ERROR:  could not determine data type of parameter $1
-CONTEXT:  PL/pgSQL function refcursor_test2(integer,integer) line 7 at FETCH
+ Should be false | Should be true 
+-----------------+----------------
+ f               | t
+(1 row)
+
 --
 -- tests for cursors with named parameter arguments
 --
@@ -2314,8 +2317,11 @@ end
 $$ language plpgsql;
 select namedparmcursor_test1(20000, 20000) as "Should be false",
        namedparmcursor_test1(20, 20) as "Should be true";
-ERROR:  could not determine data type of parameter $1
-CONTEXT:  PL/pgSQL function namedparmcursor_test1(integer,integer) line 7 at FETCH
+ Should be false | Should be true 
+-----------------+----------------
+ f               | t
+(1 row)
+
 -- mixing named and positional argument notations
 create function namedparmcursor_test2(int, int) returns boolean as $$
 declare
@@ -2333,8 +2339,11 @@ begin
 end
 $$ language plpgsql;
 select namedparmcursor_test2(20, 20);
-ERROR:  could not determine data type of parameter $1
-CONTEXT:  PL/pgSQL function namedparmcursor_test2(integer,integer) line 7 at FETCH
+ namedparmcursor_test2 
+-----------------------
+ t
+(1 row)
+
 -- mixing named and positional: param2 is given twice, once in named notation
 -- and second time in positional notation. Should throw an error at parse time
 create function namedparmcursor_test3() returns void as $$
@@ -3125,9 +3134,9 @@ begin
   raise notice 'x.f1 = %, x.f2 = %', x.f1, x.f2;
 end$$ language plpgsql;
 select footest();
-ERROR:  could not determine data type of parameter $1
-CONTEXT:  SQL statement "select * from foo where f1 = p1 and f1::text = p3"
-PL/pgSQL function footest() line 8 at SQL statement
+ERROR:  query returned no rows
+DETAIL:  parameters: p1 = '2', p3 = 'foo'
+CONTEXT:  PL/pgSQL function footest() line 8 at SQL statement
 create or replace function footest() returns void as $$
 declare
 x record;
@@ -3139,9 +3148,9 @@ begin
   raise notice 'x.f1 = %, x.f2 = %', x.f1, x.f2;
 end$$ language plpgsql;
 select footest();
-ERROR:  could not determine data type of parameter $1
-CONTEXT:  SQL statement "select * from foo where f1 > p1 or f1::text = p3"
-PL/pgSQL function footest() line 8 at SQL statement
+ERROR:  query returned more than one row
+DETAIL:  parameters: p1 = '2', p3 = 'foo'
+CONTEXT:  PL/pgSQL function footest() line 8 at SQL statement
 create or replace function footest() returns void as $$
 declare x record;
 begin
@@ -3160,9 +3169,9 @@ begin
   raise notice 'x.f1 = %, x.f2 = %', x.f1, x.f2;
 end$$ language plpgsql;
 select footest();
-ERROR:  there is no parameter $1
-CONTEXT:  SQL statement "select * from foo where f1 = $1 or f1::text = $2"
-PL/pgSQL function footest() line 5 at EXECUTE
+ERROR:  query returned no rows
+DETAIL:  parameters: $1 = '0', $2 = 'foo'
+CONTEXT:  PL/pgSQL function footest() line 5 at EXECUTE
 create or replace function footest() returns void as $$
 declare x record;
 begin
@@ -3171,9 +3180,9 @@ begin
   raise notice 'x.f1 = %, x.f2 = %', x.f1, x.f2;
 end$$ language plpgsql;
 select footest();
-ERROR:  there is no parameter $1
-CONTEXT:  SQL statement "select * from foo where f1 > $1"
-PL/pgSQL function footest() line 5 at EXECUTE
+ERROR:  query returned more than one row
+DETAIL:  parameters: $1 = '1'
+CONTEXT:  PL/pgSQL function footest() line 5 at EXECUTE
 create or replace function footest() returns void as $$
 declare x record;
 begin
@@ -3197,9 +3206,8 @@ begin
   raise notice 'x.f1 = %, x.f2 = %', x.f1, x.f2;
 end$$ language plpgsql;
 select footest();
-ERROR:  could not determine data type of parameter $1
-CONTEXT:  SQL statement "select * from foo where f1 > p1 or f1::text = p3"
-PL/pgSQL function footest() line 10 at SQL statement
+ERROR:  query returned more than one row
+CONTEXT:  PL/pgSQL function footest() line 10 at SQL statement
 reset plpgsql.print_strict_params;
 create or replace function footest() returns void as $$
 -- override the global
@@ -3214,9 +3222,9 @@ begin
   raise notice 'x.f1 = %, x.f2 = %', x.f1, x.f2;
 end$$ language plpgsql;
 select footest();
-ERROR:  could not determine data type of parameter $1
-CONTEXT:  SQL statement "select * from foo where f1 > p1 or f1::text = p3"
-PL/pgSQL function footest() line 10 at SQL statement
+ERROR:  query returned more than one row
+DETAIL:  parameters: p1 = '2', p3 = 'foo'
+CONTEXT:  PL/pgSQL function footest() line 10 at SQL statement
 -- test warnings and errors
 set plpgsql.extra_warnings to 'all';
 ERROR:  syntax error at or near "all"
@@ -4873,8 +4881,15 @@ end;
 $$ language plpgsql;
 -- PGXCTODO: This is failing due to issue 3522907, complicated SELECT queries in plpgsql functions
 select * from conflict_test() order by 1,2;
-ERROR:  could not determine data type of parameter $1
-CONTEXT:  PL/pgSQL function conflict_test() line 6 at FOR over SELECT rows
+ q1 |        q2         
+----+-------------------
+ 42 | -4567890123456789
+ 42 |               123
+ 42 |               456
+ 42 |  4567890123456789
+ 42 |  4567890123456789
+(5 rows)
+
 create or replace function conflict_test() returns setof int8_tbl as $$
 #variable_conflict use_column
 declare r record;