Fix misc issues with two-phase commit protocol and cleaning up of outstanding

transactions

When a two-phase commit protocol is interrupted mid-way, for example because of
a server crash, it can leave behind unresolved prepared transactions which must
be resolved when the node comes back. Postgres-XL provides a pgxc_clean utility
to lookup list of prepared transactions and resolve them based on status of
such a transaction on every node. But there were many issues with the utility
because of which it would either fail to resolve all transactions, or worse
resolve it in a wrong manner. This commit fixes all such issues discovered
during some simple crash recovery testing.

One of the problem with the current approach was that the utility would not
know which all nodes were involved in a transaction. So if it sees a
transaction as prepared on a subset of nodes, but does not exist on other
subset, it would not know if originally it was executed on nodes other than
where its prepared, but other nodes failed before they could prepare the
transaction. If it was indeed executed on other nodes, such transaction must be
aborted. Whereas if it was only executed on the set of nodes where its
currently prepared, then it can safely be committed.

We now store the information about nodes partcipating in a 2PC directly in the
GID. This of course has a downside of increasing the GIDSIZE which implies for
shared memory requirement. But with today's server sizes, it should not be a
very big concern. Sure, we could also look at possibility of storing this
information externally, such as on the GTM. But the fix seems good enough for
now.

14 files changed, 317 insertions, 22 deletions

diff --git a/contrib/pgxc_clean/pgxc_clean.c b/contrib/pgxc_clean/pgxc_clean.c
index a0cfe29d96..911bebc520 100644
--- a/contrib/pgxc_clean/pgxc_clean.c
+++ b/contrib/pgxc_clean/pgxc_clean.c
@@ -467,6 +467,7 @@ recover2PC(PGconn *conn, txn_info *txn)
 	switch (txn_stat)
 	{
 		case TXN_STATUS_FAILED:
+		case TXN_STATUS_UNKNOWN:
 			if (verbose_opt)
 				fprintf(outf, "        Recovery not needed.\n");
 			return;
@@ -480,6 +481,9 @@ recover2PC(PGconn *conn, txn_info *txn)
 		case TXN_STATUS_ABORTED:
 			do_abort(conn, txn);
 			return;
+		case TXN_STATUS_INPROGRESS:
+			fprintf(stderr, "        Can't recover a running transaction.\n");
+			exit(1);
 		default:
 			fprintf(stderr, "        Unknown TXN status, pgxc_clean error.\n");
 			exit(1);
@@ -642,18 +646,27 @@ getTxnStatus(PGconn *conn, GlobalTransactionId gxid, int node_idx)
 	char *res_s;
 
 	static const char *STMT_FORM = "EXECUTE DIRECT ON (%s) 'SELECT pgxc_is_committed(''%d''::xid);'";
+	static const char *STMT_FORM_RUNNING = "EXECUTE DIRECT ON (%s) 'SELECT pgxc_is_inprogress(''%d''::xid);'";
 
 	node_name = pgxc_clean_node_info[node_idx].node_name;
 	sprintf(stmt, STMT_FORM, node_name, gxid);
 
 	res = PQexec(conn, stmt);
-	if (res == NULL || PQresultStatus(res) != PGRES_TUPLES_OK)
+	if (res == NULL || PQresultStatus(res) != PGRES_TUPLES_OK ||
+			PQgetisnull(res, 0, 0))
 	{
-		fprintf(stderr, "Could not obtain transaction status for node %s, gxid %d\n", node_name, gxid);
-		exit(1);
-	}
-	if (PQgetisnull(res, 0, 0))
+		PQclear(res);
+		sprintf(stmt, STMT_FORM_RUNNING, node_name, gxid);
+		res = PQexec(conn, stmt);
+		if (res == NULL || PQresultStatus(res) != PGRES_TUPLES_OK)
+			return TXN_STATUS_UNKNOWN;
+		if (PQgetisnull(res, 0, 0))
+			return TXN_STATUS_UNKNOWN;
+		res_s = PQgetvalue(res, 0, 0);
+		if (strcmp(res_s, "t") == 0)
+			return TXN_STATUS_INPROGRESS;
 		return TXN_STATUS_UNKNOWN;
+	}
 	res_s = PQgetvalue(res, 0, 0);
 	if (strcmp(res_s, "t") == 0)
 		return TXN_STATUS_COMMITTED;
@@ -801,7 +814,8 @@ getNodeList(PGconn *conn)
 	PGresult *res;
 
 	/* SQL Statement */
-	static const char *STMT_GET_NODE_INFO = "SELECT NODE_NAME, NODE_TYPE, NODE_PORT, NODE_HOST FROM PGXC_NODE;";
+	static const char *STMT_GET_NODE_INFO = "SELECT NODE_NAME, NODE_TYPE, "
+				 "NODE_PORT, NODE_HOST, NODE_ID FROM PGXC_NODE;";
 
 	res = PQexec(conn, STMT_GET_NODE_INFO);
 	if (res == NULL || PQresultStatus(res) != PGRES_TUPLES_OK)
@@ -825,6 +839,7 @@ getNodeList(PGconn *conn)
 		NODE_TYPE node_type;
 		int  port;
 		char *host;
+		int	 nodeid;
 
 		node_name = strdup(PQgetvalue(res, ii, 0));
 		node_type_c = strdup(PQgetvalue(res, ii, 1));
@@ -845,7 +860,8 @@ getNodeList(PGconn *conn)
 		}
 		port = atoi(PQgetvalue(res, ii, 2));
 		host = strdup(PQgetvalue(res, ii, 3));
-		set_node_info(node_name, port, host, node_type, ii);
+		nodeid = atoi(PQgetvalue(res, ii, 4));
+		set_node_info(node_name, port, host, node_type, nodeid, ii);
 
 		if (node_name)
 			free(node_name);
diff --git a/contrib/pgxc_clean/txninfo.c b/contrib/pgxc_clean/txninfo.c
index d8f8b8657f..dae047e745 100644
--- a/contrib/pgxc_clean/txninfo.c
+++ b/contrib/pgxc_clean/txninfo.c
@@ -3,6 +3,9 @@
 static int check_xid_is_implicit(char *xid);
 static txn_info *find_txn(TransactionId gxid);
 static txn_info *make_txn_info(char *dbname, TransactionId gxid, char *xid, char *owner);
+static void find_txn_participant_nodes(txn_info *txn);
+
+#define XIDPREFIX "_$XC$"
 
 database_info *find_database_info(char *database_name)
 {
@@ -47,7 +50,8 @@ database_info *add_database_info(char *database_name)
 	}
 }
 
-int set_node_info(char *node_name, int port, char *host, NODE_TYPE type, int index)
+int set_node_info(char *node_name, int port, char *host, NODE_TYPE type,
+		int nodeid, int index)
 {
 	node_info *cur_node_info;
 
@@ -66,6 +70,7 @@ int set_node_info(char *node_name, int port, char *host, NODE_TYPE type, int ind
 	if (cur_node_info->host == NULL)
 		return -1;
 	cur_node_info->type = type;
+	cur_node_info->nodeid = nodeid;
 	return 0;
 }
 
@@ -82,6 +87,17 @@ node_info *find_node_info(char *node_name)
 	return(NULL);
 }
 
+node_info *find_node_info_by_nodeid(int nodeid)
+{
+	int i;
+	for (i = 0; i < pgxc_clean_node_count; i++)
+	{
+		if (pgxc_clean_node_info[i].nodeid == nodeid)
+			return &pgxc_clean_node_info[i];
+	}
+	return(NULL);
+}
+
 int find_node_index(char *node_name)
 {
 	int i;
@@ -95,6 +111,17 @@ int find_node_index(char *node_name)
 	return  -1;
 }
 
+int find_node_index_by_nodeid(int nodeid)
+{
+	int i;
+	for (i = 0; i < pgxc_clean_node_count; i++)
+	{
+		if (pgxc_clean_node_info[i].nodeid == nodeid)
+			return i;
+	}
+	return  -1;
+}
+
 int add_txn_info(char *dbname, char *node_name, TransactionId gxid, char *xid, char *owner, TXN_STATUS status)
 {
 	txn_info *txn;
@@ -236,6 +263,92 @@ TXN_STATUS check_txn_global_status_gxid(TransactionId gxid)
 	return(check_txn_global_status(find_txn(gxid)));
 }
 
+static void find_txn_participant_nodes(txn_info *txn)
+{
+	int ii;
+	char *xid;
+	char *val;
+
+	if (txn == NULL)
+		return;
+
+	if ((txn->xid == NULL || *txn->xid == '\0'))
+		return;
+
+	xid = strdup(txn->xid);
+
+#define SEP	":"
+	val = strtok(xid, SEP);
+	if (strncmp(val, XIDPREFIX, strlen(XIDPREFIX)) != 0)
+	{
+		fprintf(stderr, "Invalid format for implicit XID (%s).\n", txn->xid);
+		exit(1);
+	}
+
+	/* Get originating coordinator name */
+	val = strtok(NULL, SEP);
+	if (val == NULL)
+	{
+		fprintf(stderr, "Invalid format for implicit XID (%s).\n", txn->xid);
+		exit(1);
+	}
+	txn->origcoord = strdup(val);
+
+	/* Get if the originating coordinator was involved in the txn */
+	val = strtok(NULL, SEP);
+	if (val == NULL)
+	{
+		fprintf(stderr, "Invalid format for implicit XID (%s).\n", txn->xid);
+		exit(1);
+	}
+	txn->isorigcoord_part = atoi(val);
+
+	/* Get participating datanode count */
+	val = strtok(NULL, SEP);
+	if (val == NULL)
+	{
+		fprintf(stderr, "Invalid format for implicit XID (%s).\n", txn->xid);
+		exit(1);
+	}
+	txn->num_dnparts = atoi(val);
+
+	/* Get participating coordinator count */
+	val = strtok(NULL, SEP);
+	if (val == NULL)
+	{
+		fprintf(stderr, "Invalid format for implicit XID (%s).\n", txn->xid);
+		exit(1);
+	}
+	txn->num_coordparts = atoi(val);
+
+	txn->dnparts = (int *) malloc(sizeof (int) * txn->num_dnparts);
+	txn->coordparts = (int *) malloc(sizeof (int) * txn->num_coordparts);
+
+	for (ii = 0; ii < txn->num_dnparts; ii++)
+	{
+		val = strtok(NULL, SEP);
+		if (val == NULL)
+		{
+			fprintf(stderr, "Invalid format for implicit XID (%s).\n", txn->xid);
+			exit(1);
+		}
+		txn->dnparts[ii] = atoi(val);
+	}
+
+	for (ii = 0; ii < txn->num_coordparts; ii++)
+	{
+		val = strtok(NULL, SEP);
+		if (val == NULL)
+		{
+			fprintf(stderr, "Invalid format for implicit XID (%s).\n", txn->xid);
+			exit(1);
+		}
+		txn->coordparts[ii] = atoi(val);
+	}
+
+	return;
+}
+
 TXN_STATUS check_txn_global_status(txn_info *txn)
 {
 #define TXN_PREPARED 	0x0001
@@ -244,22 +357,59 @@ TXN_STATUS check_txn_global_status(txn_info *txn)
 
 	int ii;
 	int check_flag = 0;
+	int nodeindx;
 
 	if (txn == NULL)
 		return TXN_STATUS_INITIAL;
-	for (ii = 0; ii < pgxc_clean_node_count; ii++)
+
+	find_txn_participant_nodes(txn);
+
+	for (ii = 0; ii < txn->num_dnparts; ii++)
 	{
-		if (txn->txn_stat[ii] == TXN_STATUS_INITIAL || txn->txn_stat[ii] == TXN_STATUS_UNKNOWN)
-			continue;
-		else if (txn->txn_stat[ii] == TXN_STATUS_PREPARED)
+		nodeindx = find_node_index_by_nodeid(txn->dnparts[ii]);
+		if (nodeindx == -1)
+		{
+			fprintf(stderr, "Participant datanode %d not reachable. Can't "
+					"resolve the transaction %s", txn->dnparts[ii], txn->xid);
+				return TXN_STATUS_FAILED;
+		}
+
+		if (txn->txn_stat[nodeindx] == TXN_STATUS_INITIAL ||
+				txn->txn_stat[nodeindx] == TXN_STATUS_UNKNOWN)
+			check_flag |= TXN_ABORTED;
+		else if (txn->txn_stat[nodeindx] == TXN_STATUS_PREPARED)
 			check_flag |= TXN_PREPARED;
-		else if (txn->txn_stat[ii] == TXN_STATUS_COMMITTED)
+		else if (txn->txn_stat[nodeindx] == TXN_STATUS_COMMITTED)
 			check_flag |= TXN_COMMITTED;
-		else if (txn->txn_stat[ii] == TXN_STATUS_ABORTED)
+		else if (txn->txn_stat[nodeindx] == TXN_STATUS_ABORTED)
 			check_flag |= TXN_ABORTED;
 		else
 			return TXN_STATUS_FAILED;
 	}
+
+	for (ii = 0; ii < txn->num_coordparts; ii++)
+	{
+		nodeindx = find_node_index_by_nodeid(txn->coordparts[ii]);
+		if (nodeindx == -1)
+		{
+			fprintf(stderr, "Participant datanode %d not reachable. Can't "
+					"resolve the transaction %s", txn->coordparts[ii], txn->xid);
+				return TXN_STATUS_FAILED;
+		}
+
+		if (txn->txn_stat[nodeindx] == TXN_STATUS_INITIAL ||
+				txn->txn_stat[nodeindx] == TXN_STATUS_UNKNOWN)
+			check_flag |= TXN_ABORTED;
+		else if (txn->txn_stat[nodeindx] == TXN_STATUS_PREPARED)
+			check_flag |= TXN_PREPARED;
+		else if (txn->txn_stat[nodeindx] == TXN_STATUS_COMMITTED)
+			check_flag |= TXN_COMMITTED;
+		else if (txn->txn_stat[nodeindx] == TXN_STATUS_ABORTED)
+			check_flag |= TXN_ABORTED;
+		else
+			return TXN_STATUS_FAILED;
+	}
+
 	if ((check_flag & TXN_PREPARED) == 0)
 		/* Should be at least one "prepared statement" in nodes */
 		return TXN_STATUS_FAILED;
@@ -287,8 +437,6 @@ TXN_STATUS check_txn_global_status(txn_info *txn)
  */
 static int check_xid_is_implicit(char *xid)
 {
-#define XIDPREFIX "_$XC$"
-
 	if (strncmp(xid, XIDPREFIX, strlen(XIDPREFIX)) != 0)
 		return 0;
 	for(xid += strlen(XIDPREFIX); *xid; xid++)
diff --git a/contrib/pgxc_clean/txninfo.h b/contrib/pgxc_clean/txninfo.h
index 10798712ee..fc0e8eca65 100644
--- a/contrib/pgxc_clean/txninfo.h
+++ b/contrib/pgxc_clean/txninfo.h
@@ -22,6 +22,7 @@ typedef enum TXN_STATUS
 	TXN_STATUS_PREPARED,
 	TXN_STATUS_COMMITTED,
 	TXN_STATUS_ABORTED,
+	TXN_STATUS_INPROGRESS,
 	TXN_STATUS_FAILED		/* Error detected while interacting with the node */
 } TXN_STATUS;
 
@@ -38,6 +39,7 @@ typedef struct node_info
 	int		port;
 	char	*host;
 	NODE_TYPE type;
+	int		nodeid;
 } node_info;
 
 extern node_info *pgxc_clean_node_info;
@@ -49,6 +51,13 @@ typedef struct txn_info
 	TransactionId	gxid;
 	char			*xid;		/* xid used in prepare */
 	char			*owner;
+	char			*origcoord;	/* Original coordinator who initiated the txn */
+	bool			isorigcoord_part;	/* Is original coordinator a
+										   participant? */
+	int				num_dnparts;		/* Number of participant datanodes */
+	int				num_coordparts;		/* Number of participant coordinators */
+	int				*dnparts;	/* Whether a node was participant in the txn */
+	int				*coordparts;
 	TXN_STATUS		*txn_stat;	/* Array for each nodes */
 	char			*msg;		/* Notice message for this txn. */
 } txn_info;
@@ -73,8 +82,11 @@ extern int set_txn_status(TransactionId gxid, char *node_name, TXN_STATUS status
 extern database_info *find_database_info(char *database_name);
 extern database_info *add_database_info(char *database_name);
 extern node_info *find_node_info(char *node_name);
+extern node_info *find_node_info_by_nodeid(int nodeid);
 extern int find_node_index(char *node_name);
-extern int set_node_info(char *node_name, int port, char *host, NODE_TYPE type, int index);
+extern int find_node_index_by_nodeid(int nodeid);
+extern int set_node_info(char *node_name, int port, char *host, NODE_TYPE type,
+		int nodeid, int index);
 extern TXN_STATUS check_txn_global_status(txn_info *txn);
 extern TXN_STATUS check_txn_global_status_gxid(TransactionId gxid);
 extern bool check2PCExists(void);
diff --git a/src/backend/access/transam/transam.c b/src/backend/access/transam/transam.c
index 17279dd425..71e3015d37 100644
--- a/src/backend/access/transam/transam.c
+++ b/src/backend/access/transam/transam.c
@@ -44,6 +44,7 @@ static XidStatus TransactionLogFetch(TransactionId transactionId);
 #ifdef PGXC
 /* It is not really necessary to make it appear in header file */
 Datum pgxc_is_committed(PG_FUNCTION_ARGS);
+Datum pgxc_is_inprogress(PG_FUNCTION_ARGS);
 #endif
 
 /* ----------------------------------------------------------------
@@ -120,6 +121,25 @@ pgxc_is_committed(PG_FUNCTION_ARGS)
 	else
 		PG_RETURN_NULL();
 }
+
+/*
+ * For given Transaction ID, check if transaction is committed or aborted
+ */
+Datum
+pgxc_is_inprogress(PG_FUNCTION_ARGS)
+{
+	TransactionId	tid = (TransactionId) PG_GETARG_UINT32(0);
+	XidStatus   xidstatus;
+
+	xidstatus = TransactionLogFetch(tid);
+
+	if (xidstatus == TRANSACTION_STATUS_COMMITTED)
+		PG_RETURN_BOOL(false);
+	else if (xidstatus == TRANSACTION_STATUS_ABORTED)
+		PG_RETURN_BOOL(false);
+	else
+		PG_RETURN_BOOL(TransactionIdIsInProgress(tid));
+}
 #endif
 
 /* ----------------------------------------------------------------
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 74a0e5e9d2..a7f226b5fd 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -71,6 +71,7 @@
 #include "utils/timestamp.h"
 #ifdef PGXC
 #include "pgxc/xc_maintenance_mode.h"
+#include "pgxc/nodemgr.h"
 #endif
 
 
@@ -115,7 +116,7 @@ int			max_prepared_xacts = 0;
  * typedef struct GlobalTransactionData *GlobalTransaction appears in
  * twophase.h
  */
-#define GIDSIZE 200
+#define GIDSIZE (200 + (MAX_COORDINATORS + MAX_DATANODES) * 15)
 
 typedef struct GlobalTransactionData
 {
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 6c5f95944e..6f84a48f03 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -2308,9 +2308,8 @@ CommitTransaction(void)
 		 */
 		if (!IsOnCommitActions() && IsTwoPhaseCommitRequired(XactWriteLocalNode))
 		{
-			prepareGID = MemoryContextAlloc(TopTransactionContext, 256);
-			sprintf(prepareGID, implicit2PC_head"%u", GetTopTransactionId());
 
+			prepareGID = GetImplicit2PCGID(implicit2PC_head, XactWriteLocalNode);
 			savePrepareGID = MemoryContextStrdup(TopMemoryContext, prepareGID);
 
 			if (XactWriteLocalNode)
diff --git a/src/backend/pgxc/pool/execRemote.c b/src/backend/pgxc/pool/execRemote.c
index f5433ea860..197fad12a2 100644
--- a/src/backend/pgxc/pool/execRemote.c
+++ b/src/backend/pgxc/pool/execRemote.c
@@ -106,6 +106,8 @@ static PGXCNodeAllHandles *get_exec_connections(RemoteQueryState *planstate,
 static bool pgxc_start_command_on_connection(PGXCNodeHandle *connection,
 					RemoteQueryState *remotestate, Snapshot snapshot);
 
+static void pgxc_node_remote_count(int *dnCount, int dnNodeIds[],
+		int *coordCount, int coordNodeIds[]);
 static char *pgxc_node_remote_prepare(char *prepareGID, bool localNode);
 static bool pgxc_node_remote_finish(char *prepareGID, bool commit,
 						char *nodestring, GlobalTransactionId gxid,
@@ -1996,6 +1998,54 @@ pgxc_node_remote_cleanup_all(void)
 	pfree_pgxc_all_handles(handles);
 }
 
+/*
+ * Count how many coordinators and datanodes are involved in this transaction
+ * so that we can save that information in the GID
+ */
+static void
+pgxc_node_remote_count(int *dnCount, int dnNodeIds[],
+		int *coordCount, int coordNodeIds[])
+{
+	int i;
+	PGXCNodeAllHandles *handles = get_current_handles();
+
+	*dnCount = *coordCount = 0;
+	for (i = 0; i < handles->dn_conn_count; i++)
+	{
+		PGXCNodeHandle *conn = handles->datanode_handles[i];
+		/*
+		 * Skip empty slots
+		 */
+		if (conn->sock == NO_SOCKET)
+			continue;
+		else if (conn->transaction_status == 'T')
+		{
+			if (!conn->read_only)
+			{
+				dnNodeIds[*dnCount] = conn->nodeid;
+				*dnCount = *dnCount + 1;
+			}
+		}
+	}
+
+	for (i = 0; i < handles->co_conn_count; i++)
+	{
+		PGXCNodeHandle *conn = handles->coord_handles[i];
+		/*
+		 * Skip empty slots
+		 */
+		if (conn->sock == NO_SOCKET)
+			continue;
+		else if (conn->transaction_status == 'T')
+		{
+			if (!conn->read_only)
+			{
+				coordNodeIds[*coordCount] = conn->nodeid;
+				*coordCount = *coordCount + 1;
+			}
+		}
+	}
+}
 
 /*
  * Prepare nodes which ran write operations during the transaction.
@@ -6201,3 +6251,41 @@ void AtEOXact_DBCleanup(bool isCommit)
 		dbcleanup_info.fparams = NULL;
 	}
 }
+
+char *
+GetImplicit2PCGID(const char *implicit2PC_head, bool localWrite)
+{
+	int dnCount = 0, coordCount = 0;
+	int dnNodeIds[MaxDataNodes];
+	int coordNodeIds[MaxCoords];
+	MemoryContext oldContext = CurrentMemoryContext;
+	StringInfoData str;
+	int i;
+
+	oldContext = MemoryContextSwitchTo(TopTransactionContext);
+	initStringInfo(&str);
+	/*
+	 * Check how many coordinators and datanodes are involved in this
+	 * transaction
+	 */
+	pgxc_node_remote_count(&dnCount, dnNodeIds, &coordCount, coordNodeIds);
+	appendStringInfo(&str, "%s%u:%s:%c:%d:%d",
+			implicit2PC_head,
+			GetTopTransactionId(),
+			PGXCNodeName,
+			localWrite ? 'T' : 'F',
+			dnCount,
+			coordCount + (localWrite ? 1 : 0));
+
+	for (i = 0; i < dnCount; i++)
+		appendStringInfo(&str, ":%d", dnNodeIds[i]);
+	for (i = 0; i < coordCount; i++)
+		appendStringInfo(&str, ":%d", coordNodeIds[i]);
+
+	if (localWrite)
+		appendStringInfo(&str, ":%d", PGXCNodeIdentifier);
+
+	MemoryContextSwitchTo(oldContext);
+
+	return str.data;
+}
diff --git a/src/backend/pgxc/pool/pgxcnode.c b/src/backend/pgxc/pool/pgxcnode.c
index c4c69cdf4e..73ac6bb4af 100644
--- a/src/backend/pgxc/pool/pgxcnode.c
+++ b/src/backend/pgxc/pool/pgxcnode.c
@@ -211,11 +211,13 @@ InitMultinodeExecutor(bool is_force)
 	{
 		init_pgxc_handle(&dn_handles[count]);
 		dn_handles[count].nodeoid = dnOids[count];
+		dn_handles[count].nodeid = get_pgxc_node_id(dnOids[count]);
 	}
 	for (count = 0; count < NumCoords; count++)
 	{
 		init_pgxc_handle(&co_handles[count]);
 		co_handles[count].nodeoid = coOids[count];
+		co_handles[count].nodeid = get_pgxc_node_id(coOids[count]);
 	}
 
 	datanode_count = 0;
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index b67df832b1..0101f4b6ef 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -2933,7 +2933,7 @@ static struct config_int ConfigureNamesInt[] =
 						 "this maximum number.")
 		},
 		&MaxDataNodes,
-		16, 2, 65535,
+		16, 2, MAX_DATANODES,
 		NULL, NULL, NULL
 	},
 
@@ -2944,7 +2944,7 @@ static struct config_int ConfigureNamesInt[] =
 						 "this maximum number.")
 		},
 		&MaxCoords,
-		16, 2, 65535,
+		16, 2, MAX_COORDINATORS,
 		NULL, NULL, NULL
 	},
 
diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h
index a55a587271..9c1d8d5d4e 100644
--- a/src/include/catalog/pg_proc.h
+++ b/src/include/catalog/pg_proc.h
@@ -5288,6 +5288,8 @@ DATA(insert OID = 7009 ( pgxc_node_str		PGNSP PGUID 12 1 0 0 0 f f f f t f s 0 0
 DESCR("get the name of the node");
 DATA(insert OID = 7010 (  pgxc_is_committed	PGNSP PGUID 12 1 1 0 0 f f f f t t s 1 0 16 "28" _null_ _null_ _null_ _null_ _null_ pgxc_is_committed _null_ _null_ _null_ ));
 DESCR("is given GXID committed or aborted?");
+DATA(insert OID = 7024 (  pgxc_is_inprogress	PGNSP PGUID 12 1 1 0 0 f f f f t t s 1 0 16 "28" _null_ _null_ _null_ _null_ _null_ pgxc_is_inprogress _null_ _null_ _null_ ));
+DESCR("is given GXID in progress?");
 DATA(insert OID = 7011 ( pgxc_lock_for_backup PGNSP PGUID 12 1 0 0 0 f f f f t f v 0 0 16 "" _null_ _null_ _null_ _null_ _null_ pgxc_lock_for_backup _null_ _null_ _null_ ));
 DESCR("lock the cluster for taking backup");
 #ifdef XCP
diff --git a/src/include/pgxc/execRemote.h b/src/include/pgxc/execRemote.h
index 069218da9e..b4c3d1adc4 100644
--- a/src/include/pgxc/execRemote.h
+++ b/src/include/pgxc/execRemote.h
@@ -286,6 +286,7 @@ extern bool	PreAbort_Remote(void);
 extern void AtEOXact_Remote(void);
 extern bool IsTwoPhaseCommitRequired(bool localWrite);
 extern bool FinishRemotePreparedTransaction(char *prepareGID, bool commit);
+extern char *GetImplicit2PCGID(const char *implicit2PC_head, bool localWrite);
 
 extern void pgxc_all_success_nodes(ExecNodes **d_nodes, ExecNodes **c_nodes, char **failednodes_msg);
 extern void AtEOXact_DBCleanup(bool isCommit);
diff --git a/src/include/pgxc/nodemgr.h b/src/include/pgxc/nodemgr.h
index 2604af6f73..33c20d2c85 100644
--- a/src/include/pgxc/nodemgr.h
+++ b/src/include/pgxc/nodemgr.h
@@ -18,6 +18,10 @@
 
 #define PGXC_NODENAME_LENGTH	64
 
+/* Compile time max limits on number of coordinators and datanodes */
+#define MAX_COORDINATORS		64
+#define MAX_DATANODES			256
+
 /* GUC parameters, limit for number of nodes */
 extern int 	MaxDataNodes;
 extern int 	MaxCoords;
diff --git a/src/include/pgxc/pgxcnode.h b/src/include/pgxc/pgxcnode.h
index 1cd4162a34..78639f949c 100644
--- a/src/include/pgxc/pgxcnode.h
+++ b/src/include/pgxc/pgxcnode.h
@@ -62,6 +62,7 @@ typedef enum
 struct pgxc_node_handle
 {
 	Oid			nodeoid;
+	int			nodeid;
 
 	/* fd of the connection */
 	int		sock;
diff --git a/src/include/utils/builtins.h b/src/include/utils/builtins.h
index 3bee7695ef..667ff5af9a 100644
--- a/src/include/utils/builtins.h
+++ b/src/include/utils/builtins.h
@@ -1313,6 +1313,7 @@ extern Datum stormdb_promote_standby(PG_FUNCTION_ARGS);
 
 /* backend/access/transam/transam.c */
 extern Datum pgxc_is_committed(PG_FUNCTION_ARGS);
+extern Datum pgxc_is_inprogress(PG_FUNCTION_ARGS);
 #endif
 
 #endif   /* BUILTINS_H */