Add a facility to track waited-for XIDs for a transaction

A transaction may wait for one or more transactions to finish before proceeding
with its operation. For example, an UPDATEing transaction may wait for other
transaction if it has already updated/deleted the tuple and decide the next
action based on other transaction's outcome as well as its own isolation level.
Sometimes it may happen a transaction is marked as committed on a datanode, but
GTM has not yet received a message to this effect. We have seen that this can
lead to breakage in MVCC properties when more than one tuple version may
satisfy MVCC checks. For specifically, when a transaction which is already
committed on the datanode is still seen as in-progress, but a later transaction
which again updated the same tuple is seen as committed as per a snapshot
obtained from GTM. Such snapshots can see both, most old and most recent
versions of a tuple as visible.

This patch adds an ability to track XIDs on which a transaction may have waited
and later sends that list to GTM. GTM must not commit a transaction unless all
such transactions on which it has waited for are also finished. Till such time,
GTM will send back STATUS_DELAYED response to the client. The client must retry
commit until its done. We believe the window is extremely small and its a
corner case. So such retries should not add much overhead to the system.

14 files changed, 512 insertions, 68 deletions

diff --git a/src/backend/access/transam/gtm.c b/src/backend/access/transam/gtm.c
index 347dc480d1..2a8e78547a 100644
--- a/src/backend/access/transam/gtm.c
+++ b/src/backend/access/transam/gtm.c
@@ -187,7 +187,8 @@ BeginTranAutovacuumGTM(void)
 }
 
 int
-CommitTranGTM(GlobalTransactionId gxid)
+CommitTranGTM(GlobalTransactionId gxid, int waited_xid_count,
+		GlobalTransactionId *waited_xids)
 {
 	int ret;
 
@@ -198,7 +199,7 @@ CommitTranGTM(GlobalTransactionId gxid)
 	ret = -1;
 	if (conn)
 #endif
-	ret = commit_transaction(conn, gxid);
+	ret = commit_transaction(conn, gxid, waited_xid_count, waited_xids);
 
 	/*
 	 * If something went wrong (timeout), try and reset GTM connection.
@@ -211,7 +212,7 @@ CommitTranGTM(GlobalTransactionId gxid)
 		InitGTM();
 #ifdef XCP
 		if (conn)
-			ret = commit_transaction(conn, gxid);
+			ret = commit_transaction(conn, gxid, waited_xid_count, waited_xids);
 #endif
 	}
 
@@ -228,7 +229,9 @@ CommitTranGTM(GlobalTransactionId gxid)
  * and for COMMIT PREPARED.
  */
 int
-CommitPreparedTranGTM(GlobalTransactionId gxid, GlobalTransactionId prepared_gxid)
+CommitPreparedTranGTM(GlobalTransactionId gxid,
+		GlobalTransactionId prepared_gxid, int waited_xid_count,
+		GlobalTransactionId *waited_xids)
 {
 	int ret = 0;
 
@@ -239,7 +242,8 @@ CommitPreparedTranGTM(GlobalTransactionId gxid, GlobalTransactionId prepared_gxi
 	ret = -1;
 	if (conn)
 #endif
-	ret = commit_prepared_transaction(conn, gxid, prepared_gxid);
+	ret = commit_prepared_transaction(conn, gxid, prepared_gxid,
+			waited_xid_count, waited_xids);
 
 	/*
 	 * If something went wrong (timeout), try and reset GTM connection.
@@ -253,7 +257,8 @@ CommitPreparedTranGTM(GlobalTransactionId gxid, GlobalTransactionId prepared_gxi
 		InitGTM();
 #ifdef XCP
 		if (conn)
-			ret = commit_prepared_transaction(conn, gxid, prepared_gxid);
+			ret = commit_prepared_transaction(conn, gxid, prepared_gxid,
+					waited_xid_count, waited_xids);
 #endif
 	}
 	currentGxid = InvalidGlobalTransactionId;
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 30e36576cb..c72dd98d92 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -73,6 +73,9 @@
 #include "storage/procarray.h"
 #include "storage/sinvaladt.h"
 #include "storage/smgr.h"
+#ifdef XCP
+#include "tcop/tcopprot.h"
+#endif
 #include "utils/builtins.h"
 #include "utils/catcache.h"
 #include "utils/combocid.h"
@@ -223,6 +226,10 @@ typedef struct TransactionStateData
 	bool		didLogXid;		/* has xid been included in WAL record? */
 	int			parallelModeLevel;		/* Enter/ExitParallelMode counter */
 	struct TransactionStateData *parent;		/* back link to parent */
+#ifdef XCP
+	int				waitedForXidsCount;	/* count of xids we waited to finish */
+	TransactionId	*waitedForXids;		/* xids we waited to finish */
+#endif
 } TransactionStateData;
 
 typedef TransactionStateData *TransactionState;
@@ -426,6 +433,14 @@ static void AtSubCommit_Memory(void);
 static void AtSubStart_Memory(void);
 static void AtSubStart_ResourceOwner(void);
 
+#ifdef XCP
+static void AtSubCommit_WaitedXids(void);
+static void AtSubAbort_WaitedXids(void);
+static void AtEOXact_WaitedXids(void);
+static void TransactionRecordXidWait_Internal(TransactionState s,
+		TransactionId xid);
+#endif
+
 static void ShowTransactionState(const char *str);
 static void ShowTransactionStateRec(TransactionState state);
 static const char *BlockStateAsString(TBlockState blockState);
@@ -2498,6 +2513,10 @@ CommitTransaction(void)
 	}
 #endif
 
+#ifdef XCP
+	AtEOXact_WaitedXids();
+#endif
+
 	/* Prevent cancel/die interrupt while cleaning up */
 	HOLD_INTERRUPTS();
 
@@ -2712,11 +2731,15 @@ AtEOXact_GlobalTxn(bool commit)
 			if (GlobalTransactionIdIsValid(s->auxilliaryTransactionId) &&
 				GlobalTransactionIdIsValid(s->topGlobalTransansactionId))
 				CommitPreparedTranGTM(s->topGlobalTransansactionId,
-						s->auxilliaryTransactionId);
+						s->auxilliaryTransactionId,
+						s->waitedForXidsCount,
+						s->waitedForXids);
 			else if (GlobalTransactionIdIsValid(s->topGlobalTransansactionId))
-				CommitTranGTM(s->topGlobalTransansactionId);
+				CommitTranGTM(s->topGlobalTransansactionId,
+						s->waitedForXidsCount,
+						s->waitedForXids);
 			else if (GlobalTransactionIdIsValid(s->auxilliaryTransactionId))
-				CommitTranGTM(s->auxilliaryTransactionId);
+				CommitTranGTM(s->auxilliaryTransactionId, 0, NULL);
 		}
 		else
 		{
@@ -2740,7 +2763,7 @@ AtEOXact_GlobalTxn(bool commit)
 	{
 		IsXidFromGTM = false;
 		if (commit)
-			CommitTranGTM(s->topGlobalTransansactionId);
+			CommitTranGTM(s->topGlobalTransansactionId, 0, NULL);
 		else
 			RollbackTranGTM(s->topGlobalTransansactionId);
 		
@@ -2774,6 +2797,14 @@ AtEOXact_GlobalTxn(bool commit)
 	s->topGlobalTransansactionId = InvalidGlobalTransactionId;
 	s->auxilliaryTransactionId = InvalidGlobalTransactionId;
 
+	if (IS_PGXC_LOCAL_COORDINATOR)
+	{
+		if (s->waitedForXids)
+			pfree(s->waitedForXids);
+	}
+	s->waitedForXids = NULL;
+	s->waitedForXidsCount = 0;
+
 	SetNextTransactionId(InvalidTransactionId);
 }
 
@@ -2994,6 +3025,10 @@ PrepareTransaction(void)
 	AtPrepare_MultiXact();
 	AtPrepare_RelationMap();
 
+#ifdef XCP
+	AtEOXact_WaitedXids();
+#endif
+
 	/*
 	 * Here is where we really truly prepare.
 	 *
@@ -3138,6 +3173,16 @@ PrepareTransaction(void)
 			s->topGlobalTransansactionId = InvalidGlobalTransactionId;
 		ForgetTransactionLocalNode();
 	}
+#ifdef XCP
+	if (IS_PGXC_LOCAL_COORDINATOR)
+	{
+		if (s->waitedForXids)
+			pfree(s->waitedForXids);
+	}
+	s->waitedForXids = NULL;
+	s->waitedForXidsCount = 0;
+#endif
+
 	SetNextTransactionId(InvalidTransactionId);
 
 	/*
@@ -3421,6 +3466,16 @@ CleanupTransaction(void)
 	XactTopTransactionId = InvalidTransactionId;
 	nParallelCurrentXids = 0;
 
+#ifdef XCP
+	if (IS_PGXC_LOCAL_COORDINATOR)
+	{
+		if (s->waitedForXids)
+			pfree(s->waitedForXids);
+	}
+	s->waitedForXids = NULL;
+	s->waitedForXidsCount = 0;
+#endif
+
 	/*
 	 * done with abort processing, set current transaction state back to
 	 * default
@@ -5369,6 +5424,9 @@ CommitSubTransaction(void)
 	AtEOSubXact_HashTables(true, s->nestingLevel);
 	AtEOSubXact_PgStat(true, s->nestingLevel);
 	AtSubCommit_Snapshot(s->nestingLevel);
+#ifdef XCP
+	AtSubCommit_WaitedXids();
+#endif
 
 	/*
 	 * We need to restore the upper transaction's read-only state, in case the
@@ -5516,6 +5574,9 @@ AbortSubTransaction(void)
 		AtEOSubXact_HashTables(false, s->nestingLevel);
 		AtEOSubXact_PgStat(false, s->nestingLevel);
 		AtSubAbort_Snapshot(s->nestingLevel);
+#ifdef XCP
+		AtSubAbort_WaitedXids();
+#endif
 	}
 
 	/*
@@ -6666,4 +6727,142 @@ IsPGXCNodeXactDatanodeDirect(void)
 #endif
 		   !IsConnFromCoord();
 }
+
+#ifdef XCP
+static void
+TransactionRecordXidWait_Internal(TransactionState s, TransactionId xid)
+{
+	int i;
+
+	if (s->waitedForXids == NULL)
+	{
+		/*
+		 * XIDs recorded on the local coordinator, which are mostly collected
+		 * from various remote nodes, must survive across transaction
+		 * boundaries since they are sent to the GTM after local transaction is
+		 * committed. So we track them in the TopMemoryContext and make extra
+		 * efforts to free that memory later. Note that we do this only when we
+		 * are running in the top-level transaction. For subtranctions, they
+		 * will be copied to the parent transaction at the commit time. So at
+		 * subtranction level, they can be tracked in a transaction-local
+		 * memory without any problem
+		 */
+		if (IS_PGXC_LOCAL_COORDINATOR && (s->parent == NULL))
+			s->waitedForXids = (TransactionId *)
+				MemoryContextAlloc(TopMemoryContext, sizeof (TransactionId) *
+						MaxConnections);
+		else
+			s->waitedForXids = (TransactionId *)
+				MemoryContextAlloc(CurTransactionContext, sizeof (TransactionId) *
+						MaxConnections);
+
+		s->waitedForXidsCount = 0;
+	}
+
+	elog(DEBUG2, "TransactionRecordXidWait_Internal - recording %d", xid);
+
+	for (i = 0; i < s->waitedForXidsCount; i++)
+	{
+		if (TransactionIdEquals(xid, s->waitedForXids[i]))
+		{
+			elog(DEBUG2, "TransactionRecordXidWait_Internal - xid %d already recorded", xid);
+			return;
+		}
+	}
+
+	/*
+	 * We track maximum MaxConnections xids. In case of overflow, we forget the
+	 * earliest recorded xid. That should be enough for all practical purposes
+	 * since what we are guarding against is a very small window where a
+	 * transaction which is already committed on a datanode has not yet got an
+	 * opportunity to send its status to the GTM. Such transactions can only be
+	 * running on a different coordinator session. So tracking MaxConnections
+	 * worth xids seems like more than enough
+	 */
+	if (s->waitedForXidsCount == MaxConnections)
+	{
+		memmove(&s->waitedForXids[0],
+				&s->waitedForXids[1],
+				(s->waitedForXidsCount - 1) * sizeof (TransactionId));
+		s->waitedForXids[s->waitedForXidsCount - 1] = xid;
+	}
+	else
+		s->waitedForXids[s->waitedForXidsCount++] = xid;
+
+}
+
+void
+TransactionRecordXidWait(TransactionId xid)
+{
+	TransactionRecordXidWait_Internal(CurrentTransactionState, xid);
+}
+
+static void
+AtSubCommit_WaitedXids()
+{
+	TransactionState s = CurrentTransactionState;
+	int i;
+
+	Assert(s->parent != NULL);
+
+	/*
+	 * Move the recorded XIDs to the parent structure
+	 */
+	for (i = 0; i < s->waitedForXidsCount; i++)
+		TransactionRecordXidWait_Internal(s->parent, s->waitedForXids[i]);
+
+	/* And we can safely free them now */
+	if (s->waitedForXids != NULL)
+		pfree(s->waitedForXids);
+	s->waitedForXids = NULL;
+	s->waitedForXidsCount = 0;
+}
+
+static void
+AtSubAbort_WaitedXids()
+{
+	TransactionState s = CurrentTransactionState;
+
+	if (s->waitedForXids != NULL)
+		pfree(s->waitedForXids);
+	s->waitedForXids = NULL;
+	s->waitedForXidsCount = 0;
+
+}
+
+static void
+AtEOXact_WaitedXids(void)
+{
+	TransactionState s = CurrentTransactionState;
+	TransactionId *sendXids;
+
+	/*
+	 * Report the set of XIDs this transaction (and its committed
+	 * subtransactions) had waited-for to the coordinator. The coordinator will
+	 * then forward the list to the GTM who ensures that the logical ordering
+	 * between these transactions and this transaction is correctly followed.
+	 */
+	if (whereToSendOutput == DestRemote && !IS_PGXC_LOCAL_COORDINATOR)
+	{
+		if (s->waitedForXidsCount > 0)
+		{
+			int i;
+
+			/*
+			 * Convert the XIDs in network order and send to the client
+			 */
+			sendXids = (TransactionId *) palloc (sizeof (TransactionId) *
+					s->waitedForXidsCount);
+			for (i = 0; i < s->waitedForXidsCount; i++)
+				sendXids[i] = htonl(s->waitedForXids[i]);
+
+			pq_putmessage('W',
+					(const char *)sendXids,
+					s->waitedForXidsCount * sizeof (TransactionId));
+			pfree(sendXids);
+		}
+	}
+
+}
+#endif
 #endif
diff --git a/src/backend/pgxc/pool/execRemote.c b/src/backend/pgxc/pool/execRemote.c
index d6c9e7176e..0bb7df6cdc 100644
--- a/src/backend/pgxc/pool/execRemote.c
+++ b/src/backend/pgxc/pool/execRemote.c
@@ -1193,6 +1193,31 @@ HandleDatanodeCommandId(RemoteQueryState *combiner, char *msg_body, size_t len)
 }
 
 /*
+ * Record waited-for XIDs received from the remote nodes into the transaction
+ * state
+ */
+static void
+HandleWaitXids(char *msg_body, size_t len)
+{
+	int xid_count;
+	uint32		n32;
+	int cur;
+	int i;
+
+	/* Get the xid count */
+	xid_count = len / sizeof (TransactionId);
+
+	cur = 0;
+	for (i = 0; i < xid_count; i++)
+	{
+		Assert(cur < len);
+		memcpy(&n32, &msg_body[cur], sizeof (TransactionId));
+		cur = cur + sizeof (TransactionId);
+		TransactionRecordXidWait(ntohl(n32));
+	}
+}
+
+/*
  * Examine the specified combiner state and determine if command was completed
  * successfully
  */
@@ -2207,6 +2232,9 @@ pgxc_node_receive_responses(const int conn_count, PGXCNodeHandle ** connections,
 				case RESPONSE_ERROR:
 					/* no handling needed, just wait for ReadyForQuery */
 					break;
+
+				case RESPONSE_WAITXIDS:
+					break;
 #endif
 				default:
 					/* Inconsistent responses */
@@ -2370,6 +2398,11 @@ handle_response(PGXCNodeHandle *conn, ResponseCombiner *combiner)
 				return RESPONSE_BARRIER_OK;
 			case 'I':			/* EmptyQuery */
 				return RESPONSE_COMPLETE;
+#ifdef XCP
+			case 'W':
+				HandleWaitXids(msg, msg_len);	
+				return RESPONSE_WAITXIDS;
+#endif
 			default:
 				/* sync lost? */
 				elog(WARNING, "Received unsupported message type: %c", msg_type);
@@ -7098,7 +7131,7 @@ FinishRemotePreparedTransaction(char *prepareGID, bool commit)
 		if (commit)
 		{
 			pgxc_node_remote_commit();
-			CommitPreparedTranGTM(prepare_gxid, gxid);
+			CommitPreparedTranGTM(prepare_gxid, gxid, 0, NULL);
 		}
 		else
 		{
@@ -7133,7 +7166,12 @@ FinishRemotePreparedTransaction(char *prepareGID, bool commit)
 
 	if (commit)
 	{
-		CommitPreparedTranGTM(prepare_gxid, gxid);
+		/*
+		 * XXX For explicit 2PC, there will be enough delay for any
+		 * waited-committed transactions to send a final COMMIT message to the
+		 * GTM.
+		 */
+		CommitPreparedTranGTM(prepare_gxid, gxid, 0, NULL);
 	}
 	else
 	{
diff --git a/src/backend/storage/lmgr/lmgr.c b/src/backend/storage/lmgr/lmgr.c
index c052949749..763f1cfdce 100644
--- a/src/backend/storage/lmgr/lmgr.c
+++ b/src/backend/storage/lmgr/lmgr.c
@@ -542,6 +542,10 @@ XactLockTableWait(TransactionId xid, Relation rel, ItemPointer ctid,
 		error_context_stack = &callback;
 	}
 
+#ifdef XCP
+	TransactionRecordXidWait(xid);	
+#endif
+
 	for (;;)
 	{
 		Assert(TransactionIdIsValid(xid));
diff --git a/src/gtm/client/fe-protocol.c b/src/gtm/client/fe-protocol.c
index d18c531eff..3e43036140 100644
--- a/src/gtm/client/fe-protocol.c
+++ b/src/gtm/client/fe-protocol.c
@@ -396,6 +396,7 @@ gtmpqParseSuccess(GTM_Conn *conn, GTM_Result *result)
 		case TXN_BEGIN_GETGXID_AUTOVACUUM_RESULT:
 		case TXN_PREPARE_RESULT:
 		case TXN_START_PREPARED_RESULT:
+		case TXN_ROLLBACK_RESULT:
 			if (gtmpqGetnchar((char *)&result->gr_resdata.grd_gxid,
 						   sizeof (GlobalTransactionId), conn))
 				result->gr_status = GTM_RESULT_ERROR;
@@ -403,10 +404,12 @@ gtmpqParseSuccess(GTM_Conn *conn, GTM_Result *result)
 
 		case TXN_COMMIT_RESULT:
 		case TXN_COMMIT_PREPARED_RESULT:
-		case TXN_ROLLBACK_RESULT:
-			if (gtmpqGetnchar((char *)&result->gr_resdata.grd_gxid,
+			if (gtmpqGetnchar((char *)&result->gr_resdata.grd_eof_txn.gxid,
 						   sizeof (GlobalTransactionId), conn))
 				result->gr_status = GTM_RESULT_ERROR;
+			if (gtmpqGetnchar((char *)&result->gr_resdata.grd_eof_txn.status,
+						   sizeof (int), conn))
+				result->gr_status = GTM_RESULT_ERROR;
 			break;
 
 		case TXN_GET_GXID_RESULT:
diff --git a/src/gtm/client/gtm_client.c b/src/gtm/client/gtm_client.c
index 4f77731a4c..8808735c6c 100644
--- a/src/gtm/client/gtm_client.c
+++ b/src/gtm/client/gtm_client.c
@@ -48,6 +48,8 @@ void GTM_FreeResult(GTM_Result *result, GTM_PGXCNodeType remote_type);
 static GTM_Result *makeEmptyResultIfIsNull(GTM_Result *oldres);
 static int commit_prepared_transaction_internal(GTM_Conn *conn,
 												GlobalTransactionId gxid, GlobalTransactionId prepared_gxid,
+												int waited_xid_count,
+												GlobalTransactionId *waited_xids,
 												bool is_backup);
 static int start_prepared_transaction_internal(GTM_Conn *conn, GlobalTransactionId gxid, char *gid,
 											   char *nodestring, bool is_backup);
@@ -70,7 +72,10 @@ static GTM_Sequence get_next_internal(GTM_Conn *conn, GTM_SequenceKey key, bool
 static int set_val_internal(GTM_Conn *conn, GTM_SequenceKey key, GTM_Sequence nextval, bool iscalled, bool is_backup);
 #endif
 static int reset_sequence_internal(GTM_Conn *conn, GTM_SequenceKey key, bool is_backup);
-static int commit_transaction_internal(GTM_Conn *conn, GlobalTransactionId gxid, bool is_backup);
+static int commit_transaction_internal(GTM_Conn *conn, GlobalTransactionId gxid,
+		int waited_xid_count,
+		GlobalTransactionId *waited_xids,
+		bool is_backup);
 static int close_sequence_internal(GTM_Conn *conn, GTM_SequenceKey key, bool is_backup);
 static int rename_sequence_internal(GTM_Conn *conn, GTM_SequenceKey key, GTM_SequenceKey newkey, bool is_backup);
 static int alter_sequence_internal(GTM_Conn *conn, GTM_SequenceKey key, GTM_Sequence increment,
@@ -596,29 +601,52 @@ send_failed:
 int
 bkup_commit_transaction(GTM_Conn *conn, GlobalTransactionId gxid)
 {
-	return commit_transaction_internal(conn, gxid, true);
+	return commit_transaction_internal(conn, gxid, 0, NULL, true);
 }
 
 
 int
-commit_transaction(GTM_Conn *conn, GlobalTransactionId gxid)
+commit_transaction(GTM_Conn *conn, GlobalTransactionId gxid,
+		int waited_xid_count, GlobalTransactionId *waited_xids)
 {
-	return commit_transaction_internal(conn, gxid, false);
+	if (waited_xid_count == 0)
+	{
+		int txn_count_out;
+		int status_out;
+		int status;
+		status = commit_transaction_multi(conn, 1, &gxid, &txn_count_out,
+				&status_out);
+		Assert(txn_count_out == 1);
+		return status;
+	}
+	else
+		return commit_transaction_internal(conn, gxid, waited_xid_count,
+				waited_xids, false);
 }
 
 
 static int
-commit_transaction_internal(GTM_Conn *conn, GlobalTransactionId gxid, bool is_backup)
+commit_transaction_internal(GTM_Conn *conn, GlobalTransactionId gxid,
+		int waited_xid_count, GlobalTransactionId *waited_xids,
+		bool is_backup)
 {
 	GTM_Result *res = NULL;
 	time_t finish_time;
 
+retry:
 	 /* Start the message. */
 	if (gtmpqPutMsgStart('C', true, conn) ||
 		gtmpqPutInt(is_backup ? MSG_BKUP_TXN_COMMIT : MSG_TXN_COMMIT, sizeof (GTM_MessageType), conn) ||
-		gtmpqPutnchar((char *)&gxid, sizeof (GlobalTransactionId), conn))
+		gtmpqPutnchar((char *)&gxid, sizeof (GlobalTransactionId), conn) ||
+		gtmpqPutInt(waited_xid_count, sizeof (int), conn))
 		goto send_failed;
 
+	if (waited_xid_count > 0)
+	{
+		if (gtmpqPutnchar((char *) waited_xids, waited_xid_count * sizeof (GlobalTransactionId), conn))
+			goto send_failed;
+	}
+
 	/* Finish the message. */
 	if (gtmpqPutMsgEnd(conn))
 		goto send_failed;
@@ -641,6 +669,33 @@ commit_transaction_internal(GTM_Conn *conn, GlobalTransactionId gxid, bool is_ba
 		{
 			Assert(res->gr_type == TXN_COMMIT_RESULT);
 			Assert(res->gr_resdata.grd_gxid == gxid);
+
+			if (waited_xid_count > 0)
+			{
+				if (res->gr_resdata.grd_eof_txn.status == STATUS_DELAYED)
+				{
+					/*
+					 * GTM may decide to delay the transaction commit if one or
+					 * more of the XIDs we had waited to finish for hasn't yet
+					 * made to the GTM. While this window is very small, we
+					 * need to guard against that to ensure that a transaction
+					 * which is already seen as committed by datanodes is not
+					 * reported as in-progress by GTM. Also, we don't wait at
+					 * the GTM for other transactions to finish because that
+					 * could potentially lead to deadlocks. So instead just
+					 * sleep for a while (1ms right now) and retry the
+					 * operation.
+					 *
+					 * Since the transactions we are waiting for are in fact
+					 * already committed and hence we don't see a reason why we
+					 * might end up in an inifinite loop. Nevertheless, it
+					 * might make sense to flash a warning and proceed after
+					 * certain number of retries
+					 */
+					pg_usleep(1000);
+					goto retry;
+				}
+			}
 		}
 
 		return res->gr_status;
@@ -655,30 +710,49 @@ send_failed:
 }
 
 int
-commit_prepared_transaction(GTM_Conn *conn, GlobalTransactionId gxid, GlobalTransactionId prepared_gxid)
+commit_prepared_transaction(GTM_Conn *conn,
+		GlobalTransactionId gxid,
+		GlobalTransactionId prepared_gxid,
+		int waited_xid_count,
+		GlobalTransactionId *waited_xids)
 {
-	return commit_prepared_transaction_internal(conn, gxid, prepared_gxid, false);
+	return commit_prepared_transaction_internal(conn, gxid, prepared_gxid,
+			waited_xid_count, waited_xids, false);
 }
 
 int
 bkup_commit_prepared_transaction(GTM_Conn *conn, GlobalTransactionId gxid, GlobalTransactionId prepared_gxid)
 {
-	return commit_prepared_transaction_internal(conn, gxid, prepared_gxid, true);
+	return commit_prepared_transaction_internal(conn, gxid, prepared_gxid, 0,
+			NULL, true);
 }
 
 static int
-commit_prepared_transaction_internal(GTM_Conn *conn, GlobalTransactionId gxid, GlobalTransactionId prepared_gxid, bool is_backup)
+commit_prepared_transaction_internal(GTM_Conn *conn,
+		GlobalTransactionId gxid,
+		GlobalTransactionId prepared_gxid,
+		int waited_xid_count,
+		GlobalTransactionId *waited_xids,
+		bool is_backup)
 {
 	GTM_Result *res = NULL;
 	time_t finish_time;
 
+retry:
 	/* Start the message */
 	if (gtmpqPutMsgStart('C', true, conn) ||
 		gtmpqPutInt(is_backup ? MSG_BKUP_TXN_COMMIT_PREPARED : MSG_TXN_COMMIT_PREPARED, sizeof (GTM_MessageType), conn) ||
 		gtmpqPutnchar((char *)&gxid, sizeof (GlobalTransactionId), conn) ||
-		gtmpqPutnchar((char *)&prepared_gxid, sizeof (GlobalTransactionId), conn))
+		gtmpqPutnchar((char *)&prepared_gxid, sizeof (GlobalTransactionId), conn) ||
+		gtmpqPutInt(waited_xid_count, sizeof (int), conn))
 		goto send_failed;
 
+	if (waited_xid_count > 0)
+	{
+		if (gtmpqPutnchar((char *) waited_xids, waited_xid_count * sizeof (GlobalTransactionId), conn))
+			goto send_failed;
+	}
+
 	/* Finish the message */
 	if (gtmpqPutMsgEnd(conn))
 		goto send_failed;
@@ -701,6 +775,15 @@ commit_prepared_transaction_internal(GTM_Conn *conn, GlobalTransactionId gxid, G
 		{
 			Assert(res->gr_type == TXN_COMMIT_PREPARED_RESULT);
 			Assert(res->gr_resdata.grd_gxid == gxid);
+			if (waited_xid_count > 0)
+			{
+				if (res->gr_resdata.grd_eof_txn.status == STATUS_DELAYED)
+				{
+					/* See comments in commit_transaction_internal() */
+					pg_usleep(1000);
+					goto retry;
+				}
+			}
 		}
 
 		return res->gr_status;
@@ -2022,7 +2105,8 @@ send_failed:
 }
 
 int
-bkup_commit_transaction_multi(GTM_Conn *conn, int txn_count, GlobalTransactionId *gxid)
+bkup_commit_transaction_multi(GTM_Conn *conn, int txn_count,
+		GlobalTransactionId *gxid)
 {
 	int ii;
 
@@ -2035,8 +2119,7 @@ bkup_commit_transaction_multi(GTM_Conn *conn, int txn_count, GlobalTransactionId
 
 	for (ii = 0; ii < txn_count; ii++)
 	{
-		if (gtmpqPutnchar((char *)&gxid[ii],
-				  sizeof (GlobalTransactionId), conn))
+		if (gtmpqPutnchar((char *)&gxid[ii], sizeof (GlobalTransactionId), conn))
 			  goto send_failed;
 	}
 
diff --git a/src/gtm/main/gtm_txn.c b/src/gtm/main/gtm_txn.c
index 8f09d7c128..cd55018e22 100644
--- a/src/gtm/main/gtm_txn.c
+++ b/src/gtm/main/gtm_txn.c
@@ -149,8 +149,8 @@ GlobalTransactionIdGetStatus(GlobalTransactionId transactionId)
 /*
  * Given the GXID, find the corresponding transaction handle.
  */
-GTM_TransactionHandle
-GTM_GXIDToHandle(GlobalTransactionId gxid)
+static GTM_TransactionHandle
+GTM_GXIDToHandle_Internal(GlobalTransactionId gxid, bool warn)
 {
 	gtm_ListCell *elem = NULL;
    	GTM_TransactionInfo *gtm_txninfo = NULL;
@@ -174,13 +174,26 @@ GTM_GXIDToHandle(GlobalTransactionId gxid)
 		return gtm_txninfo->gti_handle;
 	else
 	{
-		ereport(WARNING,
+		if (warn)
+			ereport(WARNING,
 				(ERANGE, errmsg("No transaction handle for gxid: %d",
 								gxid)));
 		return InvalidTransactionHandle;
 	}
 }
 
+GTM_TransactionHandle
+GTM_GXIDToHandle(GlobalTransactionId gxid)
+{
+	return GTM_GXIDToHandle_Internal(gxid, true);
+}
+
+bool
+GTM_IsGXIDInProgress(GlobalTransactionId gxid)
+{
+	return (GTM_GXIDToHandle_Internal(gxid, false) !=
+			InvalidTransactionHandle);
+}
 /*
  * Given the GID (for a prepared transaction), find the corresponding
  * transaction handle.
@@ -979,20 +992,27 @@ int
 GTM_CommitTransactionGXID(GlobalTransactionId gxid)
 {
 	GTM_TransactionHandle txn = GTM_GXIDToHandle(gxid);
-	return GTM_CommitTransaction(txn);
+	return GTM_CommitTransaction(txn, 0, NULL);
 }
 
 /*
  * Commit multiple transactions in one go
  */
 int
-GTM_CommitTransactionMulti(GTM_TransactionHandle txn[], int txn_count, int status[])
+GTM_CommitTransactionMulti(GTM_TransactionHandle txn[], int txn_count,
+		int waited_xid_count, GlobalTransactionId *waited_xids,
+		int status[])
 {
 	GTM_TransactionInfo *gtm_txninfo[txn_count];
+	GTM_TransactionInfo *remove_txninfo[txn_count];
+	int remove_count = 0;
 	int ii;
 
 	for (ii = 0; ii < txn_count; ii++)
 	{
+		int jj;
+		bool waited_xid_running = false;
+
 		gtm_txninfo[ii] = GTM_HandleToTransactionInfo(txn[ii]);
 
 		if (gtm_txninfo[ii] == NULL)
@@ -1000,6 +1020,28 @@ GTM_CommitTransactionMulti(GTM_TransactionHandle txn[], int txn_count, int statu
 			status[ii] = STATUS_ERROR;
 			continue;
 		}
+
+		/*
+		 * If any of the waited_xids is still running, we must delay commit for
+		 * this transaction till all such waited_xids are finished
+		 */
+		for (jj = 0; jj < waited_xid_count; jj++)
+		{
+			if (GTM_IsGXIDInProgress(waited_xids[jj]))
+			{
+				elog(DEBUG1, "Xact %d not yet finished, xact %d will be delayed",
+						waited_xids[jj], gtm_txninfo[ii]->gti_gxid);
+				waited_xid_running = true;
+				break;
+			}
+		}
+
+		if (waited_xid_running) 
+		{
+			status[ii] = STATUS_DELAYED;
+			continue;
+		}
+
 		/*
 		 * Mark the transaction as being aborted
 		 */
@@ -1007,11 +1049,13 @@ GTM_CommitTransactionMulti(GTM_TransactionHandle txn[], int txn_count, int statu
 		gtm_txninfo[ii]->gti_state = GTM_TXN_COMMIT_IN_PROGRESS;
 		GTM_RWLockRelease(&gtm_txninfo[ii]->gti_lock);
 		status[ii] = STATUS_OK;
+
+		remove_txninfo[remove_count++] = gtm_txninfo[ii];
 	}
 
-	GTM_RemoveTransInfoMulti(gtm_txninfo, txn_count);
+	GTM_RemoveTransInfoMulti(remove_txninfo, remove_count);
 
-	return txn_count;
+	return remove_count;
 }
 
 /*
@@ -1041,10 +1085,11 @@ GTM_PrepareTransaction(GTM_TransactionHandle txn)
  * Commit a transaction
  */
 int
-GTM_CommitTransaction(GTM_TransactionHandle txn)
+GTM_CommitTransaction(GTM_TransactionHandle txn, int waited_xid_count,
+		GlobalTransactionId *waited_xids)
 {
 	int status;
-	GTM_CommitTransactionMulti(&txn, 1, &status);
+	GTM_CommitTransactionMulti(&txn, 1, waited_xid_count, waited_xids, &status);
 	return status;
 }
 
@@ -1714,6 +1759,9 @@ ProcessCommitTransactionCommand(Port *myport, StringInfo message, bool is_backup
 	GlobalTransactionId gxid;
 	MemoryContext oldContext;
 	int status = STATUS_OK;
+	int waited_xid_count;
+	GlobalTransactionId *waited_xids;
+
 	const char *data = pq_getmsgbytes(message, sizeof (gxid));
 
 	if (data == NULL)
@@ -1723,6 +1771,13 @@ ProcessCommitTransactionCommand(Port *myport, StringInfo message, bool is_backup
 	memcpy(&gxid, data, sizeof (gxid));
 	txn = GTM_GXIDToHandle(gxid);
 
+	waited_xid_count = pq_getmsgint(message, sizeof (int));
+	if (waited_xid_count > 0)
+	{
+		waited_xids = pq_getmsgbytes(message,
+				waited_xid_count * sizeof (GlobalTransactionId));
+	}
+
 	pq_getmsgend(message);
 
 	oldContext = MemoryContextSwitchTo(TopMemoryContext);
@@ -1730,13 +1785,17 @@ ProcessCommitTransactionCommand(Port *myport, StringInfo message, bool is_backup
 	/*
 	 * Commit the transaction
 	 */
-	status = GTM_CommitTransaction(txn);
+	status = GTM_CommitTransaction(txn, waited_xid_count, waited_xids);
 
 	MemoryContextSwitchTo(oldContext);
 
 	if(!is_backup)
 	{
-		if (GetMyThreadInfo->thr_conn->standby)
+		/*
+		 * If the transaction is successfully committed on the GTM master then
+		 * send a backup message to the GTM slave to redo the action locally
+		 */
+		if ((GetMyThreadInfo->thr_conn->standby) && (status == STATUS_OK))
 		{
 			/*
 			 * Backup first
@@ -1769,7 +1828,7 @@ ProcessCommitTransactionCommand(Port *myport, StringInfo message, bool is_backup
 			pq_sendbytes(&buf, (char *)&proxyhdr, sizeof (GTM_ProxyMsgHeader));
 		}
 		pq_sendbytes(&buf, (char *)&gxid, sizeof(gxid));
-		pq_sendint(&buf, status, sizeof(status));
+		pq_sendbytes(&buf, (char *)&status, sizeof(status));
 		pq_endmessage(myport, &buf);
 
 		if (myport->remote_type != GTM_NODE_GTM_PROXY)
@@ -1801,6 +1860,8 @@ ProcessCommitPreparedTransactionCommand(Port *myport, StringInfo message, bool i
 	MemoryContext oldContext;
 	int status[txn_count];
 	int ii;
+	int waited_xid_count;
+	GlobalTransactionId *waited_xids;
 
 	for (ii = 0; ii < txn_count; ii++)
 	{
@@ -1814,6 +1875,13 @@ ProcessCommitPreparedTransactionCommand(Port *myport, StringInfo message, bool i
 		elog(DEBUG1, "ProcessCommitTransactionCommandMulti: gxid(%u), handle(%u)", gxid[ii], txn[ii]);
 	}
 
+	waited_xid_count = pq_getmsgint(message, sizeof (int));
+	if (waited_xid_count > 0)
+	{
+		waited_xids = pq_getmsgbytes(message,
+				waited_xid_count * sizeof (GlobalTransactionId));
+	}
+
 	pq_getmsgend(message);
 
 	oldContext = MemoryContextSwitchTo(TopMemoryContext);
@@ -1823,13 +1891,26 @@ ProcessCommitPreparedTransactionCommand(Port *myport, StringInfo message, bool i
 	/*
 	 * Commit the prepared transaction.
 	 */
-	GTM_CommitTransactionMulti(txn, txn_count, status);
+	GTM_CommitTransactionMulti(txn, txn_count, waited_xid_count,
+			waited_xids, status);
 
 	MemoryContextSwitchTo(oldContext);
 
 	if (!is_backup)
 	{
-		if (GetMyThreadInfo->thr_conn->standby)
+		/*
+		 * If we successfully committed the transaction on the GTM master, then
+		 * also send a backup message to the GTM slave to redo the action
+		 * locally
+		 *
+		 * GTM_CommitTransactionMulti() above is used to only commit the main
+		 * and the auxiliary GXID. Since we either commit or delay both of
+		 * these GXIDs together, its enough to just test for one of the GXIDs.
+		 * If the transaction commit is delayed, the backup message will be
+		 * sent when the GTM master receives COMMIT message again and
+		 * successfully commits the transaction
+		 */
+		if ((GetMyThreadInfo->thr_conn->standby) && (status[0] == STATUS_OK))
 		{
 			/* Backup first */
 			int _rc;
@@ -1862,7 +1943,7 @@ ProcessCommitPreparedTransactionCommand(Port *myport, StringInfo message, bool i
 			pq_sendbytes(&buf, (char *)&proxyhdr, sizeof (GTM_ProxyMsgHeader));
 		}
 		pq_sendbytes(&buf, (char *)&gxid[0], sizeof(GlobalTransactionId));
-		pq_sendint(&buf, status[0], 4);
+		pq_sendbytes(&buf, (char *)&status[0], 4);
 		pq_endmessage(myport, &buf);
 
 		if (myport->remote_type != GTM_NODE_GTM_PROXY)
@@ -2193,7 +2274,7 @@ ProcessCommitTransactionCommandMulti(Port *myport, StringInfo message, bool is_b
 	/*
 	 * Commit the transaction
 	 */
-	GTM_CommitTransactionMulti(txn, txn_count, status);
+	GTM_CommitTransactionMulti(txn, txn_count, 0, NULL, status);
 
 	MemoryContextSwitchTo(oldContext);
 
diff --git a/src/gtm/proxy/proxy_main.c b/src/gtm/proxy/proxy_main.c
index 0534ee4a31..33b2266f0a 100644
--- a/src/gtm/proxy/proxy_main.c
+++ b/src/gtm/proxy/proxy_main.c
@@ -1596,6 +1596,7 @@ ProcessCommand(GTMProxy_ConnectionInfo *conninfo, GTM_Conn *gtm_conn,
 		case MSG_SEQUENCE_RENAME:
 		case MSG_SEQUENCE_ALTER:
 		case MSG_BARRIER:
+		case MSG_TXN_COMMIT:
 #ifdef XCP
 		case MSG_REGISTER_SESSION:
 #endif
@@ -1610,7 +1611,7 @@ ProcessCommand(GTMProxy_ConnectionInfo *conninfo, GTM_Conn *gtm_conn,
 
 		case MSG_TXN_BEGIN:
 		case MSG_TXN_BEGIN_GETGXID:
-		case MSG_TXN_COMMIT:
+		case MSG_TXN_COMMIT_MULTI:
 		case MSG_TXN_ROLLBACK:
 		case MSG_TXN_GET_GXID:
 			ProcessTransactionCommand(conninfo, gtm_conn, mtype, input_message);
@@ -1655,7 +1656,6 @@ ProcessCommand(GTMProxy_ConnectionInfo *conninfo, GTM_Conn *gtm_conn,
 static GTM_Conn *
 HandleGTMError(GTM_Conn *gtm_conn)
 {
-	Assert(gtm_conn && gtm_conn->last_errno != 0);
 
 	elog(NOTICE,
 		 "GTM communication error was detected.  Retrying connection, interval = %d.",
@@ -1758,6 +1758,7 @@ IsProxiedMessage(GTM_MessageType mtype)
 		case MSG_SEQUENCE_RENAME:
 		case MSG_SEQUENCE_ALTER:
 		case MSG_SNAPSHOT_GET:
+		case MSG_TXN_COMMIT:
 			return true;
 
 		default:
@@ -1823,7 +1824,7 @@ ProcessResponse(GTMProxy_ThreadInfo *thrinfo, GTMProxy_CommandInfo *cmdinfo,
 			ReleaseCmdBackup(cmdinfo);
 			break;
 
-		case MSG_TXN_COMMIT:
+		case MSG_TXN_COMMIT_MULTI:
 			if (res->gr_type != TXN_COMMIT_MULTI_RESULT)
 			{
 				ReleaseCmdBackup(cmdinfo);
@@ -1842,9 +1843,13 @@ ProcessResponse(GTMProxy_ThreadInfo *thrinfo, GTMProxy_CommandInfo *cmdinfo,
 
 			if (res->gr_resdata.grd_txn_rc_multi.status[cmdinfo->ci_res_index] == STATUS_OK)
 			{
+				int txn_count = 1;
+				int status = STATUS_OK;
+
 				pq_beginmessage(&buf, 'S');
-				pq_sendint(&buf, TXN_COMMIT_RESULT, 4);
-				pq_sendbytes(&buf, (char *)&cmdinfo->ci_data.cd_rc.gxid, sizeof (GlobalTransactionId));
+				pq_sendint(&buf, TXN_COMMIT_MULTI_RESULT, 4);
+				pq_sendbytes(&buf, &txn_count, sizeof (int));
+				pq_sendbytes(&buf, &status, sizeof (int));
 				pq_endmessage(cmdinfo->ci_conn->con_port, &buf);
 				pq_flush(cmdinfo->ci_conn->con_port);
 			}
@@ -1958,6 +1963,7 @@ ProcessResponse(GTMProxy_ThreadInfo *thrinfo, GTMProxy_CommandInfo *cmdinfo,
 		case MSG_SEQUENCE_RENAME:
 		case MSG_SEQUENCE_ALTER:
 		case MSG_SNAPSHOT_GET:
+		case MSG_TXN_COMMIT:
 			Assert(IsProxiedMessage(cmdinfo->ci_mtype));
 			if ((res->gr_proxyhdr.ph_conid == InvalidGTMProxyConnID) ||
 				(res->gr_proxyhdr.ph_conid >= GTM_PROXY_MAX_CONNECTIONS) ||
@@ -2296,7 +2302,12 @@ ProcessTransactionCommand(GTMProxy_ConnectionInfo *conninfo, GTM_Conn *gtm_conn,
 			GTMProxy_CommandPending(conninfo, mtype, cmd_data);
 			break;
 
-		case MSG_TXN_COMMIT:
+		case MSG_TXN_COMMIT_MULTI:
+			{
+				int txn_count = pq_getmsgint(message, sizeof (int));
+				Assert (txn_count == 1);
+			}
+			/* fall through */
 		case MSG_TXN_ROLLBACK:
 			{
 				const char *data = pq_getmsgbytes(message,
@@ -2694,7 +2705,7 @@ GTMProxy_ProcessPendingCommands(GTMProxy_ThreadInfo *thrinfo)
 				thrinfo->thr_pending_commands[ii] = gtm_NIL;
 				break;
 
-			case MSG_TXN_COMMIT:
+			case MSG_TXN_COMMIT_MULTI:
 				if (gtmpqPutInt(MSG_TXN_COMMIT_MULTI, sizeof (GTM_MessageType), gtm_conn) ||
 					gtmpqPutInt(gtm_list_length(thrinfo->thr_pending_commands[ii]), sizeof(int), gtm_conn))
 					elog(ERROR, "Error sending data");
diff --git a/src/gtm/test2/test_txn.c b/src/gtm/test2/test_txn.c
index c29b733aa2..55097a1fba 100644
--- a/src/gtm/test2/test_txn.c
+++ b/src/gtm/test2/test_txn.c
@@ -65,7 +65,7 @@ test_txn_02()
 	rc = prepare_transaction(conn, gxid);
 	_ASSERT( rc>=0 );
 
-	rc = commit_transaction(conn, gxid);
+	rc = commit_transaction(conn, gxid, 0, NULL);
 	_ASSERT( rc>=0 );
 
 	TEARDOWN();
@@ -82,7 +82,7 @@ test_txn_03()
 	gxid = begin_transaction_autovacuum(conn, GTM_ISOLATION_SERIALIZABLE);
 	_ASSERT( gxid != InvalidGlobalTransactionId );
 
-	rc = commit_transaction(conn, gxid);
+	rc = commit_transaction(conn, gxid, 0, NULL);
 	_ASSERT( rc>=0 );
 
 	TEARDOWN();
@@ -116,7 +116,7 @@ test_txn_11()
 	printf("test_txn_11: prepared_gxid=%d\n", prepared_gxid);
 	_ASSERT( rc>=0 );
 
-	rc = commit_prepared_transaction(conn, gxid, prepared_gxid);
+	rc = commit_prepared_transaction(conn, gxid, prepared_gxid, 0, NULL);
 	_ASSERT( rc>=0 );
 
 	TEARDOWN();
@@ -159,7 +159,7 @@ test_txn_53()
 
 	SETUP();
 
-	rc = commit_transaction(conn, InvalidGlobalTransactionId);
+	rc = commit_transaction(conn, InvalidGlobalTransactionId, 0, NULL);
 	_ASSERT( rc==-1 );
 
 	TEARDOWN();
diff --git a/src/include/access/gtm.h b/src/include/access/gtm.h
index b606976276..9a376d3bdc 100644
--- a/src/include/access/gtm.h
+++ b/src/include/access/gtm.h
@@ -27,7 +27,8 @@ extern void InitGTM(void);
 extern void CloseGTM(void);
 extern GlobalTransactionId BeginTranGTM(GTM_Timestamp *timestamp);
 extern GlobalTransactionId BeginTranAutovacuumGTM(void);
-extern int CommitTranGTM(GlobalTransactionId gxid);
+extern int CommitTranGTM(GlobalTransactionId gxid, int waited_xid_count,
+		GlobalTransactionId *waited_xids);
 extern int RollbackTranGTM(GlobalTransactionId gxid);
 extern int StartPreparedTranGTM(GlobalTransactionId gxid,
 								char *gid,
@@ -38,7 +39,9 @@ extern int GetGIDDataGTM(char *gid,
 						 GlobalTransactionId *prepared_gxid,
 						 char **nodestring);
 extern int CommitPreparedTranGTM(GlobalTransactionId gxid,
-								 GlobalTransactionId prepared_gxid);
+								 GlobalTransactionId prepared_gxid,
+								 int waited_xid_count,
+								 GlobalTransactionId *waited_xids);
 
 extern GTM_Snapshot GetSnapshotGTM(GlobalTransactionId gxid, bool canbe_grouped);
 
diff --git a/src/include/c.h b/src/include/c.h
index 56aebbe026..450248a001 100644
--- a/src/include/c.h
+++ b/src/include/c.h
@@ -963,6 +963,7 @@ typedef NameData *Name;
 #define STATUS_FOUND			(1)
 #define STATUS_WAITING			(2)
 #define STATUS_NOT_FOUND		(3)
+#define STATUS_DELAYED			(4)
 
 
 /*
diff --git a/src/include/gtm/gtm_client.h b/src/include/gtm/gtm_client.h
index e154e4979c..7a6c61d811 100644
--- a/src/include/gtm/gtm_client.h
+++ b/src/include/gtm/gtm_client.h
@@ -36,12 +36,17 @@ typedef union GTM_ResultData
 		GTM_Timestamp			timestamp;
 	} grd_gxid_tp;								/* TXN_BEGIN_GETGXID */
 
-	GlobalTransactionId			grd_gxid;		/* TXN_PREPARE
-												 * TXN_START_PREPARED
-												 * TXN_COMMIT
-												 * TXN_COMMIT_PREPARED
-												 * TXN_ROLLBACK
-												 */
+	GlobalTransactionId			grd_gxid;			/* TXN_PREPARE		
+													 * TXN_START_PREPARED
+													 * TXN_ROLLBACK
+													 */
+	struct {
+		GlobalTransactionId			gxid;
+													/* TXN_COMMIT
+													 * TXN_COMMIT_PREPARED
+													 */
+		int							status;
+	} grd_eof_txn;
 
 	GlobalTransactionId			grd_next_gxid;
 
@@ -185,9 +190,14 @@ GlobalTransactionId begin_transaction_autovacuum(GTM_Conn *conn, GTM_IsolationLe
 int bkup_begin_transaction_autovacuum(GTM_Conn *conn, GlobalTransactionId gxid,
 									  GTM_IsolationLevel isolevel,
 									  uint32 client_id);
-int commit_transaction(GTM_Conn *conn, GlobalTransactionId gxid);
+int commit_transaction(GTM_Conn *conn, GlobalTransactionId gxid,
+					   int waited_xid_count,
+					   GlobalTransactionId *waited_xids);
 int bkup_commit_transaction(GTM_Conn *conn, GlobalTransactionId gxid);
-int commit_prepared_transaction(GTM_Conn *conn, GlobalTransactionId gxid, GlobalTransactionId prepared_gxid);
+int commit_prepared_transaction(GTM_Conn *conn, GlobalTransactionId gxid,
+								GlobalTransactionId prepared_gxid,
+								int waited_xid_count,
+								GlobalTransactionId *waited_xids);
 int bkup_commit_prepared_transaction(GTM_Conn *conn, GlobalTransactionId gxid, GlobalTransactionId prepared_gxid);
 int abort_transaction(GTM_Conn *conn, GlobalTransactionId gxid);
 int bkup_abort_transaction(GTM_Conn *conn, GlobalTransactionId gxid);
@@ -218,7 +228,8 @@ int
 commit_transaction_multi(GTM_Conn *conn, int txn_count, GlobalTransactionId *gxid,
 						 int *txn_count_out, int *status_out);
 int
-bkup_commit_transaction_multi(GTM_Conn *conn, int txn_count, GlobalTransactionId *gxid);
+bkup_commit_transaction_multi(GTM_Conn *conn, int txn_count,
+		GlobalTransactionId *gxid);
 int
 abort_transaction_multi(GTM_Conn *conn, int txn_count, GlobalTransactionId *gxid,
 			int *txn_count_out, int *status_out);
diff --git a/src/include/gtm/gtm_txn.h b/src/include/gtm/gtm_txn.h
index 23e2382ad6..d220e81df1 100644
--- a/src/include/gtm/gtm_txn.h
+++ b/src/include/gtm/gtm_txn.h
@@ -184,6 +184,7 @@ extern GTM_Transactions	GTMTransactions;
 GTM_TransactionInfo *GTM_HandleToTransactionInfo(GTM_TransactionHandle handle);
 GTM_TransactionHandle GTM_GXIDToHandle(GlobalTransactionId gxid);
 GTM_TransactionHandle GTM_GIDToHandle(char *gid);
+bool GTM_IsGXIDInProgress(GlobalTransactionId gxid);
 
 /* Transaction Control */
 void GTM_InitTxnManager(void);
@@ -197,8 +198,11 @@ int GTM_BeginTransactionMulti(GTM_IsolationLevel isolevel[],
 int GTM_RollbackTransaction(GTM_TransactionHandle txn);
 int GTM_RollbackTransactionMulti(GTM_TransactionHandle txn[], int txn_count, int status[]);
 int GTM_RollbackTransactionGXID(GlobalTransactionId gxid);
-int GTM_CommitTransaction(GTM_TransactionHandle txn);
-int GTM_CommitTransactionMulti(GTM_TransactionHandle txn[], int txn_count, int status[]);
+int GTM_CommitTransaction(GTM_TransactionHandle txn,
+		int waited_xid_count, GlobalTransactionId *waited_xids);
+int GTM_CommitTransactionMulti(GTM_TransactionHandle txn[], int txn_count,
+		int waited_xid_count, GlobalTransactionId *waited_xids,
+		int status[]);
 int GTM_CommitTransactionGXID(GlobalTransactionId gxid);
 int GTM_PrepareTransaction(GTM_TransactionHandle txn);
 int GTM_StartPreparedTransaction(GTM_TransactionHandle txn,
diff --git a/src/include/pgxc/execRemote.h b/src/include/pgxc/execRemote.h
index 3f4d19e858..87fc6fdedc 100644
--- a/src/include/pgxc/execRemote.h
+++ b/src/include/pgxc/execRemote.h
@@ -50,6 +50,7 @@ extern bool EnforceTwoPhaseCommit;
 #ifdef XCP
 #define RESPONSE_ERROR 6
 #define RESPONSE_READY 10
+#define RESPONSE_WAITXIDS 11
 #endif
 
 typedef enum