Refactor GTM connection management.

1. GTM keeps track of currently open transactions in a global data structure.
When a specific client disconnects from GTM, it needs to figure out if any open
transactions should be reclaimed or not. Since there is one GTM thread per
client connection, we were using pthread_id() to record this information such
that when a connection ends, all transactions with gti_thread_id ==
pthread_self() are removed. This works OK in a standalone GTM. But when things
are failed over to a standby, the pthread_id on the old master makes no sense
on the new master. So now we assign a monotonically increasing identifier to
each incoming connection. Proxy remembers this identifier and sends it back to
the new master when it reconnects. This identifier is also backed up to the
standby. Identifier wrapped around is handled, but given an identifier is
assigned for every new connection, wrap around is almost near impossible to
happen in real world.

2. The conn_id which proxy uses to identify its client connection quickly while
handling responses from GTM, is an index into an array of client connections
that proxy maintains on a per thread basis. Currently, when a client
disconnects from the proxy, we were moving connections around in that array so
as to keep it packed. But this can mess up things if there are outstanding
requests on the GTM since it will encode the old conn_id in the message
response which has now changed on the proxy. We fix this by not moving
connections around. So an array index once assigned remains the same. But to
quickly iterate through the list of open connections, we also maintain a
condensed array which acts like a map to the actual connections.

3. There was a long standing bug report about GTM running out of threads. This
was kind of surprising because even though GTM forks one thread per connection,
when clients are connected via proxy, the number of GTM threads should be
pretty small. We noticed that a call to pthread_detach() could have been called
on a thread id which is not yet set and hence thread specific resources might
not get reclaimed when a thread exits. We don't have a reproducible scenario
for this bug, but this looks like a fairly decent theory to explain the bug. So
this patch fixes that issue.

4. GTM and GTM proxy code assumes that the first few members of GTM_ThreadInfo
and GTMProxy_ThreadInfo structure have the same storage layout. This was very
easy to get messed up, like we saw during this patch development. So those
common members are now moved to #define and included via that mechanism in both
the structures. We could have instead also defined a common structure, but
unfortunately that could lead to lot of code changes. So this seems like a good
compromise for now.

18 files changed, 464 insertions, 139 deletions

diff --git a/src/gtm/client/fe-connect.c b/src/gtm/client/fe-connect.c
index 1bcb3e6a17..9e7fc0689a 100644
--- a/src/gtm/client/fe-connect.c
+++ b/src/gtm/client/fe-connect.c
@@ -62,6 +62,7 @@ static const GTMPQconninfoOption GTMPQconninfoOptions[] = {
 	{"node_name", NULL},
 	{"remote_type", NULL},
 	{"postmaster", NULL},
+	{"client_id", NULL},
 	/* Terminating entry --- MUST BE LAST */
 	{NULL, NULL}
 };
@@ -185,6 +186,8 @@ connectOptions1(GTM_Conn *conn, const char *conninfo)
 	conn->is_postmaster = tmp ? atoi(tmp) : 0;
 	tmp = conninfo_getval(connOptions, "remote_type");
 	conn->remote_type = tmp ? atoi(tmp) : GTM_NODE_DEFAULT;
+	tmp = conninfo_getval(connOptions, "client_id");
+	conn->my_id = tmp ? atoi(tmp) : 0;
 
 	/*
 	 * Free the option info - all is in conn now
@@ -687,6 +690,7 @@ keep_going:						/* We will come back to here until there is
 				strncpy(sp->sp_node_name, conn->gc_node_name, SP_NODE_NAME);
 				sp->sp_remotetype = conn->remote_type;
 				sp->sp_ispostmaster = conn->is_postmaster;
+				sp->sp_client_id = conn->my_id;
 
 				/*
 				 * Send the startup packet.
@@ -760,11 +764,27 @@ keep_going:						/* We will come back to here until there is
 				}
 
 				{
+					int msgLength;
 					/*
-					 * Server sends a dummy message body of size 4 bytes
+					 * Read the message length word
 					 */
-					int tmp_int;
-					gtmpqGetInt(&tmp_int, 4, conn);
+					if (gtmpqGetInt(&msgLength, 4, conn))
+					{
+						/* We'll come back when there is more data */
+						return PGRES_POLLING_READING;
+					}
+
+					if (msgLength != 4)
+						appendGTMPQExpBuffer(&conn->errorMessage,
+								"expected message length of 4 bytes from "
+								"server, but received %d bytes\n",
+								msgLength);
+
+					if (gtmpqGetInt(&conn->my_id, 4, conn))
+					{
+						/* We'll come back when there is more data */
+						return PGRES_POLLING_READING;
+					}
 				}
 
 				/*
diff --git a/src/gtm/client/gtm_client.c b/src/gtm/client/gtm_client.c
index d099ba6729..1a43d7aa0f 100644
--- a/src/gtm/client/gtm_client.c
+++ b/src/gtm/client/gtm_client.c
@@ -416,7 +416,7 @@ send_failed:
 
 int
 bkup_begin_transaction(GTM_Conn *conn, GTM_TransactionHandle txn, GTM_IsolationLevel isolevel,
-					   bool read_only, GTM_Timestamp timestamp)
+					   bool read_only, uint32 client_id, GTM_Timestamp timestamp)
 {
 	 /* Start the message. */
 	if (gtmpqPutMsgStart('C', true, conn) ||
@@ -424,6 +424,7 @@ bkup_begin_transaction(GTM_Conn *conn, GTM_TransactionHandle txn, GTM_IsolationL
 		gtmpqPutInt(txn, sizeof(GTM_TransactionHandle), conn) ||
 		gtmpqPutInt(isolevel, sizeof (GTM_IsolationLevel), conn) ||
 		gtmpqPutc(read_only, conn) ||
+		gtmpqPutInt(client_id, sizeof (uint32), conn) ||
 		gtmpqPutnchar((char *)&timestamp, sizeof(GTM_Timestamp), conn))
 		goto send_failed;
 
@@ -444,7 +445,8 @@ send_failed:
 
 int
 bkup_begin_transaction_gxid(GTM_Conn *conn, GTM_TransactionHandle txn, GlobalTransactionId gxid,
-							GTM_IsolationLevel isolevel, bool read_only, GTM_Timestamp timestamp)
+							GTM_IsolationLevel isolevel, bool read_only,
+							uint32 client_id, GTM_Timestamp timestamp)
 {
 	 /* Start the message. */
 	if (gtmpqPutMsgStart('C', true, conn) ||
@@ -453,6 +455,7 @@ bkup_begin_transaction_gxid(GTM_Conn *conn, GTM_TransactionHandle txn, GlobalTra
 		gtmpqPutInt(gxid, sizeof(GlobalTransactionId), conn) ||
 		gtmpqPutInt(isolevel, sizeof (GTM_IsolationLevel), conn) ||
 		gtmpqPutc(read_only, conn) ||
+		gtmpqPutInt(client_id, sizeof (uint32), conn) ||
 		gtmpqPutnchar((char *)&timestamp, sizeof(GTM_Timestamp), conn))
 		goto send_failed;
 
@@ -520,14 +523,15 @@ send_failed:
 
 int
 bkup_begin_transaction_autovacuum(GTM_Conn *conn, GTM_TransactionHandle txn, GlobalTransactionId gxid,
-								  GTM_IsolationLevel isolevel)
+								  GTM_IsolationLevel isolevel, uint32 client_id)
 {
 	 /* Start the message. */
 	if (gtmpqPutMsgStart('C', true, conn) ||
 		gtmpqPutInt(MSG_BKUP_TXN_BEGIN_GETGXID_AUTOVACUUM, sizeof (GTM_MessageType), conn) ||
 		gtmpqPutInt(txn, sizeof(GTM_TransactionHandle), conn) ||
 		gtmpqPutInt(gxid, sizeof(GlobalTransactionId), conn) ||
-		gtmpqPutInt(isolevel, sizeof (GTM_IsolationLevel), conn))
+		gtmpqPutInt(isolevel, sizeof (GTM_IsolationLevel), conn) ||
+		gtmpqPutInt(client_id, sizeof (uint32), conn))
 		goto send_failed;
 
 	/* Finish the message. */
@@ -1978,7 +1982,8 @@ send_failed:
 int
 bkup_begin_transaction_multi(GTM_Conn *conn, int txn_count,
 							 GTM_TransactionHandle *txn, GlobalTransactionId start_gxid, GTM_IsolationLevel *isolevel,
-							 bool *read_only, GTMProxy_ConnID *txn_connid)
+							 bool *read_only, uint32 *client_id,
+							 GTMProxy_ConnID *txn_connid)
 {
 	int ii;
 	GlobalTransactionId gxid = start_gxid;
@@ -1999,6 +2004,7 @@ bkup_begin_transaction_multi(GTM_Conn *conn, int txn_count,
 			gtmpqPutInt(gxid, sizeof(GlobalTransactionId), conn) ||
 			gtmpqPutInt(isolevel[ii], sizeof(GTM_IsolationLevel), conn) ||
 			gtmpqPutc(read_only[ii], conn) ||
+			gtmpqPutInt(client_id[ii], sizeof (uint32), conn) ||
 			gtmpqPutInt(txn_connid[ii], sizeof(GTMProxy_ConnID), conn))
 			goto send_failed;
 	}
diff --git a/src/gtm/common/gtm_serialize.c b/src/gtm/common/gtm_serialize.c
index a73a67b548..2a0cdb338e 100644
--- a/src/gtm/common/gtm_serialize.c
+++ b/src/gtm/common/gtm_serialize.c
@@ -174,7 +174,7 @@ gtm_get_transactioninfo_size(GTM_TransactionInfo *data)
 		return len;
 
 	len += sizeof(GTM_TransactionHandle);		/* gti_handle */
-	len += sizeof(GTM_ThreadID);			/* gti_thread_id */
+	len += sizeof(uint32);			/* gti_client_id */
 	len += sizeof(bool);				/* gti_in_use */
 	len += sizeof(GlobalTransactionId);		/* gti_gxid */
 	len += sizeof(GTM_TransactionStates);		/* gti_state */
@@ -224,9 +224,9 @@ gtm_serialize_transactioninfo(GTM_TransactionInfo *data, char *buf, size_t bufle
 	memcpy(buf + len, &(data->gti_handle), sizeof(GTM_TransactionHandle));
 	len += sizeof(GTM_TransactionHandle);
 
-	/* GTM_TransactionInfo.gti_thread_id */
-	memcpy(buf + len, &(data->gti_thread_id), sizeof(GTM_ThreadID));
-	len += sizeof(GTM_ThreadID);
+	/* GTM_TransactionInfo.gti_client_id */
+	memcpy(buf + len, &(data->gti_client_id), sizeof(uint32));
+	len += sizeof(uint32);
 
 	/* GTM_TransactionInfo.gti_in_use */
 	memcpy(buf + len, &(data->gti_in_use), sizeof(bool));
@@ -349,9 +349,9 @@ gtm_deserialize_transactioninfo(GTM_TransactionInfo *data, const char *buf, size
 	memcpy(&(data->gti_handle), buf + len, sizeof(GTM_TransactionHandle));
 	len += sizeof(GTM_TransactionHandle);
 
-	/* GTM_TransactionInfo.gti_thread_id */
-	memcpy(&(data->gti_thread_id), buf + len, sizeof(GTM_ThreadID));
-	len += sizeof(GTM_ThreadID);
+	/* GTM_TransactionInfo.gti_client_id */
+	memcpy(&(data->gti_client_id), buf + len, sizeof(uint32));
+	len += sizeof(uint32);
 
 	/* GTM_TransactionInfo.gti_in_use */
 	memcpy(&(data->gti_in_use), buf + len, sizeof(bool));
diff --git a/src/gtm/common/gtm_serialize_debug.c b/src/gtm/common/gtm_serialize_debug.c
index 10a8f05189..7832c4ee5a 100644
--- a/src/gtm/common/gtm_serialize_debug.c
+++ b/src/gtm/common/gtm_serialize_debug.c
@@ -36,7 +36,7 @@ dump_transactioninfo_elog(GTM_TransactionInfo *txn)
 
 	elog(LOG, "  ========= GTM_TransactionInfo =========");
 	elog(LOG, "gti_handle: %d", txn->gti_handle);
-	elog(LOG, "gti_thread_id: %ld", txn->gti_thread_id);
+	elog(LOG, "gti_client_id: %u", txn->gti_client_id);
 	elog(LOG, "gti_in_use: %d", txn->gti_in_use);
 	elog(LOG, "gti_gxid: %d", txn->gti_gxid);
 	elog(LOG, "gti_state: %d", txn->gti_state);
diff --git a/src/gtm/common/mcxt.c b/src/gtm/common/mcxt.c
index f032e1e55b..c8fdd19795 100644
--- a/src/gtm/common/mcxt.c
+++ b/src/gtm/common/mcxt.c
@@ -66,6 +66,8 @@ MemoryContext	TopMostMemoryContext;
 void
 MemoryContextInit(void)
 {
+	void *thrinfo;
+
 	AssertState(TopMemoryContext == NULL);
 
 	/*
@@ -90,6 +92,7 @@ MemoryContextInit(void)
 	 * Not having any other place to point CurrentMemoryContext, make it point
 	 * to TopMemoryContext.  Caller should change this soon!
 	 */
+	thrinfo = GetMyThreadInfo;
 	CurrentMemoryContext = TopMemoryContext;
 
 	/*
diff --git a/src/gtm/main/gtm_standby.c b/src/gtm/main/gtm_standby.c
index b4701a42e5..758701395f 100644
--- a/src/gtm/main/gtm_standby.c
+++ b/src/gtm/main/gtm_standby.c
@@ -139,16 +139,7 @@ gtm_standby_restore_gxid(void)
 
 		GTMTransactions.gt_transactions_array[handle].gti_handle = txn.gt_transactions_array[i].gti_handle;
 
-		/* 
-		 * Don't copy the gti_thread_id. Its the thread id of the thread
-		 * running on the GTM master and does make no sense on the standy.
-		 *
-		 * XXX How do we clean up these transaction info structures if the
-		 * connection goes away after standby is promoted? We need some
-		 * mechanism to set ownership of the in-progress transactions once a
-		 * standby is promoted
-		 */
-		GTMTransactions.gt_transactions_array[handle].gti_thread_id = -1;
+		GTMTransactions.gt_transactions_array[handle].gti_client_id = txn.gt_transactions_array[i].gti_client_id;
 		GTMTransactions.gt_transactions_array[handle].gti_in_use = txn.gt_transactions_array[i].gti_in_use;
 		GTMTransactions.gt_transactions_array[handle].gti_gxid = txn.gt_transactions_array[i].gti_gxid;
 		GTMTransactions.gt_transactions_array[handle].gti_state = txn.gt_transactions_array[i].gti_state;
diff --git a/src/gtm/main/gtm_thread.c b/src/gtm/main/gtm_thread.c
index 4612023ab8..3f96d74e20 100644
--- a/src/gtm/main/gtm_thread.c
+++ b/src/gtm/main/gtm_thread.c
@@ -97,6 +97,45 @@ GTM_ThreadAdd(GTM_ThreadInfo *thrinfo)
 			break;
 		}
 	}
+	
+	/*
+	 * Lastly, assign a unique, monotonically increasing identifier to the
+	 * remote client. This is sent back to the client and client will resend it
+	 * in case of reconnect
+	 *
+	 * Since all open transactions are tracked in a single linked list on the
+	 * GTM, we need a mechanism to identify transactions associated with a
+	 * specific client connection so that they can be removed if the client
+	 * disconnects abrubptly. We could have something like a pthread_id given
+	 * that there is one GTM thread per connection, but that is not sufficient
+	 * when GTM is failed over to a standby. The pthread_id on the old master
+	 * will make no sense on the new master and it will be hard to re-establish
+	 * the association of open transactions and the client connections (note
+	 * that all of this applies only when backends are connecting to GTM via a
+	 * GTM proxy. Otherwise those open transactions will be aborted when GTM
+	 * failover happens)
+	 *
+	 * So we use a unique identifier for each incoming connection to the GTM.
+	 * GTM assigns the identifier and also sends it back to the client as part
+	 * of the connection establishment process. In case of GTM failover, and
+	 * when GTM proxies reconnect to the new master, they also send back the
+	 * identifier issued to them by the previous master. The new GTM master
+	 * then uses that identifier to associate open transactions with the client
+	 * connection. Of course, for this to work, GTM must store client
+	 * identifier in each transaction info structure and also replicate that
+	 * information to the standby when new transactions are backed up.
+	 *
+	 * Since GTM does not backup the action of assinging new identifiers, at
+	 * failover, it may happen that the new master hasn't yet seen an
+	 * identifier which is already assigned my the old master (say because the
+	 * client has not started any transaction yet). To handle this case, we
+	 * track the latest identifier as seen my the new master upon failover. If
+	 * a client sends an identifier which is newer than that, that identifier
+	 * is discarded and new master will issue a new identifier that client will
+	 * accept.
+	 */
+	thrinfo->thr_client_id = GTMThreads->gt_next_client_id;
+	GTMThreads->gt_next_client_id = GTM_CLIENT_ID_NEXT(GTMThreads->gt_next_client_id);
 	GTM_RWLockRelease(&GTMThreads->gt_lock);
 
 	/*
@@ -220,6 +259,14 @@ GTM_ThreadCreate(GTM_ConnectionInfo *conninfo,
 				(err,
 				 errmsg("Failed to create a new thread: error %d", err)));
 
+	/*
+	 * Ensure that the resources are released when the thread exits. (We used
+	 * to do this inside GTM_ThreadMainWrapper, but thrinfo->thr_id may not set
+	 * by the time GTM_ThreadMainWrapper starts executing, this possibly
+	 * calling the function on an invalid thr_id
+	 */
+	pthread_detach(thrinfo->thr_id);
+
 	return thrinfo;
 }
 
@@ -353,8 +400,6 @@ GTM_ThreadMainWrapper(void *argp)
 {
 	GTM_ThreadInfo *thrinfo = (GTM_ThreadInfo *)argp;
 
-	pthread_detach(thrinfo->thr_id);
-
 	SetMyThreadInfo(thrinfo);
 	MemoryContextSwitchTo(TopMemoryContext);
 
@@ -403,3 +448,19 @@ GTM_DoForAllOtherThreads(void (* process_routine)(GTM_ThreadInfo *))
 			(process_routine)(GTMThreads->gt_threads[ii]);
 	}
 }
+
+/*
+ * Get the latest client identifier from the list of open transactions and set
+ * the next client identifier to be issued by us appropriately. Also remember
+ * the latest client identifier separately since it will be used to check any
+ * stale identifiers once we take over and old clients reconnect
+ */
+void
+GTM_SetInitialAndNextClientIdentifierAtPromote(void)
+{
+	GTM_RWLockAcquire(&GTMThreads->gt_lock, GTM_LOCKMODE_WRITE);
+	GTMThreads->gt_starting_client_id = GTMGetLastClientIdentifier();
+	GTMThreads->gt_next_client_id =
+		GTM_CLIENT_ID_NEXT(GTMThreads->gt_starting_client_id);
+	GTM_RWLockRelease(&GTMThreads->gt_lock);
+}
diff --git a/src/gtm/main/gtm_txn.c b/src/gtm/main/gtm_txn.c
index f55a3f16d3..e40df161bc 100644
--- a/src/gtm/main/gtm_txn.c
+++ b/src/gtm/main/gtm_txn.c
@@ -42,6 +42,7 @@ static void init_GTM_TransactionInfo(GTM_TransactionInfo *gtm_txninfo,
 									 char *coord_name,
 									 GTM_TransactionHandle txn,
 									 GTM_IsolationLevel isolevel,
+									 uint32 client_id,
 									 GTMProxy_ConnID connid,
 									 bool readonly);
 static void clean_GTM_TransactionInfo(GTM_TransactionInfo *gtm_txninfo);
@@ -273,8 +274,8 @@ GTM_RemoveTransInfoMulti(GTM_TransactionInfo *gtm_txninfo[], int txn_count)
 											   GTMTransactions.gt_latestCompletedXid))
 			GTMTransactions.gt_latestCompletedXid = gtm_txninfo[ii]->gti_gxid;
 
-		elog(DEBUG1, "GTM_RemoveTransInfoMulti: removing transaction id %u, %lu, handle (%d)",
-				gtm_txninfo[ii]->gti_gxid, gtm_txninfo[ii]->gti_thread_id,
+		elog(DEBUG1, "GTM_RemoveTransInfoMulti: removing transaction id %u, %u, handle (%d)",
+				gtm_txninfo[ii]->gti_gxid, gtm_txninfo[ii]->gti_client_id,
 				gtm_txninfo[ii]->gti_handle);
 
 		/*
@@ -294,12 +295,9 @@ GTM_RemoveTransInfoMulti(GTM_TransactionInfo *gtm_txninfo[], int txn_count)
  * Also compute the latestCompletedXid.
  */
 void
-GTM_RemoveAllTransInfos(int backend_id)
+GTM_RemoveAllTransInfos(uint32 client_id, int backend_id)
 {
 	gtm_ListCell *cell, *prev;
-	GTM_ThreadID thread_id;
-
-	thread_id = pthread_self();
 
 	/*
 	 * Scan the global list of open transactions
@@ -318,7 +316,7 @@ GTM_RemoveAllTransInfos(int backend_id)
 		if ((gtm_txninfo->gti_in_use) &&
 			(gtm_txninfo->gti_state != GTM_TXN_PREPARED) &&
 			(gtm_txninfo->gti_state != GTM_TXN_PREPARE_IN_PROGRESS) &&
-			(gtm_txninfo->gti_thread_id == thread_id) &&
+			(GTM_CLIENT_ID_EQ(gtm_txninfo->gti_client_id, client_id)) &&
 			((gtm_txninfo->gti_backend_id == backend_id) || (backend_id == -1)))
 		{
 			/* remove the entry */
@@ -330,9 +328,9 @@ GTM_RemoveAllTransInfos(int backend_id)
 												   GTMTransactions.gt_latestCompletedXid))
 				GTMTransactions.gt_latestCompletedXid = gtm_txninfo->gti_gxid;
 
-			elog(DEBUG1, "GTM_RemoveAllTransInfos: removing transaction id %u, %lu:%lu %d:%d",
-					gtm_txninfo->gti_gxid, gtm_txninfo->gti_thread_id,
-					thread_id, gtm_txninfo->gti_backend_id, backend_id);
+			elog(DEBUG1, "GTM_RemoveAllTransInfos: removing transaction id %u, %u:%u %d:%d",
+					gtm_txninfo->gti_gxid, gtm_txninfo->gti_client_id,
+					client_id, gtm_txninfo->gti_backend_id, backend_id);
 			/*
 			 * Now mark the transaction as aborted and mark the structure as not-in-use
 			 */
@@ -354,6 +352,67 @@ GTM_RemoveAllTransInfos(int backend_id)
 	GTM_RWLockRelease(&GTMTransactions.gt_TransArrayLock);
 	return;
 }
+
+/*
+ * Get the latest client identifier issued to the currently open transactions.
+ * Remember this may not be the latest identifier issued by the old master, but
+ * we won't acknowledge client identifiers larger than what we are about to
+ * compute. Any such identifiers will be overwritten the new identifier issued
+ * by the new master
+ */
+uint32
+GTMGetLastClientIdentifier(void)
+{
+	gtm_ListCell *cell, *prev;
+	uint32 last_client_id = 0;
+
+	/*
+	 * Scan the global list of open transactions
+	 */
+	GTM_RWLockAcquire(&GTMTransactions.gt_TransArrayLock, GTM_LOCKMODE_WRITE);
+
+	cell = gtm_list_head(GTMTransactions.gt_open_transactions);
+	while (cell != NULL)
+	{
+		GTM_TransactionInfo *gtm_txninfo = gtm_lfirst(cell);
+
+		if (GTM_CLIENT_ID_GT(gtm_txninfo->gti_client_id, last_client_id))
+			last_client_id = gtm_txninfo->gti_client_id;
+		cell = gtm_lnext(cell);
+	}
+
+	GTM_RWLockRelease(&GTMTransactions.gt_TransArrayLock);
+	return last_client_id;
+}
+
+/*
+ * Get the oldest client identifier issued to the currently open transactions.
+ */
+uint32
+GTMGetFirstClientIdentifier(void)
+{
+	gtm_ListCell *cell, *prev;
+	uint32 first_client_id = UINT32_MAX;
+
+	/*
+	 * Scan the global list of open transactions
+	 */
+	GTM_RWLockAcquire(&GTMTransactions.gt_TransArrayLock, GTM_LOCKMODE_WRITE);
+
+	cell = gtm_list_head(GTMTransactions.gt_open_transactions);
+	while (cell != NULL)
+	{
+		GTM_TransactionInfo *gtm_txninfo = gtm_lfirst(cell);
+
+		if (GTM_CLIENT_ID_LT(gtm_txninfo->gti_client_id, first_client_id))
+			first_client_id = gtm_txninfo->gti_client_id;
+		cell = gtm_lnext(cell);
+	}
+
+	GTM_RWLockRelease(&GTMTransactions.gt_TransArrayLock);
+	return first_client_id;
+}
+
 /*
  * GlobalTransactionIdDidCommit
  *		True iff transaction associated with the identifier did commit.
@@ -708,7 +767,8 @@ GTM_BeginTransactionMulti(char *coord_name,
 			}
 		}
 
-		init_GTM_TransactionInfo(gtm_txninfo[kk], coord_name, ii, isolevel[kk], connid[kk], readonly[kk]);
+		init_GTM_TransactionInfo(gtm_txninfo[kk], coord_name, ii, isolevel[kk],
+				GetMyThreadInfo->thr_client_id, connid[kk], readonly[kk]);
 
 		GTMTransactions.gt_lastslot = ii;
 
@@ -748,6 +808,7 @@ init_GTM_TransactionInfo(GTM_TransactionInfo *gtm_txninfo,
 						 char *coord_name,
 						 GTM_TransactionHandle txn,
 						 GTM_IsolationLevel isolevel,
+						 uint32 client_id,
 						 GTMProxy_ConnID connid,
 						 bool readonly)
 {
@@ -758,7 +819,6 @@ init_GTM_TransactionInfo(GTM_TransactionInfo *gtm_txninfo,
 
 	gtm_txninfo->gti_isolevel = isolevel;
 	gtm_txninfo->gti_readonly = readonly;
-	gtm_txninfo->gti_backend_id = connid;
 	gtm_txninfo->gti_in_use = true;
 
 	gtm_txninfo->nodestring = NULL;
@@ -766,7 +826,28 @@ init_GTM_TransactionInfo(GTM_TransactionInfo *gtm_txninfo,
 
 	gtm_txninfo->gti_handle = txn;
 	gtm_txninfo->gti_vacuum = false;
-	gtm_txninfo->gti_thread_id = pthread_self();
+
+	/*
+	 * For every new transaction that gets created, we track two important
+	 * identifiers:
+	 *
+	 * gt_client_id: is the identifier assigned to the client connected to
+	 * GTM. Whenever a connection to GTM is dropped, we must clean up all
+	 * transactions opened by that client. Since we track all open transactions
+	 * in a global data structure, this identifier helps us to identify
+	 * client-specific transactions. Also, the identifier is issued and tracked
+	 * irrespective of whether the remote client is a GTM proxy or a PG
+	 * backend.
+	 *
+	 * gti_backend_id: is the identifier assigned by the GTM proxy to its
+	 * client. Proxy sends us this identifier and we track it in the list of
+	 * open transactions. If a backend disconnects from the proxy, it sends us
+	 * a MSG_BACKEND_DISCONNECT message, along with the backend identifier. As
+	 * a response to that message, we clean up all the transactions opened by
+	 * the backend.
+	 */ 
+	gtm_txninfo->gti_client_id = client_id;
+	gtm_txninfo->gti_backend_id = connid;
 }
 
 
@@ -804,6 +885,7 @@ GTM_BkupBeginTransactionMulti(char *coord_name,
 							  GTM_TransactionHandle *txn,
 							  GTM_IsolationLevel *isolevel,
 							  bool *readonly,
+							  uint32 *client_id,
 							  GTMProxy_ConnID *connid,
 							  int	txn_count)
 {
@@ -829,7 +911,8 @@ GTM_BkupBeginTransactionMulti(char *coord_name,
 
 		elog(DEBUG1, "GTM_BkupBeginTransactionMulti: handle(%u)", txn[kk]);
 
-		init_GTM_TransactionInfo(gtm_txninfo, coord_name, txn[kk], isolevel[kk], connid[kk], readonly[kk]);
+		init_GTM_TransactionInfo(gtm_txninfo, coord_name, txn[kk],
+				isolevel[kk], client_id[kk], connid[kk], readonly[kk]);
 		GTMTransactions.gt_lastslot = txn[kk];
 		GTMTransactions.gt_open_transactions = gtm_lappend(GTMTransactions.gt_open_transactions, gtm_txninfo);
 	}
@@ -842,11 +925,13 @@ void
 GTM_BkupBeginTransaction(char *coord_name,
 						 GTM_TransactionHandle txn,
 						 GTM_IsolationLevel isolevel,
-						 bool readonly)
+						 bool readonly,
+						 uint32 client_id)
 {
 	GTMProxy_ConnID connid = -1;
 
-	GTM_BkupBeginTransactionMulti(coord_name, &txn, &isolevel, &readonly, &connid, 1);
+	GTM_BkupBeginTransactionMulti(coord_name, &txn, &isolevel, &readonly,
+			&client_id, &connid, 1);
 }
 /*
  * Same as GTM_RollbackTransaction, but takes GXID as input
@@ -1121,7 +1206,9 @@ ProcessBeginTransactionCommand(Port *myport, StringInfo message)
 	/* Backup first */
 	if (GetMyThreadInfo->thr_conn->standby)
 	{
-		bkup_begin_transaction(GetMyThreadInfo->thr_conn->standby, txn, txn_isolation_level, txn_read_only, timestamp);
+		bkup_begin_transaction(GetMyThreadInfo->thr_conn->standby, txn,
+				txn_isolation_level, txn_read_only,
+				GetMyThreadInfo->thr_client_id, timestamp);
 		/* Synch. with standby */
 		if (Backup_synchronously && (myport->remote_type != GTM_NODE_GTM_PROXY))
 			gtm_sync_standby(GetMyThreadInfo->thr_conn->standby);
@@ -1163,16 +1250,19 @@ ProcessBkupBeginTransactionCommand(Port *myport, StringInfo message)
 	bool txn_read_only;
 	GTM_Timestamp timestamp;
 	MemoryContext oldContext;
+	uint32 client_id;
 
 	txn = pq_getmsgint(message, sizeof(GTM_TransactionHandle));
 	txn_isolation_level = pq_getmsgint(message, sizeof(GTM_IsolationLevel));
 	txn_read_only = pq_getmsgbyte(message);
+	client_id = pq_getmsgint(message, sizeof (uint32));
 	memcpy(&timestamp, pq_getmsgbytes(message, sizeof(GTM_Timestamp)), sizeof(GTM_Timestamp));
 	pq_getmsgend(message);
 
 	oldContext = MemoryContextSwitchTo(TopMemoryContext);
 
-	GTM_BkupBeginTransaction("", txn, txn_isolation_level, txn_read_only);
+	GTM_BkupBeginTransaction("", txn, txn_isolation_level, txn_read_only,
+			client_id);
 
 	MemoryContextSwitchTo(oldContext);
 }
@@ -1230,7 +1320,10 @@ ProcessBeginTransactionGetGXIDCommand(Port *myport, StringInfo message)
 
 retry:
 		bkup_begin_transaction_gxid(GetMyThreadInfo->thr_conn->standby,
-									txn, gxid, txn_isolation_level, txn_read_only, timestamp);
+									txn, gxid, txn_isolation_level,
+									txn_read_only,
+									GetMyThreadInfo->thr_client_id,
+									timestamp);
 
 		if (gtm_standby_check_communication_error(&count, oldconn))
 			goto retry;
@@ -1271,6 +1364,7 @@ GTM_BkupBeginTransactionGetGXIDMulti(char *coord_name,
 									 GlobalTransactionId *gxid,
 									 GTM_IsolationLevel *isolevel,
 									 bool *readonly,
+									 uint32 *client_id,
 									 GTMProxy_ConnID *connid,
 									 int txn_count)
 {
@@ -1298,7 +1392,8 @@ GTM_BkupBeginTransactionGetGXIDMulti(char *coord_name,
 			MemoryContextSwitchTo(oldContext);
 			return;
 		}
-		init_GTM_TransactionInfo(gtm_txninfo, coord_name, txn[ii], isolevel[ii], connid[ii], readonly[ii]);
+		init_GTM_TransactionInfo(gtm_txninfo, coord_name, txn[ii],
+				isolevel[ii], client_id[ii], connid[ii], readonly[ii]);
 		GTMTransactions.gt_lastslot = txn[ii];
 		gtm_txninfo->gti_gxid = gxid[ii];
 
@@ -1345,11 +1440,13 @@ GTM_BkupBeginTransactionGetGXID(char *coord_name,
 								GTM_TransactionHandle txn,
 								GlobalTransactionId gxid,
 								GTM_IsolationLevel isolevel,
-								bool readonly)
+								bool readonly,
+								uint32 client_id)
 {
 	GTMProxy_ConnID connid = -1;
 
-	GTM_BkupBeginTransactionGetGXIDMulti(coord_name, &txn, &gxid, &isolevel, &readonly, &connid, 1);
+	GTM_BkupBeginTransactionGetGXIDMulti(coord_name, &txn, &gxid, &isolevel,
+			&readonly, &client_id, &connid, 1);
 }
 
 /*
@@ -1362,16 +1459,19 @@ ProcessBkupBeginTransactionGetGXIDCommand(Port *myport, StringInfo message)
 	GlobalTransactionId gxid;
 	GTM_IsolationLevel txn_isolation_level;
 	bool txn_read_only;
+	uint32 txn_client_id;
 	GTM_Timestamp timestamp;
 
 	txn = pq_getmsgint(message, sizeof(GTM_TransactionHandle));
 	gxid = pq_getmsgint(message, sizeof(GlobalTransactionId));
 	txn_isolation_level = pq_getmsgint(message, sizeof(GTM_IsolationLevel));
 	txn_read_only = pq_getmsgbyte(message);
+	txn_client_id = pq_getmsgint(message, sizeof (uint32));
 	memcpy(&timestamp, pq_getmsgbytes(message, sizeof(GTM_Timestamp)), sizeof(GTM_Timestamp));
 	pq_getmsgend(message);
 
-	GTM_BkupBeginTransactionGetGXID("", txn, gxid, txn_isolation_level, txn_read_only);
+	GTM_BkupBeginTransactionGetGXID("", txn, gxid, txn_isolation_level,
+			txn_read_only, txn_client_id);
 }
 
 /*
@@ -1383,13 +1483,16 @@ ProcessBkupBeginTransactionGetGXIDAutovacuumCommand(Port *myport, StringInfo mes
 	GTM_TransactionHandle txn;
 	GlobalTransactionId gxid;
 	GTM_IsolationLevel txn_isolation_level;
+	uint32 txn_client_id;
 
 	txn = pq_getmsgint(message, sizeof(GTM_TransactionHandle));
 	gxid = pq_getmsgint(message, sizeof(GlobalTransactionId));
 	txn_isolation_level = pq_getmsgint(message, sizeof(GTM_IsolationLevel));
+	txn_client_id = pq_getmsgint(message, sizeof (uint32));
 	pq_getmsgend(message);
 
-	GTM_BkupBeginTransactionGetGXID("", txn, gxid, txn_isolation_level, false);
+	GTM_BkupBeginTransactionGetGXID("", txn, gxid, txn_isolation_level,
+			false, txn_client_id);
 	GTM_SetDoVacuum(txn);
 }
 
@@ -1449,7 +1552,9 @@ ProcessBeginTransactionGetGXIDAutovacuumCommand(Port *myport, StringInfo message
 
 	retry:
 		_gxid = bkup_begin_transaction_autovacuum(GetMyThreadInfo->thr_conn->standby,
-												  txn, gxid, txn_isolation_level);
+												  txn, gxid,
+												  txn_isolation_level,
+												  GetMyThreadInfo->thr_client_id);
 
 		if (gtm_standby_check_communication_error(&count, oldconn))
 			goto retry;
@@ -1497,6 +1602,7 @@ ProcessBeginTransactionGetGXIDCommandMulti(Port *myport, StringInfo message)
 	GlobalTransactionId start_gxid, end_gxid;
 	GTM_Timestamp timestamp;
 	GTMProxy_ConnID txn_connid[GTM_MAX_GLOBAL_TRANSACTIONS];
+	uint32 txn_client_id[GTM_MAX_GLOBAL_TRANSACTIONS];
 	MemoryContext oldContext;
 	int count;
 	int ii;
@@ -1511,6 +1617,7 @@ ProcessBeginTransactionGetGXIDCommandMulti(Port *myport, StringInfo message)
 		txn_isolation_level[ii] = pq_getmsgint(message, sizeof (GTM_IsolationLevel));
 		txn_read_only[ii] = pq_getmsgbyte(message);
 		txn_connid[ii] = pq_getmsgint(message, sizeof (GTMProxy_ConnID));
+		txn_client_id[ii] = GetMyThreadInfo->thr_client_id;
 	}
 
 	oldContext = MemoryContextSwitchTo(TopMemoryContext);
@@ -1561,6 +1668,7 @@ retry:
 										   start_gxid,
 										   txn_isolation_level,
 										   txn_read_only,
+										   txn_client_id,
 										   txn_connid);
 
 		if (gtm_standby_check_communication_error(&count, oldconn))
@@ -1609,6 +1717,7 @@ ProcessBkupBeginTransactionGetGXIDCommandMulti(Port *myport, StringInfo message)
 	GTM_IsolationLevel txn_isolation_level[GTM_MAX_GLOBAL_TRANSACTIONS];
 	bool txn_read_only[GTM_MAX_GLOBAL_TRANSACTIONS];
 	GTMProxy_ConnID txn_connid[GTM_MAX_GLOBAL_TRANSACTIONS];
+	uint32 txn_client_id[GTM_MAX_GLOBAL_TRANSACTIONS];
 	int ii;
 
 	txn_count = pq_getmsgint(message, sizeof(int));
@@ -1621,10 +1730,12 @@ ProcessBkupBeginTransactionGetGXIDCommandMulti(Port *myport, StringInfo message)
 		gxid[ii] = pq_getmsgint(message, sizeof(GlobalTransactionId));
 		txn_isolation_level[ii] = pq_getmsgint(message, sizeof(GTM_IsolationLevel));
 		txn_read_only[ii] = pq_getmsgbyte(message);
+		txn_client_id[ii] = pq_getmsgint(message, sizeof(uint32));
 		txn_connid[ii] = pq_getmsgint(message, sizeof(GTMProxy_ConnID));
 	}
 
-	GTM_BkupBeginTransactionGetGXIDMulti("", txn, gxid, txn_isolation_level, txn_read_only, txn_connid, txn_count);
+	GTM_BkupBeginTransactionGetGXIDMulti("", txn, gxid, txn_isolation_level,
+			txn_read_only, txn_client_id, txn_connid, txn_count);
 
 }
 /*
diff --git a/src/gtm/main/main.c b/src/gtm/main/main.c
index 9864e66a3d..dc780c4d63 100644
--- a/src/gtm/main/main.c
+++ b/src/gtm/main/main.c
@@ -143,6 +143,13 @@ MainThreadInit()
 	GTM_RWLockInit(&GTMThreads->gt_lock);
 
 	/*
+	 * Set the next client identifier to be issued after connection
+	 * establishment
+	 */
+	GTMThreads->gt_starting_client_id = 0;
+	GTMThreads->gt_next_client_id = 1;
+
+	/*
 	 * We are called even before memory context management is setup. We must
 	 * use malloc
 	 */
@@ -1088,15 +1095,32 @@ GTM_ThreadMain(void *argp)
 
 		thrinfo->thr_conn->con_port->remote_type = sp.sp_remotetype;
 		thrinfo->thr_conn->con_port->is_postmaster = sp.sp_ispostmaster;
+
+		/*
+		 * If the client has resent the identifier assigned to it previously
+		 * (by GTM master), use that identifier. 
+		 * 
+		 * We only accept identifiers which are lesser or equal to the last
+		 * identifier we had seen when we were promoted. All other identifiers
+		 * will be overwritten by what we have assigned
+		 */
+		if ((sp.sp_client_id != 0) &&
+			(sp.sp_client_id <= GTMThreads->gt_starting_client_id))
+		{
+			thrinfo->thr_client_id = sp.sp_client_id;
+		}
 	}
 
 	{
 		/*
 		 * Send a dummy authentication request message 'R' as the client
-		 * expects that in the current protocol
+		 * expects that in the current protocol. Also send the client
+		 * identifier issued by us (or sent by the client in the startup packet
+		 * if we concluded to use the same)
 		 */
 		StringInfoData buf;
 		pq_beginmessage(&buf, 'R');
+		pq_sendint(&buf, thrinfo->thr_client_id, 4);
 		pq_endmessage(thrinfo->thr_conn->con_port, &buf);
 		pq_flush(thrinfo->thr_conn->con_port);
 
@@ -1227,7 +1251,7 @@ GTM_ThreadMain(void *argp)
 				 */
 				elog(DEBUG1, "Removing all transaction infos - qtype:EOF");
 				if (!Recovery_IsStandby())
-					GTM_RemoveAllTransInfos(-1);
+					GTM_RemoveAllTransInfos(thrinfo->thr_client_id, -1);
 
 				/* Disconnect node if necessary */
 				Recovery_PGXCNodeDisconnect(thrinfo->thr_conn->con_port);
@@ -1255,9 +1279,9 @@ GTM_ThreadMain(void *argp)
 
 			default:
 				/*
-				 * Remove all transactions opened within the thread
+				 * Remove all transactions opened by the client
 				 */
-				GTM_RemoveAllTransInfos(-1);
+				GTM_RemoveAllTransInfos(thrinfo->thr_client_id, -1);
 
 				/* Disconnect node if necessary */
 				Recovery_PGXCNodeDisconnect(thrinfo->thr_conn->con_port);
@@ -1383,16 +1407,7 @@ ProcessCommand(Port *myport, StringInfo input_message)
 
 		case MSG_BACKEND_DISCONNECT:
 			elog(DEBUG1, "MSG_BACKEND_DISCONNECT received - removing all txn infos");
-			/*
-			 * !!TODO The original code used to remove all transaction info
-			 * structures with the given ph_conid stored in gti_backend_id. But
-			 * we are seeing several issues because of that during GTM
-			 * failover. So disable the code for now as we further investigate
-			 * the code
-			 */
-#ifdef NOT_USED
-			GTM_RemoveAllTransInfos(proxyhdr.ph_conid);
-#endif
+			GTM_RemoveAllTransInfos(GetMyThreadInfo->thr_client_id, proxyhdr.ph_conid);
 			/* Mark PGXC Node as disconnected if backend disconnected is postmaster */
 			ProcessPGXCNodeBackendDisconnect(myport, input_message);
 			break;
@@ -2156,6 +2171,11 @@ PromoteToActive(void)
 	elog(LOG, "Promote signal received. Becoming an active...");
 
 	/*
+	 * Set starting and next client idendifier before promotion is complete
+	 */
+	GTM_SetInitialAndNextClientIdentifierAtPromote();
+
+	/*
 	 * Do promoting things here.
 	 */
 	Recovery_StandbySetStandby(false);
diff --git a/src/gtm/proxy/proxy_main.c b/src/gtm/proxy/proxy_main.c
index 698a61b4b2..1f1c5ee9de 100644
--- a/src/gtm/proxy/proxy_main.c
+++ b/src/gtm/proxy/proxy_main.c
@@ -1179,7 +1179,9 @@ GTMProxy_ThreadMain(void *argp)
 		{
 			for (ii = 0; ii < thrinfo->thr_conn_count; ii++)
 			{
-				GTMProxy_ConnectionInfo *conninfo = thrinfo->thr_all_conns[ii];
+				int connIndx = thrinfo->thr_conn_map[ii];
+				GTMProxy_ConnectionInfo *conninfo = thrinfo->thr_all_conns[connIndx];
+				
 				/*
 				 * Now clean up disconnected connections
 				 */
@@ -1283,7 +1285,8 @@ GTMProxy_ThreadMain(void *argp)
 				 */
 				for (ii = 0; ii < thrinfo->thr_conn_count; ii++)
 				{
-					GTMProxy_ConnectionInfo *conninfo = thrinfo->thr_all_conns[ii];
+					int connIndx = thrinfo->thr_conn_map[ii];
+					GTMProxy_ConnectionInfo *conninfo = thrinfo->thr_all_conns[connIndx];
 
 					/*
 					 * Detect if the connection has been dropped to avoid
@@ -1384,7 +1387,8 @@ setjmp_again:
 		 */
 		for (ii = 0; ii < thrinfo->thr_conn_count; ii++)
 		{
-			GTMProxy_ConnectionInfo *conninfo = thrinfo->thr_all_conns[ii];
+			int connIndx = thrinfo->thr_conn_map[ii];
+			GTMProxy_ConnectionInfo *conninfo = thrinfo->thr_all_conns[connIndx];
 			thrinfo->thr_conn = conninfo;
 
 			if (thrinfo->thr_poll_fds[ii].revents & POLLHUP)
@@ -1399,7 +1403,7 @@ setjmp_again:
 				continue;
 			}
 
-			if ((thrinfo->thr_any_backup[ii]) ||
+			if ((thrinfo->thr_any_backup[connIndx]) ||
 				(thrinfo->thr_poll_fds[ii].revents & POLLIN))
 			{
 				/*
@@ -1517,7 +1521,8 @@ setjmp_again:
 		 */
 		for (ii = 0; ii < thrinfo->thr_conn_count; ii++)
 		{
-			GTMProxy_ConnectionInfo *conninfo = thrinfo->thr_all_conns[ii];
+			int connIndx = thrinfo->thr_conn_map[ii];
+			GTMProxy_ConnectionInfo *conninfo = thrinfo->thr_all_conns[connIndx];
 			if (conninfo->con_disconnected)
 			{
 				GTMProxy_ThreadRemoveConnection(thrinfo, conninfo);
@@ -2556,6 +2561,7 @@ GTMProxy_HandshakeConnection(GTMProxy_ConnectionInfo *conninfo)
 	 * expects that in the current protocol
 	 */
 	pq_beginmessage(&buf, 'R');
+	pq_sendint(&buf, 0, 4);
 	pq_endmessage(conninfo->con_port, &buf);
 	pq_flush(conninfo->con_port);
 
@@ -3335,6 +3341,7 @@ workerThreadReconnectToGTM(void)
 {
 	char gtm_connect_string[1024];
 	MemoryContext   oldContext;
+	uint32 saveMyClientId = 0;
 
 	/*
 	 * First of all, we should acquire reconnect control lock in READ mode
@@ -3347,10 +3354,21 @@ workerThreadReconnectToGTM(void)
 	/* Disconnect the current connection and re-connect to the new GTM */
 	oldContext = MemoryContextSwitchTo(TopMostMemoryContext);
 
+	/*
+	 * Before cleaning the old connection, remember the client identifier
+	 * issued to us by the old GTM master. We send that identifier back to the
+	 * new master so that it can re-establish our association with any
+	 * transactions currently open by us
+	 */
 	if (GetMyThreadInfo->thr_gtm_conn)
+	{
+		saveMyClientId = GetMyThreadInfo->thr_gtm_conn->my_id;
 		GTMPQfinish(GetMyThreadInfo->thr_gtm_conn);
-	sprintf(gtm_connect_string, "host=%s port=%d node_name=%s remote_type=%d",
-			GTMServerHost, GTMServerPortNumber, GTMProxyNodeName, GTM_NODE_GTM_PROXY);
+	}
+
+	sprintf(gtm_connect_string, "host=%s port=%d node_name=%s remote_type=%d client_id=%u",
+			GTMServerHost, GTMServerPortNumber, GTMProxyNodeName,
+			GTM_NODE_GTM_PROXY, saveMyClientId);
 	elog(DEBUG1, "Worker thread connecting to %s", gtm_connect_string);
 	GetMyThreadInfo->thr_gtm_conn = PQconnectGTM(gtm_connect_string);
 
diff --git a/src/gtm/proxy/proxy_thread.c b/src/gtm/proxy/proxy_thread.c
index 4247be2d69..81fffc4a9f 100644
--- a/src/gtm/proxy/proxy_thread.c
+++ b/src/gtm/proxy/proxy_thread.c
@@ -212,6 +212,14 @@ GTMProxy_ThreadCreate(void *(* startroutine)(void *), int idx)
 				(err,
 				 errmsg("Failed to create a new thread: error %d", err)));
 
+	/*
+	 * Prepare to reclaim resources used by the thread once it exits. (We used
+	 * to do this inside GTMProxy_ThreadMainWrapper, but its not clear if
+	 * thrinfo->thr_id will be set by the time the routine starts executing.
+	 * Its safer to do that here instead
+	 */
+	pthread_detach(thrinfo->thr_id);
+
 	return thrinfo;
 }
 
@@ -332,6 +340,7 @@ GTMProxy_ThreadInfo *
 GTMProxy_ThreadAddConnection(GTMProxy_ConnectionInfo *conninfo)
 {
 	GTMProxy_ThreadInfo *thrinfo = NULL;
+	GTMProxy_ConnID connIndx, ii;
 
 	/*
 	 * Get the next thread in the queue
@@ -368,14 +377,52 @@ GTMProxy_ThreadAddConnection(GTMProxy_ConnectionInfo *conninfo)
 		elog(ERROR, "Too many connections");
 	}
 
+	connIndx = -1;
+	for (ii = 0; ii < GTM_PROXY_MAX_CONNECTIONS; ii++)
+	{
+		if (thrinfo->thr_all_conns[ii] == NULL)
+		{
+			/*
+			 * Great, found a free slot to track the connection
+			 */
+			connIndx = ii;
+			break;
+		}
+	}
+
+	if (connIndx == -1)
+	{
+		GTM_MutexLockRelease(&thrinfo->thr_lock);
+		elog(ERROR, "Too many connections - could not find a free slot");
+	}
+
 	/*
 	 * Save the array slotid in the conninfo structure. We send this to the GTM
 	 * server as an identifier which the GTM server sends us back in the
 	 * response. We use that information to route the response back to the
-	 * approrpiate connection
+	 * approrpiate connection.
+	 *
+	 * Note that the reason to use the array slotid in the messages to/from GTM
+	 * is to ensure that the corresponding connection can be quickly found
+	 * while proxying responses back to the client.
 	 */
-	conninfo->con_id = thrinfo->thr_conn_count;
-	thrinfo->thr_all_conns[thrinfo->thr_conn_count] = conninfo;
+	conninfo->con_id = connIndx;
+	thrinfo->thr_all_conns[connIndx] = conninfo;
+
+	/*
+	 * We also maintain a map of currently used array slots in a separate data
+	 * structure. This allows us to quickly iterate through all open
+	 * connections servred by a thread. So while iterating through all open
+	 * connections, the correct mechanism would be something as follow:
+	 *
+	 * for (ii = 0; ii < thrinfo->thr_conn_count; ii++)
+	 * {
+	 * 		int connIndx = thrinfo->thr_conn_map[ii];
+	 * 	 	GTMProxy_ConnectionInfo *conninfo = * 	 	thrinfo->thr_all_conns[connIndx];	
+	 * 	 	.....
+	 * }
+	 */  
+	thrinfo->thr_conn_map[thrinfo->thr_conn_count] = connIndx;
 	thrinfo->thr_conn_count++;
 
 	/*
@@ -406,6 +453,7 @@ int
 GTMProxy_ThreadRemoveConnection(GTMProxy_ThreadInfo *thrinfo, GTMProxy_ConnectionInfo *conninfo)
 {
 	int ii;
+	int connIndx;
 
 	/*
 	 * Lock the threadninfo structure to safely remove the connection from the
@@ -413,13 +461,17 @@ GTMProxy_ThreadRemoveConnection(GTMProxy_ThreadInfo *thrinfo, GTMProxy_Connectio
 	 */
 	GTM_MutexLockAcquire(&thrinfo->thr_lock);
 
-	for (ii = 0; ii < thrinfo->thr_conn_count; ii++)
+	connIndx = -1;
+	for (ii = 0; ii < GTM_PROXY_MAX_CONNECTIONS; ii++)
 	{
 		if (thrinfo->thr_all_conns[ii] == conninfo)
+		{
+			connIndx = ii;
 			break;
+		}
 	}
 
-	if (ii >= thrinfo->thr_conn_count)
+	if (connIndx == -1)
 	{
 		GTM_MutexLockRelease(&thrinfo->thr_lock);
 		elog(ERROR, "No such connection");
@@ -428,9 +480,25 @@ GTMProxy_ThreadRemoveConnection(GTMProxy_ThreadInfo *thrinfo, GTMProxy_Connectio
 	/*
 	 * Reset command backup info
 	 */
-	thrinfo->thr_any_backup[ii] = FALSE;
-	thrinfo->thr_qtype[ii] = 0;
-	resetStringInfo(&(thrinfo->thr_inBufData[ii]));
+	thrinfo->thr_any_backup[connIndx] = FALSE;
+	thrinfo->thr_qtype[connIndx] = 0;
+	resetStringInfo(&(thrinfo->thr_inBufData[connIndx]));
+	thrinfo->thr_all_conns[connIndx] = NULL;
+
+	/*
+	 * Now also removed the entry from thr_conn_map
+	 */
+	for (ii = 0; ii < thrinfo->thr_conn_count; ii++)
+	{
+		if (thrinfo->thr_conn_map[ii] == connIndx)
+			break;
+	}
+
+	if (ii >= thrinfo->thr_conn_count)
+	{
+		GTM_MutexLockRelease(&thrinfo->thr_lock);
+		elog(FATAL, "Failed to find connection mapping to %d", connIndx);
+	}
 
 	/*
 	 * If this is the last entry in the array ? If not, then copy the last
@@ -439,18 +507,15 @@ GTMProxy_ThreadRemoveConnection(GTMProxy_ThreadInfo *thrinfo, GTMProxy_Connectio
 	if ((ii + 1) < thrinfo->thr_conn_count)
 	{
 		/* Copy the last entry in this slot */
-		thrinfo->thr_all_conns[ii] = thrinfo->thr_all_conns[thrinfo->thr_conn_count - 1];
+		thrinfo->thr_conn_map[ii] = thrinfo->thr_conn_map[thrinfo->thr_conn_count - 1];
 
 		/* Mark the last slot free */
-		thrinfo->thr_all_conns[thrinfo->thr_conn_count - 1] = NULL;
-
-		/* Adjust the con_id to reflect the current slot in the array */
-		thrinfo->thr_all_conns[ii]->con_id = ii;
+		thrinfo->thr_conn_map[thrinfo->thr_conn_count - 1] = -1;
 	}
 	else
 	{
 		/* This is the last entry in the array. Just mark it free */
-		thrinfo->thr_all_conns[ii] = NULL;
+		thrinfo->thr_conn_map[ii] = -1;
 	}
 
 	thrinfo->thr_conn_count--;
diff --git a/src/include/gtm/gtm.h b/src/include/gtm/gtm.h
index 8affafd358..30580abd3b 100644
--- a/src/include/gtm/gtm.h
+++ b/src/include/gtm/gtm.h
@@ -22,6 +22,7 @@
 #include <setjmp.h>
 
 #include "gtm/gtm_c.h"
+#include "gtm/gtm_common.h"
 #include "gtm/palloc.h"
 #include "gtm/gtm_lock.h"
 #include "gtm/gtm_conn.h"
@@ -47,28 +48,17 @@ struct GTM_ConnectionInfo;
 typedef struct GTM_ThreadInfo
 {
 	/*
-	 * Thread specific information such as connection(s) served by it
+	 * Initial few members get includes from gtm_common.h. This is to make sure
+	 * that the GTMProxy_ThreadInfo and GTM_ThreadInfo structure can be
+	 * typecasted to each other and these initial members can be safely
+	 * accessed. If you need a member which should be common to both
+	 * structures, consider adding them to GTM_COMMON_THREAD_INFO
 	 */
-	GTM_ThreadID		thr_id;
-	uint32				thr_localid;
-	bool				is_main_thread;
-	void * (* thr_startroutine)(void *);
-
-	MemoryContext	thr_thread_context;
-	MemoryContext	thr_message_context;
-	MemoryContext	thr_current_context;
-	MemoryContext	thr_error_context;
-	MemoryContext	thr_parent_context;
-
-	sigjmp_buf		*thr_sigjmp_buf;
-
-	ErrorData		thr_error_data[ERRORDATA_STACK_SIZE];
-	int				thr_error_stack_depth;
-	int				thr_error_recursion_depth;
-	int				thr_criticalsec_count;
+	GTM_COMMON_THREAD_INFO
 
 	GTM_ThreadStatus	thr_status;
 	GTM_ConnectionInfo	*thr_conn;
+	uint32				thr_client_id;		/* unique client identifier */
 
 	GTM_RWLock			thr_lock;
 	gtm_List				*thr_cached_txninfo;
@@ -80,6 +70,8 @@ typedef struct GTM_Threads
 	uint32				gt_array_size;
 	bool				gt_standby_ready;
 	GTM_ThreadInfo		**gt_threads;
+	uint32				gt_starting_client_id;
+	uint32				gt_next_client_id;
 	GTM_RWLock			gt_lock;
 } GTM_Threads;
 
@@ -93,6 +85,7 @@ void ConnFree(Port *port);
 void GTM_LockAllOtherThreads(void);
 void GTM_UnlockAllOtherThreads(void);
 void GTM_DoForAllOtherThreads(void (* process_routine)(GTM_ThreadInfo *));
+void GTM_SetInitialAndNextClientIdentifierAtPromote(void);
 
 GTM_ThreadInfo *GTM_ThreadCreate(GTM_ConnectionInfo *conninfo,
 				  void *(* startroutine)(void *));
@@ -143,4 +136,13 @@ extern GTM_ThreadID						TopMostThreadID;
 		    Assert(CritSectionCount > 0); \
 		    CritSectionCount--; \
 	} while(0)
+
+#define GTM_CLIENT_ID_EQ(a, b)		\
+	((a) == (b))
+#define GTM_CLIENT_ID_LT(a, b)		\
+	(((int32)((a) - (b)) < 0) ? true : false)
+#define GTM_CLIENT_ID_GT(a, b)		\
+	(!GTM_CLIENT_ID_LT(a, b) && !GTM_CLIENT_ID_EQ(a, b))
+#define GTM_CLIENT_ID_NEXT(a)	\
+	((((a) + 1) == UINT32_MAX) ? 1 : ((a) + 1))
 #endif
diff --git a/src/include/gtm/gtm_c.h b/src/include/gtm/gtm_c.h
index b43f0edaa0..ee19587ac4 100644
--- a/src/include/gtm/gtm_c.h
+++ b/src/include/gtm/gtm_c.h
@@ -124,6 +124,7 @@ typedef struct GTM_StartupPacket {
 	char					sp_node_name[SP_NODE_NAME];
 	GTM_PGXCNodeType		sp_remotetype;
 	bool					sp_ispostmaster;
+	uint32					sp_client_id;
 } GTM_StartupPacket;
 
 typedef enum GTM_PortLastCall
diff --git a/src/include/gtm/gtm_client.h b/src/include/gtm/gtm_client.h
index 9d7e500480..19535a239b 100644
--- a/src/include/gtm/gtm_client.h
+++ b/src/include/gtm/gtm_client.h
@@ -175,13 +175,16 @@ size_t get_sequence_list(GTM_Conn *, GTM_SeqInfo **);
  */
 GlobalTransactionId begin_transaction(GTM_Conn *conn, GTM_IsolationLevel isolevel, GTM_Timestamp *timestamp);
 int bkup_begin_transaction(GTM_Conn *conn, GTM_TransactionHandle txn, GTM_IsolationLevel isolevel,
-						   bool read_only, GTM_Timestamp timestamp);
+						   bool read_only, uint32 client_id,
+						   GTM_Timestamp timestamp);
 int bkup_begin_transaction_gxid(GTM_Conn *conn, GTM_TransactionHandle txn, GlobalTransactionId gxid,
-								GTM_IsolationLevel isolevel, bool read_only, GTM_Timestamp timestamp);
+								GTM_IsolationLevel isolevel, bool read_only,
+								uint32 client_id, GTM_Timestamp timestamp);
 
 GlobalTransactionId begin_transaction_autovacuum(GTM_Conn *conn, GTM_IsolationLevel isolevel);
 int bkup_begin_transaction_autovacuum(GTM_Conn *conn, GTM_TransactionHandle txn, GlobalTransactionId gxid,
-									  GTM_IsolationLevel isolevel);
+									  GTM_IsolationLevel isolevel,
+									  uint32 client_id);
 int commit_transaction(GTM_Conn *conn, GlobalTransactionId gxid);
 int bkup_commit_transaction(GTM_Conn *conn, GlobalTransactionId gxid);
 int commit_prepared_transaction(GTM_Conn *conn, GlobalTransactionId gxid, GlobalTransactionId prepared_gxid);
@@ -209,7 +212,9 @@ begin_transaction_multi(GTM_Conn *conn, int txn_count, GTM_IsolationLevel *txn_i
 int
 bkup_begin_transaction_multi(GTM_Conn *conn, int txn_count,
 							 GTM_TransactionHandle *txn, GlobalTransactionId start_gxid, GTM_IsolationLevel *isolevel,
-							 bool *read_only, GTMProxy_ConnID *txn_connid);
+							 bool *read_only,
+							 uint32 *client_id,
+							 GTMProxy_ConnID *txn_connid);
 int
 commit_transaction_multi(GTM_Conn *conn, int txn_count, GlobalTransactionId *gxid,
 						 int *txn_count_out, int *status_out);
diff --git a/src/include/gtm/gtm_common.h b/src/include/gtm/gtm_common.h
new file mode 100644
index 0000000000..21df3486c6
--- /dev/null
+++ b/src/include/gtm/gtm_common.h
@@ -0,0 +1,35 @@
+/*
+ *
+ * gtm_common.h
+ *
+ * Portions Copyright (c) 1996-2009, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ * Portions Copyright (c) 2010-2012 Postgres-XC Development Group
+ * Portions Copyright (c) 2014 Translattice Inc
+ *
+ * $PostgreSQL$
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef _GTM_COMMON_H
+#define _GTM_COMMON_H
+
+#define GTM_COMMON_THREAD_INFO \
+	GTM_ThreadID			thr_id; \
+	uint32					thr_localid; \
+	bool					is_main_thread; \
+	void * (* thr_startroutine)(void *); \
+	MemoryContext	thr_thread_context; \
+	MemoryContext	thr_message_context; \
+	MemoryContext	thr_current_context; \
+	MemoryContext	thr_error_context; \
+	MemoryContext	thr_parent_context; \
+	sigjmp_buf		*thr_sigjmp_buf; \
+	ErrorData		thr_error_data[ERRORDATA_STACK_SIZE]; \
+	int				thr_error_stack_depth; \
+	int				thr_error_recursion_depth; \
+	int				thr_criticalsec_count;
+
+
+#endif
diff --git a/src/include/gtm/gtm_proxy.h b/src/include/gtm/gtm_proxy.h
index a11f752963..e798d2c89a 100644
--- a/src/include/gtm/gtm_proxy.h
+++ b/src/include/gtm/gtm_proxy.h
@@ -18,6 +18,7 @@
 #include <poll.h>
 
 #include "gtm/gtm_c.h"
+#include "gtm/gtm_common.h"
 #include "gtm/palloc.h"
 #include "gtm/gtm_lock.h"
 #include "gtm/gtm_conn.h"
@@ -65,37 +66,18 @@ typedef struct GTMProxy_Connections
 typedef struct GTMProxy_ThreadInfo
 {
 	/*
-	 * Thread specific information such as connection(s) served by it
+	 * Initial few members get includes from gtm_common.h. This is to make sure
+	 * that the GTMProxy_ThreadInfo and GTM_ThreadInfo structure can be
+	 * typecasted to each other and these initial members can be safely
+	 * accessed. If you need a member which should be common to both
+	 * structures, consider adding them to GTM_COMMON_THREAD_INFO
 	 */
-	GTM_ThreadID			thr_id;
-	uint32					thr_localid;
-	bool					is_main_thread;
-	void * (* thr_startroutine)(void *);
-
-	MemoryContext	thr_thread_context;
-	MemoryContext	thr_message_context;
-	MemoryContext	thr_current_context;
-	MemoryContext	thr_error_context;
-	MemoryContext	thr_parent_context;
-
-	sigjmp_buf		*thr_sigjmp_buf;
-
-	ErrorData		thr_error_data[ERRORDATA_STACK_SIZE];
-	int				thr_error_stack_depth;
-	int				thr_error_recursion_depth;
-	int				thr_criticalsec_count;
+	GTM_COMMON_THREAD_INFO
 
 	GTMProxy_ThreadStatus	thr_status;
 	GTMProxy_ConnectionInfo	*thr_conn;		/* Current set of connections from clients */
 	uint32					thr_conn_count;	/* number of connections served by this thread */
 
-
-	/*
-	 * The structure member type/sequence upto this point must match the
-	 * GTM_ThreadInfo structure in gtm.h since they are shared in some common
-	 * library routines such as elog.c. Keeping them in sync helps us use the
-	 * same library for the proxy as well as the server.
-	 */
 	GTM_MutexLock			thr_lock;
 	GTM_CV					thr_cv;
 
@@ -109,6 +91,7 @@ typedef struct GTMProxy_ThreadInfo
 
 	/* connection array */
 	GTMProxy_ConnectionInfo	*thr_all_conns[GTM_PROXY_MAX_CONNECTIONS];
+	int						thr_conn_map[GTM_PROXY_MAX_CONNECTIONS];
 	struct pollfd			thr_poll_fds[GTM_PROXY_MAX_CONNECTIONS];
 
 	/* Command backup */
diff --git a/src/include/gtm/gtm_txn.h b/src/include/gtm/gtm_txn.h
index 57a97eb1c9..c304f665ce 100644
--- a/src/include/gtm/gtm_txn.h
+++ b/src/include/gtm/gtm_txn.h
@@ -107,7 +107,7 @@ typedef enum GTM_TransactionStates
 typedef struct GTM_TransactionInfo
 {
 	GTM_TransactionHandle	gti_handle;
-	GTM_ThreadID			gti_thread_id;
+	uint32					gti_client_id;
 
 	bool					gti_in_use;
 	GlobalTransactionId		gti_gxid;
@@ -212,7 +212,8 @@ uint32 GTM_GetAllPrepared(GlobalTransactionId gxids[], uint32 gxidcnt);
 GTM_TransactionStates GTM_GetStatus(GTM_TransactionHandle txn);
 GTM_TransactionStates GTM_GetStatusGXID(GlobalTransactionId gxid);
 int GTM_GetAllTransactions(GTM_TransactionInfo txninfo[], uint32 txncnt);
-void GTM_RemoveAllTransInfos(int backend_id);
+void GTM_RemoveAllTransInfos(uint32 client_id, int backend_id);
+uint32 GTMGetLastClientIdentifier(void);
 
 GTM_Snapshot GTM_GetSnapshotData(GTM_TransactionInfo *my_txninfo,
 								 GTM_Snapshot snapshot);
@@ -226,6 +227,7 @@ void GTM_BkupBeginTransactionMulti(char *coord_name,
 								   GTM_TransactionHandle *txn,
 								   GTM_IsolationLevel *isolevel,
 								   bool *readonly,
+								   uint32 *client_id,
 								   GTMProxy_ConnID *connid,
 								   int	txn_count);
 
@@ -253,7 +255,8 @@ void GTM_RestoreTxnInfo(FILE *ctlf, GlobalTransactionId next_gxid);
 void GTM_BkupBeginTransaction(char *coord_name,
 							  GTM_TransactionHandle txn,
 							  GTM_IsolationLevel isolevel,
-							  bool readonly);
+							  bool readonly,
+							  uint32 client_id);
 void ProcessBkupBeginTransactionGetGXIDCommand(Port *myport, StringInfo message);
 void ProcessBkupBeginTransactionGetGXIDCommandMulti(Port *myport, StringInfo message);
 
diff --git a/src/include/gtm/libpq-int.h b/src/include/gtm/libpq-int.h
index 692bf201dc..b11426ad99 100644
--- a/src/include/gtm/libpq-int.h
+++ b/src/include/gtm/libpq-int.h
@@ -46,6 +46,7 @@ struct gtm_conn
 	char		*gc_node_name;		/* PGXC Node Name */
 	int			remote_type;		/* is this a connection to/from a proxy ? */
 	int			is_postmaster;		/* is this connection to/from a postmaster instance */
+	uint32		my_id;				/* unique identifier issued to us by GTM */
 
 	/* Optional file to write trace info to */
 	FILE		*Pfdebug;