postgres-xc-committers Mailing List for Postgres-XC

Project "Postgres-XC".

The branch, master has been updated
       via  ba79eded1dfbfabc51b3de4931b638853d13a30d (commit)
      from  19a8fa536779653524a1feb862c18277efa317f4 (commit)


- Log -----------------------------------------------------------------
commit ba79eded1dfbfabc51b3de4931b638853d13a30d
Author: Michael P <mic...@us...>
Date:   Wed Sep 15 08:26:30 2010 +0900

    Implementation of 2PC from applications
    
    Support for PREPARE TRANSACTION 'tid',
    ROLLBACK PREPARED 'tid'
    and COMMIT PREPARED 'tid'.
    
    When a Transaction is prepared on a Coordinator,
    the list of involved Datanodes is saved in GTM and transaction is
    put in PREPARE state. The transaction ID 'tid' is also saved on GTM.
    
    COMMIT PREPARED or ROLLBACK PREPARED can be issued from a different Coordinator
    by using the same tid. The Coordinator receiving the Commit SQL gets a list of Datanodes
    from GTM, and commits the transaction on the right nodes.
    
    This patch adds a new interface on GTM to save also the list of Coordinators
    involved in a PREPARE transaction.
    Coordinator<->Coordinator connection protocol is not implemented yet,
    so for the moment Coordinator do not create a 2PC file at PREPARE.
    This feature will be added with the implementation of DDL synchronization among Coordinators.

diff --git a/src/backend/access/transam/gtm.c b/src/backend/access/transam/gtm.c
index c7f3547..08ed2c9 100644
--- a/src/backend/access/transam/gtm.c
+++ b/src/backend/access/transam/gtm.c
@@ -122,7 +122,8 @@ CommitTranGTM(GlobalTransactionId gxid)
 	CheckConnection();
 	ret = commit_transaction(conn, gxid);
 
-	/* If something went wrong (timeout), try and reset GTM connection. 
+	/*
+	 * If something went wrong (timeout), try and reset GTM connection. 
 	 * We will close the transaction locally anyway, and closing GTM will force
 	 * it to be closed on GTM.
 	 */
@@ -134,6 +135,34 @@ CommitTranGTM(GlobalTransactionId gxid)
 	return ret;
 }
 
+/*
+ * For a prepared transaction, commit the gxid used for PREPARE TRANSACTION
+ * and for COMMIT PREPARED.
+ */
+int
+CommitPreparedTranGTM(GlobalTransactionId gxid, GlobalTransactionId prepared_gxid)
+{
+	int ret = 0;
+
+	if (!GlobalTransactionIdIsValid(gxid) || !GlobalTransactionIdIsValid(prepared_gxid))
+		return ret;
+	CheckConnection();
+	ret = commit_prepared_transaction(conn, gxid, prepared_gxid);
+
+	/*
+	 * If something went wrong (timeout), try and reset GTM connection.
+	 * We will close the transaction locally anyway, and closing GTM will force
+	 * it to be closed on GTM.
+	 */
+
+	if (ret < 0)
+	{
+		CloseGTM();
+		InitGTM();
+	}
+	return ret;
+}
+
 int
 RollbackTranGTM(GlobalTransactionId gxid)
 {
@@ -144,7 +173,37 @@ RollbackTranGTM(GlobalTransactionId gxid)
 	CheckConnection();
 	ret = abort_transaction(conn, gxid);
 
-	/* If something went wrong (timeout), try and reset GTM connection. 
+	/*
+	 * If something went wrong (timeout), try and reset GTM connection. 
+	 * We will abort the transaction locally anyway, and closing GTM will force
+	 * it to end on GTM.
+	 */
+	if (ret < 0)
+	{
+		CloseGTM();
+		InitGTM();
+	}
+	return ret;
+}
+
+int
+BeingPreparedTranGTM(GlobalTransactionId gxid,
+					char *gid,
+					int datanodecnt,
+					PGXC_NodeId datanodes[],
+					int coordcnt,
+					PGXC_NodeId coordinators[])
+{
+	int ret = 0;
+
+	if (!GlobalTransactionIdIsValid(gxid))
+		return 0;
+	CheckConnection();
+
+	ret = being_prepared_transaction(conn, gxid, gid, datanodecnt, datanodes, coordcnt, coordinators);
+
+	/*
+	 * If something went wrong (timeout), try and reset GTM connection.
 	 * We will abort the transaction locally anyway, and closing GTM will force
 	 * it to end on GTM.
 	 */
@@ -153,6 +212,61 @@ RollbackTranGTM(GlobalTransactionId gxid)
 		CloseGTM();
 		InitGTM();
 	}
+
+	return ret;
+}
+
+int
+PrepareTranGTM(GlobalTransactionId gxid)
+{
+	int ret;
+
+	if (!GlobalTransactionIdIsValid(gxid))
+		return 0;
+	CheckConnection();
+	ret = prepare_transaction(conn, gxid);
+
+	/*
+	 * If something went wrong (timeout), try and reset GTM connection. 
+	 * We will close the transaction locally anyway, and closing GTM will force
+	 * it to be closed on GTM.
+	 */
+	if (ret < 0)
+	{
+		CloseGTM();
+		InitGTM();
+	}
+	return ret;
+}
+
+
+int
+GetGIDDataGTM(char *gid,
+			  GlobalTransactionId *gxid,
+			  GlobalTransactionId *prepared_gxid,
+			  int *datanodecnt,
+			  PGXC_NodeId **datanodes,
+			  int *coordcnt,
+			  PGXC_NodeId **coordinators)
+{
+	int ret = 0;
+
+	CheckConnection();
+	ret = get_gid_data(conn, GTM_ISOLATION_RC, gid, gxid,
+					   prepared_gxid, datanodecnt, datanodes,
+					   coordcnt, coordinators);
+
+	/*
+	 * If something went wrong (timeout), try and reset GTM connection.
+	 * We will abort the transaction locally anyway, and closing GTM will force
+	 * it to end on GTM.
+	 */
+	if (ret < 0)
+	{
+		CloseGTM();
+		InitGTM();
+	}
+
 	return ret;
 }
 
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index e8cf1bf..d881078 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -929,6 +929,11 @@ EndPrepare(GlobalTransaction gxact)
 	 * critical section, though, it doesn't matter since any failure causes
 	 * PANIC anyway.
 	 */
+#ifdef PGXC
+	/* Do not write 2PC state file on Coordinator side */
+	if (IS_PGXC_DATANODE)
+	{
+#endif
 	TwoPhaseFilePath(path, xid);
 
 	fd = BasicOpenFile(path,
@@ -1001,6 +1006,9 @@ EndPrepare(GlobalTransaction gxact)
 	 * We save the PREPARE record's location in the gxact for later use by
 	 * CheckPointTwoPhase.
 	 */
+#ifdef PGXC
+	}
+#endif
 	START_CRIT_SECTION();
 
 	MyProc->inCommit = true;
@@ -1011,6 +1019,12 @@ EndPrepare(GlobalTransaction gxact)
 
 	/* If we crash now, we have prepared: WAL replay will fix things */
 
+#ifdef PGXC
+	/* Just write 2PC state file on Datanodes */
+	if (IS_PGXC_DATANODE)
+	{
+#endif
+
 	/* write correct CRC and close file */
 	if ((write(fd, &statefile_crc, sizeof(pg_crc32))) != sizeof(pg_crc32))
 	{
@@ -1024,6 +1038,9 @@ EndPrepare(GlobalTransaction gxact)
 		ereport(ERROR,
 				(errcode_for_file_access(),
 				 errmsg("could not close two-phase state file: %m")));
+#ifdef PGXC
+	}
+#endif
 
 	/*
 	 * Mark the prepared transaction as valid.	As soon as xact.c marks MyProc
@@ -1875,3 +1892,16 @@ RecordTransactionAbortPrepared(TransactionId xid,
 
 	END_CRIT_SECTION();
 }
+
+#ifdef PGXC
+/*
+ * Remove a gxact on a Coordinator,
+ * this is used to be able to prepare a commit transaction on another coordinator than the one
+ * who prepared the transaction
+ */
+void
+RemoveGXactCoord(GlobalTransaction gxact)
+{
+    RemoveGXact(gxact);
+}
+#endif
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 8a946cc..458068c 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -2133,6 +2133,17 @@ PrepareTransaction(void)
 
 	PostPrepare_Locks(xid);
 
+#ifdef PGXC
+	/*
+	 * We want to be able to commit a prepared transaction from another coordinator,
+	 * so clean up the gxact in shared memory also.
+	 */
+	if (IS_PGXC_COORDINATOR)
+	{
+		RemoveGXactCoord(gxact);
+	}
+#endif
+
 	ResourceOwnerRelease(TopTransactionResourceOwner,
 						 RESOURCE_RELEASE_LOCKS,
 						 true, true);
diff --git a/src/backend/pgxc/pool/datanode.c b/src/backend/pgxc/pool/datanode.c
index 31b5bc0..2e8ec40 100644
--- a/src/backend/pgxc/pool/datanode.c
+++ b/src/backend/pgxc/pool/datanode.c
@@ -1105,6 +1105,25 @@ get_transaction_nodes(DataNodeHandle **connections)
 }
 
 /*
+ * Collect node numbers for the given Datanode connections
+ * and return it for prepared transactions
+ */
+PGXC_NodeId*
+collect_datanode_numbers(int conn_count, DataNodeHandle **connections)
+{
+	PGXC_NodeId *datanodes = NULL;
+	int i;
+	datanodes = (PGXC_NodeId *) palloc(conn_count * sizeof(PGXC_NodeId));
+
+	for (i = 0; i < conn_count; i++)
+	{
+		datanodes[i] = connections[i]->nodenum;
+	}
+
+	return datanodes;
+}
+
+/*
  * Return those node connections that appear to be active and
  * have data to consume on them.
  */
diff --git a/src/backend/pgxc/pool/execRemote.c b/src/backend/pgxc/pool/execRemote.c
index 05dbe2e..e7ef66e 100644
--- a/src/backend/pgxc/pool/execRemote.c
+++ b/src/backend/pgxc/pool/execRemote.c
@@ -52,6 +52,14 @@ static int	data_node_begin(int conn_count, DataNodeHandle ** connections,
 				GlobalTransactionId gxid);
 static int	data_node_commit(int conn_count, DataNodeHandle ** connections);
 static int	data_node_rollback(int conn_count, DataNodeHandle ** connections);
+static int	data_node_prepare(int conn_count, DataNodeHandle ** connections,
+				char *gid);
+static int	data_node_rollback_prepared(GlobalTransactionId gxid, GlobalTransactionId prepared_gxid,
+				int conn_count, DataNodeHandle ** connections,
+				char *gid);
+static int	data_node_commit_prepared(GlobalTransactionId gxid, GlobalTransactionId prepared_gxid,
+				int conn_count, DataNodeHandle ** connections,
+				char *gid);
 
 static void clear_write_node_list();
 
@@ -531,6 +539,7 @@ HandleCommandComplete(RemoteQueryState *combiner, char *msg_body, size_t len)
 		else
 			combiner->combine_type = COMBINE_TYPE_NONE;
 	}
+
 	combiner->command_complete_count++;
 }
 
@@ -793,6 +802,7 @@ validate_combiner(RemoteQueryState *combiner)
 	/* Check if state is defined */
 	if (combiner->request_type == REQUEST_TYPE_NOT_DEFINED)
 		return false;
+
 	/* Check all nodes completed */
 	if ((combiner->request_type == REQUEST_TYPE_COMMAND
 	        || combiner->request_type == REQUEST_TYPE_QUERY)
@@ -1205,6 +1215,389 @@ DataNodeBegin(void)
 
 
 /*
+ * Prepare transaction on Datanodes involved in current transaction.
+ * GXID associated to current transaction has to be committed on GTM.
+ */
+int
+DataNodePrepare(char *gid)
+{
+	int         res = 0;
+	int         tran_count;
+	DataNodeHandle *connections[NumDataNodes];
+
+	/* gather connections to prepare */
+	tran_count = get_transaction_nodes(connections);
+
+	/*
+	 * If we do not have open transactions we have nothing to prepare just
+	 * report success
+	 */
+	if (tran_count == 0)
+	{
+		elog(WARNING, "Nothing to PREPARE on Datanodes, gid is not used");
+		goto finish;
+	}
+
+	/* TODO: data_node_prepare */
+	res = data_node_prepare(tran_count, connections, gid);
+
+finish:
+	/*
+	 * The transaction is just prepared, but Datanodes have reset,
+	 * so we'll need a new gxid for commit prepared or rollback prepared
+	 * Application is responsible for delivering the correct gid.
+	 * Release the connections for the moment.
+	 */
+	if (!autocommit)
+		stat_transaction(tran_count);
+	if (!PersistentConnections)
+		release_handles(false);
+	autocommit = true;
+	clear_write_node_list();
+	return res;
+}
+
+
+/*
+ * Prepare transaction on dedicated nodes with gid received from application
+ */
+static int
+data_node_prepare(int conn_count, DataNodeHandle ** connections, char *gid)
+{
+	int		i;
+	int		result = 0;
+	struct timeval *timeout = NULL;
+	char   *buffer = (char *) palloc0(22 + strlen(gid) + 1);
+	RemoteQueryState *combiner = NULL;
+	GlobalTransactionId gxid = InvalidGlobalTransactionId;
+	PGXC_NodeId *datanodes = NULL;
+
+	gxid = GetCurrentGlobalTransactionId();
+
+	/*
+	 * Now that the transaction has been prepared on the nodes,
+	 * Initialize to make the business on GTM
+	 */
+	datanodes = collect_datanode_numbers(conn_count, connections);
+
+	/*
+	 * Send a Prepare in Progress message to GTM.
+	 * At the same time node list is saved on GTM.
+	 */
+	result = BeingPreparedTranGTM(gxid, gid, conn_count, datanodes, 0, NULL);
+
+	if (result < 0)
+		return EOF;
+
+	sprintf(buffer, "PREPARE TRANSACTION '%s'", gid);
+
+	/* Send PREPARE */
+	for (i = 0; i < conn_count; i++)
+		if (data_node_send_query(connections[i], buffer))
+			return EOF;
+
+	combiner = CreateResponseCombiner(conn_count, COMBINE_TYPE_NONE);
+
+	/* Receive responses */
+	if (data_node_receive_responses(conn_count, connections, timeout, combiner))
+		return EOF;
+
+	result = ValidateAndCloseCombiner(combiner) ? result : EOF;
+	if (result)
+		goto finish;
+
+	/*
+	 * Prepare the transaction on GTM after everything is done.
+	 * GXID associated with PREPARE state is considered as used on Nodes,
+	 * but is still present in Snapshot.
+	 * This GXID will be discarded from Snapshot when commit prepared is
+	 * issued from another node.
+	 */
+	result = PrepareTranGTM(gxid);
+
+finish:
+	/*
+	 * An error has happened on a Datanode or GTM,
+	 * It is necessary to rollback the transaction on already prepared nodes.
+	 * But not on nodes where the error occurred.
+	 */
+	if (result)
+	{
+		GlobalTransactionId rollback_xid = InvalidGlobalTransactionId;
+		buffer = (char *) repalloc(buffer, 20 + strlen(gid) + 1);
+
+		sprintf(buffer, "ROLLBACK PREPARED '%s'", gid);
+
+		rollback_xid = BeginTranGTM(NULL);
+		for (i = 0; i < conn_count; i++)
+		{
+			if (data_node_send_gxid(connections[i], rollback_xid))
+			{
+				add_error_message(connections[i], "Can not send request");
+				return EOF;
+			}
+			if (data_node_send_query(connections[i], buffer))
+			{
+				add_error_message(connections[i], "Can not send request");
+				return EOF;
+			}
+		}
+
+		if (!combiner)
+			combiner = CreateResponseCombiner(conn_count, COMBINE_TYPE_NONE);
+
+		if (data_node_receive_responses(conn_count, connections, timeout, combiner))
+			result = EOF;
+		result = ValidateAndCloseCombiner(combiner) ? result : EOF;
+
+		/*
+		 * Don't forget to rollback also on GTM
+		 * Both GXIDs used for PREPARE and COMMIT PREPARED are discarded from GTM snapshot here.
+		 */
+		CommitPreparedTranGTM(gxid, rollback_xid);
+
+		return EOF;
+	}
+
+	return result;
+}
+
+
+/*
+ * Commit prepared transaction on Datanodes where it has been prepared.
+ * Connection to backends has been cut when transaction has been prepared,
+ * So it is necessary to send the COMMIT PREPARE message to all the nodes.
+ * We are not sure if the transaction prepared has involved all the datanodes
+ * or not but send the message to all of them.
+ * This avoid to have any additional interaction with GTM when making a 2PC transaction.
+ */
+void
+DataNodeCommitPrepared(char *gid)
+{
+	int			res = 0;
+	int			res_gtm = 0;
+	DataNodeHandle **connections;
+	List	   *nodelist = NIL;
+	int			i, tran_count;
+	PGXC_NodeId *datanodes = NULL;
+	PGXC_NodeId *coordinators = NULL;
+	int coordcnt = 0;
+	int datanodecnt = 0;
+	GlobalTransactionId gxid, prepared_gxid;
+
+	res_gtm = GetGIDDataGTM(gid, &gxid, &prepared_gxid,
+				 &datanodecnt, &datanodes, &coordcnt, &coordinators);
+
+	tran_count = datanodecnt + coordcnt;
+	if (tran_count == 0 || res_gtm < 0)
+		goto finish;
+
+	autocommit = false;
+
+	/* Build the list of nodes based on data received from GTM */
+	for (i = 0; i < datanodecnt; i++)
+	{
+		nodelist = lappend_int(nodelist,datanodes[i]);
+	}
+
+	/* Get connections */
+	connections = get_handles(nodelist);
+
+	/* Commit here the prepared transaction to all Datanodes */
+	res = data_node_commit_prepared(gxid, prepared_gxid, datanodecnt, connections, gid);
+
+finish:
+	/* In autocommit mode statistics is collected in DataNodeExec */
+	if (!autocommit)
+		stat_transaction(tran_count);
+	if (!PersistentConnections)
+		release_handles(false);
+	autocommit = true;
+	clear_write_node_list();
+
+	/* Free node list taken from GTM */
+	if (datanodes)
+		free(datanodes);
+	if (coordinators)
+		free(coordinators);
+
+	if (res_gtm < 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_INTERNAL_ERROR),
+				 errmsg("Could not get GID data from GTM")));
+	if (res != 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_INTERNAL_ERROR),
+				 errmsg("Could not commit prepared transaction on data nodes")));
+}
+
+/*
+ * Commit a prepared transaction on all nodes
+ * Prepared transaction with this gid has reset the datanodes,
+ * so we need a new gxid.
+ * An error is returned to the application only if all the Datanodes
+ * and Coordinator do not know about the gxid proposed.
+ * This permits to avoid interactions with GTM.
+ */
+static int
+data_node_commit_prepared(GlobalTransactionId gxid, GlobalTransactionId prepared_gxid, int conn_count, DataNodeHandle ** connections, char *gid)
+{
+	int result = 0;
+	int i;
+	RemoteQueryState *combiner = NULL;
+	struct timeval *timeout = NULL;
+	char        *buffer = (char *) palloc0(18 + strlen(gid) + 1);
+
+	/* GXID has been piggybacked when gid data has been received from GTM */
+	sprintf(buffer, "COMMIT PREPARED '%s'", gid);
+
+	/* Send gxid and COMMIT PREPARED message to all the Datanodes */
+	for (i = 0; i < conn_count; i++)
+	{
+		if (data_node_send_gxid(connections[i], gxid))
+		{
+			add_error_message(connections[i], "Can not send request");
+			result = EOF;
+			goto finish;
+		}
+		if (data_node_send_query(connections[i], buffer))
+		{
+			add_error_message(connections[i], "Can not send request");
+			result = EOF;
+			goto finish;
+		}
+	}
+
+	combiner = CreateResponseCombiner(conn_count, COMBINE_TYPE_NONE);
+
+	/* Receive responses */
+	if (data_node_receive_responses(conn_count, connections, timeout, combiner))
+		result = EOF;
+
+	/* Validate and close combiner */
+	result = ValidateAndCloseCombiner(combiner) ? result : EOF;
+
+finish:
+	/* Both GXIDs used for PREPARE and COMMIT PREPARED are discarded from GTM snapshot here */
+	CommitPreparedTranGTM(gxid, prepared_gxid);
+
+	return result;
+}
+
+/*
+ * Rollback prepared transaction on Datanodes involved in the current transaction
+ */
+void
+DataNodeRollbackPrepared(char *gid)
+{
+	int			res = 0;
+	int			res_gtm = 0;
+	DataNodeHandle **connections;
+	List	   *nodelist = NIL;
+	int			i, tran_count;
+
+	PGXC_NodeId *datanodes = NULL;
+	PGXC_NodeId *coordinators = NULL;
+	int coordcnt = 0;
+	int datanodecnt = 0;
+	GlobalTransactionId gxid, prepared_gxid;
+
+	res_gtm = GetGIDDataGTM(gid, &gxid, &prepared_gxid,
+				  &datanodecnt, &datanodes, &coordcnt, &coordinators);
+
+	tran_count = datanodecnt + coordcnt;
+	if (tran_count == 0 || res_gtm < 0 )
+		goto finish;
+
+	autocommit = false;
+
+	/* Build the node list based on the result got from GTM */
+	for (i = 0; i < datanodecnt; i++)
+	{
+		nodelist = lappend_int(nodelist,datanodes[i]);
+	}
+
+	/* Get connections */
+	connections = get_handles(nodelist);
+
+	/* Here do the real rollback to Datanodes */
+	res = data_node_rollback_prepared(gxid, prepared_gxid, datanodecnt, connections, gid);
+
+finish:
+	/* In autocommit mode statistics is collected in DataNodeExec */
+	if (!autocommit)
+		stat_transaction(tran_count);
+	if (!PersistentConnections)
+		release_handles(true);
+	autocommit = true;
+	clear_write_node_list(true);
+	if (res_gtm < 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_INTERNAL_ERROR),
+				 errmsg("Could not get GID data from GTM")));
+	if (res != 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_INTERNAL_ERROR),
+				 errmsg("Could not rollback prepared transaction on Datanodes")));
+}
+
+
+/*
+ * Rollback prepared transaction
+ * We first get the prepared informations from GTM and then do the treatment
+ * At the end both prepared GXID and GXID are committed.
+ */
+static int
+data_node_rollback_prepared(GlobalTransactionId gxid, GlobalTransactionId prepared_gxid,
+							int conn_count, DataNodeHandle ** connections, char *gid)
+{
+	int result = 0;
+	int i;
+	RemoteQueryState *combiner = NULL;
+	struct timeval *timeout = NULL;
+	char		*buffer = (char *) palloc0(20 + strlen(gid) + 1);
+
+	/* Datanodes have reset after prepared state, so get a new gxid */
+	gxid = BeginTranGTM(NULL);
+
+	sprintf(buffer, "ROLLBACK PREPARED '%s'", gid);
+
+	/* Send gxid and COMMIT PREPARED message to all the Datanodes */
+	for (i = 0; i < conn_count; i++)
+	{
+		if (data_node_send_gxid(connections[i], gxid))
+		{
+			add_error_message(connections[i], "Can not send request");
+			result = EOF;
+			goto finish;
+		}
+
+		if (data_node_send_query(connections[i], buffer))
+		{
+			add_error_message(connections[i], "Can not send request");
+			result = EOF;
+			goto finish;
+		}
+	}
+
+	combiner = CreateResponseCombiner(conn_count, COMBINE_TYPE_NONE);
+
+	/* Receive responses */
+	if (data_node_receive_responses(conn_count, connections, timeout, combiner))
+		result = EOF;
+
+	/* Validate and close combiner */
+	result = ValidateAndCloseCombiner(combiner) ? result : EOF;
+
+finish:
+	/* Both GXIDs used for PREPARE and COMMIT PREPARED are discarded from GTM snapshot here */
+	CommitPreparedTranGTM(gxid, prepared_gxid);
+
+	return result;
+}
+
+
+/*
  * Commit current transaction on data nodes where it has been started
  */
 void
diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c
index 0c6208c..91456c4 100644
--- a/src/backend/tcop/utility.c
+++ b/src/backend/tcop/utility.c
@@ -337,6 +337,15 @@ ProcessUtility(Node *parsetree,
 						break;
 
 					case TRANS_STMT_PREPARE:
+#ifdef PGXC
+						/*
+						 * If 2PC if invoked from application, transaction is first prepared on Datanodes.
+						 * 2PC file is not written for Coordinators to keep the possiblity
+						 * of a COMMIT PREPARED on a separate Coordinator
+						 */
+						if (IS_PGXC_COORDINATOR)
+							DataNodePrepare(stmt->gid);
+#endif
 						if (!PrepareTransactionBlock(stmt->gid))
 						{
 							/* report unsuccessful commit in completionTag */
@@ -346,13 +355,46 @@ ProcessUtility(Node *parsetree,
 						break;
 
 					case TRANS_STMT_COMMIT_PREPARED:
+#ifdef PGXC
+						if (IS_PGXC_COORDINATOR)
+							DataNodeCommitPrepared(stmt->gid);
+#endif
 						PreventTransactionChain(isTopLevel, "COMMIT PREPARED");
+
+#ifdef PGXC
+						if (IS_PGXC_DATANODE)
+						{
+							/*
+							 * 2PC file of Coordinator is not flushed to disk when transaction is prepared
+							 * so just skip this part.
+							 */
+#endif
 						FinishPreparedTransaction(stmt->gid, true);
+#ifdef PGXC
+						}
+#endif
 						break;
 
 					case TRANS_STMT_ROLLBACK_PREPARED:
+#ifdef PGXC
+						if (IS_PGXC_COORDINATOR)
+							DataNodeRollbackPrepared(stmt->gid);
+#endif
+
 						PreventTransactionChain(isTopLevel, "ROLLBACK PREPARED");
+
+#ifdef PGXC
+						if (IS_PGXC_DATANODE)
+						{
+							/*
+							 * 2PC file of Coordinator is not flushed to disk when transaction is prepared
+							 * so just skip this part.
+							 */
+#endif
 						FinishPreparedTransaction(stmt->gid, false);
+#ifdef PGXC
+						}
+#endif
 						break;
 
 					case TRANS_STMT_ROLLBACK:
diff --git a/src/gtm/client/fe-protocol.c b/src/gtm/client/fe-protocol.c
index 0847b0d..ff73b8d 100644
--- a/src/gtm/client/fe-protocol.c
+++ b/src/gtm/client/fe-protocol.c
@@ -362,19 +362,18 @@ gtmpqParseSuccess(GTM_Conn *conn, GTM_Result *result)
 			break;
 		case TXN_BEGIN_GETGXID_AUTOVACUUM_RESULT:
 		case TXN_PREPARE_RESULT:
+		case TXN_BEING_PREPARED_RESULT:
 			if (gtmpqGetnchar((char *)&result->gr_resdata.grd_gxid,
 						   sizeof (GlobalTransactionId), conn))
 				result->gr_status = -1;
 			break;
 
 		case TXN_COMMIT_RESULT:
+		case TXN_COMMIT_PREPARED_RESULT:
 		case TXN_ROLLBACK_RESULT:
 			if (gtmpqGetnchar((char *)&result->gr_resdata.grd_gxid,
 						   sizeof (GlobalTransactionId), conn))
-			{
 				result->gr_status = -1;
-				break;
-			}
 			break;
 
 		case TXN_GET_GXID_RESULT:
@@ -531,6 +530,60 @@ gtmpqParseSuccess(GTM_Conn *conn, GTM_Result *result)
 		case TXN_GET_ALL_PREPARED_RESULT:
 			break;
 
+		case TXN_GET_GID_DATA_RESULT:
+			if (gtmpqGetnchar((char *)&result->gr_resdata.grd_txn_get_gid_data.gxid,
+							  sizeof (GlobalTransactionId), conn))
+			{
+				result->gr_status = -1;
+				break;
+			}
+			if (gtmpqGetnchar((char *)&result->gr_resdata.grd_txn_get_gid_data.prepared_gxid,
+							  sizeof (GlobalTransactionId), conn))
+			{
+				result->gr_status = -1;
+				break;
+			}
+			if (gtmpqGetInt(&result->gr_resdata.grd_txn_get_gid_data.datanodecnt,
+					sizeof (int32), conn))
+			{
+				result->gr_status = -1;
+				break;
+			}
+			if ((result->gr_resdata.grd_txn_get_gid_data.datanodes = (PGXC_NodeId *)
+					malloc(sizeof(PGXC_NodeId) * result->gr_resdata.grd_txn_get_gid_data.datanodecnt)) == NULL)
+			{
+				result->gr_status = -1;
+				break;
+			}
+			if (gtmpqGetnchar((char *)result->gr_resdata.grd_txn_get_gid_data.datanodes,
+					sizeof(PGXC_NodeId) * result->gr_resdata.grd_txn_get_gid_data.datanodecnt, conn))
+			{
+				result->gr_status = -1;
+				break;
+			}
+			if (gtmpqGetInt(&result->gr_resdata.grd_txn_get_gid_data.coordcnt,
+					sizeof (int32), conn))
+			{
+				result->gr_status = -1;
+				break;
+			}
+			if (result->gr_resdata.grd_txn_get_gid_data.coordcnt != 0)
+			{
+				if ((result->gr_resdata.grd_txn_get_gid_data.coordinators = (PGXC_NodeId *)
+					 malloc(sizeof(PGXC_NodeId) * result->gr_resdata.grd_txn_get_gid_data.coordcnt)) == NULL)
+				{
+					result->gr_status = -1;
+					break;
+				}
+				if (gtmpqGetnchar((char *)result->gr_resdata.grd_txn_get_gid_data.coordinators,
+								  sizeof(PGXC_NodeId) * result->gr_resdata.grd_txn_get_gid_data.coordcnt, conn))
+				{
+					result->gr_status = -1;
+					break;
+				}
+			}
+			break;
+
 		default:
 			printfGTMPQExpBuffer(&conn->errorMessage,
 							  "unexpected result type from server; result typr was \"%d\"\n",
diff --git a/src/gtm/client/gtm_client.c b/src/gtm/client/gtm_client.c
index 35f81ae..54b75fd 100644
--- a/src/gtm/client/gtm_client.c
+++ b/src/gtm/client/gtm_client.c
@@ -135,6 +135,7 @@ receive_failed:
 send_failed:
 	return InvalidGlobalTransactionId;
 }
+
 int
 commit_transaction(GTM_Conn *conn, GlobalTransactionId gxid)
 {
@@ -175,7 +176,48 @@ commit_transaction(GTM_Conn *conn, GlobalTransactionId gxid)
 receive_failed:
 send_failed:
 	return -1;
+}
+
+int
+commit_prepared_transaction(GTM_Conn *conn, GlobalTransactionId gxid, GlobalTransactionId prepared_gxid)
+{
+	GTM_Result *res = NULL;
+	time_t finish_time;
+
+	/* Start the message */
+	if (gtmpqPutMsgStart('C', true, conn) ||
+		gtmpqPutInt(MSG_TXN_COMMIT_PREPARED, sizeof (GTM_MessageType), conn) ||
+		gtmpqPutc(true, conn) ||
+		gtmpqPutnchar((char *)&gxid, sizeof (GlobalTransactionId), conn) ||
+		gtmpqPutc(true, conn) ||
+		gtmpqPutnchar((char *)&prepared_gxid, sizeof (GlobalTransactionId), conn))
+		goto send_failed;
+
+	/* Finish the message */
+	if (gtmpqPutMsgEnd(conn))
+		goto send_failed;
+
+	/* Flush to ensure backends gets it */
+	if (gtmpqFlush(conn))
+		goto send_failed;
+
+	finish_time = time(NULL) + CLIENT_GTM_TIMEOUT;
+	if (gtmpqWaitTimed(true, false, conn, finish_time) ||
+		gtmpqReadData(conn) < 0)
+		goto receive_failed;
+
+	if ((res = GTMPQgetResult(conn)) == NULL)
+		goto receive_failed;
+
+	if (res->gr_status == 0)
+	{
+		Assert(res->gr_type == TXN_COMMIT_PREPARED_RESULT);
+		Assert(res->gr_resdata.grd_gxid == gxid);
+	}
 
+send_failed:
+receive_failed:
+	return -1;
 }
 
 int 
@@ -222,19 +264,71 @@ send_failed:
 }
 
 int
-prepare_transaction(GTM_Conn *conn, GlobalTransactionId gxid,
-					int nodecnt, PGXC_NodeId nodes[])
+being_prepared_transaction(GTM_Conn *conn, GlobalTransactionId gxid, char *gid,
+					int datanodecnt, PGXC_NodeId datanodes[], int coordcnt,
+					PGXC_NodeId coordinators[])
 {
 	GTM_Result *res = NULL;
 	time_t finish_time;
 
 	 /* Start the message. */
 	if (gtmpqPutMsgStart('C', true, conn) ||
-		gtmpqPutInt(MSG_TXN_PREPARE, sizeof (GTM_MessageType), conn) ||
+		gtmpqPutInt(MSG_TXN_BEING_PREPARED, sizeof (GTM_MessageType), conn) ||
 		gtmpqPutc(true, conn) ||
 		gtmpqPutnchar((char *)&gxid, sizeof (GlobalTransactionId), conn) ||
-		gtmpqPutInt(nodecnt, sizeof (int), conn) ||
-		gtmpqPutnchar((char *)nodes, sizeof (PGXC_NodeId) * nodecnt, conn))
+		/* Send also GID for an explicit prepared transaction */
+		gtmpqPutInt(strlen(gid), sizeof (GTM_GIDLen), conn) ||
+		gtmpqPutnchar((char *) gid, strlen(gid), conn) ||
+		gtmpqPutInt(datanodecnt, sizeof (int), conn) ||
+		gtmpqPutnchar((char *)datanodes, sizeof (PGXC_NodeId) * datanodecnt, conn) ||
+		gtmpqPutInt(coordcnt, sizeof (int), conn))
+		goto send_failed;
+
+	/* Coordinator connections are not always involved in a transaction */
+	if (coordcnt != 0 && gtmpqPutnchar((char *)coordinators, sizeof (PGXC_NodeId) * coordcnt, conn))
+		goto send_failed;
+
+	/* Finish the message. */
+	if (gtmpqPutMsgEnd(conn))
+		goto send_failed;
+
+	/* Flush to ensure backend gets it. */
+	if (gtmpqFlush(conn))
+		goto send_failed;
+
+	finish_time = time(NULL) + CLIENT_GTM_TIMEOUT;
+	if (gtmpqWaitTimed(true, false, conn, finish_time) ||
+		gtmpqReadData(conn) < 0)
+		goto receive_failed;
+
+	if ((res = GTMPQgetResult(conn)) == NULL)
+		goto receive_failed;
+
+	if (res->gr_status == 0)
+	{
+		Assert(res->gr_type == TXN_BEING_PREPARED_RESULT);
+		Assert(res->gr_resdata.grd_gxid == gxid);
+	}
+
+	return res->gr_status;
+
+receive_failed:
+send_failed:
+	return -1;
+}
+
+
+int
+prepare_transaction(GTM_Conn *conn, GlobalTransactionId gxid)
+{
+	GTM_Result *res = NULL;
+	time_t finish_time;
+
+	 /* Start the message. */
+	if (gtmpqPutMsgStart('C', true, conn) ||
+		gtmpqPutInt(MSG_TXN_PREPARE, sizeof (GTM_MessageType), conn) ||
+		gtmpqPutc(true, conn) ||
+		gtmpqPutnchar((char *)&gxid, sizeof (GlobalTransactionId), conn))
 		goto send_failed;
 
 	/* Finish the message. */
@@ -266,6 +360,64 @@ send_failed:
 	return -1;
 }
 
+int
+get_gid_data(GTM_Conn *conn,
+			 GTM_IsolationLevel isolevel,
+			 char *gid,
+			 GlobalTransactionId *gxid,
+			 GlobalTransactionId *prepared_gxid,
+			 int *datanodecnt,
+			 PGXC_NodeId **datanodes,
+			 int *coordcnt,
+			 PGXC_NodeId **coordinators)
+{
+	bool txn_read_only = false;
+	GTM_Result *res = NULL;
+	time_t finish_time;
+
+	/* Start the message */
+	if (gtmpqPutMsgStart('C', true, conn) ||
+		gtmpqPutInt(MSG_TXN_GET_GID_DATA, sizeof (GTM_MessageType), conn) ||
+		gtmpqPutInt(isolevel, sizeof (GTM_IsolationLevel), conn) ||
+		gtmpqPutc(txn_read_only, conn) ||
+		/* Send also GID for an explicit prepared transaction */
+		gtmpqPutInt(strlen(gid), sizeof (GTM_GIDLen), conn) ||
+		gtmpqPutnchar((char *) gid, strlen(gid), conn))
+		goto send_failed;
+
+	/* Finish the message */
+	if (gtmpqPutMsgEnd(conn))
+		goto send_failed;
+
+	/* Flush to ensure backend gets it. */
+	if (gtmpqFlush(conn))
+		goto send_failed;
+
+	finish_time = time(NULL) + CLIENT_GTM_TIMEOUT;
+	if (gtmpqWaitTimed(true, false, conn, finish_time) ||
+		gtmpqReadData(conn) < 0)
+		goto receive_failed;
+
+	if ((res = GTMPQgetResult(conn)) == NULL)
+		goto receive_failed;
+
+	if (res->gr_status == 0)
+	{
+		*gxid = res->gr_resdata.grd_txn_get_gid_data.gxid;
+		*prepared_gxid = res->gr_resdata.grd_txn_get_gid_data.prepared_gxid;
+		*datanodes = res->gr_resdata.grd_txn_get_gid_data.datanodes;
+		*coordinators = res->gr_resdata.grd_txn_get_gid_data.coordinators;
+		*datanodecnt = res->gr_resdata.grd_txn_get_gid_data.datanodecnt;
+		*coordcnt = res->gr_resdata.grd_txn_get_gid_data.coordcnt;
+	}
+
+	return res->gr_status;
+
+receive_failed:
+send_failed:
+	return -1;
+}
+
 /*
  * Snapshot Management API
  */
diff --git a/src/gtm/main/gtm_txn.c b/src/gtm/main/gtm_txn.c
index 2205167..949c123 100644
--- a/src/gtm/main/gtm_txn.c
+++ b/src/gtm/main/gtm_txn.c
@@ -149,6 +149,35 @@ GTM_GXIDToHandle(GlobalTransactionId gxid)
 }
 
 /*
+ * Given the GID (for a prepared transaction), find the corresponding
+ * transaction handle.
+ */
+GTM_TransactionHandle
+GTM_GIDToHandle(char *gid)
+{
+	ListCell *elem = NULL;
+	GTM_TransactionInfo *gtm_txninfo = NULL;
+
+	GTM_RWLockAcquire(&GTMTransactions.gt_TransArrayLock, GTM_LOCKMODE_READ);
+
+	foreach(elem, GTMTransactions.gt_open_transactions)
+	{
+		gtm_txninfo = (GTM_TransactionInfo *)lfirst(elem);
+		if (gtm_txninfo->gti_gid && strcmp(gid,gtm_txninfo->gti_gid) == 0)
+			break;
+		gtm_txninfo = NULL;
+	}
+
+	GTM_RWLockRelease(&GTMTransactions.gt_TransArrayLock);
+
+	if (gtm_txninfo != NULL)
+		return gtm_txninfo->gti_handle;
+	else
+		return InvalidTransactionHandle;
+}
+
+
+/*
  * Given the transaction handle, find the corresponding transaction info
  * structure
  *
@@ -159,7 +188,7 @@ GTM_GXIDToHandle(GlobalTransactionId gxid)
 GTM_TransactionInfo *
 GTM_HandleToTransactionInfo(GTM_TransactionHandle handle)
 {
-   	GTM_TransactionInfo *gtm_txninfo = NULL;
+	GTM_TransactionInfo *gtm_txninfo = NULL;
 
 	if ((handle < 0) || (handle > GTM_MAX_GLOBAL_TRANSACTIONS))
 	{
@@ -180,6 +209,7 @@ GTM_HandleToTransactionInfo(GTM_TransactionHandle handle)
 	return gtm_txninfo;
 }
 
+
 /*
  * Remove the given transaction info structures from the global array. If the
  * calling thread does not have enough cached structures, we in fact keep the
@@ -220,9 +250,27 @@ GTM_RemoveTransInfoMulti(GTM_TransactionInfo *gtm_txninfo[], int txn_count)
 		 * Now mark the transaction as aborted and mark the structure as not-in-use
 		 */
 		gtm_txninfo[ii]->gti_state = GTM_TXN_ABORTED;
-		gtm_txninfo[ii]->gti_nodecount = 0;
+		gtm_txninfo[ii]->gti_datanodecount = 0;
+		gtm_txninfo[ii]->gti_coordcount = 0;
 		gtm_txninfo[ii]->gti_in_use = false;
 		gtm_txninfo[ii]->gti_snapshot_set = false;
+
+		/* Clean-up also structures that were used for prepared transactions */
+		if (gtm_txninfo[ii]->gti_gid)
+		{
+			pfree(gtm_txninfo[ii]->gti_gid);
+			gtm_txninfo[ii]->gti_gid = NULL;
+		}
+		if (gtm_txninfo[ii]->gti_coordinators)
+		{
+			pfree(gtm_txninfo[ii]->gti_coordinators);
+			gtm_txninfo[ii]->gti_coordinators = NULL;
+		}
+		if (gtm_txninfo[ii]->gti_datanodes)
+		{
+			pfree(gtm_txninfo[ii]->gti_datanodes);
+			gtm_txninfo[ii]->gti_datanodes = NULL;
+		}
 	}
 
 	GTM_RWLockRelease(&GTMTransactions.gt_TransArrayLock);
@@ -252,15 +300,21 @@ GTM_RemoveAllTransInfos(int backend_id)
 	while (cell != NULL)
 	{
 		GTM_TransactionInfo *gtm_txninfo = lfirst(cell);
-		/* check if current entry is associated with the thread */
+		/*
+		 * Check if current entry is associated with the thread
+		 * A transaction in prepared state has to be kept alive in the structure.
+		 * It will be committed by another thread than this one.
+		 */
 		if ((gtm_txninfo->gti_in_use) &&
+			(gtm_txninfo->gti_state != GTM_TXN_PREPARED) &&
+			(gtm_txninfo->gti_state != GTM_TXN_PREPARE_IN_PROGRESS) &&
 			(gtm_txninfo->gti_thread_id == thread_id) &&
 			((gtm_txninfo->gti_backend_id == backend_id) || (backend_id == -1)))
 		{
 			/* remove the entry */
 			GTMTransactions.gt_open_transactions = list_delete_cell(GTMTransactions.gt_open_transactions, cell, prev);
 
-			/* update the latestComletedXid */
+			/* update the latestCompletedXid */
 			if (GlobalTransactionIdIsNormal(gtm_txninfo->gti_gxid) &&
 				GlobalTransactionIdFollowsOrEquals(gtm_txninfo->gti_gxid,
 												   GTMTransactions.gt_latestCompletedXid))
@@ -272,10 +326,27 @@ GTM_RemoveAllTransInfos(int backend_id)
 			 * Now mark the transaction as aborted and mark the structure as not-in-use
 			 */
 			gtm_txninfo->gti_state = GTM_TXN_ABORTED;
-			gtm_txninfo->gti_nodecount = 0;
+			gtm_txninfo->gti_datanodecount = 0;
+			gtm_txninfo->gti_coordcount = 0;
 			gtm_txninfo->gti_in_use = false;
 			gtm_txninfo->gti_snapshot_set = false;
-			
+
+			if (gtm_txninfo->gti_gid)
+			{
+				pfree(gtm_txninfo->gti_gid);
+				gtm_txninfo->gti_gid = NULL;
+			}
+			if (gtm_txninfo->gti_coordinators)
+			{
+				pfree(gtm_txninfo->gti_coordinators);
+				gtm_txninfo->gti_coordinators = NULL;
+			}
+			if (gtm_txninfo->gti_datanodes)
+			{
+				pfree(gtm_txninfo->gti_datanodes);
+				gtm_txninfo->gti_datanodes = NULL;
+			}
+
 			/* move to next cell in the list */
 			if (prev)
 				cell = lnext(prev);
@@ -583,7 +654,7 @@ GTM_BeginTransactionMulti(GTM_CoordinatorId coord_id,
 	 * without removing the corresponding references from the global array
 	 */
 	oldContext = MemoryContextSwitchTo(TopMostMemoryContext);
-	
+
 	for (kk = 0; kk < txn_count; kk++)
 	{
 		int ii, jj, startslot;
@@ -627,10 +698,16 @@ GTM_BeginTransactionMulti(GTM_CoordinatorId coord_id,
 		gtm_txninfo[kk]->gti_backend_id = connid[kk];
 		gtm_txninfo[kk]->gti_in_use = true;
 
+		gtm_txninfo[kk]->gti_coordcount = 0;
+		gtm_txninfo[kk]->gti_datanodes = 0;
+		gtm_txninfo[kk]->gti_gid = NULL;
+		gtm_txninfo[kk]->gti_coordinators = NULL;
+		gtm_txninfo[kk]->gti_datanodes = NULL;
+
 		gtm_txninfo[kk]->gti_handle = ii;
 		gtm_txninfo[kk]->gti_vacuum = false;
 		gtm_txninfo[kk]->gti_thread_id = pthread_self();
-		GTMTransactions.gt_lastslot = ii;		
+		GTMTransactions.gt_lastslot = ii;
 
 		txns[kk] = ii;
 
@@ -761,6 +838,29 @@ GTM_CommitTransactionMulti(GTM_TransactionHandle txn[], int txn_count, int statu
 }
 
 /*
+ * Prepare a transaction
+ */
+int
+GTM_PrepareTransaction(GTM_TransactionHandle txn)
+{
+	GTM_TransactionInfo *gtm_txninfo = NULL;
+
+	gtm_txninfo = GTM_HandleToTransactionInfo(txn);
+
+	if (gtm_txninfo == NULL)
+		return STATUS_ERROR;
+
+	/*
+	 * Mark the transaction as prepared
+	 */
+	GTM_RWLockAcquire(&gtm_txninfo->gti_lock, GTM_LOCKMODE_WRITE);
+	gtm_txninfo->gti_state = GTM_TXN_PREPARED;
+	GTM_RWLockRelease(&gtm_txninfo->gti_lock);
+
+	return STATUS_OK;
+}
+
+/*
  * Commit a transaction
  */
 int
@@ -775,9 +875,12 @@ GTM_CommitTransaction(GTM_TransactionHandle txn)
  * Prepare a transaction
  */
 int
-GTM_PrepareTransaction(GTM_TransactionHandle txn,
-					   uint32 nodecnt,
-					   PGXC_NodeId nodes[])
+GTM_BeingPreparedTransaction(GTM_TransactionHandle txn,
+							 char *gid,
+							 uint32 datanodecnt,
+							 PGXC_NodeId datanodes[],
+							 uint32 coordcnt,
+							 PGXC_NodeId coordinators[])
 {
 	GTM_TransactionInfo *gtm_txninfo = GTM_HandleToTransactionInfo(txn);
 
@@ -785,15 +888,27 @@ GTM_PrepareTransaction(GTM_TransactionHandle txn,
 		return STATUS_ERROR;
 
 	/*
-	 * Mark the transaction as being aborted
+	 * Mark the transaction as being prepared
 	 */
 	GTM_RWLockAcquire(&gtm_txninfo->gti_lock, GTM_LOCKMODE_WRITE);
-	
+
 	gtm_txninfo->gti_state = GTM_TXN_PREPARE_IN_PROGRESS;
-	gtm_txninfo->gti_nodecount = nodecnt;
-	if (gtm_txninfo->gti_nodes == NULL)
-		gtm_txninfo->gti_nodes = (PGXC_NodeId *)MemoryContextAlloc(TopMostMemoryContext, sizeof (PGXC_NodeId) * GTM_MAX_2PC_NODES);
-	memcpy(gtm_txninfo->gti_nodes, nodes, sizeof (PGXC_NodeId) * nodecnt);
+	gtm_txninfo->gti_datanodecount = datanodecnt;
+	gtm_txninfo->gti_coordcount = coordcnt;
+
+	if (gtm_txninfo->gti_datanodes == NULL)
+		gtm_txninfo->gti_datanodes = (PGXC_NodeId *)MemoryContextAlloc(TopMostMemoryContext, sizeof (PGXC_NodeId) * GTM_MAX_2PC_NODES);
+	memcpy(gtm_txninfo->gti_datanodes, datanodes, sizeof (PGXC_NodeId) * datanodecnt);
+
+	/* It is possible that no coordinator is involved in a transaction */
+	if (coordcnt != 0 && gtm_txninfo->gti_coordinators == NULL)
+		gtm_txninfo->gti_coordinators = (PGXC_NodeId *)MemoryContextAlloc(TopMostMemoryContext, sizeof (PGXC_NodeId) * GTM_MAX_2PC_NODES);
+	if (coordcnt != 0)
+		memcpy(gtm_txninfo->gti_coordinators, coordinators, sizeof (PGXC_NodeId) * coordcnt);
+
+	if (gtm_txninfo->gti_gid == NULL)
+		gtm_txninfo->gti_gid = (char *)MemoryContextAlloc(TopMostMemoryContext, GTM_MAX_GID_LEN);
+	memcpy(gtm_txninfo->gti_gid, gid, strlen(gid));
 
 	GTM_RWLockRelease(&gtm_txninfo->gti_lock);
 
@@ -804,12 +919,53 @@ GTM_PrepareTransaction(GTM_TransactionHandle txn,
  * Same as GTM_PrepareTransaction but takes GXID as input
  */
 int
-GTM_PrepareTransactionGXID(GlobalTransactionId gxid,
-					   uint32 nodecnt,
-					   PGXC_NodeId nodes[])
+GTM_BeingPreparedTransactionGXID(GlobalTransactionId gxid,
+						   char *gid,
+						   uint32 datanodecnt,
+						   PGXC_NodeId datanodes[],
+						   uint32 coordcnt,
+						   PGXC_NodeId coordinators[])
 {
 	GTM_TransactionHandle txn = GTM_GXIDToHandle(gxid);
-	return GTM_PrepareTransaction(txn, nodecnt, nodes);
+	return GTM_BeingPreparedTransaction(txn, gid, datanodecnt, datanodes, coordcnt, coordinators);
+}
+
+int
+GTM_GetGIDData(GTM_TransactionHandle prepared_txn,
+			   GlobalTransactionId *prepared_gxid,
+			   int *datanodecnt,
+			   PGXC_NodeId **datanodes,
+			   int *coordcnt,
+			   PGXC_NodeId **coordinators)
+{
+	GTM_TransactionInfo *gtm_txninfo = NULL;
+	MemoryContext oldContext;
+
+	oldContext = MemoryContextSwitchTo(TopMostMemoryContext);
+
+	gtm_txninfo = GTM_HandleToTransactionInfo(prepared_txn);
+	if (gtm_txninfo == NULL)
+		return STATUS_ERROR;
+
+	/* then get the necessary Data */
+	*prepared_gxid = gtm_txninfo->gti_gxid;
+	*datanodecnt = gtm_txninfo->gti_datanodecount;
+	*coordcnt = gtm_txninfo->gti_coordcount;
+
+	*datanodes = (PGXC_NodeId *) palloc(sizeof (PGXC_NodeId) * gtm_txninfo->gti_datanodecount);
+	memcpy(*datanodes, gtm_txninfo->gti_datanodes,
+				sizeof (PGXC_NodeId) * gtm_txninfo->gti_datanodecount);
+
+	if (coordcnt != 0)
+	{
+		*coordinators = (PGXC_NodeId *) palloc(sizeof (PGXC_NodeId) * gtm_txninfo->gti_coordcount);
+		memcpy(*coordinators, gtm_txninfo->gti_coordinators,
+					sizeof (PGXC_NodeId) * gtm_txninfo->gti_coordcount);
+	}
+
+	MemoryContextSwitchTo(oldContext);
+
+	return STATUS_OK;
 }
 
 /*
@@ -1146,6 +1302,174 @@ ProcessCommitTransactionCommand(Port *myport, StringInfo message)
 }
 
 /*
+ * Process MSG_TXN_COMMIT_PREPARED_MSG
+ * Commit a prepared transaction
+ * Here the GXID used for PREPARE and COMMIT PREPARED are both committed
+ */
+void
+ProcessCommitPreparedTransactionCommand(Port *myport, StringInfo message)
+{
+	StringInfoData buf;
+	int	txn_count = 2; /* PREPARE and COMMIT PREPARED gxid's */
+	GTM_TransactionHandle txn[txn_count];
+	GlobalTransactionId gxid[txn_count];
+	MemoryContext oldContext;
+	int status[txn_count];
+	int isgxid[txn_count];
+	int ii, count;
+
+	for (ii = 0; ii < txn_count; ii++)
+	{
+		isgxid[ii] = pq_getmsgbyte(message);
+		if (isgxid[ii])
+		{
+			const char *data = pq_getmsgbytes(message, sizeof (gxid[ii]));
+			if (data == NULL)
+				ereport(ERROR,
+						(EPROTO,
+						 errmsg("Message does not contain valid GXID")));
+			memcpy(&gxid[ii], data, sizeof (gxid[ii]));
+			txn[ii] = GTM_GXIDToHandle(gxid[ii]);
+			elog(DEBUG1, "ProcessCommitTransactionCommandMulti: gxid(%u), handle(%u)", gxid[ii], txn[ii]);
+		}
+		else
+		{
+			const char *data = pq_getmsgbytes(message, sizeof (txn[ii]));
+			if (data == NULL)
+				ereport(ERROR,
+						(EPROTO,
+						 errmsg("Message does not contain valid Transaction Handle")));
+			memcpy(&txn[ii], data, sizeof (txn[ii]));
+			elog(DEBUG1, "ProcessCommitTransactionCommandMulti: handle(%u)", txn[ii]);
+		}
+	}
+
+	pq_getmsgend(message);
+
+	oldContext = MemoryContextSwitchTo(TopMemoryContext);
+
+	/*
+	 * Commit the prepared transaction.
+	 */
+	count = GTM_CommitTransactionMulti(txn, txn_count, status);
+
+	MemoryContextSwitchTo(oldContext);
+
+	pq_beginmessage(&buf, 'S');
+	pq_sendint(&buf, TXN_COMMIT_PREPARED_RESULT, 4);
+	if (myport->is_proxy)
+	{
+		GTM_ProxyMsgHeader proxyhdr;
+		proxyhdr.ph_conid = myport->conn_id;
+		pq_sendbytes(&buf, (char *)&proxyhdr, sizeof (GTM_ProxyMsgHeader));
+	}
+	pq_sendbytes(&buf, (char *)&gxid[0], sizeof(GlobalTransactionId));
+	pq_sendint(&buf, status[0], 4);
+	pq_endmessage(myport, &buf);
+
+	if (!myport->is_proxy)
+		pq_flush(myport);
+	return;
+}
+
+
+/*
+ * Process MSG_TXN_GET_GID_DATA
+ * This message is used after at the beginning of a COMMIT PREPARED
+ * or a ROLLBACK PREPARED.
+ * For a given GID the following info is returned:
+ * - a fresh GXID,
+ * - GXID of the transaction that made the prepare
+ * - datanode and coordinator node list involved in the prepare
+ */
+void
+ProcessGetGIDDataTransactionCommand(Port *myport, StringInfo message)
+{
+	StringInfoData buf;
+	char *gid;
+	int gidlen;
+	GTM_IsolationLevel txn_isolation_level;
+	bool txn_read_only;
+	MemoryContext oldContext;
+	GTM_TransactionHandle txn, prepared_txn;
+	/* Data to be sent back to client */
+	GlobalTransactionId gxid, prepared_gxid;
+	PGXC_NodeId *coordinators = NULL;
+	PGXC_NodeId *datanodes = NULL;
+	int datanodecnt,coordcnt;
+
+	/* take the isolation level and read_only instructions */
+	txn_isolation_level = pq_getmsgint(message, sizeof (GTM_IsolationLevel));
+	txn_read_only = pq_getmsgbyte(message);
+
+	/* receive GID */
+	gidlen = pq_getmsgint(message, sizeof (GTM_GIDLen));
+	gid = (char *)pq_getmsgbytes(message, gidlen);
+
+	pq_getmsgend(message);
+
+	prepared_txn = GTM_GIDToHandle(gid);
+	if (prepared_txn == InvalidTransactionHandle)
+		ereport(ERROR,
+				(EINVAL,
+				 errmsg("Failed to get GID Data for prepared transaction")));
+
+	oldContext = MemoryContextSwitchTo(TopMemoryContext);
+
+	/* First get the GXID for the new transaction */
+	txn = GTM_BeginTransaction(0, txn_isolation_level, txn_read_only);
+	if (txn == InvalidTransactionHandle)
+		ereport(ERROR,
+			(EINVAL,
+			 errmsg("Failed to start a new transaction")));
+
+	gxid = GTM_GetGlobalTransactionId(txn);
+	if (gxid == InvalidGlobalTransactionId)
+		ereport(ERROR,
+				(EINVAL,
+				 errmsg("Failed to get a new transaction id")));
+
+	/*
+	 * Make the internal process, get the prepared information from GID.
+	 */
+	if (GTM_GetGIDData(prepared_txn, &prepared_gxid, &datanodecnt, &datanodes, &coordcnt, &coordinators) != STATUS_OK)
+	{
+		ereport(ERROR,
+				(EINVAL,
+				 errmsg("Failed to get the information of prepared transaction")));
+	}
+
+	MemoryContextSwitchTo(oldContext);
+
+	/*
+	 * Send a SUCCESS message back to the client
+	 */
+	pq_beginmessage(&buf, 'S');
+	pq_sendint(&buf, TXN_GET_GID_DATA_RESULT, 4);
+	if (myport->is_proxy)
+	{
+		GTM_ProxyMsgHeader proxyhdr;
+		proxyhdr.ph_conid = myport->conn_id;
+		pq_sendbytes(&buf, (char *)&proxyhdr, sizeof (GTM_ProxyMsgHeader));
+	}
+	/* Send the two GXIDs */
+	pq_sendbytes(&buf, (char *)&gxid, sizeof(GlobalTransactionId));
+	pq_sendbytes(&buf, (char *)&prepared_gxid, sizeof(GlobalTransactionId));
+	/* Then send the data linked to nodes involved in prepare */
+	pq_sendint(&buf, datanodecnt, 4);
+	pq_sendbytes(&buf, (char *)datanodes, sizeof(PGXC_NodeId) * datanodecnt);
+	pq_sendint(&buf, coordcnt, 4);
+	if (coordcnt != 0)
+		pq_sendbytes(&buf, (char *)coordinators, sizeof(PGXC_NodeId) * coordcnt);
+
+	pq_endmessage(myport, &buf);
+
+	if (!myport->is_proxy)
+		pq_flush(myport);
+	return;
+}
+
+/*
  * Process MSG_TXN_ROLLBACK message
  */
 void
@@ -1352,18 +1676,21 @@ ProcessRollbackTransactionCommandMulti(Port *myport, StringInfo message)
 }
 
 /*
- * Process MSG_TXN_PREPARE message
+ * Process MSG_TXN_BEING_PREPARED message
  */
 void
-ProcessPrepareTransactionCommand(Port *myport, StringInfo message)
+ProcessBeingPreparedTransactionCommand(Port *myport, StringInfo message)
 {
 	StringInfoData buf;
 	GTM_TransactionHandle txn;
 	GlobalTransactionId gxid;
 	int isgxid = 0;
-	int nodecnt;
-	PGXC_NodeId *nodes;
+	int datanodecnt,coordcnt;
+	GTM_GIDLen gidlen;
+	PGXC_NodeId *coordinators = NULL;
+	PGXC_NodeId *datanodes = NULL;
 	MemoryContext oldContext;
+	char *gid;
 
 	isgxid = pq_getmsgbyte(message);
 
@@ -1387,26 +1714,104 @@ ProcessPrepareTransactionCommand(Port *myport, StringInfo message)
 		memcpy(&txn, data, sizeof (txn));
 	}
 
-	nodecnt = pq_getmsgint(message, sizeof (nodecnt));
-	nodes = (PGXC_NodeId *) palloc(sizeof (PGXC_NodeId) * nodecnt);
-	memcpy(nodes, pq_getmsgbytes(message, sizeof (PGXC_NodeId) * nodecnt),
-			sizeof (PGXC_NodeId) * nodecnt);
+	/* get GID */
+	gidlen = pq_getmsgint(message, sizeof (GTM_GIDLen));
+	gid = (char *)pq_getmsgbytes(message, gidlen);
 
+	/* Get Datanode Data */
+	datanodecnt = pq_getmsgint(message, 4);
+	datanodes = (PGXC_NodeId *) palloc(sizeof (PGXC_NodeId) * datanodecnt);
+	memcpy(datanodes, pq_getmsgbytes(message, sizeof (PGXC_NodeId) * datanodecnt),
+			sizeof (PGXC_NodeId) * datanodecnt);
+
+	/* Get Coordinator Data, can be possibly NULL */
+	coordcnt = pq_getmsgint(message, 4);
+	if (coordcnt != 0)
+	{
+		coordinators = (PGXC_NodeId *) palloc(sizeof (PGXC_NodeId) * coordcnt);
+		memcpy(coordinators, pq_getmsgbytes(message, sizeof (PGXC_NodeId) * coordcnt),
+			sizeof (PGXC_NodeId) * coordcnt);
+	}
 	pq_getmsgend(message);
 
-	oldContext = MemoryContextSwitchTo(TopMemoryContext);
+	oldContext = MemoryContextSwitchTo(TopMostMemoryContext);
 
 	/*
 	 * Prepare the transaction
 	 */
-	if (GTM_PrepareTransaction(txn, nodecnt, nodes) != STATUS_OK)
+	if (GTM_BeingPreparedTransaction(txn, gid, datanodecnt, datanodes, coordcnt, coordinators) != STATUS_OK)
 		ereport(ERROR,
 				(EINVAL,
-				 errmsg("Failed to commit the transaction")));
+				 errmsg("Failed to prepare the transaction")));
 
 	MemoryContextSwitchTo(oldContext);
 
-	pfree(nodes);
+	if (datanodes)
+		pfree(datanodes);
+	if (coordinators)
+		pfree(coordinators);
+
+	pq_beginmessage(&buf, 'S');
+	pq_sendint(&buf, TXN_BEING_PREPARED_RESULT, 4);
+	if (myport->is_proxy)
+	{
+		GTM_ProxyMsgHeader proxyhdr;
+		proxyhdr.ph_conid = myport->conn_id;
+		pq_sendbytes(&buf, (char *)&proxyhdr, sizeof (GTM_ProxyMsgHeader));
+	}
+	pq_sendbytes(&buf, (char *)&gxid, sizeof(GlobalTransactionId));
+	pq_endmessage(myport, &buf);
+
+	if (!myport->is_proxy)
+		pq_flush(myport);
+	return;
+}
+
+/*
+ * Process MSG_TXN_PREPARE message
+ */
+void
+ProcessPrepareTransactionCommand(Port *myport, StringInfo message)
+{
+	StringInfoData buf;
+	GTM_TransactionHandle txn;
+	GlobalTransactionId gxid;
+	int isgxid = 0;
+	MemoryContext oldContext;
+	int status = STATUS_OK;
+
+	isgxid = pq_getmsgbyte(message);
+
+	if (isgxid)
+	{
+		const char *data = pq_getmsgbytes(message, sizeof (gxid));
+		if (data == NULL)
+			ereport(ERROR,
+					(EPROTO,
+					 errmsg("Message does not contain valid GXID")));
+		memcpy(&gxid, data, sizeof (gxid));
+		txn = GTM_GXIDToHandle(gxid);
+	}
+	else
+	{
+		const char *data = pq_getmsgbytes(message, sizeof (txn));
+		if (data == NULL)
+			ereport(ERROR,
+					(EPROTO,
+					 errmsg("Message does not contain valid Transaction Handle")));
+		memcpy(&txn, data, sizeof (txn));
+	}
+
+	pq_getmsgend(message);
+
+	oldContext = MemoryContextSwitchTo(TopMostMemoryContext);
+
+	/*
+	 * Commit the transaction
+	 */
+	status = GTM_PrepareTransaction(txn);
+
+	MemoryContextSwitchTo(oldContext);
 
 	pq_beginmessage(&buf, 'S');
 	pq_sendint(&buf, TXN_PREPARE_RESULT, 4);
@@ -1424,6 +1829,7 @@ ProcessPrepareTransactionCommand(Port *myport, StringInfo message)
 	return;
 }
 
+
 /*
  * Process MSG_TXN_GET_GXID message
  */
diff --git a/src/gtm/main/main.c b/src/gtm/main/main.c
index 667967a..1a6e546 100644
--- a/src/gtm/main/main.c
+++ b/src/gtm/main/main.c
@@ -769,12 +769,15 @@ ProcessCommand(Port *myport, StringInfo input_message)
 		case MSG_TXN_BEGIN_GETGXID:
 		case MSG_TXN_BEGIN_GETGXID_AUTOVACUUM:
 		case MSG_TXN_PREPARE:
+		case MSG_TXN_BEING_PREPARED:
 		case MSG_TXN_COMMIT:
+		case MSG_TXN_COMMIT_PREPARED:
 		case MSG_TXN_ROLLBACK:
 		case MSG_TXN_GET_GXID:
 		case MSG_TXN_BEGIN_GETGXID_MULTI:
 		case MSG_TXN_COMMIT_MULTI:
 		case MSG_TXN_ROLLBACK_MULTI:
+		case MSG_TXN_GET_GID_DATA:
 			ProcessTransactionCommand(myport, mtype, input_message);
 			break;
 
@@ -795,7 +798,7 @@ ProcessCommand(Port *myport, StringInfo input_message)
 		case MSG_SEQUENCE_ALTER:
 			ProcessSequenceCommand(myport, mtype, input_message);
 			break;
-		
+
 		case MSG_TXN_GET_STATUS:
 		case MSG_TXN_GET_ALL_PREPARED:
 			ProcessQueryCommand(myport, mtype, input_message);
@@ -938,39 +941,47 @@ ProcessTransactionCommand(Port *myport, GTM_MessageType mtype, StringInfo messag
 
 	switch (mtype)
 	{
-		case MSG_TXN_BEGIN:	
+		case MSG_TXN_BEGIN:
 			ProcessBeginTransactionCommand(myport, message);
 			break;
 
-		case MSG_TXN_BEGIN_GETGXID:	
+		case MSG_TXN_BEGIN_GETGXID:
 			ProcessBeginTransactionGetGXIDCommand(myport, message);
 			break;
 
-		case MSG_TXN_BEGIN_GETGXID_AUTOVACUUM:	
+		case MSG_TXN_BEGIN_GETGXID_AUTOVACUUM:
 			ProcessBeginTransactionGetGXIDAutovacuumCommand(myport, message);
 			break;
 
-		case MSG_TXN_BEGIN_GETGXID_MULTI:	
+		case MSG_TXN_BEGIN_GETGXID_MULTI:
 			ProcessBeginTransactionGetGXIDCommandMulti(myport, message);
 			break;
 
-		case MSG_TXN_PREPARE:		
+		case MSG_TXN_BEING_PREPARED:
+			ProcessBeingPreparedTransactionCommand(myport, message);
+			break;
+
+		case MSG_TXN_PREPARE:
 			ProcessPrepareTransactionCommand(myport, message);
 			break;
 
-		case MSG_TXN_COMMIT:		
+		case MSG_TXN_COMMIT:
 			ProcessCommitTransactionCommand(myport, message);
 			break;
 
-		case MSG_TXN_ROLLBACK:		
+		case MSG_TXN_COMMIT_PREPARED:
+			ProcessCommitPreparedTransactionCommand(myport, message);
+			break;
+
+		case MSG_TXN_ROLLBACK:
 			ProcessRollbackTransactionCommand(myport, message);
 			break;
 
-		case MSG_TXN_COMMIT_MULTI:		
+		case MSG_TXN_COMMIT_MULTI:
 			ProcessCommitTransactionCommandMulti(myport, message);
 			break;
 
-		case MSG_TXN_ROLLBACK_MULTI:		
+		case MSG_TXN_ROLLBACK_MULTI:
 			ProcessRollbackTransactionCommandMulti(myport, message);
 			break;
 
@@ -978,6 +989,9 @@ ProcessTransactionCommand(Port *myport, GTM_MessageType mtype, StringInfo messag
 			ProcessGetGXIDTransactionCommand(myport, message);
 			break;
 
+		case MSG_TXN_GET_GID_DATA:
+			ProcessGetGIDDataTransactionCommand(myport, message);
+
 		default:
 			Assert(0);			/* Shouldn't come here.. keep compiler quite */
 	}
diff --git a/src/gtm/proxy/proxy_main.c b/src/gtm/proxy/proxy_main.c
index 66b1594..d9ca329 100644
--- a/src/gtm/proxy/proxy_main.c
+++ b/src/gtm/proxy/proxy_main.c
@@ -949,9 +949,12 @@ ProcessCommand(GTMProxy_ConnectionInfo *conninfo, GTM_Conn *gtm_conn,
 		case MSG_TXN_BEGIN_GETGXID:
 		case MSG_TXN_BEGIN_GETGXID_AUTOVACUUM:
 		case MSG_TXN_PREPARE:
+		case MSG_TXN_BEING_PREPARED:
 		case MSG_TXN_COMMIT:
+		case MSG_TXN_COMMIT_PREPARED:
 		case MSG_TXN_ROLLBACK:
 		case MSG_TXN_GET_GXID:
+		case MSG_TXN_GET_GID_DATA:
 			ProcessTransactionCommand(conninfo, gtm_conn, mtype, input_message);
 			break;
 
@@ -1115,7 +1118,11 @@ ProcessResponse(GTMProxy_ThreadInfo *thrinfo, GTMProxy_CommandInfo *cmdinfo,
 		case MSG_TXN_BEGIN:
 		case MSG_TXN_BEGIN_GETGXID_AUTOVACUUM:
 		case MSG_TXN_PREPARE:
+		case MSG_TXN_BEING_PREPARED:
+		/* There are not so many 2PC from application messages, so just proxy it. */
+		case MSG_TXN_COMMIT_PREPARED:
 		case MSG_TXN_GET_GXID:
+		case MSG_TXN_GET_GID_DATA:
 		case MSG_SNAPSHOT_GXID_GET:
 		case MSG_SEQUENCE_INIT:
 		case MSG_SEQUENCE_GET_CURRENT:
@@ -1165,8 +1172,6 @@ ProcessResponse(GTMProxy_ThreadInfo *thrinfo, GTMProxy_CommandInfo *cmdinfo,
 					 errmsg("invalid frontend message type %d",
 							cmdinfo->ci_mtype)));
 	}
-
-
 }
 
 /* ----------------
@@ -1302,7 +1307,10 @@ ProcessTransactionCommand(GTMProxy_ConnectionInfo *conninfo, GTM_Conn *gtm_conn,
 			break;
 
 		case MSG_TXN_BEGIN_GETGXID_AUTOVACUUM:
-		case MSG_TXN_PREPARE:		
+		case MSG_TXN_PREPARE:
+		case MSG_TXN_BEING_PREPARED:
+		case MSG_TXN_GET_GID_DATA:
+		case MSG_TXN_COMMIT_PREPARED:
 			GTMProxy_ProxyCommand(conninfo, gtm_conn, mtype, message);
 			break;
 
diff --git a/src/include/access/gtm.h b/src/include/access/gtm.h
index 4878d92..6740c86 100644
--- a/src/include/access/gtm.h
+++ b/src/include/access/gtm.h
@@ -24,6 +24,23 @@ extern GlobalTransactionId BeginTranGTM(GTM_Timestamp *timestamp);
 extern GlobalTransactionId BeginTranAutovacuumGTM(void);
 extern int CommitTranGTM(GlobalTransactionId gxid);
 extern int RollbackTranGTM(GlobalTransactionId gxid);
+extern int BeingPreparedTranGTM(GlobalTransactionId gxid,
+								char *gid,
+								int datanodecnt,
+								PGXC_NodeId datanodes[],
+								int coordcount,
+								PGXC_NodeId coordinators[]);
+extern int PrepareTranGTM(GlobalTransactionId gxid);
+extern int GetGIDDataGTM(char *gid,
+						 GlobalTransactionId *gxid,
+						 GlobalTransactionId *prepared_gxid,
+						 int *datanodecnt,
+						 PGXC_NodeId **datanodes,
+						 int *coordcnt,
+						 PGXC_NodeId **coordinators);
+extern int CommitPreparedTranGTM(GlobalTransactionId gxid,
+								 GlobalTransactionId prepared_gxid);
+
 extern GTM_Snapshot GetSnapshotGTM(GlobalTransactionId gxid, bool canbe_grouped);
 
 /* Sequence interface APIs with GTM */
diff --git a/src/include/access/twophase.h b/src/include/access/twophase.h
index a3a1492..485f7fa 100644
--- a/src/include/access/twophase.h
+++ b/src/include/access/twophase.h
@@ -19,6 +19,10 @@
 #include "storage/proc.h"
 #include "utils/timestamp.h"
 
+#ifdef PGXC
+#include "pgxc/pgxc.h"
+#endif
+
 /*
  * GlobalTransactionData is defined in twophase.c; other places have no
  * business knowing the internal definition.
@@ -38,6 +42,10 @@ extern GlobalTransaction MarkAsPreparing(TransactionId xid, const char *gid,
 				TimestampTz prepared_at,
 				Oid owner, Oid databaseid);
 
+#ifdef PGXC
+extern void RemoveGXactCoord(GlobalTransaction gxact);
+#endif
+
 extern void StartPrepare(GlobalTransaction gxact);
 extern void EndPrepare(GlobalTransaction gxact);
 
diff --git a/src/include/gtm/gtm_c.h b/src/include/gtm/gtm_c.h
index 0a4c941..da15df3 100644
--- a/src/include/gtm/gtm_c.h
+++ b/src/include/gtm/gtm_c.h
@@ -38,6 +38,7 @@ typedef uint32	GlobalTransactionId;		/* 32-bit global transaction ids */
 typedef uint32	PGXC_NodeId;
 typedef uint32	GTM_CoordinatorId;
 typedef int16	GTMProxy_ConnID;
+typedef uint32	GTM_GIDLen;
 
 #define InvalidGTMProxyConnID	-1
 
diff --git a/src/include/gtm/gtm_client.h b/src/include/gtm/gtm_client.h
index 9db6884..4fe4bcf 100644
--- a/src/include/gtm/gtm_client.h
+++ b/src/include/gtm/gtm_client.h
@@ -29,7 +29,9 @@ typedef union GTM_ResultData
 	} grd_gxid_tp;							/* TXN_BEGIN_GETGXID */
 
 	GlobalTransactionId		grd_gxid;		/* TXN_PREPARE
+											 * TXN_BEING_PREPARED
 											 * TXN_COMMIT
+											 * TXN_COMMIT_PREPARED
 											 * TXN_ROLLBACK
 											 */
 	
@@ -70,6 +72,16 @@ typedef union GTM_ResultData
 		int						status[GTM_MAX_GLOBAL_TRANSACTIONS];
 	} grd_txn_snap_multi;
 
+	struct
+	{
+		GlobalTransactionId		gxid;
+		GlobalTransactionId		prepared_gxid;
+		int						datanodecnt;
+		int						coordcnt;
+		PGXC_NodeId			   *datanodes;
+		PGXC_NodeId			   *coordinators;
+	} grd_txn_get_gid_data;					/* TXN_GET_GID_DATA_RESULT */
+
 	/*
 	 * TODO
 	 * 	TXN_GET_STATUS
@@ -111,9 +123,16 @@ void disconnect_gtm(GTM_Conn *conn);
 GlobalTransactionId begin_transaction(GTM_Conn *conn, GTM_IsolationLevel isolevel, GTM_Timestamp *timestamp);
 GlobalTransactionId begin_transaction_autovacuum(GTM_Conn *conn, GTM_IsolationLevel isolevel);
 int commit_transaction(GTM_Conn *conn, GlobalTransactionId gxid);
+int commit_prepared_transaction(GTM_Conn *conn, GlobalTransactionId gxid, GlobalTransactionId prepared_gxid);
 int abort_transaction(GTM_Conn *conn, GlobalTransactionId gxid);
-int prepare_transaction(GTM_Conn *conn, GlobalTransactionId gxid,
-						int nodecnt, PGXC_NodeId nodes[]);
+int being_prepared_transaction(GTM_Conn *conn, GlobalTransactionId gxid, char *gid,
+							   int datanodecnt, PGXC_NodeId datanodes[],
+							   int coordcnt, PGXC_NodeId coordinators[]);
+int prepare_transaction(GTM_Conn *conn, GlobalTransactionId gxid);
+int get_gid_data(GTM_Conn *conn, GTM_IsolationLevel isolevel, char *gid,
+				 GlobalTransactionId *gxid, GlobalTransactionId *prepared_gxid,
+				 int *datanodecnt, PGXC_NodeId **datanodes, int *coordcnt,
+				 PGXC_NodeId **coordinators);
 
 /*
  * Snapshot Management API
diff --git a/src/include/gtm/gtm_msg.h b/src/include/gtm/gtm_msg.h
index e76e762..e1730eb 100644
--- a/src/include/gtm/gtm_msg.h
+++ b/src/include/gtm/gtm_msg.h
@@ -22,11 +22,14 @@ typedef enum GTM_MessageType
 	MSG_TXN_BEGIN,			/* Start a new transaction */
 	MSG_TXN_BEGIN_GETGXID,	/* Start a new transaction and get GXID */
 	MSG_TXN_BEGIN_GETGXID_MULTI,	/* Start multiple new transactions and get GXIDs */
-	MSG_TXN_PREPARE,		/* Prepare a transation for commit */
+	MSG_TXN_BEING_PREPARED,		/* Begins to prepare a transation for commit */
 	MSG_TXN_COMMIT,			/* Commit a running or prepared transaction */
 	MSG_TXN_COMMIT_MULTI,	/* Commit multiple running or prepared transactions */
+	MSG_TXN_COMMIT_PREPARED,	/* Commit a prepared transaction */
+	MSG_TXN_PREPARE,		/* Finish preparing a transaction */
 	MSG_TXN_ROLLBACK,		/* Rollback a transaction */
 	MSG_TXN_ROLLBACK_MULTI,	/* Rollback multiple transactions */
+	MSG_TXN_GET_GID_DATA,	/* Get info associated with a GID, and get a GXID */
 	MSG_TXN_GET_GXID,		/* Get a GXID for a transaction */
 	MSG_SNAPSHOT_GET,		/* Get a global snapshot */
 	MSG_SNAPSHOT_GET_MULTI,	/* Get multiple global snapshots */
@@ -59,10 +62,13 @@ typedef enum GTM_ResultType
 	TXN_BEGIN_GETGXID_RESULT,
 	TXN_BEGIN_GETGXID_MULTI_RESULT,
 	TXN_PREPARE_RESULT,
+	TXN_BEING_PREPARED_RESULT,
+	TXN_COMMIT_PREPARED_RESULT,
 	TXN_COMMIT_RESULT,
 	TXN_COMMIT_MULTI_RESULT,
 	TXN_ROLLBACK_RESULT,
 	TXN_ROLLBACK_MULTI_RESULT,
+	TXN_GET_GID_DATA_RESULT,
 	TXN_GET_GXID_RESULT,
 	SNAPSHOT_GET_RESULT,
 	SNAPSHOT_GET_MULTI_RESULT,
diff --git a/src/include/gtm/gtm_txn.h b/src/include/gtm/gtm_txn.h
index 2d78946..5e3a02c 100644
--- a/src/include/gtm/gtm_txn.h
+++ b/src/include/gtm/gtm_txn.h
@@ -116,8 +116,11 @@ typedef struct GTM_TransactionInfo
 	GTM_IsolationLevel			gti_isolevel;
 	bool...
 
[truncated message content]