Add support for GTM to backup data at BARRIER command

This patch is a combination of series of patches that were checked in the
upstream XC code

17 files changed, 428 insertions, 13 deletions

diff --git a/doc-xc/src/sgml/config.sgmlin b/doc-xc/src/sgml/config.sgmlin
index 6a614f69ca..9ed17fd759 100644
--- a/doc-xc/src/sgml/config.sgmlin
+++ b/doc-xc/src/sgml/config.sgmlin
@@ -7199,6 +7199,22 @@ Postgres-XC Specific Parameters
       </listitem>
      </varlistentry>
 
+     <varlistentry id="gtm_backup_barrier" xreflabel="backup_barrier">
+      <term><varname>gtm_backup_barrier</varname> (<type>string</type>)</term>
+      <indexterm>
+       <primary><varname>gtm_backup_barrier</> configuration parameter</primary>
+      </indexterm>
+      <listitem>
+       <para>
+        Specify if GTM backs up the restart point for each barrier.
+        Default value is "OFF".
+        Superuser may set this variable by SET command, or this can be set in the file
+        <filename>postgresql.conf</filename> file.
+       </para>
+      </listitem>
+     </varlistentry>
+
+
      <varlistentry id="guc-enforce-two-phase-commit" xreflabel="enforce_two_phase_commit">
       <term><varname>enforce_two_phase_commit</varname> (<type>boolean</type>)</term>
       <indexterm>
diff --git a/src/backend/access/transam/gtm.c b/src/backend/access/transam/gtm.c
index bcfc931b17..3415228fa5 100644
--- a/src/backend/access/transam/gtm.c
+++ b/src/backend/access/transam/gtm.c
@@ -35,7 +35,7 @@ int GtmPort = 6666;
 #ifdef XCP
 bool IsXidFromGTM = false;
 #endif
-
+bool gtm_backup_barrier = false;
 extern bool FirstSnapshotSet;
 
 static GTM_Conn *conn;
@@ -666,3 +666,21 @@ UnregisterGTM(GTM_PGXCNodeType type)
 
 	return ret;
 }
+
+/*
+ * Report BARRIER
+ */
+int
+ReportBarrierGTM(char *barrier_id)
+{
+	if (!gtm_backup_barrier)
+		return EINVAL;
+
+	CheckConnection();
+
+	if (!conn)
+		return EOF;
+
+	return(report_barrier(conn, barrier_id));
+}
+
diff --git a/src/backend/pgxc/barrier/barrier.c b/src/backend/pgxc/barrier/barrier.c
index 2c26da41f7..0c6f1afcb5 100644
--- a/src/backend/pgxc/barrier/barrier.c
+++ b/src/backend/pgxc/barrier/barrier.c
@@ -483,6 +483,8 @@ RequestBarrier(const char *id, char *completionTag)
 	 * Step three. Inform Coordinators about a successfully completed barrier
 	 */
 	EndBarrier(prepared_handles, barrier_id);
+	/* Finally report the barrier to GTM to backup its restart point */
+	ReportBarrierGTM(barrier_id);
 
 	/* Free the handles */
 	pfree_pgxc_all_handles(prepared_handles);
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 59b38e2b0a..f082f2628c 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -871,6 +871,15 @@ static struct config_bool ConfigureNamesBool[] =
 		false,
 		NULL, NULL, NULL
 	},
+	{
+		{"gtm_backup_barrier", PGC_SUSET, QUERY_TUNING_METHOD,
+			gettext_noop("Enables coordinator to report barrier id to GTM for backup."),
+			NULL
+		},
+		&gtm_backup_barrier,
+		false,
+		NULL, NULL, NULL
+	},
 #endif
 #endif
 	{
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 9bb47f967a..88e23cb68a 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -604,6 +604,8 @@
 #pgxc_node_name = ''			# Coordinator or Datanode name
 					# (change requires restart)
 
+#gtm_backup_barrier = off		# Specify to backup gtm restart point for each barrier.
+
 ##------------------------------------------------------------------------------
 # OTHER PG-XC OPTIONS
 #------------------------------------------------------------------------------
diff --git a/src/gtm/client/fe-protocol.c b/src/gtm/client/fe-protocol.c
index 7ef689042e..59467df238 100644
--- a/src/gtm/client/fe-protocol.c
+++ b/src/gtm/client/fe-protocol.c
@@ -789,7 +789,8 @@ gtmpqParseSuccess(GTM_Conn *conn, GTM_Result *result)
 
 			break;
 		}
-
+		case BARRIER_RESULT:
+			break;
 		default:
 			printfGTMPQExpBuffer(&conn->errorMessage,
 							  "unexpected result type from server; result typr was \"%d\"\n",
@@ -863,6 +864,9 @@ gtmpqFreeResultData(GTM_Result *result, GTM_PGXCNodeType remote_type)
 		case TXN_GET_ALL_PREPARED_RESULT:
 			break;
 
+		case BARRIER_RESULT:
+			break;
+
 		case SNAPSHOT_GET_RESULT:
 		case SNAPSHOT_GXID_GET_RESULT:
 			/*
diff --git a/src/gtm/client/gtm_client.c b/src/gtm/client/gtm_client.c
index 1a43d7aa0f..d937a3e430 100644
--- a/src/gtm/client/gtm_client.c
+++ b/src/gtm/client/gtm_client.c
@@ -77,6 +77,7 @@ static int alter_sequence_internal(GTM_Conn *conn, GTM_SequenceKey key, GTM_Sequ
 static int node_register_worker(GTM_Conn *conn, GTM_PGXCNodeType type, const char *host, GTM_PGXCNodePort port,
 								char *node_name, char *datafolder, GTM_PGXCNodeStatus status, bool is_backup);
 static int node_unregister_worker(GTM_Conn *conn, GTM_PGXCNodeType type, const char * node_name, bool is_backup);
+static int report_barrier_internal(GTM_Conn *conn, char *barrier_id, bool is_backup);
 /*
  * Make an empty result if old one is null.
  */
@@ -2249,6 +2250,72 @@ send_failed:
 	return -1;
 }
 
+/*
+ * Barrier
+ */
+
+int
+report_barrier(GTM_Conn *conn, char *barrier_id)
+{
+	return(report_barrier_internal(conn, barrier_id, false));
+}
+
+int
+bkup_report_barrier(GTM_Conn *conn, char *barrier_id)
+{
+	return(report_barrier_internal(conn, barrier_id, true));
+
+}
+
+static int
+report_barrier_internal(GTM_Conn *conn, char *barrier_id, bool is_backup)
+{
+	GTM_Result *res = NULL;
+	time_t finish_time;
+	int barrier_id_len = strlen(barrier_id) + 1;
+	
+
+	/* Send the message */
+	if (gtmpqPutMsgStart('C', true, conn)) /* FIXME: not proxy header --> proxy shold handle this separately */
+		goto send_failed;
+	if (gtmpqPutInt(is_backup ? MSG_BKUP_BARRIER : MSG_BARRIER, sizeof(GTM_MessageType), conn) ||
+		gtmpqPutInt(barrier_id_len, sizeof(int), conn) ||
+		gtmpqPutnchar(barrier_id, barrier_id_len, conn))
+		goto send_failed;
+	/* Flush the message */
+	if (gtmpqPutMsgEnd(conn))
+		goto send_failed;
+	/* Flush to ensure backend gets it */
+	if (gtmpqFlush(conn))
+		goto send_failed;
+
+	/* Handle the response */
+	if (!is_backup)
+	{
+		finish_time = time(NULL) + CLIENT_GTM_TIMEOUT;
+		if (gtmpqWaitTimed(true, false, conn, finish_time) ||
+			gtmpqReadData(conn) < 0)
+			goto receive_failed;
+
+		if ((res = GTMPQgetResult(conn)) == NULL)
+			goto receive_failed;
+
+		return res->gr_status;
+	}
+	return GTM_RESULT_OK;
+
+receive_failed:
+send_failed:
+	conn->result = makeEmptyResultIfIsNull(conn->result);
+	conn->result->gr_status = GTM_RESULT_COMM_ERROR;
+	return -1;
+}
+
+
+/*
+ * Backup to Standby
+ */
+
 int
 set_begin_end_backup(GTM_Conn *conn, bool begin)
 {
diff --git a/src/gtm/main/Makefile b/src/gtm/main/Makefile
index d207e32dcc..3e76afa50f 100644
--- a/src/gtm/main/Makefile
+++ b/src/gtm/main/Makefile
@@ -15,7 +15,7 @@ ifneq ($(PORTNAME), win32)
 override CFLAGS += $(PTHREAD_CFLAGS)
 endif
 
-OBJS=main.o gtm_thread.o gtm_txn.o gtm_seq.o gtm_snap.o gtm_time.o gtm_standby.o gtm_opt.o
+OBJS=main.o gtm_thread.o gtm_txn.o gtm_seq.o gtm_snap.o gtm_time.o gtm_standby.o gtm_opt.o gtm_backup.o
 
 OTHERS= ../libpq/libpqcomm.a ../path/libgtmpath.a ../recovery/libgtmrecovery.a ../client/libgtmclient.a ../common/libgtm.a ../../port/libpgport.a 
 
diff --git a/src/gtm/main/gtm_seq.c b/src/gtm/main/gtm_seq.c
index a9f2dd1b0a..121f92c915 100644
--- a/src/gtm/main/gtm_seq.c
+++ b/src/gtm/main/gtm_seq.c
@@ -31,6 +31,7 @@
 #include "gtm/libpq.h"
 #include "gtm/libpq-int.h"
 #include "gtm/pqformat.h"
+#include "gtm/gtm_backup.h"
 
 extern bool Backup_synchronously;
 
@@ -52,6 +53,8 @@ static int seq_add_seqinfo(GTM_SeqInfo *seqinfo);
 static int seq_remove_seqinfo(GTM_SeqInfo *seqinfo);
 static GTM_SequenceKey seq_copy_key(GTM_SequenceKey key);
 static int seq_drop_with_dbkey(GTM_SequenceKey nsp);
+static bool GTM_NeedSeqRestoreUpdateInternal(GTM_SeqInfo *seqinfo);
+
 #ifdef XCP
 static GTM_Sequence get_rangemax(GTM_SeqInfo *seqinfo, GTM_Sequence range);
 #endif
@@ -348,12 +351,16 @@ GTM_SeqOpen(GTM_SequenceKey seqkey,
 	seqinfo->gs_last_value = seqinfo->gs_init_value;
 #endif
 
+	seqinfo->gs_backedUpValue = seqinfo->gs_value;
+
 	if ((errcode = seq_add_seqinfo(seqinfo)))
 	{
 		GTM_RWLockDestroy(&seqinfo->gs_lock);
 		pfree(seqinfo->gs_key);
 		pfree(seqinfo);
 	}
+	GTM_SetNeedBackup();
+
 	return errcode;
 }
 
@@ -465,6 +472,7 @@ GTM_SeqRestore(GTM_SequenceKey seqkey,
 	seqinfo->gs_init_value = seqinfo->gs_last_value = startval;
 #endif
 	seqinfo->gs_value = curval;
+	seqinfo->gs_backedUpValue = seqinfo->gs_value;
 
 	/*
 	 * Should we wrap around ?
@@ -652,6 +660,7 @@ GTM_SeqRename(GTM_SequenceKey seqkey, GTM_SequenceKey newseqkey)
 
 	newseqinfo->gs_init_value = seqinfo->gs_init_value;
 	newseqinfo->gs_value = seqinfo->gs_value;
+	newseqinfo->gs_backedUpValue = seqinfo->gs_backedUpValue;
 	newseqinfo->gs_cycle = seqinfo->gs_cycle;
 
 	newseqinfo->gs_state = seqinfo->gs_state;
@@ -1048,6 +1057,7 @@ GTM_SeqSetVal(GTM_SequenceKey seqkey, GTM_Sequence nextval, bool iscalled)
 
 	/* Remove the old key with the old name */
 	GTM_RWLockRelease(&seqinfo->gs_lock);
+	GTM_SetNeedBackup();
 	seq_release_seqinfo(seqinfo);
 
 	return 0;
@@ -1133,6 +1143,8 @@ GTM_SeqGetNext(GTM_SequenceKey seqkey)
 
 	}
 	GTM_RWLockRelease(&seqinfo->gs_lock);
+	if (GTM_NeedSeqRestoreUpdateInternal(seqinfo))
+		GTM_SetNeedBackup();
 	seq_release_seqinfo(seqinfo);
 	return value;
 }
@@ -1156,12 +1168,14 @@ GTM_SeqReset(GTM_SequenceKey seqkey)
 
 	GTM_RWLockAcquire(&seqinfo->gs_lock, GTM_LOCKMODE_WRITE);
 #ifdef XCP
-	seqinfo->gs_value = seqinfo->gs_init_value;
+	seqinfo->gs_value = seqinfo->gs_backedUpValue = seqinfo->gs_init_value;
 #else
 	seqinfo->gs_value = seqinfo->gs_last_value = seqinfo->gs_init_value;
 #endif
-	GTM_RWLockRelease(&seqinfo->gs_lock);
 
+	GTM_RWLockRelease(&gtm_bkup_lock);
+
+	GTM_SetNeedBackup();
 	seq_release_seqinfo(seqinfo);
 	return 0;
 }
@@ -2088,6 +2102,7 @@ ProcessSequenceRenameCommand(Port *myport, StringInfo message, bool is_backup)
 }
 
 
+
 /*
  * Escape whitespace and non-printable characters in the sequence name to
  * store it to the control file.
@@ -2201,9 +2216,57 @@ decode_seq_key(char* value, GTM_SequenceKey seqkey)
 	memcpy(seqkey->gsk_key, out, len);
 }
 
+static GTM_Sequence distanceToBackedUpSeqValue(GTM_SeqInfo *seqinfo)
+{
+	if (!SEQ_IS_CALLED(seqinfo))
+		return (GTM_Sequence)0;
+	if (SEQ_IS_ASCENDING(seqinfo))
+	{
+		if ((seqinfo->gs_backedUpValue - seqinfo->gs_value) >= 0)
+			return (seqinfo->gs_backedUpValue - seqinfo->gs_value);
+		if (SEQ_IS_CYCLE(seqinfo))
+			return((seqinfo->gs_max_value - seqinfo->gs_value) + (seqinfo->gs_backedUpValue - seqinfo->gs_min_value));
+		else
+			return(seqinfo->gs_backedUpValue - seqinfo->gs_value);
+	}
+	else
+	{
+		if ((seqinfo->gs_value - seqinfo->gs_backedUpValue) >= 0)
+			return(seqinfo->gs_backedUpValue - seqinfo->gs_value);
+		if (SEQ_IS_CYCLE(seqinfo))
+			return((seqinfo->gs_max_value - seqinfo->gs_backedUpValue) + (seqinfo->gs_value - seqinfo->gs_min_value));
+		else
+			return(seqinfo->gs_backedUpValue - seqinfo->gs_value);
+	}
+	return 0;
+}
 
-void
-GTM_SaveSeqInfo(FILE *ctlf)
+bool GTM_NeedSeqRestoreUpdate(GTM_SequenceKey seqkey)
+{
+	GTM_SeqInfo *seqinfo = seq_find_seqinfo(seqkey);
+	if (!seqinfo)
+		return FALSE;
+	return GTM_NeedSeqRestoreUpdateInternal(seqinfo);
+}
+
+static bool GTM_NeedSeqRestoreUpdateInternal(GTM_SeqInfo *seqinfo)
+{
+	GTM_Sequence distance;
+
+	if (!SEQ_IS_CALLED(seqinfo))
+		/* The first call.  Must backup */
+		return TRUE;
+	distance = distanceToBackedUpSeqValue(seqinfo);
+	if (SEQ_IS_ASCENDING(seqinfo))
+		return(distance >= seqinfo->gs_increment_by);
+	else
+		return(distance <= seqinfo->gs_increment_by);
+}
+
+
+
+static void
+GTM_SaveSeqInfo2(FILE *ctlf, bool isBackup)
 {
 	GTM_SeqInfoHashBucket *bucket;
 	gtm_ListCell *elem;
@@ -2230,13 +2293,83 @@ GTM_SaveSeqInfo(FILE *ctlf)
 
 			encode_seq_key(seqinfo->gs_key, buffer);
 			fprintf(ctlf, "%s\t%ld\t%ld\t%ld\t%ld\t%ld\t%c\t%c\t%x\n",
-					buffer, seqinfo->gs_value,
+					buffer, isBackup ? seqinfo->gs_backedUpValue : seqinfo->gs_value,
 					seqinfo->gs_init_value, seqinfo->gs_increment_by,
 					seqinfo->gs_min_value, seqinfo->gs_max_value,
 					(seqinfo->gs_cycle ? 't' : 'f'),
 					(seqinfo->gs_called ? 't' : 'f'),
 					seqinfo->gs_state);
 
+				GTM_RWLockRelease(&seqinfo->gs_lock);
+		}
+		GTM_RWLockRelease(&bucket->shb_lock);
+	}
+
+}
+
+void GTM_SaveSeqInfo(FILE *ctlf)
+{
+	GTM_SaveSeqInfo2(ctlf, FALSE);
+}
+
+static void advance_gs_value(GTM_SeqInfo *seqinfo)
+{
+	
+	GTM_Sequence distance;
+
+	distance = seqinfo->gs_increment_by * RestoreDuration;
+	if (SEQ_IS_ASCENDING(seqinfo))
+	{
+		if ((seqinfo->gs_max_value - seqinfo->gs_value) >= distance)
+			seqinfo->gs_backedUpValue = seqinfo->gs_value + distance;
+		else
+		{
+			if (SEQ_IS_CYCLE(seqinfo))
+				seqinfo->gs_backedUpValue = seqinfo->gs_min_value + (distance - (seqinfo->gs_max_value - seqinfo->gs_value));
+			else
+				seqinfo->gs_backedUpValue = seqinfo->gs_max_value;
+		}
+	}
+	else
+	{
+		if ((seqinfo->gs_min_value - seqinfo->gs_value) >= distance)
+			seqinfo->gs_backedUpValue = seqinfo->gs_value + distance;
+		else
+		{
+			if (SEQ_IS_CYCLE(seqinfo))
+				seqinfo->gs_backedUpValue = seqinfo->gs_max_value + (distance - (seqinfo->gs_min_value - seqinfo->gs_value));
+			else
+				seqinfo->gs_backedUpValue = seqinfo->gs_min_value;
+		}
+	}
+}
+
+
+static void
+GTM_UpdateRestorePointSeq(void)
+{
+	GTM_SeqInfoHashBucket *bucket;
+	gtm_ListCell *elem;
+	GTM_SeqInfo *seqinfo = NULL;
+	int hash;
+
+	for (hash = 0; hash < SEQ_HASH_TABLE_SIZE; hash++)
+	{
+		bucket = &GTMSequences[hash];
+
+		GTM_RWLockAcquire(&bucket->shb_lock, GTM_LOCKMODE_READ);
+
+		gtm_foreach(elem, bucket->shb_list)
+		{
+			seqinfo = (GTM_SeqInfo *) gtm_lfirst(elem);
+			if (seqinfo == NULL)
+				break;
+
+			if (seqinfo->gs_state == SEQ_STATE_DELETED)
+				continue;
+
+			GTM_RWLockAcquire(&seqinfo->gs_lock, GTM_LOCKMODE_READ);
+			advance_gs_value(seqinfo);
 			GTM_RWLockRelease(&seqinfo->gs_lock);
 		}
 
@@ -2246,6 +2379,12 @@ GTM_SaveSeqInfo(FILE *ctlf)
 }
 
 
+void GTM_WriteRestorePointSeq(FILE *ctlf)
+{
+	GTM_UpdateRestorePointSeq();
+	GTM_SaveSeqInfo2(ctlf, TRUE);
+}
+
 void
 GTM_RestoreSeqInfo(FILE *ctlf)
 {
diff --git a/src/gtm/main/gtm_txn.c b/src/gtm/main/gtm_txn.c
index e40df161bc..cd2bb98261 100644
--- a/src/gtm/main/gtm_txn.c
+++ b/src/gtm/main/gtm_txn.c
@@ -32,6 +32,7 @@
 #include "gtm/libpq.h"
 #include "gtm/libpq-int.h"
 #include "gtm/pqformat.h"
+#include "gtm/gtm_backup.h"
 
 extern bool Backup_synchronously;
 
@@ -652,6 +653,8 @@ GTM_GetGlobalTransactionIdMulti(GTM_TransactionHandle handle[], int txn_count)
 	}
 #endif
 
+	if (GTM_NeedXidRestoreUpdate())
+		GTM_SetNeedBackup();
 	GTM_RWLockRelease(&GTMTransactions.gt_XidGenLock);
 
 #ifdef XCP
@@ -697,7 +700,11 @@ ReadNewGlobalTransactionId(void)
  * started. When the GTM is finally shutdown, the next to-be-assigned GXID is
  * stroed in the control file.
  *
- * XXX We don't yet handle any crash recovery. So if the GTM is shutdown
+ * XXX We don't yet handle any crash recovery. So if the GTM is no shutdown normally...
+ *
+ * This is handled by gtm_backup.c.  Anyway, because this function is to be called by
+ * GTM_RestoreTransactionId() and the backup will be performed afterwords,
+ * we don't care the new value of GTMTransactions.gt_nextXid here.
  */
 void
 SetNextGlobalTransactionId(GlobalTransactionId gxid)
@@ -1401,7 +1408,8 @@ GTM_BkupBeginTransactionGetGXIDMulti(char *coord_name,
 				gxid[ii], txn[ii]);
 
 		/*
-		 * Advance next gxid
+		 * Advance next gxid -- because this is called at slave only, we don't care the restoration point
+		 * here.  Restoration point will be created at promotion.
 		 */
 		if (GlobalTransactionIdPrecedesOrEquals(GTMTransactions.gt_nextXid, gxid[ii]))
 			GTMTransactions.gt_nextXid = gxid[ii] + 1;
@@ -2735,6 +2743,7 @@ ProcessGetNextGXIDTransactionCommand(Port *myport, StringInfo message)
 	 */
 	GTM_RWLockAcquire(&GTMTransactions.gt_XidGenLock, GTM_LOCKMODE_WRITE);
 	next_gxid = GTMTransactions.gt_nextXid;
+
 	GTM_RWLockRelease(&GTMTransactions.gt_XidGenLock);
 
 	MemoryContextSwitchTo(oldContext);
@@ -2814,6 +2823,23 @@ GTM_SaveTxnInfo(FILE *ctlf)
 
 	fprintf(ctlf, "%u\n", next_gxid);
 }
+
+bool GTM_NeedXidRestoreUpdate(void)
+{
+	return(GlobalTransactionIdPrecedesOrEquals(GTMTransactions.gt_backedUpXid, GTMTransactions.gt_nextXid));
+}
+
+void GTM_WriteRestorePointXid(FILE *f)
+{
+	if ((MaxGlobalTransactionId - GTMTransactions.gt_nextXid) <= RestoreDuration)
+		GTMTransactions.gt_backedUpXid = GTMTransactions.gt_nextXid + RestoreDuration;
+	else
+		GTMTransactions.gt_backedUpXid = FirstNormalGlobalTransactionId + (RestoreDuration - (MaxGlobalTransactionId - GTMTransactions.gt_nextXid));
+	
+	elog(LOG, "Saving transaction restoration info, backed-up gxid: %u", GTMTransactions.gt_backedUpXid);
+	fprintf(f, "%u\n", GTMTransactions.gt_backedUpXid);
+}
+
 /*
  * TODO
  */
@@ -2823,3 +2849,4 @@ int GTM_GetAllTransactions(GTM_TransactionInfo txninfo[], uint32 txncnt);
  * TODO
  */
 uint32 GTM_GetAllPrepared(GlobalTransactionId gxids[], uint32 gxidcnt);
+
diff --git a/src/gtm/main/main.c b/src/gtm/main/main.c
index dc780c4d63..240738d3b2 100644
--- a/src/gtm/main/main.c
+++ b/src/gtm/main/main.c
@@ -51,6 +51,7 @@
 #include "gtm/gtm_msg.h"
 #include "gtm/gtm_opt.h"
 #include "gtm/gtm_utils.h"
+#include "gtm/gtm_backup.h"
 
 extern int	optind;
 extern char *optarg;
@@ -125,6 +126,7 @@ static void checkDataDir(void);
 static void DeleteLockFile(const char *filename);
 static void PromoteToActive(void);
 static void ProcessSyncStandbyCommand(Port *myport, GTM_MessageType mtype, StringInfo message);
+static void ProcessBarrierCommand(Port *myport, GTM_MessageType mtype, StringInfo message);
 
 /*
  * One-time initialization. It's called immediately after the main process
@@ -138,9 +140,10 @@ MainThreadInit()
 	pthread_key_create(&threadinfo_key, NULL);
 
 	/*
-	 * Initialize the lock protecting the global threads info
+	 * Initialize the lock protecting the global threads info and backup lock info.
 	 */
 	GTM_RWLockInit(&GTMThreads->gt_lock);
+	GTM_RWLockInit(&gtm_bkup_lock);
 
 	/*
 	 * Set the next client identifier to be issued after connection
@@ -193,6 +196,9 @@ InitGTMProcess()
 	MyThreadID = pthread_self();
 	MemoryContextInit();
 
+	/* Backup the restore point */
+	GTM_WriteRestorePoint();
+
 	/*
 	 * The memory context is now set up.
 	 * Add the thrinfo structure in the global array
@@ -1405,6 +1411,11 @@ ProcessCommand(Port *myport, StringInfo input_message)
 			ProcessQueryCommand(myport, mtype, input_message);
 			break;
 
+		case MSG_BARRIER:
+		case MSG_BKUP_BARRIER:
+			ProcessBarrierCommand(myport, mtype, input_message);
+			break;
+
 		case MSG_BACKEND_DISCONNECT:
 			elog(DEBUG1, "MSG_BACKEND_DISCONNECT received - removing all txn infos");
 			GTM_RemoveAllTransInfos(GetMyThreadInfo->thr_client_id, proxyhdr.ph_conid);
@@ -1418,6 +1429,8 @@ ProcessCommand(Port *myport, StringInfo input_message)
 					 errmsg("invalid frontend message type %d",
 							mtype)));
 	}
+	if (GTM_NeedBackup())
+		GTM_WriteRestorePoint();
 }
 
 static int
@@ -2203,7 +2216,11 @@ PromoteToActive(void)
 					"%Y-%m-%d %H:%M:%S %Z",
 					localtime(&stamp_time));
 
-		fprintf(fp, "#===================================================\n# Updated due to GTM promote request\n# %s\nstartup = ACT\n#===================================================\n", strfbuf);
+		fprintf(fp, 
+				"#===================================================\n"
+				"# Updated due to GTM promote request\n"
+				"# %s\nstartup = ACT\n"
+				"#===================================================\n", strfbuf);
 		if (fclose(fp))
 			ereport(FATAL,
 					(EINVAL,
@@ -2212,3 +2229,56 @@ PromoteToActive(void)
 	}
 	return;
 }
+
+static void ProcessBarrierCommand(Port *myport, GTM_MessageType mtype, StringInfo message)
+{
+	int barrier_id_len;
+	char *barrier_id;
+	int count = 0;
+	GTM_Conn *oldconn = GetMyThreadInfo->thr_conn->standby;
+	StringInfoData buf;
+
+	barrier_id_len = pq_getmsgint(message, sizeof(int));
+	barrier_id = (char *)pq_getmsgbytes(message, barrier_id_len);
+	pq_getmsgend(message);
+
+	elog(INFO, "Processing BARRIER %s", barrier_id);
+
+	if ((mtype == MSG_BARRIER) && GetMyThreadInfo->thr_conn->standby)
+	{
+	retry:
+		bkup_report_barrier(GetMyThreadInfo->thr_conn->standby, barrier_id);
+		if (gtm_standby_check_communication_error(&count, oldconn))
+			goto retry;
+
+		if (Backup_synchronously && (myport->remote_type != GTM_NODE_GTM_PROXY))
+			gtm_sync_standby(GetMyThreadInfo->thr_conn->standby);
+	}
+
+	GTM_WriteBarrierBackup(barrier_id);
+
+	if (mtype == MSG_BARRIER)
+	{
+		/*
+		 * Send a SUCCESS message back to the client
+		 */
+		pq_beginmessage(&buf, 'S');
+		pq_sendint(&buf, BARRIER_RESULT, 4);
+		if (myport->remote_type == GTM_NODE_GTM_PROXY)
+		{
+			GTM_ProxyMsgHeader proxyhdr;
+			proxyhdr.ph_conid = myport->conn_id;
+			pq_sendbytes(&buf, (char *)&proxyhdr, sizeof (GTM_ProxyMsgHeader));
+		}
+		pq_endmessage(myport, &buf);
+
+		if (myport->remote_type != GTM_NODE_GTM_PROXY)
+		{
+			/* Flush standby first */
+			if (GetMyThreadInfo->thr_conn->standby)
+				gtmpqFlush(GetMyThreadInfo->thr_conn->standby);
+			pq_flush(myport);
+		}
+	}
+}
+	
diff --git a/src/gtm/proxy/proxy_main.c b/src/gtm/proxy/proxy_main.c
index 1f1c5ee9de..d76074fbf5 100644
--- a/src/gtm/proxy/proxy_main.c
+++ b/src/gtm/proxy/proxy_main.c
@@ -162,6 +162,8 @@ static void ProcessSnapshotCommand(GTMProxy_ConnectionInfo *conninfo,
 		GTM_Conn *gtm_conn, GTM_MessageType mtype, StringInfo message);
 static void ProcessSequenceCommand(GTMProxy_ConnectionInfo *conninfo,
 		GTM_Conn *gtm_conn, GTM_MessageType mtype, StringInfo message);
+static void ProcessBarrierCommand(GTMProxy_ConnectionInfo *conninfo,
+		GTM_Conn *gtm_conn, GTM_MessageType mtype, StringInfo message);
 
 static void GTMProxy_RegisterPGXCNode(GTMProxy_ConnectionInfo *conninfo,
 									  char *node_name,
@@ -1614,6 +1616,9 @@ ProcessCommand(GTMProxy_ConnectionInfo *conninfo, GTM_Conn *gtm_conn,
 		case MSG_SEQUENCE_ALTER:
 			ProcessSequenceCommand(conninfo, gtm_conn, mtype, input_message);
 			break;
+		case MSG_BARRIER:
+			ProcessBarrierCommand(conninfo, gtm_conn, mtype, input_message);
+			break;
 
 		default:
 			ereport(FATAL,
@@ -1950,6 +1955,28 @@ ProcessResponse(GTMProxy_ThreadInfo *thrinfo, GTMProxy_CommandInfo *cmdinfo,
 			ReleaseCmdBackup(cmdinfo);
 			break;
 
+		case MSG_BARRIER:
+			switch (res->gr_status)
+			{
+				case GTM_RESULT_OK:
+					pq_beginmessage(&buf, 'S');
+					pq_sendint(&buf, res->gr_type, 4);
+					pq_sendbytes(&buf, res->gr_proxy_data, res->gr_msglen);
+					pq_endmessage(cmdinfo->ci_conn->con_port, &buf);
+					pq_flush(cmdinfo->ci_conn->con_port);
+					break;
+
+				default:
+					pq_beginmessage(&buf, 'E');
+					pq_sendbytes(&buf, res->gr_proxy_data, res->gr_msglen);
+					pq_endmessage(cmdinfo->ci_conn->con_port, &buf);
+					pq_flush(cmdinfo->ci_conn->con_port);
+					break;
+			}
+			cmdinfo->ci_conn->con_pending_msg = MSG_TYPE_INVALID;
+			ReleaseCmdBackup(cmdinfo);
+			break;
+			
 		default:
 			ReleaseCmdBackup(cmdinfo);
 			ereport(FATAL,
@@ -2343,6 +2370,21 @@ ProcessSequenceCommand(GTMProxy_ConnectionInfo *conninfo, GTM_Conn *gtm_conn,
 	return GTMProxy_ProxyCommand(conninfo, gtm_conn, mtype, message);
 }
 
+static void
+ProcessBarrierCommand(GTMProxy_ConnectionInfo *conninfo, GTM_Conn *gtm_conn,
+		GTM_MessageType mtype, StringInfo message)
+{
+	/*
+	 * We proxy the Barrier messages as they are. Just add the connection
+	 * identifier to it so that the response can be quickly sent back to the
+	 * right backend.
+	 *
+	 * Write the message, but don't flush it just yet.
+	 */
+	return GTMProxy_ProxyCommand(conninfo, gtm_conn, mtype, message);
+}
+
+
 /*
  * Proxy the incoming message to the GTM server after adding our own identifier
  * to it. The rest of the message is forwarded as it is without even reading
diff --git a/src/include/access/gtm.h b/src/include/access/gtm.h
index 5c70872a1b..b606976276 100644
--- a/src/include/access/gtm.h
+++ b/src/include/access/gtm.h
@@ -15,6 +15,7 @@
 /* Configuration variables */
 extern char *GtmHost;
 extern int GtmPort;
+extern bool gtm_backup_barrier;
 
 #ifdef XCP
 extern bool IsXidFromGTM;
@@ -62,4 +63,6 @@ extern int AlterSequenceGTM(char *seqname, GTM_Sequence increment,
 							GTM_Sequence lastval, bool cycle, bool is_restart);
 extern int DropSequenceGTM(char *name, GTM_SequenceKeyType type);
 extern int RenameSequenceGTM(char *seqname, const char *newseqname);
+/* Barrier */
+extern int ReportBarrierGTM(char *barrier_id);
 #endif /* ACCESS_GTM_H */
diff --git a/src/include/gtm/gtm_client.h b/src/include/gtm/gtm_client.h
index 19535a239b..e8116e4b6b 100644
--- a/src/include/gtm/gtm_client.h
+++ b/src/include/gtm/gtm_client.h
@@ -309,6 +309,12 @@ int reset_sequence(GTM_Conn *conn, GTM_SequenceKey key);
 int bkup_reset_sequence(GTM_Conn *conn, GTM_SequenceKey key);
 
 /*
+ * Barrier
+ */
+int report_barrier(GTM_Conn *conn, char *barier_id);
+int bkup_report_barrier(GTM_Conn *conn, char *barrier_id);
+
+/*
  * GTM-Standby
  */
 int set_begin_end_backup(GTM_Conn *conn, bool begin);
diff --git a/src/include/gtm/gtm_msg.h b/src/include/gtm/gtm_msg.h
index 560c4428f6..6bf56ad972 100644
--- a/src/include/gtm/gtm_msg.h
+++ b/src/include/gtm/gtm_msg.h
@@ -92,6 +92,8 @@ typedef enum GTM_MessageType
 	MSG_BKUP_TXN_BEGIN_GETGXID_AUTOVACUUM,	/* Backup of MSG_TXN_BEGIN_GETGXID_AUTOVACUUM */
 	MSG_DATA_FLUSH,					/* flush pending data */
 	MSG_BACKEND_DISCONNECT,			/* tell GTM that the backend diconnected from the proxy */
+	MSG_BARRIER,				/* Tell the barrier was issued */
+	MSG_BKUP_BARRIER,			/* Backup barrier to standby */
 
 	/*
 	 * Must be at the end
@@ -146,6 +148,7 @@ typedef enum GTM_ResultType
 	TXN_GET_STATUS_RESULT,
 	TXN_GET_ALL_PREPARED_RESULT,
 	TXN_BEGIN_GETGXID_AUTOVACUUM_RESULT,
+	BARRIER_RESULT,
 	RESULT_TYPE_COUNT
 } GTM_ResultType;
 
diff --git a/src/include/gtm/gtm_seq.h b/src/include/gtm/gtm_seq.h
index c849dbc884..10ba3d83c0 100644
--- a/src/include/gtm/gtm_seq.h
+++ b/src/include/gtm/gtm_seq.h
@@ -40,6 +40,7 @@ typedef struct GTM_SeqInfo
 {
 	GTM_SequenceKey	gs_key;
 	GTM_Sequence	gs_value;
+	GTM_Sequence	gs_backedUpValue;
 	GTM_Sequence	gs_init_value;
 #ifdef XCP
 	int32			gs_max_lastvals;
@@ -107,7 +108,6 @@ int GTM_SeqSetVal(GTM_SequenceKey seqkey, GTM_Sequence nextval, bool iscalled);
 #endif
 int GTM_SeqReset(GTM_SequenceKey seqkey);
 
-
 void ProcessSequenceInitCommand(Port *myport, StringInfo message, bool is_backup);
 void ProcessSequenceGetCurrentCommand(Port *myport, StringInfo message);
 void ProcessSequenceGetNextCommand(Port *myport, StringInfo message, bool is_backup);
@@ -135,4 +135,6 @@ int GTM_SeqRestore(GTM_SequenceKey seqkey,
 void GTM_CleanupSeqSession(char *coord_name, int coord_procid);
 #endif
 
+bool GTM_NeedSeqRestoreUpdate(GTM_SequenceKey seqkey);
+void GTM_WriteRestorePointSeq(FILE *f);
 #endif
diff --git a/src/include/gtm/gtm_txn.h b/src/include/gtm/gtm_txn.h
index c304f665ce..2b338d570d 100644
--- a/src/include/gtm/gtm_txn.h
+++ b/src/include/gtm/gtm_txn.h
@@ -84,6 +84,10 @@ extern void SetGlobalTransactionIdLimit(GlobalTransactionId oldest_datfrozenxid)
 extern void SetNextGlobalTransactionId(GlobalTransactionId gxid);
 extern void GTM_SetShuttingDown(void);
 
+/* For restoration point backup */
+extern bool GTM_NeedXidRestoreUpdate(void);
+extern void GTM_WriteRestorePointXid(FILE *f);
+
 typedef enum GTM_States
 {
 	GTM_STARTING,
@@ -145,6 +149,7 @@ typedef struct GTM_Transactions
 	 * These fields are protected by XidGenLock
 	 */
 	GlobalTransactionId gt_nextXid;		/* next XID to assign */
+	GlobalTransactionId gt_backedUpXid;	/* backed up, restoration point */
 
 	GlobalTransactionId gt_oldestXid;	/* cluster-wide minimum datfrozenxid */
 	GlobalTransactionId gt_xidVacLimit;	/* start forcing autovacuums here */