Save global_xmin in the GTM control file and use that when its restarted

The control file now also have a version identifier so that we can read and
interpret older versions while keeping flexibility to change the format

9 files changed, 248 insertions, 155 deletions

diff --git a/src/gtm/main/gtm_backup.c b/src/gtm/main/gtm_backup.c
index c1cba50086..8906160d53 100644
--- a/src/gtm/main/gtm_backup.c
+++ b/src/gtm/main/gtm_backup.c
@@ -32,6 +32,7 @@ void GTM_WriteRestorePoint(void)
 	}
 	gtm_need_bkup = FALSE;
 	GTM_RWLockRelease(&gtm_bkup_lock);
+	GTM_WriteRestorePointVersion(f);
 	GTM_WriteRestorePointXid(f);
 	GTM_WriteRestorePointSeq(f);
 	fclose(f);
@@ -56,28 +57,12 @@ void GTM_WriteBarrierBackup(char *barrier_id)
 	GTM_RWLockAcquire(&gtm_bkup_lock, GTM_LOCKMODE_WRITE);
 	gtm_need_bkup = FALSE;
 	GTM_RWLockRelease(&gtm_bkup_lock);
+	GTM_WriteRestorePointVersion(f);
 	GTM_WriteRestorePointXid(f);
 	GTM_WriteRestorePointSeq(f);
 	fclose(f);
 }
 	
-
-void GTM_MakeBackup(char *path)
-{
-	FILE *f = fopen(path, "w");
-
-	if (f == NULL)
-	{
-		ereport(LOG, (errno,
-					  errmsg("Cannot open backup file %s", path),
-					  errhint("%s", strerror(errno))));
-		return;
-	}
-	GTM_SaveTxnInfo(f);
-	GTM_SaveSeqInfo(f);
-	fclose(f);
-}
-
 void GTM_SetNeedBackup(void)
 {
 	GTM_RWLockAcquire(&gtm_bkup_lock, GTM_LOCKMODE_READ);
diff --git a/src/gtm/main/gtm_seq.c b/src/gtm/main/gtm_seq.c
index cd90cbafb9..2fa6bf297a 100644
--- a/src/gtm/main/gtm_seq.c
+++ b/src/gtm/main/gtm_seq.c
@@ -1929,7 +1929,7 @@ encode_seq_key(GTM_SequenceKey seqkey, char *buffer)
 /*
  * Decode the string encoded by the encode_seq_key function
  */
-static void
+void
 decode_seq_key(char* value, GTM_SequenceKey seqkey)
 {
 	char   *in;
@@ -2148,82 +2148,6 @@ void GTM_WriteRestorePointSeq(FILE *ctlf)
 	GTM_SaveSeqInfo2(ctlf, TRUE);
 }
 
-void
-GTM_RestoreSeqInfo(FILE *ctlf)
-{
-	char seqname[1024];
-
-	if (ctlf == NULL)
-		return;
-
-	while (fscanf(ctlf, "%s", seqname) == 1)
-	{
-		GTM_SequenceKeyData seqkey;
-		GTM_Sequence increment_by;
-		GTM_Sequence minval;
-		GTM_Sequence maxval;
-		GTM_Sequence startval;
-		GTM_Sequence curval;
-		int32 state;
-		bool cycle;
-		bool called;
-		char boolval[16];
-
-		decode_seq_key(seqname, &seqkey);
-
-		if (fscanf(ctlf, "%ld", &curval) != 1)
-		{
-			elog(WARNING, "Corrupted control file");
-			return;
-		}
-		if (fscanf(ctlf, "%ld", &startval) != 1)
-		{
-			elog(WARNING, "Corrupted control file");
-			return;
-		}
-		if (fscanf(ctlf, "%ld", &increment_by) != 1)
-		{
-			elog(WARNING, "Corrupted control file");
-			return;
-		}
-		if (fscanf(ctlf, "%ld", &minval) != 1)
-		{
-			elog(WARNING, "Corrupted control file");
-			return;
-		}
-		if (fscanf(ctlf, "%ld", &maxval) != 1)
-		{
-			elog(WARNING, "Corrupted control file");
-			return;
-		}
-		if (fscanf(ctlf, "%s", boolval) == 1)
-		{
-			cycle = (*boolval == 't');
-		}
-		else
-		{
-			elog(WARNING, "Corrupted control file");
-			return;
-		}
-		if (fscanf(ctlf, "%s", boolval) == 1)
-		{
-			called = (*boolval == 't');
-		}
-		else
-		{
-			elog(WARNING, "Corrupted control file");
-			return;
-		}
-		if (fscanf(ctlf, "%x", &state) != 1)
-		{
-			elog(WARNING, "Corrupted control file");
-			return;
-		}
-		GTM_SeqRestore(&seqkey, increment_by, minval, maxval, startval, curval,
-					   state, cycle, called);
-	}
-}
-
 /*
  * Remove all current values allocated for the specified session from all
  * sequences.
diff --git a/src/gtm/main/gtm_standby.c b/src/gtm/main/gtm_standby.c
index cc5fea758f..141495d7a0 100644
--- a/src/gtm/main/gtm_standby.c
+++ b/src/gtm/main/gtm_standby.c
@@ -68,7 +68,7 @@ gtm_standby_restore_next_gxid(void)
 	GlobalTransactionId next_gxid = InvalidGlobalTransactionId;
 
 	next_gxid = get_next_gxid(GTM_ActiveConn);
-	GTM_RestoreTxnInfo(NULL, next_gxid);
+	GTM_RestoreTxnInfo(NULL, next_gxid, NULL);
 
 	elog(DEBUG1, "Restoring the next GXID done.");
 	return 1;
diff --git a/src/gtm/main/gtm_txn.c b/src/gtm/main/gtm_txn.c
index 5e82bdc25f..6e5973dc3f 100644
--- a/src/gtm/main/gtm_txn.c
+++ b/src/gtm/main/gtm_txn.c
@@ -32,6 +32,8 @@
 
 extern bool Backup_synchronously;
 
+#define GTM_CONTROL_VERSION	20160302
+
 /* Local functions */
 static XidStatus GlobalTransactionIdGetStatus(GlobalTransactionId transactionId);
 static bool GTM_SetDoVacuum(GTM_TransactionHandle handle);
@@ -93,6 +95,9 @@ GTM_InitTxnManager(void)
 	 */
 	GTMTransactions.gt_latestCompletedXid = FirstNormalGlobalTransactionId;
 
+	/* Initialise gt_recent_global_xmin */
+	GTMTransactions.gt_recent_global_xmin = FirstNormalGlobalTransactionId;
+
 	/*
 	 * Initialize the locks to protect various XID fields as well as the linked
 	 * list of transactions
@@ -684,6 +689,11 @@ SetNextGlobalTransactionId(GlobalTransactionId gxid)
 	return;
 }
 
+void
+SetControlXid(GlobalTransactionId gxid)
+{
+	ControlXid = gxid;
+}
 
 /* Transaction Control */
 int
@@ -2776,62 +2786,11 @@ GTM_SetShuttingDown(void)
 	GTM_RWLockRelease(&GTMTransactions.gt_XidGenLock);
 }
 
-void
-GTM_RestoreTxnInfo(FILE *ctlf, GlobalTransactionId next_gxid)
-{
-	GlobalTransactionId saved_gxid;
-
-	if (ctlf)
-	{
-		if ((fscanf(ctlf, "%u", &saved_gxid) != 1) &&
-			(!GlobalTransactionIdIsValid(next_gxid)))
-			next_gxid = InitialGXIDValue_Default;
-		else if (!GlobalTransactionIdIsValid(next_gxid))
-		{
-			/* Add in extra amount in case we had not gracefully stopped */
-			next_gxid = saved_gxid + CONTROL_INTERVAL;
-			ControlXid = next_gxid;
-		}
-	}
-	else if (!GlobalTransactionIdIsValid(next_gxid))
-		next_gxid = InitialGXIDValue_Default;
-
-	elog(LOG, "Restoring last GXID to %u\n", next_gxid);
-
-	if (GlobalTransactionIdIsValid(next_gxid))
-		SetNextGlobalTransactionId(next_gxid);
-	/* Set this otherwise a strange snapshot might be returned for the first one */
-	GTMTransactions.gt_latestCompletedXid = next_gxid - 1;
-	return;
-}
-
-void
-GTM_SaveTxnInfo(FILE *ctlf)
-{
-	GlobalTransactionId next_gxid;
-
-	next_gxid = ReadNewGlobalTransactionId();
-
-	elog(DEBUG1, "Saving transaction info - next_gxid: %u", next_gxid);
-
-	fprintf(ctlf, "%u\n", next_gxid);
-}
-
 bool GTM_NeedXidRestoreUpdate(void)
 {
 	return(GlobalTransactionIdPrecedesOrEquals(GTMTransactions.gt_backedUpXid, GTMTransactions.gt_nextXid));
 }
 
-void GTM_WriteRestorePointXid(FILE *f)
-{
-	if ((MaxGlobalTransactionId - GTMTransactions.gt_nextXid) <= RestoreDuration)
-		GTMTransactions.gt_backedUpXid = GTMTransactions.gt_nextXid + RestoreDuration;
-	else
-		GTMTransactions.gt_backedUpXid = FirstNormalGlobalTransactionId + (RestoreDuration - (MaxGlobalTransactionId - GTMTransactions.gt_nextXid));
-	
-	elog(DEBUG1, "Saving transaction restoration info, backed-up gxid: %u", GTMTransactions.gt_backedUpXid);
-	fprintf(f, "%u\n", GTMTransactions.gt_backedUpXid);
-}
 
 GlobalTransactionId
 GTM_GetLatestCompletedXID(void)
diff --git a/src/gtm/main/main.c b/src/gtm/main/main.c
index c5e1947948..8186770ddd 100644
--- a/src/gtm/main/main.c
+++ b/src/gtm/main/main.c
@@ -94,6 +94,8 @@ static int	ListenSocket[MAXLISTEN];
 pthread_key_t	threadinfo_key;
 static bool		GTMAbortPending = false;
 
+static void GTM_SaveVersion(FILE *ctlf);
+
 static Port *ConnCreate(int serverFd);
 static int ServerLoop(void);
 static int initMasks(fd_set *rmask);
@@ -323,6 +325,7 @@ SaveControlInfo(void)
 		return;
 	}
 
+	GTM_SaveVersion(ctlf);
 	GTM_SaveTxnInfo(ctlf);
 	GTM_SaveSeqInfo(ctlf);
 	fclose(ctlf);
@@ -648,6 +651,7 @@ main(int argc, char *argv[])
 	else
 	{
 		GTM_MutexLockAcquire(&control_lock);
+		GTM_RestoreContext restoreContext;
 
 		ctlf = fopen(GTMControlFile, "r");
 
@@ -691,8 +695,10 @@ main(int argc, char *argv[])
 				elog(ERROR, "Could not open %s, errno %d - aborting GTM start",
 						GTMControlFile, errno);
 		}
-		GTM_RestoreTxnInfo(ctlf, next_gxid);
-		GTM_RestoreSeqInfo(ctlf);
+
+		GTM_RestoreStart(ctlf, &restoreContext);
+		GTM_RestoreTxnInfo(ctlf, next_gxid, &restoreContext);
+		GTM_RestoreSeqInfo(ctlf, &restoreContext);
 		if (ctlf)
 			fclose(ctlf);
 
@@ -2284,3 +2290,210 @@ static void ProcessBarrierCommand(Port *myport, GTM_MessageType mtype, StringInf
 	}
 }
 	
+void
+GTM_RestoreStart(FILE *ctlf, struct GTM_RestoreContext *context)
+{
+	int version;
+
+	Assert(ctlf);
+
+	if (fscanf(ctlf, "version: %d\n", &version) == 1)
+	{
+		elog(LOG, "Read control file version %d", version);
+		context->version = version;
+	}
+	else
+	{
+		elog(LOG, "Failed to read file version");
+		context->version = -1;
+	}
+}
+
+void
+GTM_RestoreTxnInfo(FILE *ctlf, GlobalTransactionId next_gxid,
+		struct GTM_RestoreContext *context)
+{
+	GlobalTransactionId saved_gxid;
+	GlobalTransactionId saved_global_xmin;
+
+	if (ctlf)
+	{
+		/*
+		 * If the control file version is 20160302, then we expect to see
+		 * next_xid and global_xmin saved as first two lines. For older
+		 * versions, just the next_xid is stored
+		 */
+		if (context && context->version == 20160302)
+		{
+			if (fscanf(ctlf, "next_xid: %u\n", &saved_gxid) != 1)
+				saved_gxid = InvalidGlobalTransactionId;
+
+			if (fscanf(ctlf, "global_xmin: %u\n", &saved_global_xmin) != 1)
+				saved_global_xmin = InvalidGlobalTransactionId;
+		}
+		else
+		{
+			if (fscanf(ctlf, "%u\n", &saved_gxid) != 1)
+				saved_gxid = InvalidGlobalTransactionId;
+			saved_global_xmin = InvalidGlobalTransactionId;
+		}
+	}
+
+	/*
+	 * If the caller has supplied an explicit XID to restore, just use that.
+	 * This is typically only be used during initdb and in some exception
+	 * circumstances to recover from failures. But otherwise we must start with
+	 * the XIDs saved in the control file
+	 *
+	 * If the global_xmin was saved (which should be unless we are dealing with
+	 * an old control file), use that. Otherwise set it saved_gxid/next_xid
+	 * whatever is available. If we don't used the value incremented by
+	 * CONTROL_INTERVAL because its better to start with a conservative value
+	 * for the GlobalXmin
+	 */
+	if (!GlobalTransactionIdIsValid(next_gxid))
+	{
+		if (GlobalTransactionIdIsValid(saved_gxid))
+		{
+			/* 
+			 * Add in extra amount in case we had not gracefully stopped
+			 */
+			next_gxid = saved_gxid + CONTROL_INTERVAL;
+			SetControlXid(next_gxid);
+		}
+		else
+			saved_gxid = next_gxid = InitialGXIDValue_Default;
+
+		if (GlobalTransactionIdIsValid(saved_global_xmin))
+			GTMTransactions.gt_recent_global_xmin = saved_global_xmin;
+		else
+			GTMTransactions.gt_recent_global_xmin = saved_gxid;
+	}
+	else
+		GTMTransactions.gt_recent_global_xmin = next_gxid;
+
+	SetNextGlobalTransactionId(next_gxid);
+	elog(LOG, "Restoring last GXID to %u\n", next_gxid);
+	elog(LOG, "Restoring global xmin to %u\n",
+			GTMTransactions.gt_recent_global_xmin);
+
+	/* Set this otherwise a strange snapshot might be returned for the first one */
+	GTMTransactions.gt_latestCompletedXid = next_gxid - 1;
+	return;
+}
+
+static void
+GTM_SaveVersion(FILE *ctlf)
+{
+	fprintf(ctlf, "version: %d\n", GTM_CONTROL_VERSION);
+}
+
+void
+GTM_SaveTxnInfo(FILE *ctlf)
+{
+	GlobalTransactionId next_gxid;
+	GlobalTransactionId global_xmin = GTMTransactions.gt_recent_global_xmin;
+
+	next_gxid = ReadNewGlobalTransactionId();
+
+	elog(DEBUG1, "Saving transaction info - next_gxid: %u, global_xmin: %u",
+			next_gxid, global_xmin);
+
+	fprintf(ctlf, "next_xid: %u\n", next_gxid);
+	fprintf(ctlf, "global_xmin: %u\n", global_xmin);
+}
+
+void
+GTM_WriteRestorePointXid(FILE *f)
+{
+	if ((MaxGlobalTransactionId - GTMTransactions.gt_nextXid) <= RestoreDuration)
+		GTMTransactions.gt_backedUpXid = GTMTransactions.gt_nextXid + RestoreDuration;
+	else
+		GTMTransactions.gt_backedUpXid = FirstNormalGlobalTransactionId + (RestoreDuration - (MaxGlobalTransactionId - GTMTransactions.gt_nextXid));
+	
+	elog(DEBUG1, "Saving transaction restoration info, backed-up gxid: %u", GTMTransactions.gt_backedUpXid);
+	fprintf(f, "next_xid: %u\n", GTMTransactions.gt_backedUpXid);
+	fprintf(f, "global_xmin: %u\n", GTMTransactions.gt_backedUpXid);
+}
+
+void
+GTM_WriteRestorePointVersion(FILE *f)
+{
+	GTM_SaveVersion(f);
+}
+
+void
+GTM_RestoreSeqInfo(FILE *ctlf, struct GTM_RestoreContext *context)
+{
+	char seqname[1024];
+
+	if (ctlf == NULL)
+		return;
+
+	while (fscanf(ctlf, "%s", seqname) == 1)
+	{
+		GTM_SequenceKeyData seqkey;
+		GTM_Sequence increment_by;
+		GTM_Sequence minval;
+		GTM_Sequence maxval;
+		GTM_Sequence startval;
+		GTM_Sequence curval;
+		int32 state;
+		bool cycle;
+		bool called;
+		char boolval[16];
+
+		decode_seq_key(seqname, &seqkey);
+
+		if (fscanf(ctlf, "%ld", &curval) != 1)
+		{
+			elog(WARNING, "Corrupted control file");
+			return;
+		}
+		if (fscanf(ctlf, "%ld", &startval) != 1)
+		{
+			elog(WARNING, "Corrupted control file");
+			return;
+		}
+		if (fscanf(ctlf, "%ld", &increment_by) != 1)
+		{
+			elog(WARNING, "Corrupted control file");
+			return;
+		}
+		if (fscanf(ctlf, "%ld", &minval) != 1)
+		{
+			elog(WARNING, "Corrupted control file");
+			return;
+		}
+		if (fscanf(ctlf, "%ld", &maxval) != 1)
+		{
+			elog(WARNING, "Corrupted control file");
+			return;
+		}
+		if (fscanf(ctlf, "%s", boolval) == 1)
+		{
+			cycle = (*boolval == 't');
+		}
+		else
+		{
+			elog(WARNING, "Corrupted control file");
+			return;
+		}
+		if (fscanf(ctlf, "%s", boolval) == 1)
+		{
+			called = (*boolval == 't');
+		}
+		else
+		{
+			elog(WARNING, "Corrupted control file");
+			return;
+		}
+		if (fscanf(ctlf, "%x", &state) != 1)
+		{
+			elog(WARNING, "Corrupted control file");
+			return;
+		}
+		GTM_SeqRestore(&seqkey, increment_by, minval, maxval, startval, curval,
+					   state, cycle, called);
+	}
+}
diff --git a/src/gtm/recovery/register_common.c b/src/gtm/recovery/register_common.c
index 0d73685d1c..5d5c8d658a 100644
--- a/src/gtm/recovery/register_common.c
+++ b/src/gtm/recovery/register_common.c
@@ -57,8 +57,9 @@ static int NodeRegisterMagic = 0xeaeaeaea;
 static int NodeUnregisterMagic = 0xebebebeb;
 static int NodeEndMagic = 0xefefefef;
 
+#define GTM_GlobalXmin GTMTransactions.gt_recent_global_xmin
+
 static GTM_PGXCNodeInfoHashBucket GTM_PGXCNodes[NODE_HASH_TABLE_SIZE];
-static GlobalTransactionId GTM_GlobalXmin = FirstNormalGlobalTransactionId;
 static GTM_Timestamp GTM_GlobalXminComputedTime;
 
 static GTM_PGXCNodeInfo *pgxcnode_find_info(GTM_PGXCNodeType type, char *node_name);
@@ -1025,10 +1026,6 @@ GTM_HandleGlobalXmin(GTM_PGXCNodeType type, char *node_name,
 		*errcode = GTM_ERRCODE_TOO_OLD_XMIN;
 
 		mynodeinfo->joining = true;
-		mynodeinfo->reported_xmin_time = GTM_TimestampGetCurrent();
-		mynodeinfo->reported_xmin = GTM_GlobalXmin;
-
-		GTM_RWLockRelease(&mynodeinfo->node_lock);
 
 		/*
 		 * When node registers from the first time, the reported_xmin is set
@@ -1042,6 +1039,11 @@ GTM_HandleGlobalXmin(GTM_PGXCNodeType type, char *node_name,
 			elog(LOG, "GTM_ERRCODE_TOO_OLD_XMIN - node_name %s, reported_xmin %d, "
 					"previously reported_xmin %d, GTM_GlobalXmin %d", node_name,
 					reported_xmin, mynodeinfo->reported_xmin, GTM_GlobalXmin);
+
+		mynodeinfo->reported_xmin_time = GTM_TimestampGetCurrent();
+		mynodeinfo->reported_xmin = GTM_GlobalXmin;
+
+		GTM_RWLockRelease(&mynodeinfo->node_lock);
 		return InvalidGlobalTransactionId;
 	}
 
diff --git a/src/include/gtm/gtm.h b/src/include/gtm/gtm.h
index 200e09c668..8dfa77b139 100644
--- a/src/include/gtm/gtm.h
+++ b/src/include/gtm/gtm.h
@@ -73,6 +73,10 @@ typedef struct GTM_Threads
 
 extern GTM_Threads *GTMThreads;
 
+typedef struct GTM_RestoreContext {
+	int	version;
+} GTM_RestoreContext;
+
 int GTM_ThreadAdd(GTM_ThreadInfo *thrinfo);
 int GTM_ThreadRemove(GTM_ThreadInfo *thrinfo);
 int GTM_ThreadJoin(GTM_ThreadInfo *thrinfo);
@@ -88,6 +92,7 @@ GTM_ThreadInfo *GTM_ThreadCreate(GTM_ConnectionInfo *conninfo,
 GTM_ThreadInfo * GTM_GetThreadInfo(GTM_ThreadID thrid);
 #ifdef XCP
 extern void SaveControlInfo(void);
+void GTM_RestoreSeqInfo(FILE *ctlf, struct GTM_RestoreContext *context);
 #define CONTROL_INTERVAL		50000
 #endif
 
@@ -143,5 +148,6 @@ extern GTM_ThreadID						TopMostThreadID;
 	((((a) + 1) == UINT32_MAX) ? 1 : ((a) + 1))
 
 #define GTM_CONTROL_FILE		"gtm.control"
+#define GTM_CONTROL_VERSION		20160302
 
 #endif
diff --git a/src/include/gtm/gtm_seq.h b/src/include/gtm/gtm_seq.h
index aee08a7621..870ac56b94 100644
--- a/src/include/gtm/gtm_seq.h
+++ b/src/include/gtm/gtm_seq.h
@@ -101,8 +101,8 @@ void ProcessSequenceAlterCommand(Port *myport, StringInfo message, bool is_backu
 
 void ProcessSequenceListCommand(Port *myport, StringInfo message);
 
+void decode_seq_key(char* value, GTM_SequenceKey seqkey);
 void GTM_SaveSeqInfo(FILE *ctlf);
-void GTM_RestoreSeqInfo(FILE *ctlf);
 int GTM_SeqRestore(GTM_SequenceKey seqkey,
 			   GTM_Sequence increment_by,
 			   GTM_Sequence minval,
diff --git a/src/include/gtm/gtm_txn.h b/src/include/gtm/gtm_txn.h
index 0a7b38b29c..86c7986811 100644
--- a/src/include/gtm/gtm_txn.h
+++ b/src/include/gtm/gtm_txn.h
@@ -28,6 +28,7 @@ typedef int XidStatus;
 #define TRANSACTION_STATUS_COMMITTED        0x01
 #define TRANSACTION_STATUS_ABORTED          0x02
 
+struct GTM_RestoreContext;
 /*
  * prototypes for functions in transam/transam.c
  */
@@ -226,8 +227,11 @@ void ProcessBeginTransactionGetGXIDCommandMulti(Port *myport, StringInfo message
 void ProcessCommitTransactionCommandMulti(Port *myport, StringInfo message, bool is_backup);
 void ProcessRollbackTransactionCommandMulti(Port *myport, StringInfo message, bool is_backup) ;
 
+void GTM_WriteRestorePointVersion(FILE *f);
+void GTM_RestoreStart(FILE *ctlf, struct GTM_RestoreContext *context);
 void GTM_SaveTxnInfo(FILE *ctlf);
-void GTM_RestoreTxnInfo(FILE *ctlf, GlobalTransactionId next_gxid);
+void GTM_RestoreTxnInfo(FILE *ctlf, GlobalTransactionId next_gxid,
+		struct GTM_RestoreContext *context);
 void GTM_BkupBeginTransaction(GTM_IsolationLevel isolevel,
 							  bool readonly,
 							  const char *global_sessionid,