Enable Hot Standby on the replicas

We had an issue with tracking knownXids on the standby and it was overflowing
the allocated array in the shared memory. It turned out that the primary reason
for this is that the GTM leaves behind a hole in XID allocation when it's
restarted. The standby oblivious to this, was complaining about array overflow
and thus die.

We now fix this by allocating array which can hold CONTROL_INTERVAL worth
additional XIDs. This would mostly be a waste because the XIDs are never
allocated. But this seems like a quick fix to further test the Hot standby. The
good thing is that we might just waste memory, but not have any impact on the
performance because of larger array since we only loop for numKnownXids which
will be more accurate.

With this change, also fix the defaults for datanode and coordinator standbys
and make them Hot Standbys. The wal_level is changed too.

3 files changed, 17 insertions, 14 deletions

diff --git a/contrib/pgxc_ctl/coord_cmd.c b/contrib/pgxc_ctl/coord_cmd.c
index 00afd1e52e..b1cd50fd4e 100644
--- a/contrib/pgxc_ctl/coord_cmd.c
+++ b/contrib/pgxc_ctl/coord_cmd.c
@@ -158,7 +158,7 @@ cmd_t *prepare_initCoordinatorMaster(char *nodeName)
 		fprintf(f, 
 				"#========================================\n"
 				"# Addition for log shipping, %s\n"
-				"wal_level = archive\n"
+				"wal_level = hot_standby\n"
 				"archive_mode = on\n"
 				"archive_command = 'rsync %%p %s@%s:%s/%%f'\n"
 				"max_wal_senders = %s\n"
@@ -341,10 +341,10 @@ cmd_t *prepare_initCoordinatorSlave(char *nodeName)
 	fprintf(f,
 			"#==========================================\n"
 			"# Added to initialize the slave, %s\n"
-			"hot_standby = off\n"
+			"hot_standby = on\n"
 			"port = %s\n"
 			"pooler_port = %s\n"
-			"wal_level = archive\n"
+			"wal_level = hot_standby\n"
 			"archive_mode = off\n"
 			"archive_command = ''\n"
 			"max_wal_senders = 0\n"
@@ -1324,7 +1324,7 @@ int add_coordinatorSlave(char *name, char *host, int port, int pooler_port, char
 	fprintf(f, 
 			"#========================================\n"
 			"# Addition for log shipping, %s\n"
-			"wal_level = archive\n"
+			"wal_level = hot_standby\n"
 			"archive_mode = on\n"
 			"archive_command = 'rsync %%p %s@%s:%s/%%f'\n"
 			"max_wal_senders = %d\n"
@@ -1425,10 +1425,10 @@ int add_coordinatorSlave(char *name, char *host, int port, int pooler_port, char
 	fprintf(f,
 			"#==========================================\n"
 			"# Added to initialize the slave, %s\n"
-			"hot_standby = off\n"
+			"hot_standby = on\n"
 			"port = %d\n"
 			"pooler_port = %d\n"
-			"wal_level = archive\n"
+			"wal_level = hot_standby\n"
 			"archive_mode = off\n"		/* No archive mode */
 			"archive_command = ''\n"	/* No archive mode */
 			"max_wal_senders = 0\n"		/* Minimum WAL senders */
diff --git a/contrib/pgxc_ctl/datanode_cmd.c b/contrib/pgxc_ctl/datanode_cmd.c
index 4a3b83b325..5d39e709a5 100644
--- a/contrib/pgxc_ctl/datanode_cmd.c
+++ b/contrib/pgxc_ctl/datanode_cmd.c
@@ -178,7 +178,7 @@ cmd_t *prepare_initDatanodeMaster(char *nodeName)
 			return(NULL);
 		}
 		fprintf(f,
-				"wal_level = archive\n"
+				"wal_level = hot_standby\n"
 				"archive_mode = on\n"
 				"archive_command = 'rsync %%p %s@%s:%s/%%f'\n"
 				"max_wal_senders = %s\n"
@@ -368,7 +368,7 @@ cmd_t *prepare_initDatanodeSlave(char *nodeName)
 	fprintf(f,
 			"#==========================================\n"
 			"# Added to startup the slave, %s\n"
-			"hot_standby = off\n"
+			"hot_standby = on\n"
 			"port = %s\n"
 			"pooler_port = %s\n"
 			"# End of addition\n",
@@ -1383,7 +1383,7 @@ int add_datanodeSlave(char *name, char *host, int port, int pooler, char *dir,
 	fprintf(f, 
 			"#========================================\n"
 			"# Addition for log shipping, %s\n"
-			"wal_level = archive\n"
+			"wal_level = hot_standby\n"
 			"archive_mode = on\n"
 			"archive_command = 'rsync %%p %s@%s:%s/%%f'\n"
 			"max_wal_senders = %d\n"
@@ -1498,10 +1498,10 @@ int add_datanodeSlave(char *name, char *host, int port, int pooler, char *dir,
 	fprintf(f,
 			"#==========================================\n"
 			"# Added to initialize the slave, %s\n"
-			"hot_standby = off\n"
+			"hot_standby = on\n"
 			"port = %s\n"
 			"pooler_port = %s\n"
-			"wal_level = archive\n"
+			"wal_level = hot_standby\n"
 			"archive_mode = off\n"		/* No archive mode */
 			"archive_command = ''\n"	/* No archive mode */
 			"max_wal_senders = 0\n"		/* Minimum WAL senders */
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index 3b317a8928..a53a182ea3 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -74,6 +74,7 @@
 #include "utils/snapmgr.h"
 #ifdef PGXC
 #include "pgxc/pgxc.h"
+#include "gtm/gtm.h"
 #include "access/gtm.h"
 #include "storage/ipc.h"
 #include "utils/guc.h"
@@ -290,7 +291,8 @@ CreateSharedProcArray(void)
 		procArray->numProcs = 0;
 		procArray->maxProcs = PROCARRAY_MAXPROCS;
 		procArray->replication_slot_xmin = InvalidTransactionId;
-		procArray->maxKnownAssignedXids = TOTAL_MAX_CACHED_SUBXIDS;
+		procArray->maxKnownAssignedXids = TOTAL_MAX_CACHED_SUBXIDS +
+			CONTROL_INTERVAL;
 		procArray->numKnownAssignedXids = 0;
 		procArray->tailKnownAssignedXids = 0;
 		procArray->headKnownAssignedXids = 0;
@@ -307,11 +309,12 @@ CreateSharedProcArray(void)
 		KnownAssignedXids = (TransactionId *)
 			ShmemInitStruct("KnownAssignedXids",
 							mul_size(sizeof(TransactionId),
-									 TOTAL_MAX_CACHED_SUBXIDS),
+									 procArray->maxKnownAssignedXids),
 							&found);
 		KnownAssignedXidsValid = (bool *)
 			ShmemInitStruct("KnownAssignedXidsValid",
-							mul_size(sizeof(bool), TOTAL_MAX_CACHED_SUBXIDS),
+							mul_size(sizeof(bool),
+								procArray->maxKnownAssignedXids),
 							&found);
 	}