Add a mechanism to start with an empty pgxc_ctl configuration file and build

the cluster one at a time.

A new option "prepare config empty" is now supported which sets up an almost
empty conf file and all components, including GTM, can be added one at a time

13 files changed, 683 insertions, 147 deletions

diff --git a/contrib/pgxc_ctl/Makefile b/contrib/pgxc_ctl/Makefile
index e94d72f144..25ff6ccf0f 100644
--- a/contrib/pgxc_ctl/Makefile
+++ b/contrib/pgxc_ctl/Makefile
@@ -36,7 +36,7 @@ include $(top_builddir)/src/Makefile.global
 include $(top_srcdir)/contrib/contrib-global.mk
 endif
 
-pgxc_ctl_bash.c: pgxc_ctl_conf_part_full pgxc_ctl_conf_part_minimal pgxc_ctl_bash_2
+pgxc_ctl_bash.c: pgxc_ctl_conf_part_full pgxc_ctl_conf_part_minimal pgxc_ctl_conf_part_empty pgxc_ctl_bash_2
 	./make_signature
 
 clean: clean-script
diff --git a/contrib/pgxc_ctl/config.c b/contrib/pgxc_ctl/config.c
index 7b8cc28989..2098fd786d 100644
--- a/contrib/pgxc_ctl/config.c
+++ b/contrib/pgxc_ctl/config.c
@@ -141,7 +141,7 @@ static void parse_line(char *line)
 	reset_value(newv);
 	while((line = get_word(line, &val)))
 	{
-		if (val)
+		if (val && (strcmp(val, "") != 0))
 		{
 			add_val(newv, val);
 		}
@@ -502,7 +502,8 @@ static int anyConfigErrors = FALSE;
 
 static void checkIfVarIsConfigured(char *name)
 {
-	if (!find_var(name) || !sval(name))
+	/* var could be just defined without valid contents */
+	if (!find_var(name))
 	{
 		anyConfigErrors = TRUE;
 		reportMissingVar(name);
@@ -589,7 +590,8 @@ int checkPortConflict(char *host, int port)
 	int ii;
 
 	/* GTM Master */
-	if ((strcasecmp(host, sval(VAR_gtmMasterServer)) == 0) && (atoi(sval(VAR_gtmMasterPort)) == port))
+	if (doesExist(VAR_gtmMasterServer, 0) && doesExist(VAR_gtmMasterPort, 0) &&
+	(strcasecmp(host, sval(VAR_gtmMasterServer)) == 0) && (atoi(sval(VAR_gtmMasterPort)) == port))
 		return 1;
 	/* GTM Slave */
 	if (isVarYes(VAR_gtmSlave) && (strcasecmp(host, sval(VAR_gtmSlaveServer)) == 0) && (atoi(sval(VAR_gtmSlavePort)) == port))
@@ -631,7 +633,8 @@ int checkDirConflict(char *host, char *dir)
 	if (strcasecmp(dir, "none") == 0)
 		return 0;
 	/* GTM Master */
-	if ((strcasecmp(host, sval(VAR_gtmMasterServer)) == 0) && (strcmp(dir, sval(VAR_gtmMasterDir)) == 0))
+	if (doesExist(VAR_gtmMasterServer, 0) && doesExist(VAR_gtmMasterDir, 0) &&
+	(strcasecmp(host, sval(VAR_gtmMasterServer)) == 0) && (strcmp(dir, sval(VAR_gtmMasterDir)) == 0))
 		return 1;
 	/* GTM Slave */
 	if (isVarYes(VAR_gtmSlave) && (strcasecmp(host, sval(VAR_gtmSlaveServer)) == 0) && (strcmp(dir, sval(VAR_gtmSlaveDir)) == 0))
diff --git a/contrib/pgxc_ctl/coord_cmd.c b/contrib/pgxc_ctl/coord_cmd.c
index 5d8bbbae84..78af7fb3cf 100644
--- a/contrib/pgxc_ctl/coord_cmd.c
+++ b/contrib/pgxc_ctl/coord_cmd.c
@@ -1138,7 +1138,20 @@ int add_coordinatorMaster(char *name, char *host, int port, int pooler,
 	/* find any available coordinator */
 	connCordIndx = get_any_available_coord(-1);
 	if (connCordIndx == -1)
-		return 1;
+	{
+		/*
+		 * This is the FIRST coordinator being added into
+		 * the cluster. Just start it and be done with it.
+		 *
+		 * Start the new coordinator with --coordinator option
+		 */
+		AddMember(nodelist, name);
+		start_coordinator_master(nodelist);
+		CleanArray(nodelist);
+
+		/* ALTER our own definition appropriately */
+		goto selfadd;
+	}
 
 	/* Lock ddl */
 	if ((lockf = pgxc_popen_wRaw("psql -h %s -p %s %s",
@@ -1214,6 +1227,8 @@ int add_coordinatorMaster(char *name, char *host, int port, int pooler,
 	/* Quit DDL lokkup session */
 	fprintf(lockf, "\\q\n");
 	pclose(lockf);
+
+selfadd:
 	if ((f = pgxc_popen_wRaw("psql -h %s -p %d %s", host, port, sval(VAR_defaultDatabase))) == NULL)
 		elog(ERROR, "ERROR: cannot connect to the coordinator master %s.\n", name);
 	else
diff --git a/contrib/pgxc_ctl/datanode_cmd.c b/contrib/pgxc_ctl/datanode_cmd.c
index a70506037d..a60f53c68a 100644
--- a/contrib/pgxc_ctl/datanode_cmd.c
+++ b/contrib/pgxc_ctl/datanode_cmd.c
@@ -939,8 +939,7 @@ static int failover_oneDatanode(int datanodeIdx)
  *
  *-----------------------------------------------------------------------*/
 int add_datanodeMaster(char *name, char *host, int port, int pooler, char *dir,
-		char *waldir, char *restore_dname, char *extraConf,
-		char *extraPgHbaConf)
+		char *waldir, char *extraConf, char *extraPgHbaConf)
 {
 	FILE *f, *lockf;
 	int size, idx;
@@ -952,10 +951,9 @@ int add_datanodeMaster(char *name, char *host, int port, int pooler, char *dir,
 	char *gtmPort;
 	char pgdumpall_out[MAXPATH+1];
 	char **nodelist = NULL;
-	int ii, jj, restore_dnode_idx;
+	int ii, jj, restore_dnode_idx, restore_coord_idx = -1;
 	char **confFiles = NULL;
 	char **pgHbaConfFiles = NULL;
-	char *only_globals = "-g";
 	bool wal;
 
 	if (waldir && (strcasecmp(waldir, "none") != 0))
@@ -1007,6 +1005,17 @@ int add_datanodeMaster(char *name, char *host, int port, int pooler, char *dir,
 		return 1;
 	}
 
+	/* find any available datanode */
+	restore_dnode_idx = get_any_available_datanode(-1);
+	if (restore_dnode_idx == -1)
+		restore_coord_idx = get_any_available_coord(-1);
+
+	if (restore_dnode_idx == -1 && restore_coord_idx == -1)
+	{
+		elog(ERROR, "ERROR: no valid datanode or coordinator configuration!");
+		return 1;
+	}
+
 	if ((extendVar(VAR_datanodeNames, idx + 1, "none") != 0) ||
 		(extendVar(VAR_datanodeMasterServers, idx + 1, "none")  != 0) ||
 		(extendVar(VAR_datanodePorts, idx + 1, "none")  != 0) ||
@@ -1014,11 +1023,6 @@ int add_datanodeMaster(char *name, char *host, int port, int pooler, char *dir,
 		(extendVar(VAR_datanodeMasterDirs, idx + 1, "none")  != 0) ||
 		(extendVar(VAR_datanodeMasterWALDirs, idx + 1, "none")  != 0) ||
 		(extendVar(VAR_datanodeMaxWALSenders, idx + 1, "none")  != 0) ||
-		(extendVar(VAR_datanodeSlaveServers, idx + 1, "none")  != 0) ||
-		(extendVar(VAR_datanodeSlavePorts, idx + 1, "none")  != 0) ||
-		(extendVar(VAR_datanodeSlavePoolerPorts, idx + 1, "none")  != 0) ||
-		(extendVar(VAR_datanodeSlaveDirs, idx + 1, "none")  != 0) ||
-		(extendVar(VAR_datanodeArchLogDirs, idx + 1, "none")  != 0) ||
 		(extendVar(VAR_datanodeSpecificExtraConfig, idx + 1, "none")  != 0) ||
 		(extendVar(VAR_datanodeSpecificExtraPgHba, idx + 1, "none") != 0))
 	{
@@ -1026,6 +1030,19 @@ int add_datanodeMaster(char *name, char *host, int port, int pooler, char *dir,
 		return 1;
 	}
 
+	if (isVarYes(VAR_datanodeSlave))
+	{
+		if ((extendVar(VAR_datanodeSlaveServers, idx + 1, "none")  != 0) ||
+			(extendVar(VAR_datanodeSlavePorts, idx + 1, "none")  != 0) ||
+			(extendVar(VAR_datanodeSlavePoolerPorts, idx + 1, "none")  != 0) ||
+			(extendVar(VAR_datanodeSlaveDirs, idx + 1, "none")  != 0) ||
+			(extendVar(VAR_datanodeSlaveWALDirs, idx + 1, "none")  != 0) ||
+			(extendVar(VAR_datanodeArchLogDirs, idx + 1, "none")  != 0))
+		{
+			elog(PANIC, "PANIC: Internal error, inconsistent datanode slave information\n");
+			return 1;
+		}
+	}
 
 	/*
 	 * Now reconfigure
@@ -1042,11 +1059,15 @@ int add_datanodeMaster(char *name, char *host, int port, int pooler, char *dir,
 	assign_arrayEl(VAR_datanodeMasterDirs, idx, dir, NULL);
 	assign_arrayEl(VAR_datanodeMasterWALDirs, idx, waldir, NULL);
 	assign_arrayEl(VAR_datanodeMaxWALSenders, idx, aval(VAR_datanodeMaxWALSenders)[0], NULL);	/* Could be vulnerable */
-	assign_arrayEl(VAR_datanodeSlaveServers, idx, "none", NULL);
-	assign_arrayEl(VAR_datanodeSlavePorts, idx, "-1", NULL);
-	assign_arrayEl(VAR_datanodeSlavePoolerPorts, idx, "-1", NULL);
-	assign_arrayEl(VAR_datanodeSlaveDirs, idx, "none", NULL);
-	assign_arrayEl(VAR_datanodeArchLogDirs, idx, "none", NULL);
+ 	if (isVarYes(VAR_datanodeSlave))
+	{
+		assign_arrayEl(VAR_datanodeSlaveServers, idx, "none", NULL);
+		assign_arrayEl(VAR_datanodeSlavePorts, idx, "-1", NULL);
+		assign_arrayEl(VAR_datanodeSlavePoolerPorts, idx, "-1", NULL);
+		assign_arrayEl(VAR_datanodeSlaveDirs, idx, "none", NULL);
+		assign_arrayEl(VAR_datanodeSlaveWALDirs, idx, "none", NULL);
+		assign_arrayEl(VAR_datanodeArchLogDirs, idx, "none", NULL);
+	}
 	assign_arrayEl(VAR_datanodeSpecificExtraConfig, idx, extraConf, NULL);
 	assign_arrayEl(VAR_datanodeSpecificExtraPgHba, idx, extraPgHbaConf, NULL);
 	/*
@@ -1088,11 +1109,15 @@ int add_datanodeMaster(char *name, char *host, int port, int pooler, char *dir,
 	fprintAval(f, VAR_datanodeMasterDirs);
 	fprintAval(f, VAR_datanodeMasterWALDirs);
 	fprintAval(f, VAR_datanodeMaxWALSenders);
-	fprintAval(f, VAR_datanodeSlaveServers);
-	fprintAval(f, VAR_datanodeSlavePorts);
-	fprintAval(f, VAR_datanodeSlavePoolerPorts);
-	fprintAval(f, VAR_datanodeSlaveDirs);
-	fprintAval(f, VAR_datanodeArchLogDirs);
+ 	if (isVarYes(VAR_datanodeSlave))
+	{
+		fprintAval(f, VAR_datanodeSlaveServers);
+		fprintAval(f, VAR_datanodeSlavePorts);
+		fprintAval(f, VAR_datanodeSlavePoolerPorts);
+		fprintAval(f, VAR_datanodeSlaveDirs);
+		fprintAval(f, VAR_datanodeSlaveWALDirs);
+		fprintAval(f, VAR_datanodeArchLogDirs);
+	}
 	fprintAval(f, VAR_datanodeSpecificExtraConfig);
 	fprintAval(f, VAR_datanodeSpecificExtraPgHba);
 	fprintf(f, "%s", "#----End of reconfiguration -------------------------\n");
@@ -1149,45 +1174,49 @@ int add_datanodeMaster(char *name, char *host, int port, int pooler, char *dir,
 		pclose(f);
 	}
 
-	restore_dnode_idx = -1;
-	for (ii = 0; aval(VAR_datanodeNames)[ii]; ii++)
-	{
-		if (!is_none(aval(VAR_datanodeNames)[ii]))
-		{
-			if (strcmp(aval(VAR_datanodeNames)[ii], restore_dname) == 0)
-				restore_dnode_idx = ii;
-		}
-	}
-	if (strcmp("none", restore_dname) != 0 && restore_dnode_idx == -1)
-	{
-		elog(ERROR, "ERROR: improper datanode specified to restore from, %s\n", restore_dname);
-		return 1;
-	}
-
-	if (restore_dnode_idx == -1)
-	{
-		restore_dnode_idx = 0;
-	}
-	else
-		only_globals= " ";
-
-
-	/* Lock ddl */
-	if ((lockf = pgxc_popen_wRaw("psql -h %s -p %d %s", aval(VAR_datanodeMasterServers)[restore_dnode_idx], atoi(aval(VAR_datanodePorts)[restore_dnode_idx]), sval(VAR_defaultDatabase))) == NULL)
-	{
-		elog(ERROR, "ERROR: could not open psql command, %s\n", strerror(errno));
-		return 1;
-	}
+ 	/* Lock ddl */
+ 	if (restore_dnode_idx != -1)
+ 	{
+ 		if ((lockf = pgxc_popen_wRaw("psql -h %s -p %d %s", aval(VAR_datanodeMasterServers)[restore_dnode_idx], atoi(aval(VAR_datanodePorts)[restore_dnode_idx]), sval(VAR_defaultDatabase))) == NULL)
+ 		{
+ 			elog(ERROR, "ERROR: could not open datanode psql command, %s\n", strerror(errno));
+ 			return 1;
+ 		}
+ 	}
+ 	else if (restore_coord_idx != -1)
+ 	{
+ 		if ((lockf = pgxc_popen_wRaw("psql -h %s -p %d %s", aval(VAR_coordMasterServers)[restore_coord_idx], atoi(aval(VAR_coordPorts)[restore_coord_idx]), sval(VAR_defaultDatabase))) == NULL)
+ 		{
+ 			elog(ERROR, "ERROR: could not open coordinator psql command, %s\n", strerror(errno));
+  			return 1;
+  		}
+  	}
+  	else
+  	{
+ 		elog(ERROR, "ERROR: no valid datanode or coordinator configuration!");
+  		return 1;
+  	}
+ 
 	fprintf(lockf, "select pgxc_lock_for_backup();\n");	/* Keep open until the end of the addition. */
 	fflush(lockf);
 
 	/* pg_dumpall */
 	createLocalFileName(GENERAL, pgdumpall_out, MAXPATH);
-	doImmediateRaw("pg_dumpall -p %s -h %s -s --include-nodes --dump-nodes %s >%s",
+ 	if (restore_dnode_idx != -1)
+ 		doImmediateRaw("pg_dumpall -p %s -h %s -s --include-nodes --dump-nodes >%s",
 				   aval(VAR_datanodePorts)[restore_dnode_idx],
 				   aval(VAR_datanodeMasterServers)[restore_dnode_idx],
-				   only_globals,
 				   pgdumpall_out);
+ 	else if (restore_coord_idx != -1)
+ 		doImmediateRaw("pg_dumpall -p %s -h %s -s --include-nodes --dump-nodes >%s",
+ 					   aval(VAR_coordPorts)[restore_coord_idx],
+ 					   aval(VAR_coordMasterServers)[restore_coord_idx],
+ 					   pgdumpall_out);
+ 	else
+ 	{
+ 		elog(ERROR, "ERROR: no valid datanode or coordinator configuration!");
+ 		return 1;
+ 	}
 
 	/* Start the new datanode */
 	doImmediate(host, NULL, "pg_ctl start -w -Z restoremode -D %s -o -i", dir);
@@ -1214,7 +1243,7 @@ int add_datanodeMaster(char *name, char *host, int port, int pooler, char *dir,
 		{
 			if ((f = pgxc_popen_wRaw("psql -h %s -p %s %s", aval(VAR_coordMasterServers)[ii], aval(VAR_coordPorts)[ii], sval(VAR_defaultDatabase))) == NULL)
 			{
-				elog(ERROR, "ERROR: cannot connect to the datanode master %s.\n", aval(VAR_coordNames)[ii]);
+				elog(ERROR, "ERROR: cannot connect to the coordinator master %s.\n", aval(VAR_coordNames)[ii]);
 				continue;
 			}
 			fprintf(f, "CREATE NODE %s WITH (TYPE = 'datanode', host='%s', PORT=%d);\n", name, host, port);
@@ -1224,8 +1253,10 @@ int add_datanodeMaster(char *name, char *host, int port, int pooler, char *dir,
 		}
 	}
 
-	/* find any available coordinator */
-	connCordIdx = get_any_available_coord(-1);
+ 	if (restore_coord_idx == -1)
+  		connCordIdx = get_any_available_coord(-1);
+ 	else
+ 		connCordIdx = restore_coord_idx;
 	if (connCordIdx == -1)
 		return 1;
 
@@ -1256,10 +1287,8 @@ int add_datanodeMaster(char *name, char *host, int port, int pooler, char *dir,
 	fprintf(lockf, "\\q\n");
 	pclose(lockf);
 	return 0;
-
 }
 
-
 int add_datanodeSlave(char *name, char *host, int port, int pooler, char *dir,
 		char *walDir, char *archDir)
 {
@@ -1537,7 +1566,7 @@ int remove_datanodeMaster(char *name, int clean_opt)
 	/* Check if the datanode is configured */
 	if ((idx = datanodeIdx(name)) < 0)
 	{
-		elog(ERROR, "ERROR: Coordinator %s is not configured.\n", name);
+		elog(ERROR, "ERROR: Datanode %s is not configured.\n", name);
 		return 1;
 	}
 	/* Check if all the other datanodes are running */
@@ -1619,17 +1648,25 @@ int remove_datanodeMaster(char *name, int clean_opt)
 		doImmediate(aval(VAR_datanodeMasterServers)[idx], NULL, "rm -rf %s", aval(VAR_datanodeMasterDirs)[idx]);
 	/* Update configuration and backup --> should cleanup "none" entries here */
 	replace_arrayEl(VAR_datanodeNames, idx, "none", NULL);
-	replace_arrayEl(VAR_datanodeMasterDirs, idx, "none", NULL);
-	replace_arrayEl(VAR_datanodeMasterWALDirs, idx, "none", NULL);
+ 	replace_arrayEl(VAR_datanodeMasterServers, idx, "none", NULL);
 	replace_arrayEl(VAR_datanodePorts, idx, "-1", "-1");
 	replace_arrayEl(VAR_datanodePoolerPorts, idx, "-1", "-1");
-	replace_arrayEl(VAR_datanodeMasterServers, idx, "none", NULL);
+ 	replace_arrayEl(VAR_datanodeMasterDirs, idx, "none", NULL);
+ 	replace_arrayEl(VAR_datanodeMasterWALDirs, idx, "none", NULL);
 	replace_arrayEl(VAR_datanodeMaxWALSenders, idx, "0", "0");
-	replace_arrayEl(VAR_datanodeSlaveServers, idx, "none", NULL);
-	replace_arrayEl(VAR_datanodeSlaveDirs, idx, "none", NULL);
-	replace_arrayEl(VAR_datanodeSlaveWALDirs, idx, "none", NULL);
-	replace_arrayEl(VAR_datanodeArchLogDirs, idx, "none", NULL);
-	replace_arrayEl(VAR_datanodeSpecificExtraConfig, idx, "none", NULL);
+ 	replace_arrayEl(VAR_datanodeSpecificExtraConfig, idx, "none", NULL);
+ 	replace_arrayEl(VAR_datanodeSpecificExtraPgHba, idx, "none", NULL);
+ 
+ 	if (isVarYes(VAR_datanodeSlave))
+	{
+		replace_arrayEl(VAR_datanodeSlaveServers, idx, "none", NULL);
+		replace_arrayEl(VAR_datanodeSlavePorts, idx, "none", NULL);
+		replace_arrayEl(VAR_datanodeSlavePoolerPorts, idx, "none", NULL);
+		replace_arrayEl(VAR_datanodeSlaveDirs, idx, "none", NULL);
+		replace_arrayEl(VAR_datanodeSlaveWALDirs, idx, "none", NULL);
+		replace_arrayEl(VAR_datanodeArchLogDirs, idx, "none", NULL);
+	}
+ 
 	handle_no_slaves();
 	/*
 	 * Write config files
@@ -1642,7 +1679,7 @@ int remove_datanodeMaster(char *name, int clean_opt)
 	}
 	fprintf(f, 
 			"#================================================================\n"
-			"# pgxc configuration file updated due to coodinator master removal\n"
+			"# pgxc configuration file updated due to datanode master removal\n"
 			"#        %s\n",
 			timeStampString(date, MAXTOKEN+1));
 	fprintSval(f, VAR_datanodeSlave);
@@ -1653,12 +1690,16 @@ int remove_datanodeMaster(char *name, int clean_opt)
 	fprintAval(f, VAR_datanodePoolerPorts);
 	fprintAval(f, VAR_datanodeMasterServers);
 	fprintAval(f, VAR_datanodeMaxWALSenders);
-	fprintAval(f, VAR_datanodeSlaveServers);
-	fprintAval(f, VAR_datanodeSlavePorts);
-	fprintAval(f, VAR_datanodeSlaveDirs);
-	fprintAval(f, VAR_datanodeSlaveWALDirs);
-	fprintAval(f, VAR_datanodeArchLogDirs);
+ 	if (isVarYes(VAR_datanodeSlave))
+	{
+		fprintAval(f, VAR_datanodeSlaveServers);
+		fprintAval(f, VAR_datanodeSlavePorts);
+		fprintAval(f, VAR_datanodeSlaveDirs);
+		fprintAval(f, VAR_datanodeSlaveWALDirs);
+		fprintAval(f, VAR_datanodeArchLogDirs);
+	}
 	fprintAval(f, VAR_datanodeSpecificExtraConfig);
+	fprintAval(f, VAR_datanodeSpecificExtraPgHba);
 	fclose(f);
 	backup_configuration();
 	return 0;
diff --git a/contrib/pgxc_ctl/datanode_cmd.h b/contrib/pgxc_ctl/datanode_cmd.h
index 5eee9b6a53..9a8eca58df 100644
--- a/contrib/pgxc_ctl/datanode_cmd.h
+++ b/contrib/pgxc_ctl/datanode_cmd.h
@@ -53,8 +53,7 @@ extern cmd_t *prepare_cleanDatanodeSlave(char *nodeName);
 
 #ifdef XCP
 extern int add_datanodeMaster(char *name, char *host, int port, int pooler,
-		char *dir, char *walDir, char *restore_dname, char *extraConf,
-		char *extraPgHbaConf);
+		char *dir, char *walDir, char *extraConf, char *extraPgHbaConf);
 #else
 extern int add_datanodeMaster(char *name, char *host, int port, char *dir,
 		char *restore_dname, char *extraConf, char *extraPgHbaConf);
diff --git a/contrib/pgxc_ctl/do_command.c b/contrib/pgxc_ctl/do_command.c
index c6d0cae903..035400d320 100644
--- a/contrib/pgxc_ctl/do_command.c
+++ b/contrib/pgxc_ctl/do_command.c
@@ -40,6 +40,7 @@
 
 extern char *pgxc_ctl_conf_prototype[];
 extern char *pgxc_ctl_conf_prototype_minimal[];
+extern char *pgxc_ctl_conf_prototype_empty[];
 
 int forceInit = false;
 
@@ -67,6 +68,7 @@ static void do_show_help(char *line);
 
 typedef enum ConfigType
 {
+	CONFIG_EMPTY,
 	CONFIG_MINIMAL,
 	CONFIG_COMPLETE
 } ConfigType;
@@ -102,7 +104,9 @@ static void do_prepareConfFile(char *Path, ConfigType config_type)
 		return;
 	}
 
-	if (config_type == CONFIG_MINIMAL)
+	if (config_type == CONFIG_EMPTY)
+		my_pgxc_conf_prototype = pgxc_ctl_conf_prototype_empty;
+	else if (config_type == CONFIG_MINIMAL)
 		my_pgxc_conf_prototype = pgxc_ctl_conf_prototype_minimal;
 	else
 		my_pgxc_conf_prototype = pgxc_ctl_conf_prototype;
@@ -517,7 +521,7 @@ static void do_kill_command(char *line)
 
 static void init_all(void)
 {
-	init_gtm_master();
+	init_gtm_master(true);
 	start_gtm_master();
 	if (isVarYes(VAR_gtmSlave))
 	{
@@ -574,12 +578,12 @@ static void do_init_command(char *line)
 	{
 		if (!GetToken() || (TestToken("all")))
 		{
-			init_gtm_master();
+			init_gtm_master(true);
 			if (isVarYes(VAR_gtmSlave))
 				init_gtm_slave();
 		}
 		else if (TestToken("master"))
-			init_gtm_master();
+			init_gtm_master(true);
 		else if (TestToken("slave"))
 			init_gtm_slave();
 		else
@@ -868,7 +872,6 @@ static void do_add_command(char *line)
 	char *dir;
 	char *walDir;
 	char *archDir;
-	char *dnode;
 	char *extraConf;
 	char *extraPgHbaConf;
 
@@ -880,7 +883,7 @@ static void do_add_command(char *line)
 	if (TestToken("gtm"))
 	{
 		/*
-		 * add gtm slave name host port dir
+		 * add gtm master name host port dir
 		 */
 
 		if (!GetToken())
@@ -888,16 +891,27 @@ static void do_add_command(char *line)
 			elog(ERROR, "ERROR: Specify option for add gtm command.\n");
 			return;
 		}
-		if (!TestToken("slave"))
-		{
-			elog(ERROR, "ERROR: you can specify only slave to add gtm command. %s is invalid.\n", token);
-			return;
-		}
-		GetAndSet(name, "ERROR: please specify the name of gtm slave\n");
-		GetAndSet(host, "ERROR: please specify the host name for gtm slave\n");
-		GetAndSet(port, "ERROR: please specify the port number for gtm slave\n");
-		GetAndSet(dir, "ERROR: please specify the working director for gtm slave\n");
-		add_gtmSlave(name, host, atoi(port), dir);
+ 		if (TestToken("master"))
+ 		{
+ 			GetAndSet(name, "ERROR: please specify the name of gtm master\n");
+ 			GetAndSet(host, "ERROR: please specify the host name for gtm master\n");
+ 			GetAndSet(port, "ERROR: please specify the port number for gtm master\n");
+ 			GetAndSet(dir, "ERROR: please specify the working director for gtm master\n");
+ 			add_gtmMaster(name, host, atoi(port), dir);
+ 		}
+ 		else if (TestToken("slave"))
+		{
+			GetAndSet(name, "ERROR: please specify the name of gtm slave\n");
+			GetAndSet(host, "ERROR: please specify the host name for gtm slave\n");
+			GetAndSet(port, "ERROR: please specify the port number for gtm slave\n");
+			GetAndSet(dir, "ERROR: please specify the working director for gtm slave\n");
+			add_gtmSlave(name, host, atoi(port), dir);
+		}
+ 		else
+ 		{
+ 			elog(ERROR, "ERROR: you can specify only master/slave to add gtm command. %s is invalid.\n", token);
+ 			return;
+ 		}
 		freeAndReset(name);
 		freeAndReset(host);
 		freeAndReset(port);
@@ -935,9 +949,9 @@ static void do_add_command(char *line)
 			GetAndSet(host, "ERROR: please specify the host for the coordinator masetr\n");
 			GetAndSet(port, "ERROR: please specify the port number for the coordinator master\n");
 			GetAndSet(pooler, "ERROR: please specify the pooler port number for the coordinator master.\n");
-			GetAndSet(dir, "ERROR: please specify the working director for the coordinator master\n");
-			GetAndSet(extraConf, "ERROR: please specify file to read extra configuration. Specify 'none' if nothig extra to be added.\n");
-			GetAndSet(extraPgHbaConf, "ERROR: please specify file to read extra pg_hba configuration. Specify 'none' if nothig extra to be added.\n");
+			GetAndSet(dir, "ERROR: please specify the working directory for the coordinator master\n");
+			GetAndSet(extraConf, "ERROR: please specify file to read extra configuration. Specify 'none' if nothing extra to be added.\n");
+			GetAndSet(extraPgHbaConf, "ERROR: please specify file to read extra pg_hba configuration. Specify 'none' if nothing extra to be added.\n");
 			add_coordinatorMaster(name, host, atoi(port), atoi(pooler), dir,
 					extraConf, extraPgHbaConf);
 			freeAndReset(name);
@@ -975,11 +989,10 @@ static void do_add_command(char *line)
 			GetAndSet(pooler, "ERROR: please specify the pooler port number for the datanode master.\n");
 			GetAndSet(dir, "ERROR: please specify the working director for the datanode master\n");
 			GetAndSet(walDir, "ERROR: please specify the WAL directory for the datanode master WAL. Specify 'none' for default\n");
-			GetAndSet(dnode, "ERROR: please specify name of existing datanode of which this will be a copy of. Specify 'none' for a bare datanode\n");
 			GetAndSet(extraConf, "ERROR: please specify file to read extra configuration. Specify 'none' if nothig extra to be added.\n");
 			GetAndSet(extraPgHbaConf, "ERROR: please specify file to read extra pg_hba configuration. Specify 'none' if nothig extra to be added.\n");
 			add_datanodeMaster(name, host, atoi(port), atoi(pooler), dir,
-					walDir, dnode, extraConf, extraPgHbaConf);
+					walDir, extraConf, extraPgHbaConf);
 			freeAndReset(name);
 			freeAndReset(host);
 			freeAndReset(port);
@@ -1023,24 +1036,34 @@ static void do_remove_command(char *line)
 	}
 	if (TestToken("gtm"))
 	{
-		if (!GetToken() || !TestToken("slave"))
+		if (!GetToken())
 		{
-			elog(ERROR, "ERROR: Please speciy slave to add gtm command\n");
+			elog(ERROR, "ERROR: Specify option to remove gtm command\n");
+			return;
+		}
+		if (TestToken("master"))
+		{
+			if (GetToken() && TestToken("clean"))
+				clean_opt = TRUE;
+			remove_gtmMaster(clean_opt);
+		}
+		else if (TestToken("slave"))
+		{
+			if (GetToken() && TestToken("clean"))
+				clean_opt = TRUE;
+			remove_gtmSlave(clean_opt);
+		}
+		else 
+		{
+			elog(ERROR, "ERROR: you can specify only master/slave to remove gtm command. %s is invalid.\n", token);
 			return;
 		}
-		if (GetToken() && TestToken("clean"))
-			clean_opt = TRUE;
-		remove_gtmSlave(clean_opt);
 	}
 	else if (TestToken("gtm_proxy"))
 	{
 		GetAndSet(name, "ERROR: please specify gtm proxy name to remove.\n");
-		if (TestToken("clean"))
-		{
+ 		if (GetToken() && TestToken("clean"))
 			clean_opt = TRUE;
-			freeAndReset(name);
-			GetAndSet(name, "ERROR: please specify gtm proxy name to remove.\n");
-		}
 		remove_gtmProxy(name, clean_opt	);
 		freeAndReset(name);
 	}
@@ -1054,24 +1077,16 @@ static void do_remove_command(char *line)
 		if (TestToken("master"))
 		{
 			GetAndSet(name, "ERROR: please specify the name of the coordinator master\n");
-			if (TestToken("clean"))
-			{
+			if (GetToken() && TestToken("clean"))
 				clean_opt = TRUE;
-				freeAndReset(name);
-				GetAndSet(name, "ERROR: please specify the name of the coordinator master\n");
-			}
 			remove_coordinatorMaster(name, clean_opt);
 			freeAndReset(name);
 		}
 		else
 		{
 			GetAndSet(name, "ERROR: please specify the name of the coordinator slave\n");
-			if (TestToken("clean"))
-			{
+			if (GetToken() && TestToken("clean"))
 				clean_opt = TRUE;
-				freeAndReset(name);
-				GetAndSet(name, "ERROR: please specify the name of the coordinator master\n");
-			}
 			remove_coordinatorSlave(name, clean_opt);
 			freeAndReset(name);
 		}
@@ -1086,24 +1101,16 @@ static void do_remove_command(char *line)
 		if (TestToken("master"))
 		{
 			GetAndSet(name, "ERROR: please specify the name of the datanode master\n");
-			if (TestToken("clean"))
-			{
+			if (GetToken() && TestToken("clean"))
 				clean_opt = TRUE;
-				freeAndReset(name);
-				GetAndSet(name, "ERROR: please specify the name of the coordinator master\n");
-			}
 			remove_datanodeMaster(name, clean_opt);
 			freeAndReset(name);
 		}
 		else
 		{
 			GetAndSet(name, "ERROR: please specify the name of the datanode slave\n");
-			if (TestToken("clean"))
-			{
+			if (GetToken() && TestToken("clean"))
 				clean_opt = TRUE;
-				freeAndReset(name);
-				GetAndSet(name, "ERROR: please specify the name of the coordinator master\n");
-			}
 			remove_datanodeSlave(name, clean_opt);
 			freeAndReset(name);
 		}
@@ -1113,13 +1120,6 @@ static void do_remove_command(char *line)
 	return;
 }
 		
-
-
-
-
-			
-
-
 static char *m_Option;
 
 static char *handle_m_option(char *line, char **m_option)
@@ -2336,7 +2336,9 @@ int do_singleLine(char *buf, char *wkline)
 			if (TestToken("config"))
 				GetToken();
 
-			if (TestToken("minimal"))
+			if (TestToken("empty"))
+				config_type = CONFIG_EMPTY;
+			else if (TestToken("minimal"))
 				config_type = CONFIG_MINIMAL;
 			else if (TestToken("complete"))
 				config_type = CONFIG_COMPLETE;
@@ -2974,11 +2976,64 @@ get_any_available_coord(int except)
 		if (!is_none(aval(VAR_coordMasterServers)[ii]))
 		{
 			if (pingNode(aval(VAR_coordMasterServers)[ii],
-						aval(VAR_coordPorts)[ii]) == 0)
+						 aval(VAR_coordPorts)[ii]) == 0)
+				return ii;
+		}
+	}
+
+	/*
+	 * this could be the first coordinator that is being added.
+	 * This call would happen *after* expanding the array to
+	 * accomodate the new coordinator. Hence we check for size
+	 * being more than 1
+	 */
+	if (arraySizeName(VAR_coordNames) > 1)
+	{
+		for (ii = 0; aval(VAR_coordNames)[ii]; ii++)
+		{
+			if (!is_none(aval(VAR_coordNames)[ii]))
+			{
+				elog(ERROR, "ERROR: failed to find any running coordinator");
+				return -1;
+			}
+		}
+	}
+	return -1;
+}
+ 
+int
+get_any_available_datanode(int except)
+{
+	int ii;
+	for (ii = 0; aval(VAR_datanodeMasterServers)[ii]; ii++)
+	{
+		if (ii == except)
+			continue;
+
+		if (!is_none(aval(VAR_datanodeMasterServers)[ii]))
+		{
+			if (pingNode(aval(VAR_datanodeMasterServers)[ii],
+						 aval(VAR_datanodePorts)[ii]) == 0)
 				return ii;
 		}
 	}
 
-	elog(ERROR, "ERROR: failed to find any running coordinator");
+	/*
+	 * this could be the first datanode that is being added.
+	 * This call would happen *after* expanding the array to
+	 * accomodate the new datanode. Hence we check for size
+	 * being more than 1
+	 */
+	if (arraySizeName(VAR_datanodeNames) > 1)
+	{
+		for (ii = 0; aval(VAR_datanodeNames)[ii]; ii++)
+		{
+			if (!is_none(aval(VAR_datanodeNames)[ii]))
+			{
+				elog(ERROR, "ERROR: failed to find any running datanode");
+				return -1;
+			}
+		}
+	}
 	return -1;
 }
diff --git a/contrib/pgxc_ctl/do_command.h b/contrib/pgxc_ctl/do_command.h
index b24d64a8ef..b39a272516 100644
--- a/contrib/pgxc_ctl/do_command.h
+++ b/contrib/pgxc_ctl/do_command.h
@@ -15,4 +15,5 @@ extern int forceInit;
 extern void do_command(FILE *inf, FILE *outf);
 extern int  do_singleLine(char *buf, char *wkline);
 extern int get_any_available_coord(int except);
+extern int get_any_available_datanode(int except);
 #endif /* DO_COMMAND_H */
diff --git a/contrib/pgxc_ctl/gtm_cmd.c b/contrib/pgxc_ctl/gtm_cmd.c
index 69faf037a6..5f92fb7965 100644
--- a/contrib/pgxc_ctl/gtm_cmd.c
+++ b/contrib/pgxc_ctl/gtm_cmd.c
@@ -47,13 +47,12 @@ static char date[MAXTOKEN+1];
 /*
  * Init gtm master -----------------------------------------------------------------
  */
-cmd_t *prepare_initGtmMaster(void)
+cmd_t *prepare_initGtmMaster(bool stop)
 {
 	cmd_t *cmdInitGtmMaster, *cmdGtmConf, *cmdGxid;
 	char date[MAXTOKEN+1];
 	FILE *f;
 	char **fileList = NULL;
-	int result;
 	char remoteDirCheck[MAXPATH * 2 + 128];
 
 	remoteDirCheck[0] = '\0';
@@ -111,13 +110,19 @@ cmd_t *prepare_initGtmMaster(void)
 	/* Setup GTM with appropriate GXID value */
 	
 	appendCmdEl(cmdGtmConf, (cmdGxid = initCmd(sval(VAR_gtmMasterServer))));
-	snprintf(newCommand(cmdGxid), MAXLINE,
+ 	if (stop)
+		snprintf(newCommand(cmdGxid), MAXLINE,
 			 "(gtm -x 2000 -D %s &); sleep 1; gtm_ctl stop -Z gtm -D %s",
 			 sval(VAR_gtmMasterDir), sval(VAR_gtmMasterDir));
+ 	else
+ 		snprintf(newCommand(cmdGxid), MAXLINE,
+ 			 "(gtm -x 2000 -D %s &); sleep 1;",
+ 			 sval(VAR_gtmMasterDir));
 
 	return cmdInitGtmMaster;
 }
-int init_gtm_master(void)
+ 
+int init_gtm_master(bool stop)
 {
 	int rc;
 	cmdList_t *cmdList;
@@ -128,7 +133,7 @@ int init_gtm_master(void)
 
 	/* Kill current gtm, build work directory and run initgtm */
 
-	if ((cmd = prepare_initGtmMaster()))
+	if ((cmd = prepare_initGtmMaster(stop)))
 		addCmd(cmdList, cmd);
 
 	rc = doCmdList(cmdList);
@@ -138,6 +143,67 @@ int init_gtm_master(void)
 }
 
 /*
+ * Add gtm master
+ *
+ */
+int add_gtmMaster(char *name, char *host, int port, char *dir)
+{
+	char port_s[MAXTOKEN+1];
+	char date[MAXTOKEN+1];
+	FILE *f;
+	int	rc;
+
+	if (is_none(name))
+	{
+		elog(ERROR, "ERROR: Cannot add gtm master with the name \"none\".\n");
+		return 1;
+	}
+	if (is_none(host))
+	{
+		elog(ERROR, "ERROR: Cannot add gtm master with the name \"none\".\n");
+		return 1;
+	}
+	if (is_none(dir))
+	{
+		elog(ERROR, "ERROR: Cannot add gtm master with the directory \"none\".\n");
+		return 1;
+	}
+	if (checkSpecificResourceConflict(name, host, port, dir, TRUE))
+	{
+		elog(ERROR, "ERROR: New specified name:%s, host:%s, port:%d and dir:\"%s\" conflicts with existing node.\n",
+			 name, host, port, dir);
+		return 1;
+	}
+	assign_sval(VAR_gtmName, Strdup(name));
+	assign_sval(VAR_gtmMasterServer, Strdup(host));
+	snprintf(port_s, MAXTOKEN, "%d", port);
+	assign_sval(VAR_gtmMasterPort, Strdup(port_s));
+	assign_sval(VAR_gtmMasterDir, Strdup(dir));
+	makeServerList();
+	if ((f = fopen(pgxc_ctl_config_path, "a")) == NULL)
+	{
+		/* Should it be panic? */
+		elog(ERROR, "ERROR: cannot open configuration file \"%s\", %s\n", pgxc_ctl_config_path, strerror(errno));
+		return 1;
+	}
+	fprintf(f, 
+			"#===================================================\n"
+			"# pgxc configuration file updated due to GTM master addition\n"
+			"#        %s\n", 
+			timeStampString(date, MAXTOKEN+1));
+	fprintSval(f, VAR_gtmName);
+	fprintSval(f, VAR_gtmMasterServer);
+	fprintSval(f, VAR_gtmMasterPort);
+	fprintSval(f, VAR_gtmMasterDir);
+	fprintf(f, "%s","#----End of reconfiguration -------------------------\n");
+	fclose(f);
+	backup_configuration();
+	if ((rc = init_gtm_master(false)) != 0)
+		return rc;
+	return(start_gtm_master());
+}
+
+/*
  * Add gtm slave: to be used after all the configuration is done.
  *
  * This function only maintains internal configuration, updte configuration file,
@@ -208,6 +274,57 @@ int add_gtmSlave(char *name, char *host, int port, char *dir)
 	return(start_gtm_slave());
 }
 
+int remove_gtmMaster(bool clean_opt)
+{
+	FILE *f;
+
+	/* Check if gtm_slave is configured */
+	if (!sval(VAR_gtmMasterServer) || is_none(sval(VAR_gtmMasterServer)))
+	{
+		elog(ERROR, "ERROR: GTM master is not configured.\n");
+		return 1;
+	}
+
+	/* Check if gtm_master is running and stop if yes */
+	if (do_gtm_ping(sval(VAR_gtmMasterServer), atoi(sval(VAR_gtmMasterPort))) == 0)
+		stop_gtm_master();
+
+	elog(NOTICE, "Removing gtm master.\n");
+	/* Clean */
+	if (clean_opt)
+		clean_gtm_master();
+	/* Reconfigure */
+	reset_var(VAR_gtmName);
+	assign_sval(VAR_gtmName, Strdup("none"));
+	reset_var(VAR_gtmMasterServer);
+	assign_sval(VAR_gtmMasterServer, Strdup("none"));
+	reset_var(VAR_gtmMasterPort);
+	assign_sval(VAR_gtmMasterPort, Strdup("-1"));
+	reset_var(VAR_gtmMasterDir);
+	assign_sval(VAR_gtmMasterDir, Strdup("none"));
+	/* Write the configuration file and bakup it */
+	if ((f = fopen(pgxc_ctl_config_path, "a")) == NULL)
+	{
+		/* Should it be panic? */
+		elog(ERROR, "ERROR: cannot open configuration file \"%s\", %s\n", pgxc_ctl_config_path, strerror(errno));
+		return 1;
+	}
+	fprintf(f, 
+			"#===================================================\n"
+			"# pgxc configuration file updated due to GTM master removal\n"
+			"#        %s\n",
+			timeStampString(date, MAXTOKEN+1));
+	fprintSval(f, VAR_gtmName);
+	fprintSval(f, VAR_gtmMasterServer);
+	fprintSval(f, VAR_gtmMasterPort);
+	fprintSval(f, VAR_gtmMasterDir);
+	fprintf(f, "%s", "#----End of reconfiguration -------------------------\n");
+	fclose(f);
+	backup_configuration();
+	elog(NOTICE, "Done.\n");
+	return 0;
+}
+
 int remove_gtmSlave(bool clean_opt)
 {
 	FILE *f;
diff --git a/contrib/pgxc_ctl/gtm_cmd.h b/contrib/pgxc_ctl/gtm_cmd.h
index 79d0f06290..07f7f8dea9 100644
--- a/contrib/pgxc_ctl/gtm_cmd.h
+++ b/contrib/pgxc_ctl/gtm_cmd.h
@@ -14,16 +14,18 @@
 #include "gtm/gtm_c.h"
 #include "utils.h"
 
-extern int init_gtm_master(void);
+extern int init_gtm_master(bool stop);
 extern int init_gtm_slave(void);
 extern int init_gtm_proxy(char **nodeList);
 extern int init_gtm_proxy_all(void);
-extern cmd_t *prepare_initGtmMaster(void);
+extern cmd_t *prepare_initGtmMaster(bool stop);
 extern cmd_t *prepare_initGtmSlave(void);
 extern cmd_t *prepare_initGtmProxy(char *nodeName);
 
+extern int add_gtmMaster(char *name, char *host, int port, char *dir);
 extern int add_gtmSlave(char *name, char *host, int port, char *dir);
 extern int add_gtmProxy(char *name, char *host, int port, char *dir);
+extern int remove_gtmMaster(bool clean_opt);
 extern int remove_gtmSlave(bool clean_opt);
 extern int remove_gtmProxy(char *name, bool clean_opt);
 
diff --git a/contrib/pgxc_ctl/make_signature b/contrib/pgxc_ctl/make_signature
index 713cd61897..09990714a9 100755
--- a/contrib/pgxc_ctl/make_signature
+++ b/contrib/pgxc_ctl/make_signature
@@ -169,3 +169,34 @@ NULL
 EOF
 
 rm pgxc_ctl_conf_part.wk
+
+cp pgxc_ctl_conf_part_empty pgxc_ctl_conf_empty.wk
+
+ex pgxc_ctl_conf_empty.wk <<EOF
+%s/"/\\\"/ge
+w
+%s/^\(.*\)$/"\1",/e
+%s/^"#ifdef XCP",$/#ifdef XCP/e
+%s/^"#endif",$/#endif/e
+wq
+EOF
+
+cat >> pgxc_ctl_bash.c <<EOF
+/*
+ * Prototype of pgxc_ctl configuration file.
+ *
+ * It should be self descripting.   Can be extracted to your pgxc_ctl
+ * work directory with 'prepare empty' command.
+ */
+
+char *pgxc_ctl_conf_prototype_empty[] = {
+EOF
+
+cat pgxc_ctl_conf_empty.wk >> pgxc_ctl_bash.c
+
+cat >> pgxc_ctl_bash.c <<EOF
+NULL
+};
+EOF
+
+rm pgxc_ctl_conf_empty.wk
diff --git a/contrib/pgxc_ctl/monitor.c b/contrib/pgxc_ctl/monitor.c
index bc7608ad5c..1483504269 100644
--- a/contrib/pgxc_ctl/monitor.c
+++ b/contrib/pgxc_ctl/monitor.c
@@ -59,7 +59,10 @@ static void printResult(int res, char *what, char *name)
 
 static void monitor_gtm_master(void)
 {
-	return(printResult(do_gtm_ping(sval(VAR_gtmMasterServer), atoi(sval(VAR_gtmMasterPort))), "gtm master", NULL));
+	if (doesExist(VAR_gtmMasterServer, 0) && doesExist(VAR_gtmMasterPort, 0))
+		return(printResult(do_gtm_ping(sval(VAR_gtmMasterServer), atoi(sval(VAR_gtmMasterPort))), "gtm master", NULL));
+	else
+		elog(NOTICE, "GTM master not running\n");
 }
 
 static void monitor_gtm_slave(void)
diff --git a/contrib/pgxc_ctl/pgxc_ctl_conf_part_empty b/contrib/pgxc_ctl/pgxc_ctl_conf_part_empty
new file mode 100644
index 0000000000..dcc9ccb6be
--- /dev/null
+++ b/contrib/pgxc_ctl/pgxc_ctl_conf_part_empty
@@ -0,0 +1,267 @@
+#!/usr/bin/env bash
+#
+# Postgres-XC Configuration file for pgxc_ctl utility. 
+#
+# Configuration file can be specified as -c option from pgxc_ctl command.   Default is
+# $PGXC_CTL_HOME/pgxc_ctl.org.
+#
+# This is bash script so you can make any addition for your convenience to configure
+# your Postgres-XC cluster.
+#
+# Please understand that pgxc_ctl provides only a subset of configuration which pgxc_ctl
+# provide.  Here's several several assumptions/restrictions pgxc_ctl depends on.
+#
+# 1) All the resources of pgxc nodes has to be owned by the same user.   Same user means
+#    user with the same user name.  User ID may be different from server to server.
+#    This must be specified as a variable $pgxcOwner.
+#
+# 2) All the servers must be reacheable via ssh without password.   It is highly recommended
+#    to setup key-based authentication among all the servers.
+#
+# 3) All the databases in coordinator/datanode has at least one same superuser.  Pgxc_ctl
+#    uses this user to connect to coordinators and datanodes.   Again, no password should
+#    be used to connect.  You have many options to do this, pg_hba.conf, pg_ident.conf and
+#    others.  Pgxc_ctl provides a way to configure pg_hba.conf but not pg_ident.conf.   This
+#    will be implemented in the later releases.
+#
+# 4) Gtm master and slave can have different port to listen, while coordinator and datanode
+#    slave should be assigned the same port number as master.
+#
+# 5) Port nuber of a coordinator slave must be the same as its master.
+#
+# 6) Master and slave are connected using synchronous replication.  Asynchronous replication
+#    have slight (almost none) chance to bring total cluster into inconsistent state.
+#    This chance is very low and may be negligible.  Support of asynchronous replication
+#    may be supported in the later release.
+#
+# 7) Each coordinator and datanode can have only one slave each.  Cascaded replication and
+#    multiple slave are not supported in the current pgxc_ctl.
+#
+# 8) Killing nodes may end up with IPC resource leak, such as semafor and shared memory.
+#    Only listening port (socket) will be cleaned with clean command.
+#
+# 9) Backup and restore are not supported in pgxc_ctl at present.   This is a big task and
+#    may need considerable resource.
+#
+#========================================================================================
+#
+#
+# pgxcInstallDir variable is needed if you invoke "deploy" command from pgxc_ctl utility.
+# If don't you don't need this variable.
+pgxcInstallDir=$HOME/pgxc
+#---- OVERALL -----------------------------------------------------------------------------
+#
+pgxcOwner=$USER		# owner of the Postgres-XC databaseo cluster.  Here, we use this
+						# both as linus user and database user.  This must be
+						# the super user of each coordinator and datanode.
+pgxcUser=$pgxcOwner		# OS user of Postgres-XC owner
+
+tmpDir=/tmp					# temporary dir used in XC servers
+localTmpDir=$tmpDir			# temporary dir used here locally
+
+configBackup=n					# If you want config file backup, specify y to this value.
+configBackupHost=pgxc-linker	# host to backup config file
+configBackupDir=$HOME/pgxc		# Backup directory
+configBackupFile=pgxc_ctl.bak	# Backup file name --> Need to synchronize when original changed.
+
+dataDirRoot=$HOME/DATA/pgxl/nodes
+
+#---- GTM ------------------------------------------------------------------------------------
+
+# GTM is mandatory.  You must have at least (and only) one GTM master in your Postgres-XC cluster.
+# If GTM crashes and you need to reconfigure it, you can do it by pgxc_update_gtm command to update
+# GTM master with others.   Of course, we provide pgxc_remove_gtm command to remove it.  This command
+# will not stop the current GTM.  It is up to the operator.
+
+#---- Overall -------
+gtmName=()
+
+#---- GTM Master -----------------------------------------------
+
+#---- Overall ----
+gtmMasterServer=()
+gtmMasterPort=()
+gtmMasterDir=()
+
+#---- Configuration ---
+gtmExtraConfig=() # Will be added gtm.conf for both Master and Slave (done at initilization only)
+gtmMasterSpecificExtraConfig=()	# Will be added to Master's gtm.conf (done at initialization only)
+
+#---- GTM Slave -----------------------------------------------
+
+# Because GTM is a key component to maintain database consistency, you may want to configure GTM slave
+# for backup.
+
+#---- Overall ------
+gtmSlave=n					# Specify y if you configure GTM Slave.   Otherwise, GTM slave will not be configured and
+							# all the following variables will be reset.
+gtmSlaveName=()
+gtmSlaveServer=()		# value none means GTM slave is not available.  Give none if you don't configure GTM Slave.
+gtmSlavePort=()			# Not used if you don't configure GTM slave.
+gtmSlaveDir=()	# Not used if you don't configure GTM slave.
+# Please note that when you have GTM failover, then there will be no slave available until you configure the slave
+# again. (pgxc_add_gtm_slave function will handle it)
+
+#---- Configuration ----
+gtmSlaveSpecificExtraConfig=() # Will be added to Slave's gtm.conf (done at initialization only)
+
+#---- GTM Proxy -------------------------------------------------------------------------------------------------------
+# GTM proxy will be selected based upon which server each component runs on.
+# When fails over to the slave, the slave inherits its master's gtm proxy.  It should be
+# reconfigured based upon the new location.
+#
+# To do so, slave should be restarted.   So pg_ctl promote -> (edit postgresql.conf and recovery.conf) -> pg_ctl restart
+#
+# You don't have to configure GTM Proxy if you dont' configure GTM slave or you are happy if every component connects
+# to GTM Master directly.  If you configure GTL slave, you must configure GTM proxy too.
+
+#---- Shortcuts ------
+gtmProxyDir=()
+
+#---- Overall -------
+gtmProxy=()				# Specify y if you conifugre at least one GTM proxy.   You may not configure gtm proxies
+						# only when you dont' configure GTM slaves.
+						# If you specify this value not to y, the following parameters will be set to default empty values.
+						# If we find there're no valid Proxy server names (means, every servers are specified
+						# as none), then gtmProxy value will be set to "n" and all the entries will be set to
+						# empty values.
+gtmProxyNames=()	# No used if it is not configured
+gtmProxyServers=()			# Specify none if you dont' configure it.
+gtmProxyPorts=()				# Not used if it is not configured.
+gtmProxyDirs=()	# Not used if it is not configured.
+
+#---- Configuration ----
+gtmPxyExtraConfig=n		# Extra configuration parameter for gtm_proxy.  Coordinator section has an example.
+
+#---- Coordinators ----------------------------------------------------------------------------------------------------
+
+#---- shortcuts ----------
+coordMasterDir=$dataDirRoot/coord_master
+coordSlaveDir=$HOME/coord_slave
+coordArchLogDir=$HOME/coord_archlog
+
+#---- Overall ------------
+coordNames=()		# Master and slave use the same name
+coordPorts=()			# Master server listening ports
+poolerPorts=()			# Master pooler ports
+coordPgHbaEntries=(::1/128)	# Assumes that all the coordinator (master/slave) accepts
+												# the same connection
+												# This entry allows only $pgxcOwner to connect.
+												# If you'd like to setup another connection, you should
+												# supply these entries through files specified below.
+#coordPgHbaEntries=(127.0.0.1/32)	# Same as above but for IPv4 connections
+
+#---- Master -------------
+coordMasterServers=()		# none means this master is not available
+coordMasterDirs=()
+coordMaxWALsender=5	# max_wal_senders: needed to configure slave. If zero value is specified,
+						# it is expected to supply this parameter explicitly by external files
+						# specified in the following.	If you don't configure slaves, leave this value to zero.
+coordMaxWALSenders=()
+						# max_wal_senders configuration for each coordinator.
+
+#---- Slave -------------
+coordSlave=n			# Specify y if you configure at least one coordiantor slave.  Otherwise, the following
+						# configuration parameters will be set to empty values.
+						# If no effective server names are found (that is, every servers are specified as none),
+						# then coordSlave value will be set to n and all the following values will be set to
+						# empty values.
+coordSlaveSync=n		# Specify to connect with synchronized mode.
+coordSlaveServers=()			# none means this slave is not available
+coordSlavePorts=()			# coordinator slave listening ports
+coordSlavePoolerPorts=()			# coordinator slave pooler ports
+coordSlaveDirs=()
+coordArchLogDirs=()
+
+#---- Configuration files---
+# Need these when you'd like setup specific non-default configuration 
+# These files will go to corresponding files for the master.
+# You may supply your bash script to setup extra config lines and extra pg_hba.conf entries 
+# Or you may supply these files manually.
+coordExtraConfig=coordExtraConfig	# Extra configuration file for coordinators.  
+						# This file will be added to all the coordinators'
+						# postgresql.conf
+# Pleae note that the following sets up minimum parameters which you may want to change.
+# You can put your postgresql.conf lines here.
+cat > $coordExtraConfig <<EOF
+#================================================
+# Added to all the coordinator postgresql.conf
+# Original: $coordExtraConfig
+log_destination = 'stderr'
+logging_collector = on
+log_directory = 'pg_log'
+listen_addresses = '*'
+max_connections = 100
+hot_standby = off
+EOF
+
+# Additional Configuration file for specific coordinator master.
+# You can define each setting by similar means as above.
+coordSpecificExtraConfig=()
+coordSpecificExtraPgHba=()
+
+#---- Datanodes -------------------------------------------------------------------------------------------------------
+
+#---- Shortcuts --------------
+datanodeMasterDir=$dataDirRoot/dn_master
+datanodeSlaveDir=$dataDirRoot/dn_slave
+datanodeArchLogDir=$dataDirRoot/datanode_archlog
+
+#---- Overall ---------------
+primaryDatanode= # Primary Node.
+datanodeNames=()
+datanodePorts=()	# Master and slave use the same port!
+datanodePoolerPorts=()	# Master and slave use the same port!
+datanodePgHbaEntries=(::1/128)	# Assumes that all the coordinator (master/slave) accepts
+										# the same connection
+										# This list sets up pg_hba.conf for $pgxcOwner user.
+										# If you'd like to setup other entries, supply them
+										# through extra configuration files specified below.
+#datanodePgHbaEntries=(127.0.0.1/32)	# Same as above but for IPv4 connections
+
+#---- Master ----------------
+datanodeMasterServers=()	# none means this master is not available.
+													# This means that there should be the master but is down.
+													# The cluster is not operational until the master is
+													# recovered and ready to run.	
+datanodeMasterDirs=()
+datanodeMaxWalSender=5								# max_wal_senders: needed to configure slave. If zero value is 
+													# specified, it is expected this parameter is explicitly supplied
+													# by external configuration files.
+													# If you don't configure slaves, leave this value zero.
+datanodeMaxWALSenders=()
+						# max_wal_senders configuration for each datanode
+
+#---- Slave -----------------
+datanodeSlave=n			# Specify y if you configure at least one coordiantor slave.  Otherwise, the following
+						# configuration parameters will be set to empty values.
+						# If no effective server names are found (that is, every servers are specified as none),
+						# then datanodeSlave value will be set to n and all the following values will be set to
+						# empty values.
+datanodeSlaveServers=()	# value none means this slave is not available
+datanodeSlavePorts=()	# Master and slave use the same port!
+datanodeSlavePoolerPorts=()	# Master and slave use the same port!
+#datanodeSlaveSync=y		# If datanode slave is connected in synchronized mode
+datanodeSlaveDirs=()
+datanodeArchLogDirs=()
+
+# ---- Configuration files ---
+# You may supply your bash script to setup extra config lines and extra pg_hba.conf entries here.
+# These files will go to corresponding files for the master.
+# Or you may supply these files manually.
+datanodeExtraConfig=datanodeExtraConfig	
+cat > $datanodeExtraConfig <<EOF
+#================================================
+# Added to all the datanode postgresql.conf
+# Original: $datanodeExtraConfig
+log_destination = 'stderr'
+logging_collector = on
+log_directory = 'pg_log'
+listen_addresses = '*'
+max_connections = 100
+hot_standby = off
+EOF
+# Additional Configuration file for specific datanode master.
+# You can define each setting by similar means as above.
+datanodeSpecificExtraConfig=()
+datanodeSpecificExtraPgHba=()
diff --git a/contrib/pgxc_ctl/variables.h b/contrib/pgxc_ctl/variables.h
index 002336c6a3..d408c86fe1 100644
--- a/contrib/pgxc_ctl/variables.h
+++ b/contrib/pgxc_ctl/variables.h
@@ -74,6 +74,8 @@ void var_assign(char **dest, char *src);
 char  *listValue(char *name);
 int extendVar(char *name, int newSize, char *def_value);
 int doesExist(char *name, int idx);
+void assign_arrayEl_internal(char *name, int idx, char *val, char *pad,
+		int extend);
 
 #define AddMember(a, b) do{if((a) == NULL) (a) = Malloc0(sizeof(char *)); (a) = add_member((a), (b));}while(0)
 void clean_array(char **array);