Allow separate settings for coordinator slave server and pooler ports.

Existing pgxc_ctl mechanism does not allow user to set different ports (server
and pooler) on standby and standby must use the same configuration as the
master. While this definitely makes management and failover easier, its also
very constraining, especially during testing.

In passing, also fix assorted bugs with coordinator failover and failback.

12 files changed, 168 insertions, 31 deletions

diff --git a/contrib/pgxc_ctl/config.c b/contrib/pgxc_ctl/config.c
index 92e28398c2..6a4b2c9af3 100644
--- a/contrib/pgxc_ctl/config.c
+++ b/contrib/pgxc_ctl/config.c
@@ -350,6 +350,8 @@ static void emptyCoordSlaves()
 
 	reset_var_val(VAR_coordSlave, "n");
 	reset_var(VAR_coordSlaveServers);
+	reset_var(VAR_coordSlavePorts);
+	reset_var(VAR_coordSlavePoolerPorts);
 	reset_var(VAR_coordSlaveDirs);
 	reset_var(VAR_coordArchLogDirs);
 	for (ii = 0; ii < arraySizeName(VAR_coordNames); ii++)
@@ -625,7 +627,7 @@ int checkPortConflict(char *host, int port)
 	if (isVarYes(VAR_coordSlave))
 		for (ii = 0; aval(VAR_coordNames)[ii]; ii++)
 			if (doesExist(VAR_coordSlaveServers, ii) && !is_none(aval(VAR_coordSlaveServers)[ii]) && 
-				(strcasecmp(host, aval(VAR_coordSlaveServers)[ii]) == 0) && (atoi(aval(VAR_coordPorts)[ii]) == port))
+				(strcasecmp(host, aval(VAR_coordSlaveServers)[ii]) == 0) && (atoi(aval(VAR_coordSlavePorts)[ii]) == port))
 				return 1;
 	/* Datanode Master */
 	for (ii = 0; aval(VAR_datanodeNames)[ii]; ii++)
@@ -840,6 +842,8 @@ static void verifyResource(void)
 							   NULL};
 	char *coordSlaveVars[] = {VAR_coordNames, 
 							  VAR_coordSlaveServers, 
+							  VAR_coordSlavePorts, 
+							  VAR_coordSlavePoolerPorts, 
 							  VAR_coordSlaveDirs, 
 							  VAR_coordArchLogDirs, 
 							  NULL};
@@ -933,7 +937,7 @@ static void verifyResource(void)
 	/* GTM and coordinator slaves, if any */
 	if (isVarYes(VAR_coordSlave))
 		checkResourceConflict(VAR_gtmName, VAR_gtmMasterServer, VAR_gtmMasterPort, NULL, VAR_gtmMasterDir,
-							  VAR_coordNames, VAR_coordSlaveServers, VAR_coordPorts, NULL, VAR_coordSlaveDirs, TRUE, TRUE);
+							  VAR_coordNames, VAR_coordSlaveServers, VAR_coordSlavePorts, NULL, VAR_coordSlaveDirs, TRUE, TRUE);
 	/* GTM and datanode masters */
 	checkResourceConflict(VAR_gtmName, VAR_gtmMasterServer, VAR_gtmMasterPort, NULL, VAR_gtmMasterDir,
 						  VAR_datanodeNames, VAR_datanodeMasterServers, VAR_datanodePorts, NULL, VAR_datanodeMasterDirs, TRUE, TRUE);
@@ -959,7 +963,8 @@ static void verifyResource(void)
 		/* GTM slave and coordinator slaves, if any */
 		if (isVarYes(VAR_coordSlave))
 			checkResourceConflict(VAR_gtmName, VAR_gtmSlaveServer, VAR_gtmSlavePort, NULL, VAR_gtmSlaveDir,
-								  VAR_coordNames, VAR_coordSlaveServers, VAR_coordPorts, VAR_poolerPorts, VAR_coordSlaveDirs,
+								  VAR_coordNames, VAR_coordSlaveServers,
+								  VAR_coordSlavePorts, VAR_coordSlavePoolerPorts, VAR_coordSlaveDirs,
 								  TRUE, TRUE);
 		/* GTM slave and datanode masters */
 		checkResourceConflict(VAR_gtmName, VAR_gtmSlaveServer, VAR_gtmSlavePort, NULL, VAR_gtmSlaveDir,
@@ -983,7 +988,7 @@ static void verifyResource(void)
 		/* GTM proxy and coordinator slaves, if any */
 		if (sval(VAR_coordSlave) && (strcmp(sval(VAR_coordSlave), "y") == 0))
 			checkResourceConflict(VAR_gtmProxyNames, VAR_gtmProxyServers, VAR_gtmProxyPorts, NULL, VAR_gtmProxyDirs,
-								  VAR_coordNames, VAR_coordSlaveServers, VAR_coordPorts, VAR_poolerPorts, VAR_coordSlaveDirs,
+								  VAR_coordNames, VAR_coordSlaveServers, VAR_coordSlavePorts, VAR_coordSlavePoolerPorts, VAR_coordSlaveDirs,
 								  TRUE, TRUE);
 		/* GTM proxy and datanode masters */
 			checkResourceConflict(VAR_gtmProxyNames, VAR_gtmProxyServers, VAR_gtmProxyPorts, NULL, VAR_gtmProxyDirs,
@@ -1001,7 +1006,8 @@ static void verifyResource(void)
 	/* Coordinator master and coordinator slaves, if any */
 	if (isVarYes(VAR_coordSlave))
 		checkResourceConflict(VAR_coordNames, VAR_coordMasterServers, VAR_coordPorts, VAR_poolerPorts, VAR_coordMasterDirs,
-							  VAR_coordNames, VAR_coordSlaveServers, VAR_coordPorts, VAR_poolerPorts, VAR_coordSlaveDirs,
+							  VAR_coordNames, VAR_coordSlaveServers,
+							  VAR_coordSlavePorts, VAR_coordSlavePoolerPorts, VAR_coordSlaveDirs,
 							  TRUE, FALSE);
 	/* Coordinator masters and datanode masters */
 	checkResourceConflict(VAR_coordNames, VAR_coordMasterServers, VAR_coordPorts, VAR_poolerPorts, VAR_coordMasterDirs,
@@ -1019,12 +1025,12 @@ static void verifyResource(void)
 	if (isVarYes(VAR_coordSlave))
 	{
 		/* Coordinator slave and datanode masters */
-		checkResourceConflict(VAR_coordNames, VAR_coordSlaveServers, VAR_coordPorts, VAR_poolerPorts, VAR_coordSlaveDirs,
+		checkResourceConflict(VAR_coordNames, VAR_coordSlaveServers, VAR_coordSlavePorts, VAR_coordSlavePoolerPorts, VAR_coordSlaveDirs,
 							  VAR_datanodeNames, VAR_datanodeSlaveServers, VAR_datanodePorts, NULL, VAR_datanodeSlaveDirs,
 							  FALSE, TRUE);
 		/* Coordinator slave and datanode slave, if any */
 		if (isVarYes(VAR_datanodeSlave))
-			checkResourceConflict(VAR_coordNames, VAR_coordSlaveServers, VAR_coordPorts, VAR_poolerPorts, VAR_coordSlaveDirs,
+			checkResourceConflict(VAR_coordNames, VAR_coordSlaveServers, VAR_coordSlavePorts, VAR_coordSlavePoolerPorts, VAR_coordSlaveDirs,
 								  VAR_datanodeNames, VAR_datanodeSlaveServers,
 								  VAR_datanodeSlavePorts, NULL, VAR_datanodeSlaveDirs,
 								  TRUE, TRUE);
diff --git a/contrib/pgxc_ctl/coord_cmd.c b/contrib/pgxc_ctl/coord_cmd.c
index f4da1c9c84..7ffd1e1460 100644
--- a/contrib/pgxc_ctl/coord_cmd.c
+++ b/contrib/pgxc_ctl/coord_cmd.c
@@ -311,12 +311,15 @@ cmd_t *prepare_initCoordinatorSlave(char *nodeName)
 			"# Added to initialize the slave, %s\n"
 			"hot_standby = on\n"
 			"port = %s\n"
+			"pooler_port = %s\n"
 			"wal_level = minimal\n"
 			"archive_mode = off\n"
 			"archive_command = ''\n"
 			"max_wal_senders = 0\n"
 			"# End of Addition\n",
-			timeStampString(timestamp, MAXTOKEN), aval(VAR_coordPorts)[idx]);
+			timeStampString(timestamp, MAXTOKEN),
+			aval(VAR_coordSlavePorts)[idx],
+			aval(VAR_coordSlavePoolerPorts)[idx]);
 	fclose(f);
 	cmdPgConf->localStdin = Strdup(localStdin);
 	snprintf(newCommand(cmdPgConf), MAXLINE,
@@ -774,7 +777,7 @@ cmd_t *prepare_killCoordinatorSlave(char *nodeName)
 	{
 		elog(WARNING, "WARNING: pid for coordinator slave \"%s\" was not found.  Remove socket only.\n", nodeName);
 		snprintf(newCommand(cmd), MAXLINE,
-			 "rm -f /tmp/.s.'*'%d'*'", atoi(aval(VAR_coordPorts)[idx]));
+			 "rm -f /tmp/.s.'*'%d'*'", atoi(aval(VAR_coordSlavePorts)[idx]));
 	}
 	return(cmd);
 }
@@ -877,7 +880,7 @@ cmd_t *prepare_cleanCoordinatorSlave(char *nodeName)
 	snprintf(newCommand(cmd), MAXLINE,
 			 "rm -rf %s;mkdir -p %s;chmod 0700 %s; rm -f /tmp/.s.*%d*; rm -f /tmp/.s.*%d*",
 			 aval(VAR_coordSlaveDirs)[idx], aval(VAR_coordSlaveDirs)[idx], aval(VAR_coordSlaveDirs)[idx],
-			 atoi(aval(VAR_coordPorts)[idx]), atoi(aval(VAR_poolerPorts)[idx]));
+			 atoi(aval(VAR_coordSlavePorts)[idx]), atoi(aval(VAR_coordSlavePoolerPorts)[idx]));
 	return cmd;
 }
 
@@ -983,6 +986,8 @@ int add_coordinatorMaster(char *name, char *host, int port, int pooler, char *di
 	assign_arrayEl(VAR_coordMasterDirs, idx, dir, NULL);
 	assign_arrayEl(VAR_coordMaxWALSenders, idx, aval(VAR_coordMaxWALSenders)[0], "-1");	/* Could be vulnerable */
 	assign_arrayEl(VAR_coordSlaveServers, idx, "none", NULL);
+	assign_arrayEl(VAR_coordSlavePorts, idx, "none", NULL);
+	assign_arrayEl(VAR_coordSlavePoolerPorts, idx, "none", NULL);
 	assign_arrayEl(VAR_coordSlaveDirs, idx, "none", NULL);
 	assign_arrayEl(VAR_coordArchLogDirs, idx, "none", NULL);
 	assign_arrayEl(VAR_coordSpecificExtraConfig, idx, "none", NULL);
@@ -1020,6 +1025,8 @@ int add_coordinatorMaster(char *name, char *host, int port, int pooler, char *di
 	fprintAval(f, VAR_coordMaxWALSenders);
 	fprintSval(f, VAR_coordSlave);
 	fprintAval(f, VAR_coordSlaveServers);
+	fprintAval(f, VAR_coordSlavePorts);
+	fprintAval(f, VAR_coordSlavePoolerPorts);
 	fprintAval(f, VAR_coordSlaveDirs);
 	fprintAval(f, VAR_coordArchLogDirs);
 	fprintAval(f, VAR_coordSpecificExtraConfig);
@@ -1149,10 +1156,12 @@ int add_coordinatorMaster(char *name, char *host, int port, int pooler, char *di
 	return 0;
 }
 
-int add_coordinatorSlave(char *name, char *host, char *dir, char *archDir)
+int add_coordinatorSlave(char *name, char *host, int port, int pooler_port, char *dir, char *archDir)
 {
 	int idx;
 	FILE *f;
+	char port_s[MAXTOKEN+1];
+	char pooler_s[MAXTOKEN+1];
 
 	/* Check if the name is valid coordinator */
 	if ((idx = coordIdx(name)) < 0)
@@ -1176,7 +1185,7 @@ int add_coordinatorSlave(char *name, char *host, char *dir, char *archDir)
 	 * We don't check the name conflict here because acquiring valid coordiinator index means that
 	 * there's no name conflict.
 	 */
-	if (checkPortConflict(host, atoi(aval(VAR_coordPorts)[idx])))
+	if (checkPortConflict(host, port))
 	{
 		elog(ERROR, "ERROR: the port %s has already been used in the host %s.\n",  aval(VAR_coordPorts)[idx], host);
 		return 1;
@@ -1192,6 +1201,10 @@ int add_coordinatorSlave(char *name, char *host, char *dir, char *archDir)
 		elog(ERROR, "ERROR: Coordinator master %s is not running.\n", name);
 		return 1;
 	}
+
+	snprintf(port_s, MAXTOKEN, "%d", port);
+	snprintf(pooler_s, MAXTOKEN, "%d", pooler_port);
+	
 	/* Prepare the resources (directories) */
 	doImmediate(host, NULL, "rm -rf %s; mkdir -p %s;chmod 0700 %s", dir, dir, dir);
 	doImmediate(host, NULL, "rm -rf %s; mkdir -p %s;chmod 0700 %s", archDir, archDir, archDir);
@@ -1234,6 +1247,8 @@ int add_coordinatorSlave(char *name, char *host, char *dir, char *archDir)
 	/* Need an API to expand the array to desired size */
 	if ((extendVar(VAR_coordSlaveServers, idx, "none") != 0) ||
 		(extendVar(VAR_coordSlaveDirs, idx, "none")  != 0) ||
+		(extendVar(VAR_coordSlavePorts, idx, "none")  != 0) ||
+		(extendVar(VAR_coordSlavePoolerPorts, idx, "none")  != 0) ||
 		(extendVar(VAR_coordArchLogDirs, idx, "none") != 0))
 	{
 		elog(PANIC, "PANIC: Internal error, inconsitent coordinator information\n");
@@ -1242,6 +1257,8 @@ int add_coordinatorSlave(char *name, char *host, char *dir, char *archDir)
 	if (!isVarYes(VAR_coordSlave))
 		assign_sval(VAR_coordSlave, "y");
 	assign_arrayEl(VAR_coordSlaveServers, idx, host, NULL);
+	assign_arrayEl(VAR_coordSlavePorts, idx, port_s, NULL);
+	assign_arrayEl(VAR_coordSlavePoolerPorts, idx, pooler_s, NULL);
 	assign_arrayEl(VAR_coordSlaveDirs, idx, dir, NULL);
 	assign_arrayEl(VAR_coordArchLogDirs, idx, archDir, NULL);
 	/* Update the configuration file and backup it */
@@ -1258,6 +1275,8 @@ int add_coordinatorSlave(char *name, char *host, char *dir, char *archDir)
 			timeStampString(date, MAXTOKEN+1));
 	fprintSval(f, VAR_coordSlave);
 	fprintAval(f, VAR_coordSlaveServers);
+	fprintAval(f, VAR_coordSlavePorts);
+	fprintAval(f, VAR_coordSlavePoolerPorts);
 	fprintAval(f, VAR_coordArchLogDirs);
 	fprintAval(f, VAR_coordSlaveDirs);
 	fprintf(f, "%s", "#----End of reconfiguration -------------------------\n");
@@ -1294,12 +1313,15 @@ int add_coordinatorSlave(char *name, char *host, char *dir, char *archDir)
 			"# Added to initialize the slave, %s\n"
 			"hot_standby = on\n"
 			"port = %d\n"
+			"pooler_port = %d\n"
 			"wal_level = minimal\n"		/* WAL level --- minimal.   No cascade slave so far. */
 			"archive_mode = off\n"		/* No archive mode */
 			"archive_command = ''\n"	/* No archive mode */
 			"max_wal_senders = 0\n"		/* Minimum WAL senders */
 			"# End of Addition\n",
-			timeStampString(date, MAXTOKEN), atoi(aval(VAR_coordPorts)[idx]));
+			timeStampString(date, MAXTOKEN),
+			atoi(aval(VAR_coordSlavePorts)[idx]),
+			atoi(aval(VAR_coordSlavePoolerPorts)[idx]));
 	pclose(f);
 	/* Update the slave recovery.conf */
 	if ((f = pgxc_popen_w(host, "cat >> %s/recovery.conf", dir)) == NULL)
@@ -1467,6 +1489,8 @@ int remove_coordinatorMaster(char *name, int clean_opt)
 	assign_arrayEl(VAR_coordMasterServers, idx, "none", NULL);
 	assign_arrayEl(VAR_coordMaxWALSenders, idx, "0", "0");
 	assign_arrayEl(VAR_coordSlaveServers, idx, "none", NULL);
+	assign_arrayEl(VAR_coordSlavePorts, idx, "none", NULL);
+	assign_arrayEl(VAR_coordSlavePoolerPorts, idx, "none", NULL);
 	assign_arrayEl(VAR_coordSlaveDirs, idx, "none", NULL);
 	assign_arrayEl(VAR_coordArchLogDirs, idx, "none", NULL);
 	assign_arrayEl(VAR_coordSpecificExtraConfig, idx, "none", NULL);
@@ -1493,6 +1517,8 @@ int remove_coordinatorMaster(char *name, int clean_opt)
 	fprintAval(f, VAR_coordMasterServers);
 	fprintAval(f, VAR_coordMaxWALSenders);
 	fprintAval(f, VAR_coordSlaveServers);
+	fprintAval(f, VAR_coordSlavePorts);
+	fprintAval(f, VAR_coordSlavePoolerPorts);
 	fprintAval(f, VAR_coordSlaveDirs);
 	fprintAval(f, VAR_coordArchLogDirs);
 	fprintAval(f, VAR_coordSpecificExtraConfig);
@@ -1524,7 +1550,7 @@ int remove_coordinatorSlave(char *name, int clean_opt)
 		return 1;
 	}
 	AddMember(nodelist, name);
-	if (pingNode(aval(VAR_coordSlaveServers)[idx], aval(VAR_coordPorts)[idx]) == 0)
+	if (pingNode(aval(VAR_coordSlaveServers)[idx], aval(VAR_coordSlavePorts)[idx]) == 0)
 		stop_coordinator_slave(nodelist, "immediate");
 	{
 		FILE *f;
@@ -1552,6 +1578,8 @@ int remove_coordinatorSlave(char *name, int clean_opt)
 	 * Maintain variables
 	 */
 	assign_arrayEl(VAR_coordSlaveServers, idx, "none", NULL);
+	assign_arrayEl(VAR_coordSlavePorts, idx, "none", NULL);
+	assign_arrayEl(VAR_coordSlavePoolerPorts, idx, "none", NULL);
 	assign_arrayEl(VAR_coordSlaveDirs, idx, "none", NULL);
 	assign_arrayEl(VAR_coordArchLogDirs, idx, "none", NULL);
 	handle_no_slaves();
@@ -1571,6 +1599,8 @@ int remove_coordinatorSlave(char *name, int clean_opt)
 			timeStampString(date, MAXTOKEN));
 	fprintSval(f, VAR_coordSlave);
 	fprintAval(f, VAR_coordSlaveServers);
+	fprintAval(f, VAR_coordSlavePorts);
+	fprintAval(f, VAR_coordSlavePoolerPorts);
 	fprintAval(f, VAR_coordSlaveDirs);
 	fprintAval(f, VAR_coordArchLogDirs);
 	fclose(f);
@@ -1813,7 +1843,7 @@ cmd_t *prepare_stopCoordinatorSlave(char *nodeName, char *immediate)
 		elog(WARNING, "WARNING: %s is not a coordinator.\n", nodeName);
 		return(NULL);
 	}
-	if (pingNode(aval(VAR_coordMasterServers)[idx], aval(VAR_coordPorts)[idx]) == 0)
+	if (pingNode(aval(VAR_coordMasterServers)[idx], aval(VAR_coordSlavePorts)[idx]) == 0)
 	{
 		/* Master is running.  Need to switch log shipping to asynchronous mode. */
 		cmd = cmdMasterReload = initCmd(aval(VAR_coordMasterServers)[idx]);
@@ -1933,6 +1963,12 @@ static int failover_oneCoordinator(int coordIdx)
 	char *gtmPort;
 	FILE *f;
 	char timestamp[MAXTOKEN+1];
+
+#ifdef XCP	
+	char cmd[MAXLINE];
+	int	 cmdlen;
+	bool dnReconfigured;
+#endif	
 	
 #define checkRc() do{if(WEXITSTATUS(rc_local) > rc) rc = WEXITSTATUS(rc_local);}while(0)
 
@@ -1985,7 +2021,11 @@ static int failover_oneCoordinator(int coordIdx)
 	
 	/* Update the configuration variable */
 	var_assign(&(aval(VAR_coordMasterServers)[coordIdx]), Strdup(aval(VAR_coordSlaveServers)[coordIdx]));
+	var_assign(&(aval(VAR_coordPorts)[coordIdx]), Strdup(aval(VAR_coordSlavePorts)[coordIdx]));
+	var_assign(&(aval(VAR_poolerPorts)[coordIdx]), Strdup(aval(VAR_coordSlavePoolerPorts)[coordIdx]));
 	var_assign(&(aval(VAR_coordSlaveServers)[coordIdx]), Strdup("none"));
+	var_assign(&(aval(VAR_coordSlavePorts)[coordIdx]), Strdup("none"));
+	var_assign(&(aval(VAR_coordSlavePoolerPorts)[coordIdx]), Strdup("none"));
 	var_assign(&(aval(VAR_coordMasterDirs)[coordIdx]), Strdup(aval(VAR_coordSlaveDirs)[coordIdx]));
 	var_assign(&(aval(VAR_coordSlaveDirs)[coordIdx]), Strdup("none"));
 
@@ -1998,14 +2038,22 @@ static int failover_oneCoordinator(int coordIdx)
 			"#=====================================================\n"
 			"# Updated due to the coordinator failover, %s, %s\n"
 			"coordMasterServers=( %s )\n"
+			"coordPorts =( %s )\n"
+			"poolerPorts =( %s )\n"
 			"coordMasterDirs=( %s )\n"
 			"coordSlaveServers=( %s )\n"
+			"coordSlavePorts=( %s )\n"
+			"coordSlavePoolerPorts=( %s )\n"
 			"coordSlaveDirs=( %s )\n"
 			"# End of the update\n",
 			aval(VAR_coordNames)[coordIdx], timeStampString(timestamp, MAXTOKEN),
 			listValue(VAR_coordMasterServers),
+			listValue(VAR_coordPorts),
+			listValue(VAR_poolerPorts),
 			listValue(VAR_coordMasterDirs),
 			listValue(VAR_coordSlaveServers),
+			listValue(VAR_coordSlavePorts),
+			listValue(VAR_coordSlavePoolerPorts),
 			listValue(VAR_coordSlaveDirs));
 	fclose(f);
 
@@ -2016,6 +2064,44 @@ static int failover_oneCoordinator(int coordIdx)
 		checkRc();
 	}
 
+#ifdef XCP	
+	cmdlen = 0;
+	cmd[0] = '\0';
+	/*
+	 * Reconfigure datanodes with the new datanode. We prepare the commands and
+	 * pass them to the first coordinator we reconfigure later
+	 */
+	for (jj = 0; aval(VAR_datanodeNames)[jj]; jj++)
+	{
+		int len;
+
+		if (is_none(aval(VAR_datanodeMasterServers)[jj]))
+			continue;
+			
+		if (pingNode(aval(VAR_datanodeMasterServers)[jj], aval(VAR_datanodePorts)[jj]) != 0)
+		{
+			elog(ERROR, "Datanode %s is not running.  Skip reconfiguration for this datanode.\n",
+				 aval(VAR_coordNames)[jj]);
+			continue;
+		}
+		
+		len = snprintf(cmd + cmdlen, MAXLINE - cmdlen, "EXECUTE DIRECT ON (%s) 'ALTER NODE %s WITH (HOST=''%s'', PORT=%s)';\n"
+				"EXECUTE DIRECT ON (%s) 'select pgxc_pool_reload()';\n",
+								 aval(VAR_datanodeNames)[jj],
+								 aval(VAR_coordNames)[coordIdx],
+								 aval(VAR_coordMasterServers)[coordIdx],
+								 aval(VAR_coordPorts)[coordIdx],
+								 aval(VAR_datanodeNames)[jj]);
+		if (len > (MAXLINE - cmdlen))
+		{
+			elog(ERROR, "Datanode command exceeds the maximum allowed length");
+			return -1;
+		}
+		cmdlen += len;
+	}
+	dnReconfigured = false;
+#endif
+
 	/*
 	 * Reconfigure coordinators with new coordinator
 	 */
@@ -2043,8 +2129,15 @@ static int failover_oneCoordinator(int coordIdx)
 		fprintf(f,
 				"ALTER NODE %s WITH (HOST='%s', PORT=%s);\n"
 				"select pgxc_pool_reload();\n"
+#ifdef XCP				
+				"%s"
+#endif				
 				"\\q\n",
-				aval(VAR_coordNames)[coordIdx], aval(VAR_coordMasterServers)[coordIdx], aval(VAR_coordPorts)[coordIdx]);
+				aval(VAR_coordNames)[coordIdx], aval(VAR_coordMasterServers)[coordIdx], aval(VAR_coordPorts)[coordIdx]
+#ifdef XCP				
+				,dnReconfigured ? "" : cmd
+#endif				
+				);
 		pclose(f);
 	}
 	return(rc);
@@ -2169,7 +2262,8 @@ int show_config_coordSlave(int flag, int idx, char *hostname)
 	if (outBuf[0])
 		elog(NOTICE, "%s", outBuf);
 	elog(NOTICE,"    Nodename: '%s', port: %s, pooler port: %s\n",
-		 aval(VAR_coordNames)[idx], aval(VAR_coordPorts)[idx], aval(VAR_poolerPorts)[idx]);
+		 aval(VAR_coordNames)[idx], aval(VAR_coordSlavePorts)[idx],
+		 aval(VAR_coordSlavePoolerPorts)[idx]);
 	elog(NOTICE, "    Dir: '%s', Archive Log Dir: '%s'\n",
 		 aval(VAR_coordSlaveDirs)[idx], aval(VAR_coordArchLogDirs)[idx]);
 	unlockLogFile();
diff --git a/contrib/pgxc_ctl/coord_cmd.h b/contrib/pgxc_ctl/coord_cmd.h
index 673e2c82f5..e1f2a25579 100644
--- a/contrib/pgxc_ctl/coord_cmd.h
+++ b/contrib/pgxc_ctl/coord_cmd.h
@@ -53,7 +53,7 @@ extern cmd_t *prepare_stopCoordinatorMaster(char *nodeName, char *immediate);
 extern cmd_t *prepare_stopCoordinatorSlave(char *nodeName, char *immediate);
 
 extern int add_coordinatorMaster(char *name, char *host, int port, int pooler, char *dir);
-extern int add_coordinatorSlave(char *name, char *host, char *dir, char *archDir);
+extern int add_coordinatorSlave(char *name, char *host, int port, int pooler, char *dir, char *archDir);
 extern int remove_coordinatorMaster(char *name, int clean_opt);
 extern int remove_coordinatorSlave(char *name, int clean_opt);
 
diff --git a/contrib/pgxc_ctl/do_command.c b/contrib/pgxc_ctl/do_command.c
index bfc7517f92..31724d61a5 100644
--- a/contrib/pgxc_ctl/do_command.c
+++ b/contrib/pgxc_ctl/do_command.c
@@ -929,9 +929,11 @@ static void do_add_command(char *line)
 		{
 			GetAndSet(name, "ERROR: please specify the name of the coordinator slave\n");
 			GetAndSet(host, "ERROR: please specify the host for the coordinator slave\n");
+			GetAndSet(port, "ERROR: please specify the port number for the coordinator slave\n");
+			GetAndSet(pooler, "ERROR: please specify the pooler port number for the coordinator slave.\n");
 			GetAndSet(dir, "ERROR: please specify the working director for coordinator slave\n");
 			GetAndSet(archDir, "ERROR: please specify WAL archive directory for coordinator slave\n");
-			add_coordinatorSlave(name, host, dir, archDir);
+			add_coordinatorSlave(name, host, atoi(port), atoi(pooler), dir, archDir);
 			freeAndReset(name);
 			freeAndReset(host);
 			freeAndReset(dir);
diff --git a/contrib/pgxc_ctl/do_shell.c b/contrib/pgxc_ctl/do_shell.c
index 65c6fa68f8..77cccfeb17 100644
--- a/contrib/pgxc_ctl/do_shell.c
+++ b/contrib/pgxc_ctl/do_shell.c
@@ -180,6 +180,7 @@ FILE *pgxc_popen_w(char *host, const char *cmd_fmt, ...)
 	va_start(arg, cmd_fmt);
 	vsnprintf(actualCmd, MAXLINE, cmd_fmt, arg);
 	va_end(arg);
+
 	snprintf(sshCmd, MAXLINE, "ssh %s@%s \" %s \"", sval(VAR_pgxcUser), host, actualCmd);
 	if ((f = popen(sshCmd, "w")) == NULL)
 		elog(ERROR, "ERROR: could not open the command \"%s\" to write, %s\n", sshCmd, strerror(errno));
@@ -212,6 +213,12 @@ int doImmediate(char *host, char *stdIn, const char *cmd_fmt, ...)
 	{
 		int rc1;
 		/* Remote case */
+		if (strcmp(host, "none") == 0)
+		{
+			int *ip = NULL;
+			*ip = 1;
+		}
+
 		snprintf(actualCmd, MAXLINE, "ssh %s@%s \"( %s ) > %s 2>&1\" < %s > /dev/null 2>&1",
 				 sval(VAR_pgxcUser), host, cmd_wk, 
 				 createRemoteFileName(STDOUT, remoteStdout, MAXPATH),
@@ -324,6 +331,12 @@ int doCmdEl(cmd_t *cmd)
 	}
 	if (cmd->host)
 	{
+		if (strcmp(cmd->host, "none") == 0)
+		{
+			int *ip = NULL;
+			*ip = 1;
+		}
+
 		/* Build actual command */
 		snprintf(allocActualCmd(cmd), MAXLINE,
 				 "ssh %s@%s \"( %s ) > %s 2>&1\" < %s > /dev/null 2>&1",
diff --git a/contrib/pgxc_ctl/monitor.c b/contrib/pgxc_ctl/monitor.c
index 92c01adbed..f64bbec4e5 100644
--- a/contrib/pgxc_ctl/monitor.c
+++ b/contrib/pgxc_ctl/monitor.c
@@ -130,7 +130,7 @@ static void monitor_coordinator_slave(char **nodeList)
 		if (!doesExist(VAR_coordSlaveServers, idx) || is_none(aval(VAR_coordSlaveServers)[idx]))
 			elog(ERROR, "ERROR: coordinator slave %s is not configured\n", actualNodeList[ii]);
 		else
-			printResult(pingNode(aval(VAR_coordSlaveServers)[idx], aval(VAR_coordPorts)[idx]), 
+			printResult(pingNode(aval(VAR_coordSlaveServers)[idx], aval(VAR_coordSlavePorts)[idx]), 
 						"coordinator slave", actualNodeList[ii]);
 	}
 }
@@ -152,8 +152,8 @@ static void monitor_coordinator(char **nodeList)
 		printResult(pingNode(aval(VAR_coordMasterServers)[idx], aval(VAR_coordPorts)[idx]), 
 					"coordinator master", actualNodeList[ii]);
 		if (doesExist(VAR_coordSlaveServers, idx) && !is_none(aval(VAR_coordSlaveServers)[idx]))
-			printResult(pingNode(aval(VAR_coordSlaveServers)[idx], aval(VAR_coordPorts)[idx]),
-						"coordinatr slave", actualNodeList[ii]);
+			printResult(pingNode(aval(VAR_coordSlaveServers)[idx], aval(VAR_coordSlavePorts)[idx]),
+						"coordinator slave", actualNodeList[ii]);
 	}
 }
 static void monitor_datanode_master(char **nodeList)
diff --git a/contrib/pgxc_ctl/pgxc_ctl_bash.c b/contrib/pgxc_ctl/pgxc_ctl_bash.c
index 30fae0c445..40fb10f6a9 100644
--- a/contrib/pgxc_ctl/pgxc_ctl_bash.c
+++ b/contrib/pgxc_ctl/pgxc_ctl_bash.c
@@ -115,6 +115,8 @@ char *pgxc_ctl_bash_script[] = {
 "	echo coordSlave $coordSlave",
 "	echo coordSlaveSync $coordSlaveSync",
 "	print_array coordSlaveServers",
+"	print_array coordSlavePorts",
+"	print_array coordSlavePoolerPorts",
 "	print_array coordSlaveDirs",
 "	print_array coordArchLogDirs",
 "",
@@ -493,8 +495,8 @@ char *pgxc_ctl_conf_prototype[] = {
 "",
 "#---- Overall ------------",
 "coordNames=(coord1 coord2 coord3 coord4)		# Master and slave use the same name",
-"coordPorts=(20004 20005 20004 20005)			# Master and slave use the same port",
-"poolerPorts=(20010 20011 20010 20011)			# Master and slave use the same pooler port",
+"coordPorts=(20004 20005 20004 20005)			# Master ports",
+"poolerPorts=(20010 20011 20010 20011)			# Master pooler ports",
 "coordPgHbaEntries=(192.168.1.0/24)				# Assumes that all the coordinator (master/slave) accepts",
 "												# the same connection",
 "												# This entry allows only $pgxcOwner to connect.",
@@ -521,6 +523,8 @@ char *pgxc_ctl_conf_prototype[] = {
 "						# empty values.",
 "coordSlaveSync=y		# Specify to connect with synchronized mode.",
 "coordSlaveServers=(node07 node08 node09 node06)			# none means this slave is not available",
+"coordSlavePorts=(20004 20005 20004 20005)			# Master ports",
+"coordSlavePoolerPorts=(20010 20011 20010 20011)			# Master pooler ports",
 "coordSlaveDirs=($coordSlaveDir $coordSlaveDir $coordSlaveDir $coordSlaveDir)",
 "coordArchLogDirs=($coordArchLogDir $coordArchLogDir $coordArchLogDir $coordArchLogDir)",
 "",
@@ -583,9 +587,9 @@ char *pgxc_ctl_conf_prototype[] = {
 "# without this feature.",
 "primaryDatanode=datanode1				# Primary Node.",
 "datanodeNames=(datanode1 datanode2 datanode3 datanode4)",
-"datanodePorts=(20008 20009 20008 20009)	# Master and slave use the same port!",
+"datanodePorts=(20008 20009 20008 20009)	# Master ports",
 #ifdef XCP
-"datanodePoolerPorts=(20012 20013 20012 20013)	# Master and slave use the same port!",
+"datanodePoolerPorts=(20012 20013 20012 20013)	# Master pooler ports",
 #endif
 "datanodePgHbaEntries=(192.168.1.0/24)	# Assumes that all the coordinator (master/slave) accepts",
 "										# the same connection",
@@ -616,6 +620,8 @@ char *pgxc_ctl_conf_prototype[] = {
 "						# then datanodeSlave value will be set to n and all the following values will be set to",
 "						# empty values.",
 "datanodeSlaveServers=(node07 node08 node09 node06)	# value none means this slave is not available",
+"datanodeSlavePorts=(20008 20009 20008 20009)	# value none means this slave is not available",
+"datanodeSlavePoolerPorts=(20012 20013 20012 20013)	# value none means this slave is not available",
 "datanodeSlaveSync=y		# If datanode slave is connected in synchronized mode",
 "datanodeSlaveDirs=($datanodeSlaveDir $datanodeSlaveDir $datanodeSlaveDir $datanodeSlaveDir)",
 "datanodeArchLogDirs=( $datanodeArchLogDir $datanodeArchLogDir $datanodeArchLogDir $datanodeArchLogDir )",
diff --git a/contrib/pgxc_ctl/pgxc_ctl_bash_2 b/contrib/pgxc_ctl/pgxc_ctl_bash_2
index c98864ba0b..21839eae46 100755
--- a/contrib/pgxc_ctl/pgxc_ctl_bash_2
+++ b/contrib/pgxc_ctl/pgxc_ctl_bash_2
@@ -82,6 +82,8 @@ function print_values
 	echo coordSlave $coordSlave
 	echo coordSlaveSync $coordSlaveSync
 	print_array coordSlaveServers
+	print_array coordSlavePorts
+	print_array coordSlavePoolerPorts
 	print_array coordSlaveDirs
 	print_array coordArchLogDirs
 
diff --git a/contrib/pgxc_ctl/pgxc_ctl_conf_part b/contrib/pgxc_ctl/pgxc_ctl_conf_part
index 7f2ae47fb5..9640dc9c41 100755
--- a/contrib/pgxc_ctl/pgxc_ctl_conf_part
+++ b/contrib/pgxc_ctl/pgxc_ctl_conf_part
@@ -140,8 +140,8 @@ coordArchLogDir=$HOME/pgxc/nodes/coord_archlog
 
 #---- Overall ------------
 coordNames=(coord1 coord2 coord3 coord4)		# Master and slave use the same name
-coordPorts=(20004 20005 20004 20005)			# Master and slave use the same port
-poolerPorts=(20010 20011 20010 20011)			# Master and slave use the same pooler port
+coordPorts=(20004 20005 20004 20005)			# Master ports
+poolerPorts=(20010 20011 20010 20011)			# Master pooler ports
 coordPgHbaEntries=(192.168.1.0/24)				# Assumes that all the coordinator (master/slave) accepts
 												# the same connection
 												# This entry allows only $pgxcOwner to connect.
@@ -168,6 +168,8 @@ coordSlave=y			# Specify y if you configure at least one coordiantor slave.  Oth
 						# empty values.
 coordSlaveSync=y		# Specify to connect with synchronized mode.
 coordSlaveServers=(node07 node08 node09 node06)			# none means this slave is not available
+coordSlavePorts=(20004 20005 20004 20005)			# Master ports
+coordSlavePoolerPorts=(20010 20011 20010 20011)			# Master pooler ports
 coordSlaveDirs=($coordSlaveDir $coordSlaveDir $coordSlaveDir $coordSlaveDir)
 coordArchLogDirs=($coordArchLogDir $coordArchLogDir $coordArchLogDir $coordArchLogDir)
 
@@ -230,9 +232,9 @@ datanodeArchLogDir=$HOME/pgxc/nodes/datanode_archlog
 # without this feature.
 primaryDatanode=datanode1				# Primary Node.
 datanodeNames=(datanode1 datanode2 datanode3 datanode4)
-datanodePorts=(20008 20009 20008 20009)	# Master and slave use the same port!
+datanodePorts=(20008 20009 20008 20009)	# Master ports
 #ifdef XCP
-datanodePoolerPorts=(20012 20013 20012 20013)	# Master and slave use the same port!
+datanodePoolerPorts=(20012 20013 20012 20013)	# Master pooler ports
 #endif
 datanodePgHbaEntries=(192.168.1.0/24)	# Assumes that all the coordinator (master/slave) accepts
 										# the same connection
@@ -263,6 +265,8 @@ datanodeSlave=y			# Specify y if you configure at least one coordiantor slave.
 						# then datanodeSlave value will be set to n and all the following values will be set to
 						# empty values.
 datanodeSlaveServers=(node07 node08 node09 node06)	# value none means this slave is not available
+datanodeSlavePorts=(20008 20009 20008 20009)	# value none means this slave is not available
+datanodeSlavePoolerPorts=(20012 20013 20012 20013)	# value none means this slave is not available
 datanodeSlaveSync=y		# If datanode slave is connected in synchronized mode
 datanodeSlaveDirs=($datanodeSlaveDir $datanodeSlaveDir $datanodeSlaveDir $datanodeSlaveDir)
 datanodeArchLogDirs=( $datanodeArchLogDir $datanodeArchLogDir $datanodeArchLogDir $datanodeArchLogDir )
diff --git a/contrib/pgxc_ctl/signature.h b/contrib/pgxc_ctl/signature.h
index 8484b398b7..4268c99d7b 100644
--- a/contrib/pgxc_ctl/signature.h
+++ b/contrib/pgxc_ctl/signature.h
@@ -11,5 +11,5 @@
 #ifndef SIGNATURE_H
 #define SIGNATURE_H
 /* Signature file to identify the make */
-#define signature "140725_1610_N"
+#define signature "140731_1647_N"
 #endif /* SIGNATURE_H */
diff --git a/contrib/pgxc_ctl/varnames.h b/contrib/pgxc_ctl/varnames.h
index 4bb54a32f2..e2774c57e8 100644
--- a/contrib/pgxc_ctl/varnames.h
+++ b/contrib/pgxc_ctl/varnames.h
@@ -69,6 +69,8 @@
 /* Coordinators slave */
 #define VAR_coordSlave		"coordSlave"
 #define VAR_coordSlaveServers	"coordSlaveServers"
+#define VAR_coordSlavePorts	"coordSlavePorts"
+#define VAR_coordSlavePoolerPorts	"coordSlavePoolerPorts"
 #define VAR_coordSlaveSync	"coordSlaveSync"
 #define VAR_coordSlaveDirs	"coordSlaveDirs"
 #define VAR_coordArchLogDirs	"coordArchLogDirs"
diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c
index cc3daecd62..b0a3a17149 100644
--- a/src/backend/tcop/utility.c
+++ b/src/backend/tcop/utility.c
@@ -3874,6 +3874,14 @@ IsStmtAllowedInLockedMode(Node *parsetree, const char *queryString)
 														 * advisory lock on the crashed node.
 														 */
 
+		case T_AlterNodeStmt:							/*
+														 * This has to be
+														 * allowed so that
+														 * ALTER NODE can be
+														 * issued in case a
+														 * datanode or
+														 * coordinator failover
+														 */  
 		case T_TransactionStmt:
 		case T_PlannedStmt:
 		case T_ClosePortalStmt: