Use whichever coordinator available for running cluster commands

The utility had assumed that the first coordinator at index 0 will always be
available. But that may not be the case when we have actually removed the first
coordinator using "pgxc_ctl remove coordinator master <name>"

5 files changed, 84 insertions, 19 deletions

diff --git a/contrib/pgxc_ctl/config.c b/contrib/pgxc_ctl/config.c
index 414046e21d..a3f9a4abb6 100644
--- a/contrib/pgxc_ctl/config.c
+++ b/contrib/pgxc_ctl/config.c
@@ -336,6 +336,8 @@ static void emptyCoordSlaves()
 		add_val(find_var(VAR_coordSlaveServers), "none");
 		add_val(find_var(VAR_coordSlaveDirs), "none");
 		add_val(find_var(VAR_coordArchLogDirs), "none");
+		add_val(find_var(VAR_coordSlavePorts), "none");
+		add_val(find_var(VAR_coordSlavePoolerPorts), "none");
 	}
 }
 
@@ -355,8 +357,10 @@ static void emptyDatanodeSlaves()
 	for (ii = 0; ii < arraySizeName(VAR_datanodeSlaveServers); ii++)
 	{
 		add_val(find_var(VAR_datanodeSlaveServers), "none");
-		add_val(find_var(VAR_coordSlaveDirs), "none");
-		add_val(find_var(VAR_coordArchLogDirs), "none");
+		add_val(find_var(VAR_datanodeSlaveDirs), "none");
+		add_val(find_var(VAR_datanodeArchLogDirs), "none");
+		add_val(find_var(VAR_datanodeSlavePorts), "-1");
+		add_val(find_var(VAR_datanodeSlavePoolerPorts), "-1");
 	}
 }
 
diff --git a/contrib/pgxc_ctl/coord_cmd.c b/contrib/pgxc_ctl/coord_cmd.c
index f34f6e4768..dc1c74edd7 100644
--- a/contrib/pgxc_ctl/coord_cmd.c
+++ b/contrib/pgxc_ctl/coord_cmd.c
@@ -104,9 +104,11 @@ cmd_t *prepare_initCoordinatorMaster(char *nodeName)
 	appendCmdEl(cmdInitdb, (cmdPgConf = initCmd(aval(VAR_coordMasterServers)[jj])));
 	snprintf(newCommand(cmdPgConf), MAXLINE,
 			 "cat >> %s/postgresql.conf", aval(VAR_coordMasterDirs)[jj]);
-	if (!is_none(sval(VAR_coordExtraConfig)))
+	if (doesExist(VAR_coordExtraConfig, 0) &&
+		!is_none(sval(VAR_coordExtraConfig)))
 		AddMember(confFiles, sval(VAR_coordExtraConfig));
-	if (!is_none(aval(VAR_coordSpecificExtraConfig)[jj]))
+	if (doesExist(VAR_coordSpecificExtraConfig, jj) &&
+		!is_none(aval(VAR_coordSpecificExtraConfig)[jj]))
 		AddMember(confFiles, aval(VAR_coordSpecificExtraConfig)[jj]);
 	if ((f = prepareLocalStdin((cmdPgConf->localStdin = Malloc(MAXPATH+1)), MAXPATH, confFiles)) == NULL)
 	{
@@ -534,6 +536,7 @@ static cmd_t *prepare_configureDataNode(char *nodeName)
 	int ii;
 	int jj;
 	int idx;
+	int connCordIndx;
 	FILE *f;
 	bool is_preferred;
 
@@ -545,11 +548,16 @@ static cmd_t *prepare_configureDataNode(char *nodeName)
 	if (is_none(aval(VAR_datanodeMasterServers)[idx]))
 		return NULL;
 	cmd = initCmd(NULL);
+
 	/* We use one of the coordinators to send queries to datanodes */
+	connCordIndx = get_any_available_coord(-1);
+	if (connCordIndx == -1)
+		return NULL;
+
 	snprintf(newCommand(cmd), MAXLINE,
 			 "psql -p %d -h %s -a %s %s",
-			 atoi(aval(VAR_coordPorts)[0]),
-			 aval(VAR_coordMasterServers)[0],
+			 atoi(aval(VAR_coordPorts)[connCordIndx]),
+			 aval(VAR_coordMasterServers)[connCordIndx],
 			 sval(VAR_defaultDatabase),
 			 sval(VAR_pgxcOwner));
 	if ((f = prepareLocalStdin(newFilename(cmd->localStdin), MAXPATH, NULL)) == NULL)
@@ -937,6 +945,7 @@ int add_coordinatorMaster(char *name, char *host, int port, int pooler,
 	char port_s[MAXTOKEN+1];
 	char pooler_s[MAXTOKEN+1];
 	int gtmPxyIdx;
+	int connCordIndx;
 	char *gtmHost;
 	char *gtmPort;
 	char pgdumpall_out[MAXPATH+1];
@@ -1119,8 +1128,16 @@ int add_coordinatorMaster(char *name, char *host, int port, int pooler,
 		pclose(f);
 	}
 
+	/* find any available coordinator */
+	connCordIndx = get_any_available_coord(-1);
+	if (connCordIndx == -1)
+		return 1;
+
 	/* Lock ddl */
-	if ((lockf = pgxc_popen_wRaw("psql -h %s -p %s %s", aval(VAR_coordMasterServers)[0], aval(VAR_coordPorts)[0], sval(VAR_defaultDatabase))) == NULL)
+	if ((lockf = pgxc_popen_wRaw("psql -h %s -p %s %s",
+					aval(VAR_coordMasterServers)[connCordIndx],
+					aval(VAR_coordPorts)[connCordIndx],
+					sval(VAR_defaultDatabase))) == NULL)
 	{
 		elog(ERROR, "ERROR: could not open psql command, %s\n", strerror(errno));
 		return 1;
@@ -1131,7 +1148,8 @@ int add_coordinatorMaster(char *name, char *host, int port, int pooler,
 	/* pg_dumpall */
 	createLocalFileName(GENERAL, pgdumpall_out, MAXPATH);
 	doImmediateRaw("pg_dumpall -p %s -h %s -s --include-nodes --dump-nodes --file=%s",
-				   aval(VAR_coordPorts)[0], aval(VAR_coordMasterServers)[0], pgdumpall_out);
+				   aval(VAR_coordPorts)[connCordIndx],
+				   aval(VAR_coordMasterServers)[connCordIndx], pgdumpall_out);
 
 	/* Start the new coordinator */
 	doImmediate(host, NULL, "pg_ctl start -Z restoremode -D %s -o -i", dir);
@@ -1172,7 +1190,10 @@ int add_coordinatorMaster(char *name, char *host, int port, int pooler,
 	{
 		if (!is_none(aval(VAR_datanodeNames)[ii]))
 		{
-			if ((f = pgxc_popen_wRaw("psql -h %s -p %d %s", aval(VAR_coordMasterServers)[0], atoi(aval(VAR_coordPorts)[0]), sval(VAR_defaultDatabase))) == NULL)
+			if ((f = pgxc_popen_wRaw("psql -h %s -p %d %s",
+							aval(VAR_coordMasterServers)[connCordIndx],
+							atoi(aval(VAR_coordPorts)[connCordIndx]),
+							sval(VAR_defaultDatabase))) == NULL)
 			{
 				elog(ERROR, "ERROR: cannot connect to the coordinator master %s.\n", aval(VAR_coordNames)[ii]);
 				continue;
@@ -1499,12 +1520,9 @@ int remove_coordinatorMaster(char *name, int clean_opt)
 	{
 		if (doesExist(VAR_datanodeNames, ii) && !is_none(aval(VAR_datanodeNames)[ii]))
 		{
-			int coord_idx;
-
-			if (idx == 0)
-				coord_idx = 1;
-			else
-				coord_idx  = 0;
+			int coord_idx = get_any_available_coord(idx);
+			if (coord_idx == -1)
+				return 1;
 
 			f = pgxc_popen_wRaw("psql -p %d -h %s %s", atoi(aval(VAR_coordPorts)[coord_idx]), aval(VAR_coordMasterServers)[coord_idx], sval(VAR_defaultDatabase));
 			if (f == NULL)
diff --git a/contrib/pgxc_ctl/datanode_cmd.c b/contrib/pgxc_ctl/datanode_cmd.c
index 96baab8bfd..48b01206ca 100644
--- a/contrib/pgxc_ctl/datanode_cmd.c
+++ b/contrib/pgxc_ctl/datanode_cmd.c
@@ -101,9 +101,11 @@ cmd_t *prepare_initDatanodeMaster(char *nodeName)
 			"listen_addresses = '*'\n"
 			"max_connections = 100\n",
 			timeStampString(timeStamp, MAXTOKEN));
-	if (!is_none(sval(VAR_datanodeExtraConfig)))
+	if (doesExist(VAR_datanodeExtraConfig, 0) &&
+		!is_none(sval(VAR_datanodeExtraConfig)))
 		AddMember(fileList, sval(VAR_datanodeExtraConfig));
-	if (!is_none(aval(VAR_datanodeSpecificExtraConfig)[idx]))
+	if (doesExist(VAR_datanodeSpecificExtraConfig, idx) &&
+		!is_none(aval(VAR_datanodeSpecificExtraConfig)[idx]))
 		AddMember(fileList, aval(VAR_datanodeSpecificExtraConfig)[idx]);
 	appendFiles(f, fileList);
 	CleanArray(fileList);
@@ -915,6 +917,7 @@ int add_datanodeMaster(char *name, char *host, int port, int pooler, char *dir,
 	char port_s[MAXTOKEN+1];
     char pooler_s[MAXTOKEN+1];
 	int gtmPxyIdx;
+	int connCordIdx;
 	char *gtmHost;
 	char *gtmPort;
 	char pgdumpall_out[MAXPATH+1];
@@ -1173,12 +1176,20 @@ int add_datanodeMaster(char *name, char *host, int port, int pooler, char *dir,
 		}
 	}
 
+	/* find any available coordinator */
+	connCordIdx = get_any_available_coord(-1);
+	if (connCordIdx == -1)
+		return 1;
+
 	/* Issue CREATE NODE  on datanodes */
 	for (ii = 0; aval(VAR_datanodeNames)[ii]; ii++)
 	{
 		if (!is_none(aval(VAR_datanodeNames)[ii]))
 		{
-			if ((f = pgxc_popen_wRaw("psql -h %s -p %s %s", aval(VAR_coordMasterServers)[0], aval(VAR_coordPorts)[0], sval(VAR_defaultDatabase))) == NULL)
+			if ((f = pgxc_popen_wRaw("psql -h %s -p %s %s",
+							aval(VAR_coordMasterServers)[connCordIdx],
+							aval(VAR_coordPorts)[connCordIdx],
+							sval(VAR_defaultDatabase))) == NULL)
 			{
 				elog(ERROR, "ERROR: cannot connect to the coordinator %s.\n", aval(VAR_coordNames)[0]);
 				continue;
@@ -1453,6 +1464,7 @@ int remove_datanodeMaster(char *name, int clean_opt)
 	 */
 
 	int idx;
+	int connCordIdx;
 	int ii;
 	FILE *f;
 	char **namelist = NULL;
@@ -1505,13 +1517,22 @@ int remove_datanodeMaster(char *name, int clean_opt)
 			pclose(f);
 		}
 	}
+
+	/* find any available coordinator */
+	connCordIdx = get_any_available_coord(-1);
+	if (connCordIdx == -1)
+		return 1;
+
 	/* Issue DROP NODE  on datanodes */
 	for (ii = 0; aval(VAR_datanodeNames)[ii]; ii++)
 	{
 		if (!is_none(aval(VAR_datanodeNames)[ii]) &&
 			strcmp(aval(VAR_datanodeNames)[ii], name) != 0)
 		{
-			if ((f = pgxc_popen_wRaw("psql -h %s -p %s %s", aval(VAR_coordMasterServers)[0], aval(VAR_coordPorts)[0], sval(VAR_defaultDatabase))) == NULL)
+			if ((f = pgxc_popen_wRaw("psql -h %s -p %s %s",
+							aval(VAR_coordMasterServers)[connCordIdx],
+							aval(VAR_coordPorts)[connCordIdx],
+							sval(VAR_defaultDatabase))) == NULL)
 			{
 				elog(ERROR, "ERROR: cannot connect to the coordinator %s.\n", aval(VAR_coordNames)[0]);
 				continue;
diff --git a/contrib/pgxc_ctl/do_command.c b/contrib/pgxc_ctl/do_command.c
index d2d8333baf..9ff5f10344 100644
--- a/contrib/pgxc_ctl/do_command.c
+++ b/contrib/pgxc_ctl/do_command.c
@@ -2955,3 +2955,24 @@ do_show_help(char *line)
 			  );
 	}
 }
+
+int
+get_any_available_coord(int except)
+{
+	int ii;
+	for (ii = 0; aval(VAR_coordMasterServers)[ii]; ii++)
+	{
+		if (ii == except)
+			continue;
+
+		if (!is_none(aval(VAR_coordMasterServers)[ii]))
+		{
+			if (pingNode(aval(VAR_coordMasterServers)[ii],
+						aval(VAR_coordPorts)[ii]) == 0)
+				return ii;
+		}
+	}
+
+	elog(ERROR, "ERROR: failed to find any running coordinator");
+	return -1;
+}
diff --git a/contrib/pgxc_ctl/do_command.h b/contrib/pgxc_ctl/do_command.h
index 5c9c685991..b24d64a8ef 100644
--- a/contrib/pgxc_ctl/do_command.h
+++ b/contrib/pgxc_ctl/do_command.h
@@ -14,4 +14,5 @@
 extern int forceInit;
 extern void do_command(FILE *inf, FILE *outf);
 extern int  do_singleLine(char *buf, char *wkline);
+extern int get_any_available_coord(int except);
 #endif /* DO_COMMAND_H */