Add support to specify separate XLOG dirs for datanode masters and datanode

slaves in pgxc_ctl.conf file as well as corresponding "add" commands.

Recent releases of Postgres now allow users to specify a separate XLOG dir and
initdb time and we extend the same facility to pgxc_ctl.

6 files changed, 220 insertions, 25 deletions

diff --git a/contrib/pgxc_ctl/config.c b/contrib/pgxc_ctl/config.c
index a3f9a4abb6..0355971c3a 100644
--- a/contrib/pgxc_ctl/config.c
+++ b/contrib/pgxc_ctl/config.c
@@ -625,6 +625,9 @@ int checkDirConflict(char *host, char *dir)
 {
 	int ii;
 
+	/* "none" conflictd with nothing */
+	if (strcasecmp(dir, "none") == 0)
+		return 0;
 	/* GTM Master */
 	if ((strcasecmp(host, sval(VAR_gtmMasterServer)) == 0) && (strcmp(dir, sval(VAR_gtmMasterDir)) == 0))
 		return 1;
@@ -649,11 +652,22 @@ int checkDirConflict(char *host, char *dir)
 	for (ii = 0; aval(VAR_datanodeNames)[ii]; ii++)
 		if ((strcasecmp(host, aval(VAR_datanodeMasterServers)[ii]) == 0) && (strcmp(dir, aval(VAR_datanodeMasterDirs)[ii]) == 0))
 			return 1;
+	/* Datanode Master WAL Dirs */
+	for (ii = 0; aval(VAR_datanodeNames)[ii]; ii++)
+		if ((strcasecmp(host, aval(VAR_datanodeMasterServers)[ii]) == 0) &&
+				(strcmp(dir, aval(VAR_datanodeMasterWALDirs)[ii]) == 0))
+			return 1;
 	/* Datanode Slave */
 	if (isVarYes(VAR_datanodeSlave))
 		if (doesExist(VAR_datanodeSlaveServers, ii) && doesExist(VAR_datanodeSlaveDirs, ii) &&
 			(strcasecmp(host, aval(VAR_datanodeSlaveServers)[ii]) == 0) && (strcmp(dir, aval(VAR_datanodeSlaveDirs)[ii]) == 0))
 			return 1;
+	/* Datanode Slave WAL Dirs */
+	if (isVarYes(VAR_datanodeSlave))
+		if (doesExist(VAR_datanodeSlaveServers, ii) && doesExist(VAR_datanodeSlaveDirs, ii) &&
+			(strcasecmp(host, aval(VAR_datanodeSlaveServers)[ii]) == 0) &&
+			(strcmp(dir, aval(VAR_datanodeSlaveWALDirs)[ii]) == 0))
+			return 1;
 	return 0;
 }
 
@@ -842,6 +856,7 @@ static void verifyResource(void)
 								  VAR_datanodePoolerPorts, 
 								  VAR_datanodeMasterServers,
 								  VAR_datanodeMasterDirs, 
+								  VAR_datanodeMasterWALDirs, 
 								  VAR_datanodeMaxWALSenders, 
 								  NULL};
 	char *datanodeSlaveVars[] = {VAR_datanodeNames,
@@ -849,6 +864,7 @@ static void verifyResource(void)
 								 VAR_datanodeSlavePorts,
 								 VAR_datanodeSlavePoolerPorts, 
 								 VAR_datanodeSlaveDirs,
+								 VAR_datanodeSlaveWALDirs,
 								 VAR_datanodeArchLogDirs,
 								 NULL};
 #if 0
@@ -920,11 +936,21 @@ static void verifyResource(void)
 	/* GTM and datanode masters */
 	checkResourceConflict(VAR_gtmName, VAR_gtmMasterServer, VAR_gtmMasterPort, NULL, VAR_gtmMasterDir,
 						  VAR_datanodeNames, VAR_datanodeMasterServers, VAR_datanodePorts, NULL, VAR_datanodeMasterDirs, TRUE, TRUE);
+	checkResourceConflict(VAR_gtmName, VAR_gtmMasterServer, VAR_gtmMasterPort, NULL, VAR_gtmMasterDir,
+						  VAR_datanodeNames, VAR_datanodeMasterServers,
+						  VAR_datanodePorts, NULL, VAR_datanodeMasterWALDirs, TRUE, TRUE);
 	/* GTM and datanode slaves, if any */
 	if(isVarYes(VAR_datanodeSlave))
+	{
 		checkResourceConflict(VAR_gtmName, VAR_gtmMasterServer, VAR_gtmMasterPort, NULL, VAR_gtmMasterDir,
 							  VAR_datanodeNames, VAR_datanodeMasterServers, VAR_datanodeSlavePorts, NULL, VAR_datanodeSlaveDirs,
 							  TRUE, TRUE);
+		checkResourceConflict(VAR_gtmName, VAR_gtmMasterServer, VAR_gtmMasterPort, NULL, VAR_gtmMasterDir,
+							  VAR_datanodeNames, VAR_datanodeMasterServers,
+							  VAR_datanodeSlavePorts, NULL,
+							  VAR_datanodeSlaveWALDirs,
+							  TRUE, TRUE);
+	}
 	/* 
 	 * GTM slave and others ------------
 	 */
@@ -955,11 +981,22 @@ static void verifyResource(void)
 		checkResourceConflict(VAR_gtmSlaveName, VAR_gtmSlaveServer, VAR_gtmSlavePort, NULL, VAR_gtmSlaveDir,
 							  VAR_datanodeNames, VAR_datanodeMasterServers, VAR_datanodePorts, NULL, VAR_datanodeMasterDirs,
 							  TRUE, TRUE);
+		checkResourceConflict(VAR_gtmSlaveName, VAR_gtmSlaveServer, VAR_gtmSlavePort, NULL, VAR_gtmSlaveDir,
+							  VAR_datanodeNames, VAR_datanodeMasterServers,
+							  VAR_datanodePorts, NULL, VAR_datanodeMasterWALDirs,
+							  TRUE, TRUE);
 		/* GTM slave and datanode slave, if any */
 		if (isVarYes(VAR_datanodeSlave))
+		{
 			checkResourceConflict(VAR_gtmSlaveName, VAR_gtmSlaveServer, VAR_gtmSlavePort, NULL, VAR_gtmSlaveDir,
 								  VAR_datanodeNames, VAR_datanodeSlaveServers, VAR_datanodeSlavePorts, NULL, VAR_datanodeSlaveDirs,
 								  TRUE, TRUE);
+			checkResourceConflict(VAR_gtmSlaveName, VAR_gtmSlaveServer, VAR_gtmSlavePort, NULL, VAR_gtmSlaveDir,
+								  VAR_datanodeNames, VAR_datanodeSlaveServers,
+								  VAR_datanodeSlavePorts, NULL,
+								  VAR_datanodeSlaveWALDirs,
+								  TRUE, TRUE);
+		}
 	}
 	/* 
 	 * GTM proxy and others ---------
@@ -979,11 +1016,22 @@ static void verifyResource(void)
 			checkResourceConflict(VAR_gtmProxyNames, VAR_gtmProxyServers, VAR_gtmProxyPorts, NULL, VAR_gtmProxyDirs,
 							  VAR_datanodeNames, VAR_datanodeMasterServers, VAR_datanodePorts, NULL, VAR_datanodeMasterDirs,
 							  TRUE, TRUE);
+			checkResourceConflict(VAR_gtmProxyNames, VAR_gtmProxyServers, VAR_gtmProxyPorts, NULL, VAR_gtmProxyDirs,
+							  VAR_datanodeNames, VAR_datanodeMasterServers,
+							  VAR_datanodePorts, NULL, VAR_datanodeMasterWALDirs,
+							  TRUE, TRUE);
 		/* GTM proxy and datanode slave, if any */
 		if (sval(VAR_datanodeSlave) && (strcmp(sval(VAR_datanodeSlave), "y") == 0))
+		{
 			checkResourceConflict(VAR_gtmProxyNames, VAR_gtmProxyServers, VAR_gtmProxyPorts, NULL, VAR_gtmProxyDirs,
 								  VAR_datanodeNames, VAR_datanodeSlaveServers, VAR_datanodeSlavePorts, NULL, VAR_datanodeSlaveDirs,
 								  TRUE, TRUE);
+			checkResourceConflict(VAR_gtmProxyNames, VAR_gtmProxyServers, VAR_gtmProxyPorts, NULL, VAR_gtmProxyDirs,
+								  VAR_datanodeNames, VAR_datanodeSlaveServers,
+								  VAR_datanodeSlavePorts, NULL,
+								  VAR_datanodeSlaveWALDirs,
+								  TRUE, TRUE);
+		}
 	}
 	/* 
 	 * Coordinator Masters and others
@@ -998,12 +1046,23 @@ static void verifyResource(void)
 	checkResourceConflict(VAR_coordNames, VAR_coordMasterServers, VAR_coordPorts, VAR_poolerPorts, VAR_coordMasterDirs,
 						  VAR_datanodeNames, VAR_datanodeMasterServers, VAR_datanodePorts, NULL, VAR_datanodeMasterDirs,
 						  FALSE, TRUE);
+	checkResourceConflict(VAR_coordNames, VAR_coordMasterServers, VAR_coordPorts, VAR_poolerPorts, VAR_coordMasterDirs,
+						  VAR_datanodeNames, VAR_datanodeMasterServers,
+						  VAR_datanodePorts, NULL, VAR_datanodeMasterWALDirs,
+						  FALSE, TRUE);
 	/* Coordinator masters and datanode slave, if any */
 	if (isVarYes(VAR_datanodeSlave))
+	{
 		checkResourceConflict(VAR_coordNames, VAR_coordMasterServers, VAR_coordPorts, VAR_poolerPorts, VAR_coordMasterDirs,
 							  VAR_datanodeNames, VAR_datanodeSlaveServers,
 							  VAR_datanodeSlavePorts, NULL, VAR_datanodeSlaveDirs,
 							  TRUE, TRUE);
+		checkResourceConflict(VAR_coordNames, VAR_coordMasterServers, VAR_coordPorts, VAR_poolerPorts, VAR_coordMasterDirs,
+							  VAR_datanodeNames, VAR_datanodeSlaveServers,
+							  VAR_datanodeSlavePorts, NULL,
+							  VAR_datanodeSlaveWALDirs,
+							  TRUE, TRUE);
+	}
 	/* 
 	 * Coordinator slaves and others
 	 */
@@ -1011,14 +1070,26 @@ static void verifyResource(void)
 	{
 		/* Coordinator slave and datanode masters */
 		checkResourceConflict(VAR_coordNames, VAR_coordSlaveServers, VAR_coordSlavePorts, VAR_coordSlavePoolerPorts, VAR_coordSlaveDirs,
-							  VAR_datanodeNames, VAR_datanodeSlaveServers, VAR_datanodePorts, NULL, VAR_datanodeSlaveDirs,
+							  VAR_datanodeNames, VAR_datanodeMasterServers,
+							  VAR_datanodePorts, NULL, VAR_datanodeMasterDirs,
+							  FALSE, TRUE);
+		checkResourceConflict(VAR_coordNames, VAR_coordSlaveServers, VAR_coordSlavePorts, VAR_coordSlavePoolerPorts, VAR_coordSlaveDirs,
+							  VAR_datanodeNames, VAR_datanodeMasterServers,
+							  VAR_datanodePorts, NULL, VAR_datanodeMasterWALDirs,
 							  FALSE, TRUE);
 		/* Coordinator slave and datanode slave, if any */
 		if (isVarYes(VAR_datanodeSlave))
+		{
 			checkResourceConflict(VAR_coordNames, VAR_coordSlaveServers, VAR_coordSlavePorts, VAR_coordSlavePoolerPorts, VAR_coordSlaveDirs,
 								  VAR_datanodeNames, VAR_datanodeSlaveServers,
 								  VAR_datanodeSlavePorts, NULL, VAR_datanodeSlaveDirs,
 								  TRUE, TRUE);
+			checkResourceConflict(VAR_coordNames, VAR_coordSlaveServers, VAR_coordSlavePorts, VAR_coordSlavePoolerPorts, VAR_coordSlaveDirs,
+								  VAR_datanodeNames, VAR_datanodeSlaveServers,
+								  VAR_datanodeSlavePorts, NULL,
+								  VAR_datanodeSlaveWALDirs,
+								  TRUE, TRUE);
+		}
 	}
 	/* 
 	 * Datanode masters and others ---
@@ -1027,12 +1098,23 @@ static void verifyResource(void)
 	checkResourceConflict(VAR_datanodeNames, VAR_datanodeMasterServers, VAR_datanodePorts, NULL, VAR_datanodeMasterDirs,
 						  NULL, NULL, NULL, NULL, NULL,
 						  FALSE, TRUE);
+	checkResourceConflict(VAR_datanodeNames, VAR_datanodeMasterServers,
+			VAR_datanodePorts, NULL, VAR_datanodeMasterWALDirs,
+						  NULL, NULL, NULL, NULL, NULL,
+						  FALSE, TRUE);
 	/* Datanode master and datanode slave, if any */
 	if (sval(VAR_datanodeSlave) && (strcmp(sval(VAR_datanodeSlave), "y") == 0))
+	{
 		checkResourceConflict(VAR_datanodeNames, VAR_datanodeMasterServers, VAR_datanodePorts, NULL, VAR_datanodeMasterDirs,
 							  VAR_datanodeNames, VAR_datanodeSlaveServers,
 							  VAR_datanodeSlavePorts, NULL, VAR_datanodeSlaveDirs,
 							  TRUE, FALSE);
+		checkResourceConflict(VAR_datanodeNames, VAR_datanodeMasterServers, VAR_datanodePorts, NULL, VAR_datanodeMasterDirs,
+							  VAR_datanodeNames, VAR_datanodeSlaveServers,
+							  VAR_datanodeSlavePorts, NULL,
+							  VAR_datanodeSlaveWALDirs,
+							  TRUE, FALSE);
+	}
 	if (anyConfigErrors)
 	{
 		elog(ERROR, "ERROR: Found conflicts among resources.  Exiting.\n");
diff --git a/contrib/pgxc_ctl/datanode_cmd.c b/contrib/pgxc_ctl/datanode_cmd.c
index cbc12400ad..e681cec289 100644
--- a/contrib/pgxc_ctl/datanode_cmd.c
+++ b/contrib/pgxc_ctl/datanode_cmd.c
@@ -65,10 +65,17 @@ cmd_t *prepare_initDatanodeMaster(char *nodeName)
 	FILE *f;
 	char timeStamp[MAXTOKEN+1];
 	char remoteDirCheck[MAXPATH * 2 + 128];
+	bool wal;
 
 	if ((idx = datanodeIdx(nodeName)) < 0)
 		return(NULL);
 
+	if (doesExist(VAR_datanodeMasterWALDirs, idx) &&
+			!is_none(aval(VAR_datanodeMasterWALDirs)[idx]))
+		wal = true;
+	else
+		wal = false;
+
 	remoteDirCheck[0] = '\0';
 	if (!forceInit)
 	{
@@ -78,6 +85,15 @@ cmd_t *prepare_initDatanodeMaster(char *nodeName)
 				aval(VAR_datanodeMasterDirs)[idx],
 				aval(VAR_datanodeMasterDirs)[idx]
 			   );
+		if (wal)
+		{
+			sprintf(remoteDirCheck, "if [ \"$(ls -A %s 2> /dev/null)\" ]; then echo 'ERROR: "
+					"target directory (%s) exists and not empty. "
+					"Skip Datanode initilialization'; exit; fi",
+					aval(VAR_datanodeMasterWALDirs)[idx],
+					aval(VAR_datanodeMasterWALDirs)[idx]
+				   );
+		}
 
 	}
 
@@ -86,10 +102,13 @@ cmd_t *prepare_initDatanodeMaster(char *nodeName)
 	snprintf(newCommand(cmdInitdb), MAXLINE,
 			 "%s;"
 			 "rm -rf %s;"
-			 "mkdir -p %s; initdb --nodename %s -D %s",
+			 "mkdir -p %s; initdb --nodename %s %s %s -D %s",
 			 remoteDirCheck,
 			 aval(VAR_datanodeMasterDirs)[idx], aval(VAR_datanodeMasterDirs)[idx],
-			 aval(VAR_datanodeNames)[idx], aval(VAR_datanodeMasterDirs)[idx]);
+			 aval(VAR_datanodeNames)[idx],
+			 wal ? "-X" : "",
+			 wal ? aval(VAR_datanodeMasterWALDirs)[idx] : "",
+			 aval(VAR_datanodeMasterDirs)[idx]);
 		
 	/* Initialize postgresql.conf */
 	appendCmdEl(cmdInitdb, (cmdPgConf = initCmd(aval(VAR_datanodeMasterServers)[idx])));
@@ -919,7 +938,8 @@ static int failover_oneDatanode(int datanodeIdx)
  *
  *-----------------------------------------------------------------------*/
 int add_datanodeMaster(char *name, char *host, int port, int pooler, char *dir,
-		char *restore_dname, char *extraConf, char *extraPgHbaConf)
+		char *waldir, char *restore_dname, char *extraConf,
+		char *extraPgHbaConf)
 {
 	FILE *f, *lockf;
 	int size, idx;
@@ -935,7 +955,13 @@ int add_datanodeMaster(char *name, char *host, int port, int pooler, char *dir,
 	char **confFiles = NULL;
 	char **pgHbaConfFiles = NULL;
 	char *only_globals = "-g";
+	bool wal;
 
+	if (waldir && (strcasecmp(waldir, "none") != 0))
+		wal = true;
+	else
+		wal = false;
+	
 	/* Check if all the datanodes are running */
 	if (!check_AllDatanodeRunning())
 	{
@@ -958,6 +984,11 @@ int add_datanodeMaster(char *name, char *host, int port, int pooler, char *dir,
 		elog(ERROR, "ERROR: directory \"%s\" conflicts at host %s.\n", dir, host);
 		return 1;
 	}
+	if (checkDirConflict(host, waldir))
+	{
+		elog(ERROR, "ERROR: directory \"%s\" conflicts at host %s.\n", waldir, host);
+		return 1;
+	}
 	/*
 	 * Check if datanode masgter configuration is consistent
 	 */
@@ -966,6 +997,7 @@ int add_datanodeMaster(char *name, char *host, int port, int pooler, char *dir,
 	    (arraySizeName(VAR_datanodePoolerPorts) != size) ||
 		(arraySizeName(VAR_datanodeMasterServers) != size) ||
 		(arraySizeName(VAR_datanodeMasterDirs) != size) ||
+		(arraySizeName(VAR_datanodeMasterWALDirs) != size) ||
 		(arraySizeName(VAR_datanodeMaxWALSenders) != size) ||
 		(arraySizeName(VAR_datanodeSpecificExtraConfig) != size) ||
 		(arraySizeName(VAR_datanodeSpecificExtraPgHba) != size))
@@ -979,6 +1011,7 @@ int add_datanodeMaster(char *name, char *host, int port, int pooler, char *dir,
 		(extendVar(VAR_datanodePorts, idx + 1, "none")  != 0) ||
 		(extendVar(VAR_datanodePoolerPorts, idx + 1, "none")  != 0) ||
 		(extendVar(VAR_datanodeMasterDirs, idx + 1, "none")  != 0) ||
+		(extendVar(VAR_datanodeMasterWALDirs, idx + 1, "none")  != 0) ||
 		(extendVar(VAR_datanodeMaxWALSenders, idx + 1, "none")  != 0) ||
 		(extendVar(VAR_datanodeSlaveServers, idx + 1, "none")  != 0) ||
 		(extendVar(VAR_datanodeSlavePorts, idx + 1, "none")  != 0) ||
@@ -1006,6 +1039,7 @@ int add_datanodeMaster(char *name, char *host, int port, int pooler, char *dir,
 	assign_arrayEl(VAR_datanodePorts, idx, port_s, "-1");
 	assign_arrayEl(VAR_datanodePoolerPorts, idx, pooler_s, "-1");
 	assign_arrayEl(VAR_datanodeMasterDirs, idx, dir, NULL);
+	assign_arrayEl(VAR_datanodeMasterWALDirs, idx, waldir, NULL);
 	assign_arrayEl(VAR_datanodeMaxWALSenders, idx, aval(VAR_datanodeMaxWALSenders)[0], NULL);	/* Could be vulnerable */
 	assign_arrayEl(VAR_datanodeSlaveServers, idx, "none", NULL);
 	assign_arrayEl(VAR_datanodeSlavePorts, idx, "-1", NULL);
@@ -1051,6 +1085,7 @@ int add_datanodeMaster(char *name, char *host, int port, int pooler, char *dir,
 	fprintAval(f, VAR_datanodePorts);
 	fprintAval(f, VAR_datanodePoolerPorts);
 	fprintAval(f, VAR_datanodeMasterDirs);
+	fprintAval(f, VAR_datanodeMasterWALDirs);
 	fprintAval(f, VAR_datanodeMaxWALSenders);
 	fprintAval(f, VAR_datanodeSlaveServers);
 	fprintAval(f, VAR_datanodeSlavePorts);
@@ -1070,7 +1105,10 @@ int add_datanodeMaster(char *name, char *host, int port, int pooler, char *dir,
 	gtmPort = (gtmPxyIdx > 0) ? aval(VAR_gtmProxyPorts)[gtmPxyIdx] : sval(VAR_gtmMasterPort);
 
 	/* initdb */
-	doImmediate(host, NULL, "initdb -D %s --nodename %s", dir, name);
+	doImmediate(host, NULL, "initdb -D %s %s %s --nodename %s", dir,
+			wal ? "-X" : "",
+			wal ? waldir : "",
+			name);
 
 	/* Edit configurations */
 	if ((f = pgxc_popen_w(host, "cat >> %s/postgresql.conf", dir)))
@@ -1221,13 +1259,21 @@ int add_datanodeMaster(char *name, char *host, int port, int pooler, char *dir,
 }
 
 
-int add_datanodeSlave(char *name, char *host, int port, int pooler, char *dir, char *archDir)
+int add_datanodeSlave(char *name, char *host, int port, int pooler, char *dir,
+		char *walDir, char *archDir)
 {
 	int idx;
 	FILE *f;
 	char port_s[MAXTOKEN+1];
     char pooler_s[MAXTOKEN+1];
 	int kk;
+	bool wal;
+
+	if (walDir && (strcasecmp(walDir, "none") != 0))
+		wal = true;
+	else
+		wal = false;
+
 
 	/* Check if the name is valid datanode */
 	if ((idx = datanodeIdx(name)) < 0)
@@ -1256,9 +1302,10 @@ int add_datanodeSlave(char *name, char *host, int port, int pooler, char *dir, c
 		elog(ERROR, "ERROR: the port %s has already been used in the host %s.\n",  aval(VAR_datanodePorts)[idx], host);
 		return 1;
 	}
-	if (checkDirConflict(host, dir) || checkDirConflict(host, archDir))
+	if (checkDirConflict(host, dir) || checkDirConflict(host, archDir) ||
+			checkDirConflict(host, walDir))
 	{
-		elog(ERROR, "ERROR: directory %s or %s has already been used by other node.\n", dir, archDir);
+		elog(ERROR, "ERROR: directory %s or %s or %s has already been used by other node.\n", dir, archDir, walDir);
 		return 1;
 	}
 	/* Check if the datanode master is running */
@@ -1270,6 +1317,8 @@ int add_datanodeSlave(char *name, char *host, int port, int pooler, char *dir, c
 	/* Prepare the resources (directories) */
 	doImmediate(host, NULL, "rm -rf %s; mkdir -p %s;chmod 0700 %s", dir, dir, dir);
 	doImmediate(host, NULL, "rm -rf %s; mkdir -p %s;chmod 0700 %s", archDir, archDir, archDir);
+	doImmediate(host, NULL, "rm -rf %s; mkdir -p %s;chmod 0700 %s", walDir,
+			walDir, walDir);
 	/* Reconfigure the master with WAL archive */
 	/* Update the configuration and backup the configuration file */
 	if ((f = pgxc_popen_w(aval(VAR_datanodeMasterServers)[idx], "cat >> %s/postgresql.conf", aval(VAR_datanodeMasterDirs)[idx])) == NULL)
@@ -1337,6 +1386,7 @@ int add_datanodeSlave(char *name, char *host, int port, int pooler, char *dir, c
 	assign_arrayEl(VAR_datanodeSlavePorts, idx, port_s, NULL);
 	assign_arrayEl(VAR_datanodeSlavePoolerPorts, idx, pooler_s, NULL);
 	assign_arrayEl(VAR_datanodeSlaveDirs, idx, dir, NULL);
+	assign_arrayEl(VAR_datanodeSlaveWALDirs, idx, walDir, NULL);
 	assign_arrayEl(VAR_datanodeArchLogDirs, idx, archDir, NULL);
 	/* Update the configuration file and backup it */
 	if ((f = fopen(pgxc_ctl_config_path, "a")) == NULL)
@@ -1356,6 +1406,7 @@ int add_datanodeSlave(char *name, char *host, int port, int pooler, char *dir, c
 	fprintAval(f, VAR_datanodeSlavePoolerPorts);
 	fprintAval(f, VAR_datanodeArchLogDirs);
 	fprintAval(f, VAR_datanodeSlaveDirs);
+	fprintAval(f, VAR_datanodeSlaveWALDirs);
 	fprintf(f, "%s", "#----End of reconfiguration -------------------------\n");
 	fclose(f);
 	backup_configuration();
@@ -1377,8 +1428,11 @@ int add_datanodeSlave(char *name, char *host, int port, int pooler, char *dir, c
 	doImmediate(aval(VAR_datanodeMasterServers)[idx], NULL, 
 				"pg_ctl start -w -Z datanode -D %s", aval(VAR_datanodeMasterDirs)[idx]);
 	/* pg_basebackup */
-	doImmediate(host, NULL, "pg_basebackup -p %s -h %s -D %s -x",
-				aval(VAR_datanodePorts)[idx], aval(VAR_datanodeMasterServers)[idx], dir);
+	doImmediate(host, NULL, "pg_basebackup -p %s -h %s -D %s -x %s %s",
+				aval(VAR_datanodePorts)[idx],
+				aval(VAR_datanodeMasterServers)[idx], dir,
+				wal ? "--xlogdir" : "",
+				wal ? walDir : "");
 	/* Update the slave configuration with hot standby and port */
 	if ((f = pgxc_popen_w(host, "cat >> %s/postgresql.conf", dir)) == NULL)
 	{
@@ -1565,12 +1619,14 @@ int remove_datanodeMaster(char *name, int clean_opt)
 	/* Update configuration and backup --> should cleanup "none" entries here */
 	replace_arrayEl(VAR_datanodeNames, idx, "none", NULL);
 	replace_arrayEl(VAR_datanodeMasterDirs, idx, "none", NULL);
+	replace_arrayEl(VAR_datanodeMasterWALDirs, idx, "none", NULL);
 	replace_arrayEl(VAR_datanodePorts, idx, "-1", "-1");
 	replace_arrayEl(VAR_datanodePoolerPorts, idx, "-1", "-1");
 	replace_arrayEl(VAR_datanodeMasterServers, idx, "none", NULL);
 	replace_arrayEl(VAR_datanodeMaxWALSenders, idx, "0", "0");
 	replace_arrayEl(VAR_datanodeSlaveServers, idx, "none", NULL);
 	replace_arrayEl(VAR_datanodeSlaveDirs, idx, "none", NULL);
+	replace_arrayEl(VAR_datanodeSlaveWALDirs, idx, "none", NULL);
 	replace_arrayEl(VAR_datanodeArchLogDirs, idx, "none", NULL);
 	replace_arrayEl(VAR_datanodeSpecificExtraConfig, idx, "none", NULL);
 	handle_no_slaves();
@@ -1591,6 +1647,7 @@ int remove_datanodeMaster(char *name, int clean_opt)
 	fprintSval(f, VAR_datanodeSlave);
 	fprintAval(f, VAR_datanodeNames);
 	fprintAval(f, VAR_datanodeMasterDirs);
+	fprintAval(f, VAR_datanodeMasterWALDirs);
 	fprintAval(f, VAR_datanodePorts);
 	fprintAval(f, VAR_datanodePoolerPorts);
 	fprintAval(f, VAR_datanodeMasterServers);
@@ -1598,6 +1655,7 @@ int remove_datanodeMaster(char *name, int clean_opt)
 	fprintAval(f, VAR_datanodeSlaveServers);
 	fprintAval(f, VAR_datanodeSlavePorts);
 	fprintAval(f, VAR_datanodeSlaveDirs);
+	fprintAval(f, VAR_datanodeSlaveWALDirs);
 	fprintAval(f, VAR_datanodeArchLogDirs);
 	fprintAval(f, VAR_datanodeSpecificExtraConfig);
 	fclose(f);
@@ -1658,6 +1716,7 @@ int remove_datanodeSlave(char *name, int clean_opt)
 	 */
 	replace_arrayEl(VAR_datanodeSlaveServers, idx, "none", NULL);
 	replace_arrayEl(VAR_datanodeSlaveDirs, idx, "none", NULL);
+	replace_arrayEl(VAR_datanodeSlaveWALDirs, idx, "none", NULL);
 	replace_arrayEl(VAR_datanodeArchLogDirs, idx, "none", NULL);
 	handle_no_slaves();
 	/*
@@ -1677,6 +1736,7 @@ int remove_datanodeSlave(char *name, int clean_opt)
 	fprintSval(f, VAR_datanodeSlave);
 	fprintAval(f, VAR_datanodeSlaveServers);
 	fprintAval(f, VAR_datanodeSlaveDirs);
+	fprintAval(f, VAR_datanodeSlaveWALDirs);
 	fprintAval(f, VAR_datanodeArchLogDirs);
 	fclose(f);
 	backup_configuration();
@@ -1692,16 +1752,29 @@ cmd_t *prepare_cleanDatanodeMaster(char *nodeName)
 {
 	cmd_t *cmd;
 	int idx;
-	
+	bool wal;
+
 	if ((idx = datanodeIdx(nodeName)) <  0)
 	{
 		elog(ERROR, "ERROR: %s is not a datanode\n", nodeName);
 		return(NULL);
 	}
+
+	if (doesExist(VAR_datanodeMasterWALDirs, idx) &&
+		!is_none(aval(VAR_datanodeMasterWALDirs)[idx]))
+		wal = true;
+	else
+		wal = false;
+	
 	cmd = initCmd(aval(VAR_datanodeMasterServers)[idx]);
 	snprintf(newCommand(cmd), MAXLINE,
-			 "rm -rf %s; mkdir -p %s; chmod 0700 %s; rm -f /tmp/.s.*%d*",
-			 aval(VAR_datanodeMasterDirs)[idx], aval(VAR_datanodeMasterDirs)[idx], aval(VAR_datanodeMasterDirs)[idx], atoi(aval(VAR_datanodePoolerPorts)[idx]));
+			 "rm -rf %s; %s %s ; mkdir -p %s; chmod 0700 %s; rm -f /tmp/.s.*%d*",
+			 aval(VAR_datanodeMasterDirs)[idx],
+			 wal ? "rm -rf " : "",
+			 wal ? aval(VAR_datanodeMasterWALDirs)[idx] : "",
+			 aval(VAR_datanodeMasterDirs)[idx],
+			 aval(VAR_datanodeMasterDirs)[idx],
+			 atoi(aval(VAR_datanodePoolerPorts)[idx]));
 	return(cmd);
 }
 
@@ -1740,6 +1813,7 @@ cmd_t *prepare_cleanDatanodeSlave(char *nodeName)
 {
 	cmd_t *cmd;
 	int idx;
+	bool wal;
 	
 	if ((idx = datanodeIdx(nodeName)) <  0)
 	{
@@ -1748,10 +1822,20 @@ cmd_t *prepare_cleanDatanodeSlave(char *nodeName)
 	}
 	if (!doesExist(VAR_datanodeSlaveServers, idx) || is_none(aval(VAR_datanodeSlaveServers)[idx]))
 		return NULL;
+
+	if (doesExist(VAR_datanodeSlaveWALDirs, idx) &&
+		!is_none(aval(VAR_datanodeSlaveWALDirs)[idx]))
+		wal = true;
+	else
+		wal = false;
+	
 	cmd = initCmd(aval(VAR_datanodeSlaveServers)[idx]);
 	snprintf(newCommand(cmd), MAXLINE,
-			 "rm -rf %s; mkdir -p %s; chmod 0700 %s",
-			 aval(VAR_datanodeSlaveDirs)[idx], aval(VAR_datanodeSlaveDirs)[idx], aval(VAR_datanodeSlaveDirs)[idx]);
+			 "rm -rf %s; %s %s ; mkdir -p %s; chmod 0700 %s",
+			 aval(VAR_datanodeSlaveDirs)[idx],
+			 wal ? " rm -rf " : "",
+			 wal ? aval(VAR_datanodeSlaveWALDirs)[idx] : "",
+			 aval(VAR_datanodeSlaveDirs)[idx], aval(VAR_datanodeSlaveDirs)[idx]);
 	return(cmd);
 }
 
diff --git a/contrib/pgxc_ctl/datanode_cmd.h b/contrib/pgxc_ctl/datanode_cmd.h
index 7f0224dc77..5eee9b6a53 100644
--- a/contrib/pgxc_ctl/datanode_cmd.h
+++ b/contrib/pgxc_ctl/datanode_cmd.h
@@ -53,12 +53,14 @@ extern cmd_t *prepare_cleanDatanodeSlave(char *nodeName);
 
 #ifdef XCP
 extern int add_datanodeMaster(char *name, char *host, int port, int pooler,
-		char *dir, char *restore_dname, char *extraConf, char *extraPgHbaConf);
+		char *dir, char *walDir, char *restore_dname, char *extraConf,
+		char *extraPgHbaConf);
 #else
 extern int add_datanodeMaster(char *name, char *host, int port, char *dir,
 		char *restore_dname, char *extraConf, char *extraPgHbaConf);
 #endif
-extern int add_datanodeSlave(char *name, char *host, int port, int pooler, char *dir, char *archDir);
+extern int add_datanodeSlave(char *name, char *host, int port, int pooler,
+		char *dir, char *walDir, char *archDir);
 extern int remove_datanodeMaster(char *name, int clean_opt);
 extern int remove_datanodeSlave(char *name, int clean_opt);
 
diff --git a/contrib/pgxc_ctl/do_command.c b/contrib/pgxc_ctl/do_command.c
index 9ff5f10344..c6d0cae903 100644
--- a/contrib/pgxc_ctl/do_command.c
+++ b/contrib/pgxc_ctl/do_command.c
@@ -866,6 +866,7 @@ static void do_add_command(char *line)
 	char *port;
 	char *pooler;
 	char *dir;
+	char *walDir;
 	char *archDir;
 	char *dnode;
 	char *extraConf;
@@ -973,16 +974,18 @@ static void do_add_command(char *line)
 			GetAndSet(port, "ERROR: please specify the port number for the datanode master\n");
 			GetAndSet(pooler, "ERROR: please specify the pooler port number for the datanode master.\n");
 			GetAndSet(dir, "ERROR: please specify the working director for the datanode master\n");
+			GetAndSet(walDir, "ERROR: please specify the WAL directory for the datanode master WAL. Specify 'none' for default\n");
 			GetAndSet(dnode, "ERROR: please specify name of existing datanode of which this will be a copy of. Specify 'none' for a bare datanode\n");
 			GetAndSet(extraConf, "ERROR: please specify file to read extra configuration. Specify 'none' if nothig extra to be added.\n");
 			GetAndSet(extraPgHbaConf, "ERROR: please specify file to read extra pg_hba configuration. Specify 'none' if nothig extra to be added.\n");
 			add_datanodeMaster(name, host, atoi(port), atoi(pooler), dir,
-					dnode, extraConf, extraPgHbaConf);
+					walDir, dnode, extraConf, extraPgHbaConf);
 			freeAndReset(name);
 			freeAndReset(host);
 			freeAndReset(port);
 			freeAndReset(pooler);
 			freeAndReset(dir);
+			freeAndReset(walDir);
 		}
 		else
 		{
@@ -990,15 +993,18 @@ static void do_add_command(char *line)
 			GetAndSet(host, "ERROR: please specify the host for the datanode slave\n");
 			GetAndSet(port, "ERROR: please specify the port number for the datanode slave\n");
 			GetAndSet(pooler, "ERROR: please specify the pooler port number for the datanode slave.\n");
-			GetAndSet(dir, "ERROR: please specify the working director for datanode slave\n");
+			GetAndSet(dir, "ERROR: please specify the working directory for datanode slave\n");
+			GetAndSet(walDir, "ERROR: please specify the WAL directory for datanode slave WAL. Specify 'none' for default.\n");
 			GetAndSet(archDir, "ERROR: please specify WAL archive directory for datanode slave\n");
 			
-			add_datanodeSlave(name, host, atoi(port), atoi(pooler), dir, archDir);
+			add_datanodeSlave(name, host, atoi(port), atoi(pooler), dir,
+					walDir, archDir);
 			freeAndReset(name);
 			freeAndReset(host);
 			freeAndReset(port);
 			freeAndReset(pooler);
 			freeAndReset(dir);
+			freeAndReset(walDir);
 		}
 	}
 	return;
@@ -2661,8 +2667,8 @@ do_show_help(char *line)
 				"add gtm_proxy name host port dir\n"
 				"add coordinator master name host port pooler dir extra_conf extra_pghba\n"
 				"add coordinator slave name host port pooler dir archDir\n"
-				"add datanode master name host port pooler dir restore_datanode_name extra_conf extra_pghba\n"
-				"add datanode slave name host port pooler dir archDir\n"
+				"add datanode master name host port pooler dir xlogdir restore_datanode_name extra_conf extra_pghba\n"
+				"add datanode slave name host port pooler dir xlogdir archDir\n"
 				"\n"
 				"Add the specified node to your postgres-xl cluster:\n"
 				"For more details, please see the pgxc_ctl documentation\n"
diff --git a/contrib/pgxc_ctl/varnames.h b/contrib/pgxc_ctl/varnames.h
index ca545f8790..f3e65a4420 100644
--- a/contrib/pgxc_ctl/varnames.h
+++ b/contrib/pgxc_ctl/varnames.h
@@ -98,6 +98,7 @@
 /* Datanode masters */
 #define VAR_datanodeMasterServers	"datanodeMasterServers"
 #define VAR_datanodeMasterDirs		"datanodeMasterDirs"
+#define VAR_datanodeMasterWALDirs		"datanodeMasterWALDirs"
 #define VAR_datanodeMaxWALSenders	"datanodeMaxWALSenders"
 
 /* Datanode slaves */
@@ -107,6 +108,7 @@
 #define VAR_datanodeSlavePoolerPorts		"datanodeSlavePoolerPorts"
 #define VAR_datanodeSlaveSync		"datanodeSlaveSync"
 #define VAR_datanodeSlaveDirs		"datanodeSlaveDirs"
+#define VAR_datanodeSlaveWALDirs		"datanodeSlaveWALDirs"
 #define VAR_datanodeArchLogDirs		"datanodeArchLogDirs"
 
 /* Datanode configuration files */
diff --git a/doc/src/sgml/pgxc_ctl-ref.sgml b/doc/src/sgml/pgxc_ctl-ref.sgml
index 9a2f1a9c7c..50c094c060 100644
--- a/doc/src/sgml/pgxc_ctl-ref.sgml
+++ b/doc/src/sgml/pgxc_ctl-ref.sgml
@@ -1388,6 +1388,15 @@ PGXC$ prepare config minimal my_minimal_config.conf
        </para>
       </listitem>
      </varlistentry>
+
+     <varlistentry>
+      <term><option>datanodeMasterWALDirs</option></term>
+      <listitem>
+       <para>
+        Array of Datanode masters' XLOG directories.
+       </para>
+      </listitem>
+     </varlistentry>
     
      <varlistentry>
       <term><option>datanodeMaterServers</option></term>
@@ -1489,6 +1498,15 @@ PGXC$ prepare config minimal my_minimal_config.conf
      </varlistentry>
     
      <varlistentry>
+      <term><option>datanodeSlaveWALDirs</option></term>
+      <listitem>
+       <para>
+        Array of XLOG directories for each Datanode slave.
+       </para>
+      </listitem>
+     </varlistentry>
+    
+     <varlistentry>
       <term><option>datanodeSlaveServers</option></term>
       <listitem>
        <para>
@@ -1580,8 +1598,8 @@ PGXC$ prepare config minimal my_minimal_config.conf
     <term><literal>add gtm_proxy <replaceable class="parameter">name</replaceable> <replaceable class="parameter">host</replaceable> <replaceable class="parameter">port</replaceable> <replaceable class="parameter">dir</replaceable></literal></term>
     <term><literal>add coordinator master <replaceable class="parameter">name</replaceable> <replaceable class="parameter">host</replaceable> <replaceable class="parameter">port</replaceable> <replaceable class="parameter">pooler</replaceable> <replaceable class="parameter">dir</replaceable>< <replaceable class="parameter">extraServerConf</replaceable> <replaceable class="parameter">extraPgHbaConf</replaceable></literal></term>
     <term><literal>add coordinator slave <replaceable class="parameter">name</replaceable> <replaceable class="parameter">host</replaceable> <replaceable class="parameter">port</replaceable> <replaceable class="parameter">pooler</replaceable> <replaceable class="parameter">dir</replaceable> <replaceable class="parameter">archDir</replaceable></literal></term>
-    <term><literal>add datanode master <replaceable class="parameter">name</replaceable> <replaceable class="parameter">host</replaceable>  <replaceable class="parameter">port</replaceable> <replaceable class="parameter">pooler</replaceable> <replaceable class="parameter">dir</replaceable>  <replaceable class="parameter">restoreDatanode</replaceable> <replaceable class="parameter">extraServerConf</replaceable> <replaceable class="parameter">extraPgHbaConf</replaceable></literal></term>
-    <term><literal>add datanode slave <replaceable class="parameter">name</replaceable> <replaceable class="parameter">host</replaceable>  <replaceable class="parameter">port</replaceable> <replaceable class="parameter">pooler</replaceable> <replaceable class="parameter">dir</replaceable> <replaceable class="parameter">archDir</replaceable></literal></term>
+    <term><literal>add datanode master <replaceable class="parameter">name</replaceable> <replaceable class="parameter">host</replaceable>  <replaceable class="parameter">port</replaceable> <replaceable class="parameter">pooler</replaceable>  <replaceable class="parameter">dir</replaceable> <replaceable class="parameter">xlogdir</replaceable>  <replaceable class="parameter">restoreDatanode</replaceable> <replaceable class="parameter">extraServerConf</replaceable> <replaceable class="parameter">extraPgHbaConf</replaceable></literal></term>
+    <term><literal>add datanode slave <replaceable class="parameter">name</replaceable> <replaceable class="parameter">host</replaceable>  <replaceable class="parameter">port</replaceable> <replaceable class="parameter">pooler</replaceable>  <replaceable class="parameter">dir</replaceable> <replaceable class="parameter">xlogdir</replaceable> <replaceable class="parameter">archDir</replaceable></literal></term>
     <listitem>
      <para>
       Add the specified node to your Postgres-XL cluster.  Each node needs a
@@ -1591,7 +1609,8 @@ PGXC$ prepare config minimal my_minimal_config.conf
       Datanodes.  Coordinator and Datanode slaves need a directory to receive
       WAL segments from their master. While adding a Coordinator master and a
       Datanode master, extra server configuration and extra pg_hba
-      configuration parameters can be specified in a file.
+      configuration parameters can be specified in a file. A separate XLOG
+      directory may also be specified for Datanode master and slave.
      </para>
      <para>
       When you add a Coordinator and Datanode master, node information at