Support for CLEAN CONNECTION

Utility to clean up Postgres-XC Pooler connections.
This utility is launched to all the Coordinators of the cluster

Use of CLEAN CONNECTION is limited to a super user.
It is advised to clean connections before dropping a Database.

SQL query synopsis is as follows:
CLEAN CONNECTION TO
             (COORDINATOR num | DATANODE num | ALL {FORCE})
              FOR DATABASE dbname
Connection cleaning has to be made on a chosen database called dbname.

It is also possible to clean connections of several Coordinators or Datanodes
Ex:  CLEAN CONNECTION TO DATANODE 1,5,7 FOR DATABASE template1
             CLEAN CONNECTION TO COORDINATOR 2,4,6 FOR DATABASE template1

Or even to all Coordinators/Datanodes at the same time
Ex:  CLEAN CONNECTION TO DATANODE * FOR DATABASE template1
             CLEAN CONNECTION TO COORDINATOR * FOR DATABASE template1

When FORCE is used, all the transactions using pooler connections are aborted,
and pooler connections are cleaned up.
Ex:  CLEAN CONNECTION TO ALL FORCE FOR DATABASE template1;

FORCE can only be used with TO ALL, as it takes a lock on pooler to stop requests
asking for connections, aborts all the connections in the cluster, and cleans up
pool connections

15 files changed, 803 insertions, 23 deletions

diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 9ffada513a..21e6021669 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -191,7 +191,7 @@ static TypeName *TableFuncTypeName(List *columns);
 		AlterForeignServerStmt AlterGroupStmt
 		AlterObjectSchemaStmt AlterOwnerStmt AlterSeqStmt AlterTableStmt
 		AlterUserStmt AlterUserMappingStmt AlterUserSetStmt AlterRoleStmt AlterRoleSetStmt
-		AnalyzeStmt ClosePortalStmt ClusterStmt CommentStmt
+		AnalyzeStmt CleanConnStmt ClosePortalStmt ClusterStmt CommentStmt
 		ConstraintsSetStmt CopyStmt CreateAsStmt CreateCastStmt
 		CreateDomainStmt CreateGroupStmt CreateOpClassStmt
 		CreateOpFamilyStmt AlterOpFamilyStmt CreatePLangStmt
@@ -328,7 +328,7 @@ static TypeName *TableFuncTypeName(List *columns);
 %type <boolean> opt_freeze opt_default opt_recheck
 %type <defelt>	opt_binary opt_oids copy_delimiter
 
-%type <list>	node_list
+%type <list>	data_node_list coord_list
 %type <str>		DirectStmt
 
 %type <boolean> copy_from
@@ -445,7 +445,7 @@ static TypeName *TableFuncTypeName(List *columns);
 	BOOLEAN_P BOTH BY
 
 	CACHE CALLED CASCADE CASCADED CASE CAST CATALOG_P CHAIN CHAR_P
-	CHARACTER CHARACTERISTICS CHECK CHECKPOINT CLASS CLOSE
+	CHARACTER CHARACTERISTICS CHECK CHECKPOINT CLASS CLEAN CLOSE
 	CLUSTER COALESCE COLLATE COLUMN COMMENT COMMIT
 	COMMITTED CONCURRENTLY CONFIGURATION CONNECTION CONSTRAINT CONSTRAINTS
 	CONTENT_P CONTINUE_P CONVERSION_P COORDINATOR COPY COST CREATE CREATEDB
@@ -637,6 +637,7 @@ stmt :
 			| AlterUserStmt
 			| AnalyzeStmt
 			| CheckPointStmt
+			| CleanConnStmt
 			| ClosePortalStmt
 			| ClusterStmt
 			| CommentStmt
@@ -6545,7 +6546,7 @@ ExecDirectStmt: EXECUTE DIRECT ON COORDINATOR DirectStmt
 					n->query = $5;
 					$$ = (Node *)n;
 				}
-				| EXECUTE DIRECT ON NODE node_list DirectStmt
+				| EXECUTE DIRECT ON NODE data_node_list DirectStmt
 				{
 					ExecDirectStmt *n = makeNode(ExecDirectStmt);
 					n->coordinator = FALSE;
@@ -6559,9 +6560,21 @@ DirectStmt:
 			Sconst					/* by default all are $$=$1 */
 		;
 
-node_list: 
+coord_list:
 		 	Iconst					{ $$ = list_make1(makeInteger($1)); }
-			| node_list ',' Iconst	{ $$ = lappend($1, makeInteger($3)); }
+			| coord_list ',' Iconst	{ $$ = lappend($1, makeInteger($3)); }
+			| '*'
+				{
+					int i;
+					$$ = NIL;
+					for (i=1; i<=NumCoords; i++)
+						$$ = lappend($$, makeInteger(i));
+				}
+		;
+
+data_node_list:
+		 	Iconst					{ $$ = list_make1(makeInteger($1)); }
+			| data_node_list ',' Iconst	{ $$ = lappend($1, makeInteger($3)); }
 			| '*'
 				{
 					int i;
@@ -6574,6 +6587,44 @@ node_list:
 /*****************************************************************************
  *
  *		QUERY:
+ *
+ *		CLEAN CONNECTION TO (COORDINATOR num | NODE num | ALL {FORCE})
+ *				FOR DATABASE dbname
+ *
+ *****************************************************************************/
+
+CleanConnStmt: CLEAN CONNECTION TO COORDINATOR coord_list FOR DATABASE database_name
+				{
+					CleanConnStmt *n = makeNode(CleanConnStmt);
+					n->is_coord = true;
+					n->nodes = $5;
+					n->is_force = false;
+					n->dbname = $8;
+					$$ = (Node *)n;
+				}
+				| CLEAN CONNECTION TO NODE data_node_list FOR DATABASE database_name
+				{
+					CleanConnStmt *n = makeNode(CleanConnStmt);
+					n->is_coord = false;
+					n->nodes = $5;
+					n->is_force = false;
+					n->dbname = $8;
+					$$ = (Node *)n;
+				}
+				| CLEAN CONNECTION TO ALL opt_force FOR DATABASE database_name
+				{
+					CleanConnStmt *n = makeNode(CleanConnStmt);
+					n->is_coord = true;
+					n->nodes = NIL;
+					n->is_force = $5;
+					n->dbname = $8;
+					$$ = (Node *)n;
+				}
+		;
+
+/*****************************************************************************
+ *
+ *		QUERY:
  *				PREPARE <plan_name> [(args, ...)] AS <query>
  *
  *****************************************************************************/
@@ -10259,6 +10310,7 @@ unreserved_keyword:
 			| CHARACTERISTICS
 			| CHECKPOINT
 			| CLASS
+			| CLEAN
 			| CLOSE
 			| CLUSTER
 			| COMMENT
diff --git a/src/backend/pgxc/locator/locator.c b/src/backend/pgxc/locator/locator.c
index 098e254531..845f9fc78c 100644
--- a/src/backend/pgxc/locator/locator.c
+++ b/src/backend/pgxc/locator/locator.c
@@ -466,8 +466,8 @@ GetAllDataNodes(void)
 
 /*
  * Return a list of all Coordinators
- * This is used to send DDL to all nodes
- * Do not put in the list the local Coordinator where this function is launched
+ * This is used to send DDL to all nodes and to clean up pooler connections.
+ * Do not put in the list the local Coordinator where this function is launched.
  */
 List *
 GetAllCoordNodes(void)
diff --git a/src/backend/pgxc/pool/Makefile b/src/backend/pgxc/pool/Makefile
index f8679eb10c..8c2b66a155 100644
--- a/src/backend/pgxc/pool/Makefile
+++ b/src/backend/pgxc/pool/Makefile
@@ -14,6 +14,6 @@ subdir = src/backend/pgxc/pool
 top_builddir = ../../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = pgxcnode.o execRemote.o poolmgr.o poolcomm.o postgresql_fdw.o
+OBJS = pgxcnode.o execRemote.o poolmgr.o poolcomm.o postgresql_fdw.o poolutils.o
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/pgxc/pool/pgxcnode.c b/src/backend/pgxc/pool/pgxcnode.c
index 3bc3a83aa4..cbaf68c610 100644
--- a/src/backend/pgxc/pool/pgxcnode.c
+++ b/src/backend/pgxc/pool/pgxcnode.c
@@ -268,7 +268,7 @@ pgxc_node_receive(const int conn_count,
 	FD_ZERO(&readfds);
 	for (i = 0; i < conn_count; i++)
 	{
-		/* If connection finised sending do not wait input from it */
+		/* If connection finished sending do not wait input from it */
 		if (connections[i]->state == DN_CONNECTION_STATE_IDLE
 				|| HAS_MESSAGE_BUFFERED(connections[i]))
 			continue;
diff --git a/src/backend/pgxc/pool/poolcomm.c b/src/backend/pgxc/pool/poolcomm.c
index 853b3857c7..bdb6cb16c4 100644
--- a/src/backend/pgxc/pool/poolcomm.c
+++ b/src/backend/pgxc/pool/poolcomm.c
@@ -480,10 +480,12 @@ pool_putmessage(PoolPort *port, char msgtype, const char *s, size_t len)
 
 /* message code('f'), size(8), node_count */
 #define SEND_MSG_BUFFER_SIZE 9
-
+/* message code('s'), result */
+#define SEND_RES_BUFFER_SIZE 5
+#define SEND_PID_BUFFER_SIZE (5 + (MaxConnections - 1) * 4)
 
 /*
- * Build up a message carrying file deskriptors and send them over specified
+ * Build up a message carrying file descriptors or process numbers and send them over specified
  * connection
  */
 int
@@ -639,3 +641,178 @@ failure:
 	free(cmptr);
 	return EOF;
 }
+
+/*
+ * Send result to specified connection
+ */
+int
+pool_sendres(PoolPort *port, int res)
+{
+	char		buf[SEND_RES_BUFFER_SIZE];
+	uint		n32;
+
+	/* Header */
+	buf[0] = 's';
+	/* Result */
+	n32 = htonl(res);
+	memcpy(buf + 1, &n32, 4);
+
+	if (send(Socket(*port), &buf, SEND_RES_BUFFER_SIZE, 0) != SEND_RES_BUFFER_SIZE)
+		return EOF;
+
+	return 0;
+}
+
+/*
+ * Read result from specified connection.
+ * Return 0 at success or EOF at error.
+ */
+int
+pool_recvres(PoolPort *port)
+{
+	int			r;
+	int			res = 0;
+	uint		n32;
+	char		buf[SEND_RES_BUFFER_SIZE];
+
+	r = recv(Socket(*port), &buf, SEND_RES_BUFFER_SIZE, 0);
+	if (r < 0)
+	{
+		/*
+		 * Report broken connection
+		 */
+		ereport(ERROR,
+				(errcode_for_socket_access(),
+				 errmsg("could not receive data from client: %m")));
+		goto failure;
+	}
+	else if (r == 0)
+	{
+		goto failure;
+	}
+	else if (r != SEND_RES_BUFFER_SIZE)
+	{
+		ereport(ERROR,
+				(errcode(ERRCODE_PROTOCOL_VIOLATION),
+				 errmsg("incomplete message from client")));
+		goto failure;
+	}
+
+	/* Verify response */
+	if (buf[0] != 's')
+	{
+		ereport(ERROR,
+				(errcode(ERRCODE_PROTOCOL_VIOLATION),
+				 errmsg("unexpected message code")));
+		goto failure;
+	}
+
+	memcpy(&n32, buf + 1, 4);
+	n32 = ntohl(n32);
+	if (n32 != 0)
+		return EOF;
+
+	return res;
+
+failure:
+	return EOF;
+}
+
+/*
+ * Read a message from the specified connection carrying pid numbers
+ * of transactions interacting with pooler
+ */
+int
+pool_recvpids(PoolPort *port, int **pids)
+{
+	int			r, i;
+	uint		n32;
+	char		buf[SEND_PID_BUFFER_SIZE];
+
+	/*
+	 * Buffer size is upper bounded by the maximum number of connections,
+	 * as in the pooler each connection has one Pooler Agent.
+	 */
+
+	r = recv(Socket(*port), &buf, SEND_PID_BUFFER_SIZE, 0);
+	if (r < 0)
+	{
+		/*
+		 * Report broken connection
+		 */
+		ereport(ERROR,
+				(errcode_for_socket_access(),
+				 errmsg("could not receive data from client: %m")));
+		goto failure;
+	}
+	else if (r == 0)
+	{
+		goto failure;
+	}
+	else if (r != SEND_PID_BUFFER_SIZE)
+	{
+		ereport(ERROR,
+				(errcode(ERRCODE_PROTOCOL_VIOLATION),
+				 errmsg("incomplete message from client")));
+		goto failure;
+	}
+
+	/* Verify response */
+	if (buf[0] != 'p')
+	{
+		ereport(ERROR,
+				(errcode(ERRCODE_PROTOCOL_VIOLATION),
+				 errmsg("unexpected message code")));
+		goto failure;
+	}
+
+	memcpy(&n32, buf + 1, 4);
+	n32 = ntohl(n32);
+	if (n32 == 0)
+	{
+		elog(WARNING, "No transaction to abort");
+		return n32;
+	}
+
+	*pids = (int *) palloc(sizeof(int) * n32);
+
+	for (i = 0; i < n32; i++)
+	{
+		int n;
+		memcpy(&n, buf + 5 + i * sizeof(int), sizeof(int));
+		*pids[i] = ntohl(n);
+	}
+	return n32;
+
+failure:
+	return 0;
+}
+
+/*
+ * Send a message containing pid numbers to the specified connection 
+ */
+int
+pool_sendpids(PoolPort *port, int *pids, int count)
+{
+	int res = 0;
+	int i;
+	char		buf[SEND_PID_BUFFER_SIZE];
+	uint		n32;
+
+	buf[0] = 'p';
+	n32 = htonl((uint32) count);
+	memcpy(buf + 1, &n32, 4);
+	for (i = 0; i < count; i++)
+	{
+		int n;
+		n = htonl((uint32) pids[i]);
+		memcpy(buf + 5 + i * sizeof(int), &n, 4);
+	}
+
+	if (send(Socket(*port), &buf, SEND_PID_BUFFER_SIZE,0) != SEND_PID_BUFFER_SIZE)
+	{
+		res = EOF;
+	}
+
+	return res;
+}
diff --git a/src/backend/pgxc/pool/poolmgr.c b/src/backend/pgxc/pool/poolmgr.c
index 7ebced89f9..264271bd68 100644
--- a/src/backend/pgxc/pool/poolmgr.c
+++ b/src/backend/pgxc/pool/poolmgr.c
@@ -45,6 +45,7 @@
 #include "libpq/pqformat.h"
 #include "pgxc/locator.h"
 #include "pgxc/pgxc.h"
+#include "pgxc/poolutils.h"
 #include "../interfaces/libpq/libpq-fe.h"
 #include "postmaster/postmaster.h"		/* For UnixSocketDir */
 #include <stdlib.h>
@@ -89,6 +90,7 @@ static PoolAgent **poolAgents;
 
 static PoolHandle *Handle = NULL;
 
+static int	is_pool_cleaning = false;
 static int	server_fd = -1;
 
 static void agent_init(PoolAgent *agent, const char *database);
@@ -109,6 +111,8 @@ static void destroy_slot(PGXCNodePoolSlot *slot);
 static void grow_pool(DatabasePool *dbPool, int index, char client_conn_type);
 static void destroy_node_pool(PGXCNodePool *node_pool);
 static void PoolerLoop(void);
+static int clean_connection(List *dn_discard, List *co_discard, const char *database);
+static int *abort_pids(int *count, int pid, const char *database);
 
 /* Signal handlers */
 static void pooler_die(SIGNAL_ARGS);
@@ -631,6 +635,7 @@ agent_create(void)
 	agent->pool = NULL;
 	agent->dn_connections = NULL;
 	agent->coord_connections = NULL;
+	agent->pid = 0;
 
 	/* Append new agent to the list */
 	poolAgents[agentCount++] = agent;
@@ -644,14 +649,32 @@ agent_create(void)
 void
 PoolManagerConnect(PoolHandle *handle, const char *database)
 {
+	int n32;
+	char msgtype = 'c';
+
 	Assert(handle);
 	Assert(database);
 
 	/* Save the handle */
 	Handle = handle;
 
+	/* Message type */
+	pool_putbytes(&handle->port, &msgtype, 1);
+
+	/* Message length */
+	n32 = htonl(strlen(database) + 13);
+	pool_putbytes(&handle->port, (char *) &n32, 4);
+
+	/* PID number */
+	n32 = htonl(MyProcPid);
+	pool_putbytes(&handle->port, (char *) &n32, 4);
+
+	/* Length of Database string */
+	n32 = htonl(strlen(database) + 1);
+	pool_putbytes(&handle->port, (char *) &n32, 4);
+
 	/* Send database name followed by \0 terminator */
-	pool_putmessage(&handle->port, 'c', database, strlen(database) + 1);
+	pool_putbytes(&handle->port, database, strlen(database) + 1);
 	pool_flush(&handle->port);
 }
 
@@ -779,6 +802,7 @@ PoolManagerGetConnections(List *datanodelist, List *coordlist)
 
 	pool_putmessage(&Handle->port, 'g', (char *) nodes, sizeof(int) * (totlen + 2));
 	pool_flush(&Handle->port);
+
 	/* Receive response */
 	fds = (int *) palloc(sizeof(int) * totlen);
 	if (fds == NULL)
@@ -795,6 +819,84 @@ PoolManagerGetConnections(List *datanodelist, List *coordlist)
 	return fds;
 }
 
+/*
+ * Abort active transactions using pooler.
+ * Take a lock forbidding access to Pooler for new transactions.
+ */
+int
+PoolManagerAbortTransactions(char *dbname, int **proc_pids)
+{
+	int num_proc_ids = 0;
+
+	Assert(Handle);
+
+	pool_putmessage(&Handle->port, 'a', dbname, strlen(dbname) + 1);
+
+	pool_flush(&Handle->port);
+
+	/* Then Get back Pids from Pooler */
+	num_proc_ids = pool_recvpids(&Handle->port, proc_pids);
+
+	return num_proc_ids;
+}
+
+
+/*
+ * Clean up Pooled connections
+ */
+void
+PoolManagerCleanConnection(List *datanodelist, List *coordlist, char *dbname)
+{
+	int			totlen = list_length(datanodelist) + list_length(coordlist);
+	int			nodes[totlen + 2];
+	ListCell   *nodelist_item;
+	int			i, n32;
+	char		msgtype = 'f';
+
+	nodes[0] = htonl(list_length(datanodelist));
+	i = 1;
+	if (list_length(datanodelist) != 0)
+	{
+		foreach(nodelist_item, datanodelist)
+		{
+			nodes[i++] = htonl(lfirst_int(nodelist_item));
+		}
+	}
+	/* Then with Coordinator list (can be nul) */
+	nodes[i++] = htonl(list_length(coordlist));
+	if (list_length(coordlist) != 0)
+	{
+		foreach(nodelist_item, coordlist)
+		{
+			nodes[i++] = htonl(lfirst_int(nodelist_item));
+		}
+	}
+
+	/* Message type */
+	pool_putbytes(&Handle->port, &msgtype, 1);
+
+	/* Message length */
+	n32 = htonl(sizeof(int) * (totlen + 2) + strlen(dbname) + 9);
+	pool_putbytes(&Handle->port, (char *) &n32, 4);
+
+	/* Send list of nodes */
+	pool_putbytes(&Handle->port, (char *) nodes, sizeof(int) * (totlen + 2));
+
+	/* Length of Database string */
+	n32 = htonl(strlen(dbname) + 1);
+	pool_putbytes(&Handle->port, (char *) &n32, 4);
+
+	/* Send database name, followed by \0 terminator */
+	pool_putbytes(&Handle->port, dbname, strlen(dbname) + 1);
+	pool_flush(&Handle->port);
+
+	/* Receive result message */
+	if (pool_recvres(&Handle->port) != CLEAN_CONNECTION_COMPLETED)
+		ereport(ERROR,
+				(errcode(ERRCODE_INTERNAL_ERROR),
+				 errmsg("Clean connections not completed")));
+}
+
 
 /*
  * Handle messages to agent
@@ -816,14 +918,39 @@ agent_handle_input(PoolAgent * agent, StringInfo s)
 		List	   *datanodelist = NIL;
 		List	   *coordlist = NIL;
 		int		   *fds;
-		int			i;
+		int		   *pids;
+		int			i, len, res;
+
+		/*
+		 * During a pool cleaning, Abort, Connect and Get Connections messages
+		 * are not allowed on pooler side.
+		 * It avoids to have new backends taking connections
+		 * while remaining transactions are aborted during FORCE and then
+		 * Pools are being shrinked.
+		 */
+		if (is_pool_cleaning && (qtype == 'a' ||
+								 qtype == 'c' ||
+								 qtype == 'g'))
+			elog(WARNING,"Pool operation cannot run during Pool cleaning");
 
 		switch (qtype)
 		{
-			case 'c':			/* CONNECT */
+			case 'a':			/* ABORT */
 				pool_getmessage(&agent->port, s, 0);
 				database = pq_getmsgstring(s);
-				/* 
+				pq_getmsgend(s);
+				pids = abort_pids(&len, agent->pid, database);
+
+				pool_sendpids(&agent->port, pids, len);
+				if (pids)
+					pfree(pids);
+				break;
+			case 'c':			/* CONNECT */
+				pool_getmessage(&agent->port, s, 0);
+				agent->pid = pq_getmsgint(s, 4);
+				len = pq_getmsgint(s, 4);
+				database = pq_getmsgbytes(s, len);
+				/*
 				 * Coordinator pool is not initialized.
 				 * With that it would be impossible to create a Database by default.
 				 */
@@ -835,6 +962,29 @@ agent_handle_input(PoolAgent * agent, StringInfo s)
 				agent_destroy(agent);
 				pq_getmsgend(s);
 				break;
+			case 'f':			/* CLEAN CONNECTION */
+				pool_getmessage(&agent->port, s, 0);
+				datanodecount = pq_getmsgint(s, 4);
+				/* It is possible to clean up only Coordinators connections */
+				for (i = 0; i < datanodecount; i++)
+					datanodelist = lappend_int(datanodelist, pq_getmsgint(s, 4));
+				coordcount = pq_getmsgint(s, 4);
+				/* It is possible to clean up only Datanode connections */
+				for (i = 0; i < coordcount; i++)
+					coordlist = lappend_int(coordlist, pq_getmsgint(s, 4));
+				len = pq_getmsgint(s, 4);
+				database = pq_getmsgbytes(s, len);
+				pq_getmsgend(s);
+
+				/* Clean up connections here */
+				res = clean_connection(datanodelist, coordlist, database);
+
+				list_free(datanodelist);
+				list_free(coordlist);
+
+				/* Send success result */
+				pool_sendres(&agent->port, res);
+				break;
 			case 'g':			/* GET CONNECTIONS */
 				/*
 				 * Length of message is caused by:
@@ -852,9 +1002,8 @@ agent_handle_input(PoolAgent * agent, StringInfo s)
 					datanodelist = lappend_int(datanodelist, pq_getmsgint(s, 4));
 				coordcount = pq_getmsgint(s, 4);
 				/* It is possible that no Coordinators are involved in the transaction */
-				if (coordcount != 0)
-					for (i = 0; i < coordcount; i++)
-						coordlist = lappend_int(coordlist, pq_getmsgint(s, 4));
+				for (i = 0; i < coordcount; i++)
+					coordlist = lappend_int(coordlist, pq_getmsgint(s, 4));
 				pq_getmsgend(s);
 				/*
 				 * In case of error agent_acquire_connections will log
@@ -1282,8 +1431,7 @@ insert_database_pool(DatabasePool *databasePool)
 /*
  * Find pool for specified database in the list
  */
-static DatabasePool
-*
+static DatabasePool *
 find_database_pool(const char *database)
 {
 	DatabasePool *databasePool;
@@ -1292,12 +1440,11 @@ find_database_pool(const char *database)
 	databasePool = databasePools;
 	while (databasePool)
 	{
-
 		/* if match break the loop and return */
 		if (strcmp(database, databasePool->database) == 0)
 			break;
-		databasePool = databasePool->next;
 
+		databasePool = databasePool->next;
 	}
 	return databasePool;
 }
@@ -1724,6 +1871,147 @@ PoolerLoop(void)
 	}
 }
 
+/*
+ * Clean Connection in all Database Pools for given Datanode and Coordinator list
+ */
+#define TIMEOUT_CLEAN_LOOP 10
+
+int
+clean_connection(List *dn_discard, List *co_discard, const char *database)
+{
+	DatabasePool *databasePool;
+	int			dn_len = list_length(dn_discard);
+	int			co_len = list_length(co_discard);
+	int			dn_list[list_length(dn_discard)];
+	int			co_list[list_length(co_discard)];
+	int			count, i;
+	int			res = CLEAN_CONNECTION_COMPLETED;
+	ListCell   *nodelist_item;
+	PGXCNodePool *nodePool;
+
+	/* Save in array the lists of node number */
+	count = 0;
+	foreach(nodelist_item,dn_discard)
+		dn_list[count++] = lfirst_int(nodelist_item);
+
+	count = 0;
+	foreach(nodelist_item, co_discard)
+		co_list[count++] = lfirst_int(nodelist_item);
+
+	/* Find correct Database pool to clean */
+	databasePool = find_database_pool(database);
+
+	/* Database pool has not been found */
+	if (!databasePool)
+		return CLEAN_CONNECTION_NOT_COMPLETED;
+
+	/*
+	 * Clean each Pool Correctly
+	 * First for Datanode Pool
+	 */
+	for (count = 0; count < dn_len; count++)
+	{
+		int node_num = dn_list[count];
+		nodePool = databasePool->dataNodePools[node_num - 1];
+
+		if (nodePool)
+		{
+			/* Check if connections are in use */
+			if (nodePool->freeSize != nodePool->size)
+			{
+				elog(WARNING, "Pool of Database %s is using Datanode %d connections",
+							databasePool->database, node_num);
+				res = CLEAN_CONNECTION_NOT_COMPLETED;
+			}
+
+			/* Destroy connections currently in Node Pool */
+			if (nodePool->slot)
+			{
+				for (i = 0; i < nodePool->freeSize; i++)
+					destroy_slot(nodePool->slot[i]);
+
+				/* Move slots in use at the beginning of Node Pool array */
+				for (i = nodePool->freeSize; i < nodePool->size; i++ )
+					nodePool->slot[i - nodePool->freeSize] = nodePool->slot[i];
+			}
+			nodePool->size -= nodePool->freeSize;
+			nodePool->freeSize = 0;
+		}
+	}
+
+	/* Then for Coordinators */
+	for (count = 0; count < co_len; count++)
+	{
+		int node_num = co_list[count];
+		nodePool = databasePool->coordNodePools[node_num - 1];
+
+		if (nodePool)
+		{
+			/* Check if connections are in use */
+			if (nodePool->freeSize != nodePool->size)
+			{
+				elog(WARNING, "Pool of Database %s is using Coordinator %d connections",
+							databasePool->database, node_num);
+				res = CLEAN_CONNECTION_NOT_COMPLETED;
+			}
+
+			/* Destroy connections currently in Node Pool */
+			if (nodePool->slot)
+			{
+				for (i = 0; i < nodePool->freeSize; i++)
+					destroy_slot(nodePool->slot[i]);
+
+				/* Move slots in use at the beginning of Node Pool array */
+				for (i = nodePool->freeSize; i < nodePool->size; i++ )
+					nodePool->slot[i - nodePool->freeSize] = nodePool->slot[i];
+			}
+			nodePool->size -= nodePool->freeSize;
+			nodePool->freeSize = 0;
+		}
+	}
+
+	/* Release lock on Pooler, to allow transactions to connect again. */
+	is_pool_cleaning = false;
+	return res;
+}
+
+/*
+ * Take a Lock on Pooler.
+ * Abort PIDs registered with the agents for the given database.
+ * Send back to client list of PIDs signaled to watch them.
+ */
+int *
+abort_pids(int *len, int pid, const char *database)
+{
+	int *pids = NULL;
+	int i = 0;
+	int count;
+
+	Assert(!is_pool_cleaning);
+	Assert(agentCount > 0);
+
+	is_pool_cleaning = true;
+
+	pids = (int *) palloc((agentCount - 1) * sizeof(int));
+
+	/* Send a SIGTERM signal to all processes of Pooler agents except this one */
+	for (count = 0; count < agentCount; count++)
+	{
+		if (poolAgents[count]->pid != pid &&
+			strcmp(poolAgents[count]->pool->database, database) == 0)
+		{
+			if (kill(poolAgents[count]->pid, SIGTERM) < 0)
+				elog(ERROR, "kill(%ld,%d) failed: %m",
+							(long) poolAgents[count]->pid, SIGTERM);
+
+			pids[i++] = poolAgents[count]->pid;
+		}
+	}
+
+	*len = i;
+
+	return pids;
+}
 
 /*
  *
diff --git a/src/backend/pgxc/pool/poolutils.c b/src/backend/pgxc/pool/poolutils.c
new file mode 100644
index 0000000000..e14c936ec8
--- /dev/null
+++ b/src/backend/pgxc/pool/poolutils.c
@@ -0,0 +1,187 @@
+/*-------------------------------------------------------------------------
+ *
+ * poolutils.c
+ *
+ * Utilities for Postgres-XC pooler
+ * 
+ * Portions Copyright (c) 1996-2009, PostgreSQL Global Development Group
+ * Portions Copyright (c) 2010 Nippon Telegraph and Telephone Corporation
+ *
+ * IDENTIFICATION
+ *    $$
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+#include "miscadmin.h"
+#include "libpq/pqsignal.h"
+
+#include "pgxc/pgxc.h"
+#include "pgxc/poolmgr.h"
+#include "pgxc/locator.h"
+#include "pgxc/poolutils.h"
+#include "access/gtm.h"
+#include "commands/dbcommands.h"
+
+#include "nodes/parsenodes.h"
+
+/*
+ * CleanConnection()
+ *
+ * Utility to clean up Postgres-XC Pooler connections.
+ * This utility is launched to all the Coordinators of the cluster
+ *
+ * Use of CLEAN CONNECTION is limited to a super user.
+ * It is advised to clean connections before shutting down a Node or drop a Database.
+ *
+ * SQL query synopsis is as follows:
+ * CLEAN CONNECTION TO
+ *		(COORDINATOR num | DATANODE num | ALL {FORCE})
+ *		FOR DATABASE dbname
+ *
+ * Connection cleaning has to be made on a chosen database called dbname.
+ *
+ * It is also possible to clean connections of several Coordinators or Datanodes
+ * Ex:	CLEAN CONNECTION TO DATANODE 1,5,7 FOR DATABASE template1
+ *		CLEAN CONNECTION TO COORDINATOR 2,4,6 FOR DATABASE template1
+ *
+ * Or even to all Coordinators/Datanodes at the same time
+ * Ex:	CLEAN CONNECTION TO DATANODE * FOR DATABASE template1
+ *		CLEAN CONNECTION TO COORDINATOR * FOR DATABASE template1
+ *
+ * When FORCE is used, all the transactions using pooler connections are aborted,
+ * and pooler connections are cleaned up.
+ * Ex:	CLEAN CONNECTION TO ALL FORCE FOR DATABASE template1;
+ *
+ * FORCE can only be used with TO ALL, as it takes a lock on pooler to stop requests
+ * asking for connections, aborts all the connections in the cluster, and cleans up
+ * pool connections.
+ */
+void
+CleanConnection(CleanConnStmt *stmt)
+{
+	ListCell   *nodelist_item;
+	List	   *co_list = NIL;
+	List	   *dn_list = NIL;
+	List	   *stmt_nodes = NIL;
+	char	   *dbname = stmt->dbname;
+	bool		is_coord = stmt->is_coord;
+	bool		is_force = stmt->is_force;
+	int			max_node_number = 0;
+	Oid			oid;
+
+	/* Only a DB administrator can clean pooler connections */
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 errmsg("must be superuser to clean pool connections")));
+
+	/* Check if the Database exists by getting its Oid */
+	oid = get_database_oid(dbname);
+	if (!OidIsValid(oid))
+	{
+		ereport(WARNING,
+				(errcode(ERRCODE_UNDEFINED_DATABASE),
+				 errmsg("database \"%s\" does not exist", dbname)));
+		return;
+	}
+
+	/*
+	 * FORCE is activated,
+	 * Send a SIGTERM signal to all the processes and take a lock on Pooler
+	 * to avoid backends to take new connections when cleaning.
+	 * Only Disconnect is allowed.
+	 */
+	if (is_force)
+	{
+		int loop = 0;
+		int *proc_pids = NULL;
+		int num_proc_pids, count;
+
+		num_proc_pids = PoolManagerAbortTransactions(dbname, &proc_pids);
+
+		/*
+		 * Watch the processes that received a SIGTERM.
+		 * At the end of the timestamp loop, processes are considered as not finished
+		 * and force the connection cleaning has failed
+		 */
+
+		while (num_proc_pids > 0 && loop < TIMEOUT_CLEAN_LOOP)
+		{
+			for (count = num_proc_pids - 1; count >= 0; count--)
+			{
+				switch(kill(proc_pids[count],0))
+				{
+					case 0: /* Termination not done yet */
+						break;
+
+					default:
+						/* Move tail pid in free space */
+						proc_pids[count] = proc_pids[num_proc_pids - 1];
+						num_proc_pids--;
+						break;
+				}
+			}
+			pg_usleep(1000000);
+			loop++;
+		}
+
+		if (proc_pids)
+			pfree(proc_pids);
+
+		if (loop >= TIMEOUT_CLEAN_LOOP)
+			ereport(WARNING,
+					(errcode(ERRCODE_INTERNAL_ERROR),
+					 errmsg("All Transactions have not been aborted")));
+	}
+
+	/* Check node list */
+	if (stmt->nodes && is_coord)
+		max_node_number = NumCoords;
+	else
+		max_node_number = NumDataNodes;
+
+	foreach(nodelist_item, stmt->nodes)
+	{
+		int node_num = intVal(lfirst(nodelist_item));
+		stmt_nodes = lappend_int(stmt_nodes, node_num);
+
+		if (node_num > max_node_number ||
+			node_num < 1)
+			ereport(ERROR,
+					(errcode(ERRCODE_INTERNAL_ERROR),
+					 errmsg("Node Number %d is incorrect", node_num)));
+	}
+
+	/* Build lists to be sent to Pooler Manager */
+	if (stmt->nodes && is_coord)
+		co_list = stmt_nodes;
+	else if (stmt->nodes && !is_coord)
+		dn_list = stmt_nodes;
+	else
+	{
+		co_list = GetAllCoordNodes();
+		dn_list = GetAllDataNodes();
+	}
+
+	/*
+	 * If force is launched, send a signal to all the processes
+	 * that are in transaction and take a lock.
+	 * Get back their process number and watch them locally here.
+	 * Process are checked as alive or not with pg_usleep and when all processes are down
+	 * go out of the control loop.
+	 * If at the end of the loop processes are not down send an error to client.
+	 * Then Make a clean with normal pool cleaner.
+	 * Always release the lock when calling CLEAN CONNECTION.
+	 */
+
+	/* Finish by contacting Pooler Manager */
+	PoolManagerCleanConnection(dn_list, co_list, dbname);
+
+	/* Clean up memory */
+	if (co_list)
+		list_free(co_list);
+	if (dn_list)
+		list_free(dn_list);
+}
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index c2a473fd11..fbf7596cfb 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -2625,6 +2625,7 @@ die(SIGNAL_ARGS)
 	errno = save_errno;
 }
 
+
 /*
  * Timeout or shutdown signal from postmaster during client authentication.
  * Simply exit(1).
diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c
index 68a483e357..3a67c29b4b 100644
--- a/src/backend/tcop/utility.c
+++ b/src/backend/tcop/utility.c
@@ -61,6 +61,7 @@
 #include "pgxc/locator.h"
 #include "pgxc/pgxc.h"
 #include "pgxc/planner.h"
+#include "pgxc/poolutils.h"
 
 static void ExecUtilityStmtOnNodes(const char *queryString, ExecNodes *nodes,
 								   bool force_autocommit, RemoteQueryExecType exec_type);
@@ -1402,6 +1403,14 @@ ProcessUtility(Node *parsetree,
 			if (!IsConnFromCoord())
 				ExecRemoteUtility((RemoteQuery *) parsetree);
 			break;
+
+		case T_CleanConnStmt:
+			Assert(IS_PGXC_COORDINATOR);
+			CleanConnection((CleanConnStmt *) parsetree);
+
+			if (IS_PGXC_COORDINATOR)
+				ExecUtilityStmtOnNodes(queryString, NULL, true, EXEC_ON_COORDS);
+			break;
 #endif
 		default:
 			elog(ERROR, "unrecognized node type: %d",
@@ -2406,6 +2415,9 @@ CreateCommandTag(Node *parsetree)
 		case T_ExecDirectStmt:
 			tag = "EXECUTE DIRECT";
 			break;
+		case T_CleanConnStmt:
+			tag = "CLEAN CONNECTION";
+			break;
 
 		default:
 			elog(WARNING, "unrecognized node type: %d",
@@ -2836,6 +2848,11 @@ GetCommandLogLevel(Node *parsetree)
 
 			}
 			break;
+#ifdef PGXC
+		case T_CleanConnStmt:
+			lev = LOGSTMT_DDL;
+			break;
+#endif
 
 		default:
 			elog(WARNING, "unrecognized node type: %d",
diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h
index 2d60d918d5..10b259ad85 100644
--- a/src/include/nodes/nodes.h
+++ b/src/include/nodes/nodes.h
@@ -355,6 +355,7 @@ typedef enum NodeTag
 	T_AlterUserMappingStmt,
 	T_DropUserMappingStmt,
 	T_ExecDirectStmt,
+	T_CleanConnStmt,
 
 	/*
 	 * TAGS FOR PARSE TREE NODES (parsenodes.h)
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index 355c44847e..79b5fac560 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -2416,6 +2416,18 @@ typedef struct ExecDirectStmt
 	List	   *nodes;
 	char	   *query;
 } ExecDirectStmt;
+
+/*
+ * CLEAN CONNECTION statement
+ */
+typedef struct CleanConnStmt
+{
+	NodeTag		type;
+	List	   *nodes;		/* list of nodes dropped */
+	char	   *dbname;		/* name of database to drop connections */
+	bool		is_coord;	/* type of connections dropped */
+	bool		is_force;	/* option force  */
+} CleanConnStmt;
 /* PGXC_END */
 
 #endif   /* PARSENODES_H */
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index aec7b6b3d9..62c7c0dcb9 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -75,6 +75,9 @@ PG_KEYWORD("characteristics", CHARACTERISTICS, UNRESERVED_KEYWORD)
 PG_KEYWORD("check", CHECK, RESERVED_KEYWORD)
 PG_KEYWORD("checkpoint", CHECKPOINT, UNRESERVED_KEYWORD)
 PG_KEYWORD("class", CLASS, UNRESERVED_KEYWORD)
+#ifdef PGXC
+PG_KEYWORD("clean", CLEAN, UNRESERVED_KEYWORD)
+#endif
 PG_KEYWORD("close", CLOSE, UNRESERVED_KEYWORD)
 PG_KEYWORD("cluster", CLUSTER, UNRESERVED_KEYWORD)
 PG_KEYWORD("coalesce", COALESCE, COL_NAME_KEYWORD)
diff --git a/src/include/pgxc/poolcomm.h b/src/include/pgxc/poolcomm.h
index 9e286ab292..a10af3da9e 100644
--- a/src/include/pgxc/poolcomm.h
+++ b/src/include/pgxc/poolcomm.h
@@ -45,5 +45,9 @@ extern int	pool_putbytes(PoolPort *port, const char *s, size_t len);
 extern int	pool_flush(PoolPort *port);
 extern int	pool_sendfds(PoolPort *port, int *fds, int count);
 extern int	pool_recvfds(PoolPort *port, int *fds, int count);
+extern int	pool_sendres(PoolPort *port, int res);
+extern int	pool_recvres(PoolPort *port);
+extern int	pool_sendpids(PoolPort *port, int *pids, int count);
+extern int	pool_recvpids(PoolPort *port, int **pids);
 
 #endif   /* POOLCOMM_H */
diff --git a/src/include/pgxc/poolmgr.h b/src/include/pgxc/poolmgr.h
index 3a615cc0f3..7b6e4d4758 100644
--- a/src/include/pgxc/poolmgr.h
+++ b/src/include/pgxc/poolmgr.h
@@ -63,6 +63,8 @@ typedef struct databasepool
  */
 typedef struct
 {
+	/* Process ID of postmaster child process associated to pool agent */
+	int			pid;
 	/* communication channel */
 	PoolPort	port;
 	DatabasePool *pool;
@@ -132,6 +134,12 @@ extern void PoolManagerConnect(PoolHandle *handle, const char *database);
 /* Get pooled connections */
 extern int *PoolManagerGetConnections(List *datanodelist, List *coordlist);
 
+/* Clean pool connections */
+extern void PoolManagerCleanConnection(List *datanodelist, List *coordlist, char *dbname);
+
+/* Send Abort signal to transactions being run */
+extern int	PoolManagerAbortTransactions(char *dbname, int **proc_pids);
+
 /* Return connections back to the pool, for both Coordinator and Datanode connections */
 extern void PoolManagerReleaseConnections(int dn_ndisc, int* dn_discard, int co_ndisc, int* co_discard);
 
diff --git a/src/include/pgxc/poolutils.h b/src/include/pgxc/poolutils.h
new file mode 100644
index 0000000000..0f34561253
--- /dev/null
+++ b/src/include/pgxc/poolutils.h
@@ -0,0 +1,30 @@
+/*-------------------------------------------------------------------------
+ *
+ * poolutils.h
+ *
+ *		Utilities for Postgres-XC Pooler
+ *
+ * Portions Copyright (c) 1996-2009, PostgreSQL Global Development Group
+ * Portions Copyright (c) 2010 Nippon Telegraph and Telephone Corporation
+ *
+ * IDENTIFICATION
+ *	$$
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef POOLUTILS_H
+#define POOLUTILS_H
+
+#include "nodes/parsenodes.h"
+
+#define TIMEOUT_CLEAN_LOOP 10 /* Wait 10s for all the transactions to shutdown */
+
+/* Error codes for connection cleaning */
+#define CLEAN_CONNECTION_COMPLETED			0
+#define CLEAN_CONNECTION_NOT_COMPLETED		1
+#define CLEAN_CONNECTION_TX_REMAIN			2
+#define	CLEAN_CONNECTION_EOF				-1
+
+void CleanConnection(CleanConnStmt *stmt);
+#endif