diff options
author | Michael P | 2010-11-16 00:48:51 +0000 |
---|---|---|
committer | Pavan Deolasee | 2011-05-19 16:45:23 +0000 |
commit | 0d08af45c86c8d7c628df1a905839a617afe3b80 (patch) | |
tree | 9ecf1dd1feca89ef814788c5d0d28a0ff164b7e0 | |
parent | 95cf4b8525a4793659cb613eff225016037b367d (diff) |
Support for CLEAN CONNECTION
Utility to clean up Postgres-XC Pooler connections.
This utility is launched to all the Coordinators of the cluster
Use of CLEAN CONNECTION is limited to a super user.
It is advised to clean connections before dropping a Database.
SQL query synopsis is as follows:
CLEAN CONNECTION TO
(COORDINATOR num | DATANODE num | ALL {FORCE})
FOR DATABASE dbname
Connection cleaning has to be made on a chosen database called dbname.
It is also possible to clean connections of several Coordinators or Datanodes
Ex: CLEAN CONNECTION TO DATANODE 1,5,7 FOR DATABASE template1
CLEAN CONNECTION TO COORDINATOR 2,4,6 FOR DATABASE template1
Or even to all Coordinators/Datanodes at the same time
Ex: CLEAN CONNECTION TO DATANODE * FOR DATABASE template1
CLEAN CONNECTION TO COORDINATOR * FOR DATABASE template1
When FORCE is used, all the transactions using pooler connections are aborted,
and pooler connections are cleaned up.
Ex: CLEAN CONNECTION TO ALL FORCE FOR DATABASE template1;
FORCE can only be used with TO ALL, as it takes a lock on pooler to stop requests
asking for connections, aborts all the connections in the cluster, and cleans up
pool connections
-rw-r--r-- | src/backend/parser/gram.y | 64 | ||||
-rw-r--r-- | src/backend/pgxc/locator/locator.c | 4 | ||||
-rw-r--r-- | src/backend/pgxc/pool/Makefile | 2 | ||||
-rw-r--r-- | src/backend/pgxc/pool/pgxcnode.c | 2 | ||||
-rw-r--r-- | src/backend/pgxc/pool/poolcomm.c | 181 | ||||
-rw-r--r-- | src/backend/pgxc/pool/poolmgr.c | 310 | ||||
-rw-r--r-- | src/backend/pgxc/pool/poolutils.c | 187 | ||||
-rw-r--r-- | src/backend/tcop/postgres.c | 1 | ||||
-rw-r--r-- | src/backend/tcop/utility.c | 17 | ||||
-rw-r--r-- | src/include/nodes/nodes.h | 1 | ||||
-rw-r--r-- | src/include/nodes/parsenodes.h | 12 | ||||
-rw-r--r-- | src/include/parser/kwlist.h | 3 | ||||
-rw-r--r-- | src/include/pgxc/poolcomm.h | 4 | ||||
-rw-r--r-- | src/include/pgxc/poolmgr.h | 8 | ||||
-rw-r--r-- | src/include/pgxc/poolutils.h | 30 |
15 files changed, 803 insertions, 23 deletions
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 9ffada513a..21e6021669 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -191,7 +191,7 @@ static TypeName *TableFuncTypeName(List *columns); AlterForeignServerStmt AlterGroupStmt AlterObjectSchemaStmt AlterOwnerStmt AlterSeqStmt AlterTableStmt AlterUserStmt AlterUserMappingStmt AlterUserSetStmt AlterRoleStmt AlterRoleSetStmt - AnalyzeStmt ClosePortalStmt ClusterStmt CommentStmt + AnalyzeStmt CleanConnStmt ClosePortalStmt ClusterStmt CommentStmt ConstraintsSetStmt CopyStmt CreateAsStmt CreateCastStmt CreateDomainStmt CreateGroupStmt CreateOpClassStmt CreateOpFamilyStmt AlterOpFamilyStmt CreatePLangStmt @@ -328,7 +328,7 @@ static TypeName *TableFuncTypeName(List *columns); %type <boolean> opt_freeze opt_default opt_recheck %type <defelt> opt_binary opt_oids copy_delimiter -%type <list> node_list +%type <list> data_node_list coord_list %type <str> DirectStmt %type <boolean> copy_from @@ -445,7 +445,7 @@ static TypeName *TableFuncTypeName(List *columns); BOOLEAN_P BOTH BY CACHE CALLED CASCADE CASCADED CASE CAST CATALOG_P CHAIN CHAR_P - CHARACTER CHARACTERISTICS CHECK CHECKPOINT CLASS CLOSE + CHARACTER CHARACTERISTICS CHECK CHECKPOINT CLASS CLEAN CLOSE CLUSTER COALESCE COLLATE COLUMN COMMENT COMMIT COMMITTED CONCURRENTLY CONFIGURATION CONNECTION CONSTRAINT CONSTRAINTS CONTENT_P CONTINUE_P CONVERSION_P COORDINATOR COPY COST CREATE CREATEDB @@ -637,6 +637,7 @@ stmt : | AlterUserStmt | AnalyzeStmt | CheckPointStmt + | CleanConnStmt | ClosePortalStmt | ClusterStmt | CommentStmt @@ -6545,7 +6546,7 @@ ExecDirectStmt: EXECUTE DIRECT ON COORDINATOR DirectStmt n->query = $5; $$ = (Node *)n; } - | EXECUTE DIRECT ON NODE node_list DirectStmt + | EXECUTE DIRECT ON NODE data_node_list DirectStmt { ExecDirectStmt *n = makeNode(ExecDirectStmt); n->coordinator = FALSE; @@ -6559,9 +6560,21 @@ DirectStmt: Sconst /* by default all are $$=$1 */ ; -node_list: +coord_list: Iconst { $$ = list_make1(makeInteger($1)); } - | node_list ',' Iconst { $$ = lappend($1, makeInteger($3)); } + | coord_list ',' Iconst { $$ = lappend($1, makeInteger($3)); } + | '*' + { + int i; + $$ = NIL; + for (i=1; i<=NumCoords; i++) + $$ = lappend($$, makeInteger(i)); + } + ; + +data_node_list: + Iconst { $$ = list_make1(makeInteger($1)); } + | data_node_list ',' Iconst { $$ = lappend($1, makeInteger($3)); } | '*' { int i; @@ -6574,6 +6587,44 @@ node_list: /***************************************************************************** * * QUERY: + * + * CLEAN CONNECTION TO (COORDINATOR num | NODE num | ALL {FORCE}) + * FOR DATABASE dbname + * + *****************************************************************************/ + +CleanConnStmt: CLEAN CONNECTION TO COORDINATOR coord_list FOR DATABASE database_name + { + CleanConnStmt *n = makeNode(CleanConnStmt); + n->is_coord = true; + n->nodes = $5; + n->is_force = false; + n->dbname = $8; + $$ = (Node *)n; + } + | CLEAN CONNECTION TO NODE data_node_list FOR DATABASE database_name + { + CleanConnStmt *n = makeNode(CleanConnStmt); + n->is_coord = false; + n->nodes = $5; + n->is_force = false; + n->dbname = $8; + $$ = (Node *)n; + } + | CLEAN CONNECTION TO ALL opt_force FOR DATABASE database_name + { + CleanConnStmt *n = makeNode(CleanConnStmt); + n->is_coord = true; + n->nodes = NIL; + n->is_force = $5; + n->dbname = $8; + $$ = (Node *)n; + } + ; + +/***************************************************************************** + * + * QUERY: * PREPARE <plan_name> [(args, ...)] AS <query> * *****************************************************************************/ @@ -10259,6 +10310,7 @@ unreserved_keyword: | CHARACTERISTICS | CHECKPOINT | CLASS + | CLEAN | CLOSE | CLUSTER | COMMENT diff --git a/src/backend/pgxc/locator/locator.c b/src/backend/pgxc/locator/locator.c index 098e254531..845f9fc78c 100644 --- a/src/backend/pgxc/locator/locator.c +++ b/src/backend/pgxc/locator/locator.c @@ -466,8 +466,8 @@ GetAllDataNodes(void) /* * Return a list of all Coordinators - * This is used to send DDL to all nodes - * Do not put in the list the local Coordinator where this function is launched + * This is used to send DDL to all nodes and to clean up pooler connections. + * Do not put in the list the local Coordinator where this function is launched. */ List * GetAllCoordNodes(void) diff --git a/src/backend/pgxc/pool/Makefile b/src/backend/pgxc/pool/Makefile index f8679eb10c..8c2b66a155 100644 --- a/src/backend/pgxc/pool/Makefile +++ b/src/backend/pgxc/pool/Makefile @@ -14,6 +14,6 @@ subdir = src/backend/pgxc/pool top_builddir = ../../../.. include $(top_builddir)/src/Makefile.global -OBJS = pgxcnode.o execRemote.o poolmgr.o poolcomm.o postgresql_fdw.o +OBJS = pgxcnode.o execRemote.o poolmgr.o poolcomm.o postgresql_fdw.o poolutils.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/pgxc/pool/pgxcnode.c b/src/backend/pgxc/pool/pgxcnode.c index 3bc3a83aa4..cbaf68c610 100644 --- a/src/backend/pgxc/pool/pgxcnode.c +++ b/src/backend/pgxc/pool/pgxcnode.c @@ -268,7 +268,7 @@ pgxc_node_receive(const int conn_count, FD_ZERO(&readfds); for (i = 0; i < conn_count; i++) { - /* If connection finised sending do not wait input from it */ + /* If connection finished sending do not wait input from it */ if (connections[i]->state == DN_CONNECTION_STATE_IDLE || HAS_MESSAGE_BUFFERED(connections[i])) continue; diff --git a/src/backend/pgxc/pool/poolcomm.c b/src/backend/pgxc/pool/poolcomm.c index 853b3857c7..bdb6cb16c4 100644 --- a/src/backend/pgxc/pool/poolcomm.c +++ b/src/backend/pgxc/pool/poolcomm.c @@ -480,10 +480,12 @@ pool_putmessage(PoolPort *port, char msgtype, const char *s, size_t len) /* message code('f'), size(8), node_count */ #define SEND_MSG_BUFFER_SIZE 9 - +/* message code('s'), result */ +#define SEND_RES_BUFFER_SIZE 5 +#define SEND_PID_BUFFER_SIZE (5 + (MaxConnections - 1) * 4) /* - * Build up a message carrying file deskriptors and send them over specified + * Build up a message carrying file descriptors or process numbers and send them over specified * connection */ int @@ -639,3 +641,178 @@ failure: free(cmptr); return EOF; } + +/* + * Send result to specified connection + */ +int +pool_sendres(PoolPort *port, int res) +{ + char buf[SEND_RES_BUFFER_SIZE]; + uint n32; + + /* Header */ + buf[0] = 's'; + /* Result */ + n32 = htonl(res); + memcpy(buf + 1, &n32, 4); + + if (send(Socket(*port), &buf, SEND_RES_BUFFER_SIZE, 0) != SEND_RES_BUFFER_SIZE) + return EOF; + + return 0; +} + +/* + * Read result from specified connection. + * Return 0 at success or EOF at error. + */ +int +pool_recvres(PoolPort *port) +{ + int r; + int res = 0; + uint n32; + char buf[SEND_RES_BUFFER_SIZE]; + + r = recv(Socket(*port), &buf, SEND_RES_BUFFER_SIZE, 0); + if (r < 0) + { + /* + * Report broken connection + */ + ereport(ERROR, + (errcode_for_socket_access(), + errmsg("could not receive data from client: %m"))); + goto failure; + } + else if (r == 0) + { + goto failure; + } + else if (r != SEND_RES_BUFFER_SIZE) + { + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("incomplete message from client"))); + goto failure; + } + + /* Verify response */ + if (buf[0] != 's') + { + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("unexpected message code"))); + goto failure; + } + + memcpy(&n32, buf + 1, 4); + n32 = ntohl(n32); + if (n32 != 0) + return EOF; + + return res; + +failure: + return EOF; +} + +/* + * Read a message from the specified connection carrying pid numbers + * of transactions interacting with pooler + */ +int +pool_recvpids(PoolPort *port, int **pids) +{ + int r, i; + uint n32; + char buf[SEND_PID_BUFFER_SIZE]; + + /* + * Buffer size is upper bounded by the maximum number of connections, + * as in the pooler each connection has one Pooler Agent. + */ + + r = recv(Socket(*port), &buf, SEND_PID_BUFFER_SIZE, 0); + if (r < 0) + { + /* + * Report broken connection + */ + ereport(ERROR, + (errcode_for_socket_access(), + errmsg("could not receive data from client: %m"))); + goto failure; + } + else if (r == 0) + { + goto failure; + } + else if (r != SEND_PID_BUFFER_SIZE) + { + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("incomplete message from client"))); + goto failure; + } + + /* Verify response */ + if (buf[0] != 'p') + { + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("unexpected message code"))); + goto failure; + } + + memcpy(&n32, buf + 1, 4); + n32 = ntohl(n32); + if (n32 == 0) + { + elog(WARNING, "No transaction to abort"); + return n32; + } + + *pids = (int *) palloc(sizeof(int) * n32); + + for (i = 0; i < n32; i++) + { + int n; + memcpy(&n, buf + 5 + i * sizeof(int), sizeof(int)); + *pids[i] = ntohl(n); + } + return n32; + +failure: + return 0; +} + +/* + * Send a message containing pid numbers to the specified connection + */ +int +pool_sendpids(PoolPort *port, int *pids, int count) +{ + int res = 0; + int i; + char buf[SEND_PID_BUFFER_SIZE]; + uint n32; + + buf[0] = 'p'; + n32 = htonl((uint32) count); + memcpy(buf + 1, &n32, 4); + for (i = 0; i < count; i++) + { + int n; + n = htonl((uint32) pids[i]); + memcpy(buf + 5 + i * sizeof(int), &n, 4); + } + + if (send(Socket(*port), &buf, SEND_PID_BUFFER_SIZE,0) != SEND_PID_BUFFER_SIZE) + { + res = EOF; + } + + return res; +} diff --git a/src/backend/pgxc/pool/poolmgr.c b/src/backend/pgxc/pool/poolmgr.c index 7ebced89f9..264271bd68 100644 --- a/src/backend/pgxc/pool/poolmgr.c +++ b/src/backend/pgxc/pool/poolmgr.c @@ -45,6 +45,7 @@ #include "libpq/pqformat.h" #include "pgxc/locator.h" #include "pgxc/pgxc.h" +#include "pgxc/poolutils.h" #include "../interfaces/libpq/libpq-fe.h" #include "postmaster/postmaster.h" /* For UnixSocketDir */ #include <stdlib.h> @@ -89,6 +90,7 @@ static PoolAgent **poolAgents; static PoolHandle *Handle = NULL; +static int is_pool_cleaning = false; static int server_fd = -1; static void agent_init(PoolAgent *agent, const char *database); @@ -109,6 +111,8 @@ static void destroy_slot(PGXCNodePoolSlot *slot); static void grow_pool(DatabasePool *dbPool, int index, char client_conn_type); static void destroy_node_pool(PGXCNodePool *node_pool); static void PoolerLoop(void); +static int clean_connection(List *dn_discard, List *co_discard, const char *database); +static int *abort_pids(int *count, int pid, const char *database); /* Signal handlers */ static void pooler_die(SIGNAL_ARGS); @@ -631,6 +635,7 @@ agent_create(void) agent->pool = NULL; agent->dn_connections = NULL; agent->coord_connections = NULL; + agent->pid = 0; /* Append new agent to the list */ poolAgents[agentCount++] = agent; @@ -644,14 +649,32 @@ agent_create(void) void PoolManagerConnect(PoolHandle *handle, const char *database) { + int n32; + char msgtype = 'c'; + Assert(handle); Assert(database); /* Save the handle */ Handle = handle; + /* Message type */ + pool_putbytes(&handle->port, &msgtype, 1); + + /* Message length */ + n32 = htonl(strlen(database) + 13); + pool_putbytes(&handle->port, (char *) &n32, 4); + + /* PID number */ + n32 = htonl(MyProcPid); + pool_putbytes(&handle->port, (char *) &n32, 4); + + /* Length of Database string */ + n32 = htonl(strlen(database) + 1); + pool_putbytes(&handle->port, (char *) &n32, 4); + /* Send database name followed by \0 terminator */ - pool_putmessage(&handle->port, 'c', database, strlen(database) + 1); + pool_putbytes(&handle->port, database, strlen(database) + 1); pool_flush(&handle->port); } @@ -779,6 +802,7 @@ PoolManagerGetConnections(List *datanodelist, List *coordlist) pool_putmessage(&Handle->port, 'g', (char *) nodes, sizeof(int) * (totlen + 2)); pool_flush(&Handle->port); + /* Receive response */ fds = (int *) palloc(sizeof(int) * totlen); if (fds == NULL) @@ -795,6 +819,84 @@ PoolManagerGetConnections(List *datanodelist, List *coordlist) return fds; } +/* + * Abort active transactions using pooler. + * Take a lock forbidding access to Pooler for new transactions. + */ +int +PoolManagerAbortTransactions(char *dbname, int **proc_pids) +{ + int num_proc_ids = 0; + + Assert(Handle); + + pool_putmessage(&Handle->port, 'a', dbname, strlen(dbname) + 1); + + pool_flush(&Handle->port); + + /* Then Get back Pids from Pooler */ + num_proc_ids = pool_recvpids(&Handle->port, proc_pids); + + return num_proc_ids; +} + + +/* + * Clean up Pooled connections + */ +void +PoolManagerCleanConnection(List *datanodelist, List *coordlist, char *dbname) +{ + int totlen = list_length(datanodelist) + list_length(coordlist); + int nodes[totlen + 2]; + ListCell *nodelist_item; + int i, n32; + char msgtype = 'f'; + + nodes[0] = htonl(list_length(datanodelist)); + i = 1; + if (list_length(datanodelist) != 0) + { + foreach(nodelist_item, datanodelist) + { + nodes[i++] = htonl(lfirst_int(nodelist_item)); + } + } + /* Then with Coordinator list (can be nul) */ + nodes[i++] = htonl(list_length(coordlist)); + if (list_length(coordlist) != 0) + { + foreach(nodelist_item, coordlist) + { + nodes[i++] = htonl(lfirst_int(nodelist_item)); + } + } + + /* Message type */ + pool_putbytes(&Handle->port, &msgtype, 1); + + /* Message length */ + n32 = htonl(sizeof(int) * (totlen + 2) + strlen(dbname) + 9); + pool_putbytes(&Handle->port, (char *) &n32, 4); + + /* Send list of nodes */ + pool_putbytes(&Handle->port, (char *) nodes, sizeof(int) * (totlen + 2)); + + /* Length of Database string */ + n32 = htonl(strlen(dbname) + 1); + pool_putbytes(&Handle->port, (char *) &n32, 4); + + /* Send database name, followed by \0 terminator */ + pool_putbytes(&Handle->port, dbname, strlen(dbname) + 1); + pool_flush(&Handle->port); + + /* Receive result message */ + if (pool_recvres(&Handle->port) != CLEAN_CONNECTION_COMPLETED) + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("Clean connections not completed"))); +} + /* * Handle messages to agent @@ -816,14 +918,39 @@ agent_handle_input(PoolAgent * agent, StringInfo s) List *datanodelist = NIL; List *coordlist = NIL; int *fds; - int i; + int *pids; + int i, len, res; + + /* + * During a pool cleaning, Abort, Connect and Get Connections messages + * are not allowed on pooler side. + * It avoids to have new backends taking connections + * while remaining transactions are aborted during FORCE and then + * Pools are being shrinked. + */ + if (is_pool_cleaning && (qtype == 'a' || + qtype == 'c' || + qtype == 'g')) + elog(WARNING,"Pool operation cannot run during Pool cleaning"); switch (qtype) { - case 'c': /* CONNECT */ + case 'a': /* ABORT */ pool_getmessage(&agent->port, s, 0); database = pq_getmsgstring(s); - /* + pq_getmsgend(s); + pids = abort_pids(&len, agent->pid, database); + + pool_sendpids(&agent->port, pids, len); + if (pids) + pfree(pids); + break; + case 'c': /* CONNECT */ + pool_getmessage(&agent->port, s, 0); + agent->pid = pq_getmsgint(s, 4); + len = pq_getmsgint(s, 4); + database = pq_getmsgbytes(s, len); + /* * Coordinator pool is not initialized. * With that it would be impossible to create a Database by default. */ @@ -835,6 +962,29 @@ agent_handle_input(PoolAgent * agent, StringInfo s) agent_destroy(agent); pq_getmsgend(s); break; + case 'f': /* CLEAN CONNECTION */ + pool_getmessage(&agent->port, s, 0); + datanodecount = pq_getmsgint(s, 4); + /* It is possible to clean up only Coordinators connections */ + for (i = 0; i < datanodecount; i++) + datanodelist = lappend_int(datanodelist, pq_getmsgint(s, 4)); + coordcount = pq_getmsgint(s, 4); + /* It is possible to clean up only Datanode connections */ + for (i = 0; i < coordcount; i++) + coordlist = lappend_int(coordlist, pq_getmsgint(s, 4)); + len = pq_getmsgint(s, 4); + database = pq_getmsgbytes(s, len); + pq_getmsgend(s); + + /* Clean up connections here */ + res = clean_connection(datanodelist, coordlist, database); + + list_free(datanodelist); + list_free(coordlist); + + /* Send success result */ + pool_sendres(&agent->port, res); + break; case 'g': /* GET CONNECTIONS */ /* * Length of message is caused by: @@ -852,9 +1002,8 @@ agent_handle_input(PoolAgent * agent, StringInfo s) datanodelist = lappend_int(datanodelist, pq_getmsgint(s, 4)); coordcount = pq_getmsgint(s, 4); /* It is possible that no Coordinators are involved in the transaction */ - if (coordcount != 0) - for (i = 0; i < coordcount; i++) - coordlist = lappend_int(coordlist, pq_getmsgint(s, 4)); + for (i = 0; i < coordcount; i++) + coordlist = lappend_int(coordlist, pq_getmsgint(s, 4)); pq_getmsgend(s); /* * In case of error agent_acquire_connections will log @@ -1282,8 +1431,7 @@ insert_database_pool(DatabasePool *databasePool) /* * Find pool for specified database in the list */ -static DatabasePool -* +static DatabasePool * find_database_pool(const char *database) { DatabasePool *databasePool; @@ -1292,12 +1440,11 @@ find_database_pool(const char *database) databasePool = databasePools; while (databasePool) { - /* if match break the loop and return */ if (strcmp(database, databasePool->database) == 0) break; - databasePool = databasePool->next; + databasePool = databasePool->next; } return databasePool; } @@ -1724,6 +1871,147 @@ PoolerLoop(void) } } +/* + * Clean Connection in all Database Pools for given Datanode and Coordinator list + */ +#define TIMEOUT_CLEAN_LOOP 10 + +int +clean_connection(List *dn_discard, List *co_discard, const char *database) +{ + DatabasePool *databasePool; + int dn_len = list_length(dn_discard); + int co_len = list_length(co_discard); + int dn_list[list_length(dn_discard)]; + int co_list[list_length(co_discard)]; + int count, i; + int res = CLEAN_CONNECTION_COMPLETED; + ListCell *nodelist_item; + PGXCNodePool *nodePool; + + /* Save in array the lists of node number */ + count = 0; + foreach(nodelist_item,dn_discard) + dn_list[count++] = lfirst_int(nodelist_item); + + count = 0; + foreach(nodelist_item, co_discard) + co_list[count++] = lfirst_int(nodelist_item); + + /* Find correct Database pool to clean */ + databasePool = find_database_pool(database); + + /* Database pool has not been found */ + if (!databasePool) + return CLEAN_CONNECTION_NOT_COMPLETED; + + /* + * Clean each Pool Correctly + * First for Datanode Pool + */ + for (count = 0; count < dn_len; count++) + { + int node_num = dn_list[count]; + nodePool = databasePool->dataNodePools[node_num - 1]; + + if (nodePool) + { + /* Check if connections are in use */ + if (nodePool->freeSize != nodePool->size) + { + elog(WARNING, "Pool of Database %s is using Datanode %d connections", + databasePool->database, node_num); + res = CLEAN_CONNECTION_NOT_COMPLETED; + } + + /* Destroy connections currently in Node Pool */ + if (nodePool->slot) + { + for (i = 0; i < nodePool->freeSize; i++) + destroy_slot(nodePool->slot[i]); + + /* Move slots in use at the beginning of Node Pool array */ + for (i = nodePool->freeSize; i < nodePool->size; i++ ) + nodePool->slot[i - nodePool->freeSize] = nodePool->slot[i]; + } + nodePool->size -= nodePool->freeSize; + nodePool->freeSize = 0; + } + } + + /* Then for Coordinators */ + for (count = 0; count < co_len; count++) + { + int node_num = co_list[count]; + nodePool = databasePool->coordNodePools[node_num - 1]; + + if (nodePool) + { + /* Check if connections are in use */ + if (nodePool->freeSize != nodePool->size) + { + elog(WARNING, "Pool of Database %s is using Coordinator %d connections", + databasePool->database, node_num); + res = CLEAN_CONNECTION_NOT_COMPLETED; + } + + /* Destroy connections currently in Node Pool */ + if (nodePool->slot) + { + for (i = 0; i < nodePool->freeSize; i++) + destroy_slot(nodePool->slot[i]); + + /* Move slots in use at the beginning of Node Pool array */ + for (i = nodePool->freeSize; i < nodePool->size; i++ ) + nodePool->slot[i - nodePool->freeSize] = nodePool->slot[i]; + } + nodePool->size -= nodePool->freeSize; + nodePool->freeSize = 0; + } + } + + /* Release lock on Pooler, to allow transactions to connect again. */ + is_pool_cleaning = false; + return res; +} + +/* + * Take a Lock on Pooler. + * Abort PIDs registered with the agents for the given database. + * Send back to client list of PIDs signaled to watch them. + */ +int * +abort_pids(int *len, int pid, const char *database) +{ + int *pids = NULL; + int i = 0; + int count; + + Assert(!is_pool_cleaning); + Assert(agentCount > 0); + + is_pool_cleaning = true; + + pids = (int *) palloc((agentCount - 1) * sizeof(int)); + + /* Send a SIGTERM signal to all processes of Pooler agents except this one */ + for (count = 0; count < agentCount; count++) + { + if (poolAgents[count]->pid != pid && + strcmp(poolAgents[count]->pool->database, database) == 0) + { + if (kill(poolAgents[count]->pid, SIGTERM) < 0) + elog(ERROR, "kill(%ld,%d) failed: %m", + (long) poolAgents[count]->pid, SIGTERM); + + pids[i++] = poolAgents[count]->pid; + } + } + + *len = i; + + return pids; +} /* * diff --git a/src/backend/pgxc/pool/poolutils.c b/src/backend/pgxc/pool/poolutils.c new file mode 100644 index 0000000000..e14c936ec8 --- /dev/null +++ b/src/backend/pgxc/pool/poolutils.c @@ -0,0 +1,187 @@ +/*------------------------------------------------------------------------- + * + * poolutils.c + * + * Utilities for Postgres-XC pooler + * + * Portions Copyright (c) 1996-2009, PostgreSQL Global Development Group + * Portions Copyright (c) 2010 Nippon Telegraph and Telephone Corporation + * + * IDENTIFICATION + * $$ + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" +#include "miscadmin.h" +#include "libpq/pqsignal.h" + +#include "pgxc/pgxc.h" +#include "pgxc/poolmgr.h" +#include "pgxc/locator.h" +#include "pgxc/poolutils.h" +#include "access/gtm.h" +#include "commands/dbcommands.h" + +#include "nodes/parsenodes.h" + +/* + * CleanConnection() + * + * Utility to clean up Postgres-XC Pooler connections. + * This utility is launched to all the Coordinators of the cluster + * + * Use of CLEAN CONNECTION is limited to a super user. + * It is advised to clean connections before shutting down a Node or drop a Database. + * + * SQL query synopsis is as follows: + * CLEAN CONNECTION TO + * (COORDINATOR num | DATANODE num | ALL {FORCE}) + * FOR DATABASE dbname + * + * Connection cleaning has to be made on a chosen database called dbname. + * + * It is also possible to clean connections of several Coordinators or Datanodes + * Ex: CLEAN CONNECTION TO DATANODE 1,5,7 FOR DATABASE template1 + * CLEAN CONNECTION TO COORDINATOR 2,4,6 FOR DATABASE template1 + * + * Or even to all Coordinators/Datanodes at the same time + * Ex: CLEAN CONNECTION TO DATANODE * FOR DATABASE template1 + * CLEAN CONNECTION TO COORDINATOR * FOR DATABASE template1 + * + * When FORCE is used, all the transactions using pooler connections are aborted, + * and pooler connections are cleaned up. + * Ex: CLEAN CONNECTION TO ALL FORCE FOR DATABASE template1; + * + * FORCE can only be used with TO ALL, as it takes a lock on pooler to stop requests + * asking for connections, aborts all the connections in the cluster, and cleans up + * pool connections. + */ +void +CleanConnection(CleanConnStmt *stmt) +{ + ListCell *nodelist_item; + List *co_list = NIL; + List *dn_list = NIL; + List *stmt_nodes = NIL; + char *dbname = stmt->dbname; + bool is_coord = stmt->is_coord; + bool is_force = stmt->is_force; + int max_node_number = 0; + Oid oid; + + /* Only a DB administrator can clean pooler connections */ + if (!superuser()) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("must be superuser to clean pool connections"))); + + /* Check if the Database exists by getting its Oid */ + oid = get_database_oid(dbname); + if (!OidIsValid(oid)) + { + ereport(WARNING, + (errcode(ERRCODE_UNDEFINED_DATABASE), + errmsg("database \"%s\" does not exist", dbname))); + return; + } + + /* + * FORCE is activated, + * Send a SIGTERM signal to all the processes and take a lock on Pooler + * to avoid backends to take new connections when cleaning. + * Only Disconnect is allowed. + */ + if (is_force) + { + int loop = 0; + int *proc_pids = NULL; + int num_proc_pids, count; + + num_proc_pids = PoolManagerAbortTransactions(dbname, &proc_pids); + + /* + * Watch the processes that received a SIGTERM. + * At the end of the timestamp loop, processes are considered as not finished + * and force the connection cleaning has failed + */ + + while (num_proc_pids > 0 && loop < TIMEOUT_CLEAN_LOOP) + { + for (count = num_proc_pids - 1; count >= 0; count--) + { + switch(kill(proc_pids[count],0)) + { + case 0: /* Termination not done yet */ + break; + + default: + /* Move tail pid in free space */ + proc_pids[count] = proc_pids[num_proc_pids - 1]; + num_proc_pids--; + break; + } + } + pg_usleep(1000000); + loop++; + } + + if (proc_pids) + pfree(proc_pids); + + if (loop >= TIMEOUT_CLEAN_LOOP) + ereport(WARNING, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("All Transactions have not been aborted"))); + } + + /* Check node list */ + if (stmt->nodes && is_coord) + max_node_number = NumCoords; + else + max_node_number = NumDataNodes; + + foreach(nodelist_item, stmt->nodes) + { + int node_num = intVal(lfirst(nodelist_item)); + stmt_nodes = lappend_int(stmt_nodes, node_num); + + if (node_num > max_node_number || + node_num < 1) + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("Node Number %d is incorrect", node_num))); + } + + /* Build lists to be sent to Pooler Manager */ + if (stmt->nodes && is_coord) + co_list = stmt_nodes; + else if (stmt->nodes && !is_coord) + dn_list = stmt_nodes; + else + { + co_list = GetAllCoordNodes(); + dn_list = GetAllDataNodes(); + } + + /* + * If force is launched, send a signal to all the processes + * that are in transaction and take a lock. + * Get back their process number and watch them locally here. + * Process are checked as alive or not with pg_usleep and when all processes are down + * go out of the control loop. + * If at the end of the loop processes are not down send an error to client. + * Then Make a clean with normal pool cleaner. + * Always release the lock when calling CLEAN CONNECTION. + */ + + /* Finish by contacting Pooler Manager */ + PoolManagerCleanConnection(dn_list, co_list, dbname); + + /* Clean up memory */ + if (co_list) + list_free(co_list); + if (dn_list) + list_free(dn_list); +} diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index c2a473fd11..fbf7596cfb 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -2625,6 +2625,7 @@ die(SIGNAL_ARGS) errno = save_errno; } + /* * Timeout or shutdown signal from postmaster during client authentication. * Simply exit(1). diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index 68a483e357..3a67c29b4b 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -61,6 +61,7 @@ #include "pgxc/locator.h" #include "pgxc/pgxc.h" #include "pgxc/planner.h" +#include "pgxc/poolutils.h" static void ExecUtilityStmtOnNodes(const char *queryString, ExecNodes *nodes, bool force_autocommit, RemoteQueryExecType exec_type); @@ -1402,6 +1403,14 @@ ProcessUtility(Node *parsetree, if (!IsConnFromCoord()) ExecRemoteUtility((RemoteQuery *) parsetree); break; + + case T_CleanConnStmt: + Assert(IS_PGXC_COORDINATOR); + CleanConnection((CleanConnStmt *) parsetree); + + if (IS_PGXC_COORDINATOR) + ExecUtilityStmtOnNodes(queryString, NULL, true, EXEC_ON_COORDS); + break; #endif default: elog(ERROR, "unrecognized node type: %d", @@ -2406,6 +2415,9 @@ CreateCommandTag(Node *parsetree) case T_ExecDirectStmt: tag = "EXECUTE DIRECT"; break; + case T_CleanConnStmt: + tag = "CLEAN CONNECTION"; + break; default: elog(WARNING, "unrecognized node type: %d", @@ -2836,6 +2848,11 @@ GetCommandLogLevel(Node *parsetree) } break; +#ifdef PGXC + case T_CleanConnStmt: + lev = LOGSTMT_DDL; + break; +#endif default: elog(WARNING, "unrecognized node type: %d", diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index 2d60d918d5..10b259ad85 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -355,6 +355,7 @@ typedef enum NodeTag T_AlterUserMappingStmt, T_DropUserMappingStmt, T_ExecDirectStmt, + T_CleanConnStmt, /* * TAGS FOR PARSE TREE NODES (parsenodes.h) diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 355c44847e..79b5fac560 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -2416,6 +2416,18 @@ typedef struct ExecDirectStmt List *nodes; char *query; } ExecDirectStmt; + +/* + * CLEAN CONNECTION statement + */ +typedef struct CleanConnStmt +{ + NodeTag type; + List *nodes; /* list of nodes dropped */ + char *dbname; /* name of database to drop connections */ + bool is_coord; /* type of connections dropped */ + bool is_force; /* option force */ +} CleanConnStmt; /* PGXC_END */ #endif /* PARSENODES_H */ diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h index aec7b6b3d9..62c7c0dcb9 100644 --- a/src/include/parser/kwlist.h +++ b/src/include/parser/kwlist.h @@ -75,6 +75,9 @@ PG_KEYWORD("characteristics", CHARACTERISTICS, UNRESERVED_KEYWORD) PG_KEYWORD("check", CHECK, RESERVED_KEYWORD) PG_KEYWORD("checkpoint", CHECKPOINT, UNRESERVED_KEYWORD) PG_KEYWORD("class", CLASS, UNRESERVED_KEYWORD) +#ifdef PGXC +PG_KEYWORD("clean", CLEAN, UNRESERVED_KEYWORD) +#endif PG_KEYWORD("close", CLOSE, UNRESERVED_KEYWORD) PG_KEYWORD("cluster", CLUSTER, UNRESERVED_KEYWORD) PG_KEYWORD("coalesce", COALESCE, COL_NAME_KEYWORD) diff --git a/src/include/pgxc/poolcomm.h b/src/include/pgxc/poolcomm.h index 9e286ab292..a10af3da9e 100644 --- a/src/include/pgxc/poolcomm.h +++ b/src/include/pgxc/poolcomm.h @@ -45,5 +45,9 @@ extern int pool_putbytes(PoolPort *port, const char *s, size_t len); extern int pool_flush(PoolPort *port); extern int pool_sendfds(PoolPort *port, int *fds, int count); extern int pool_recvfds(PoolPort *port, int *fds, int count); +extern int pool_sendres(PoolPort *port, int res); +extern int pool_recvres(PoolPort *port); +extern int pool_sendpids(PoolPort *port, int *pids, int count); +extern int pool_recvpids(PoolPort *port, int **pids); #endif /* POOLCOMM_H */ diff --git a/src/include/pgxc/poolmgr.h b/src/include/pgxc/poolmgr.h index 3a615cc0f3..7b6e4d4758 100644 --- a/src/include/pgxc/poolmgr.h +++ b/src/include/pgxc/poolmgr.h @@ -63,6 +63,8 @@ typedef struct databasepool */ typedef struct { + /* Process ID of postmaster child process associated to pool agent */ + int pid; /* communication channel */ PoolPort port; DatabasePool *pool; @@ -132,6 +134,12 @@ extern void PoolManagerConnect(PoolHandle *handle, const char *database); /* Get pooled connections */ extern int *PoolManagerGetConnections(List *datanodelist, List *coordlist); +/* Clean pool connections */ +extern void PoolManagerCleanConnection(List *datanodelist, List *coordlist, char *dbname); + +/* Send Abort signal to transactions being run */ +extern int PoolManagerAbortTransactions(char *dbname, int **proc_pids); + /* Return connections back to the pool, for both Coordinator and Datanode connections */ extern void PoolManagerReleaseConnections(int dn_ndisc, int* dn_discard, int co_ndisc, int* co_discard); diff --git a/src/include/pgxc/poolutils.h b/src/include/pgxc/poolutils.h new file mode 100644 index 0000000000..0f34561253 --- /dev/null +++ b/src/include/pgxc/poolutils.h @@ -0,0 +1,30 @@ +/*------------------------------------------------------------------------- + * + * poolutils.h + * + * Utilities for Postgres-XC Pooler + * + * Portions Copyright (c) 1996-2009, PostgreSQL Global Development Group + * Portions Copyright (c) 2010 Nippon Telegraph and Telephone Corporation + * + * IDENTIFICATION + * $$ + * + *------------------------------------------------------------------------- + */ + +#ifndef POOLUTILS_H +#define POOLUTILS_H + +#include "nodes/parsenodes.h" + +#define TIMEOUT_CLEAN_LOOP 10 /* Wait 10s for all the transactions to shutdown */ + +/* Error codes for connection cleaning */ +#define CLEAN_CONNECTION_COMPLETED 0 +#define CLEAN_CONNECTION_NOT_COMPLETED 1 +#define CLEAN_CONNECTION_TX_REMAIN 2 +#define CLEAN_CONNECTION_EOF -1 + +void CleanConnection(CleanConnStmt *stmt); +#endif |