Avoid throwing an error during connection establishment time at GTM

We now instead just log relevant information and allow GTM to continue
functioning. This should address concerns of GTM exiting in case of sudden
spurt of activities

Patch by Wang Diancheng

4 files changed, 44 insertions, 30 deletions

diff --git a/src/gtm/main/gtm_thread.c b/src/gtm/main/gtm_thread.c
index 3f96d74e20..c513f6a71f 100644
--- a/src/gtm/main/gtm_thread.c
+++ b/src/gtm/main/gtm_thread.c
@@ -49,7 +49,11 @@ GTM_ThreadAdd(GTM_ThreadInfo *thrinfo)
 		 * allocation.
 		 */
 		if (GTMThreads->gt_array_size == GTM_MAX_THREADS)
-			elog(ERROR, "Too many threads active");
+		{
+			elog(LOG, "Too many threads active");
+			GTM_RWLockRelease(&GTMThreads->gt_lock);
+			return -1;
+		}
 
 		if (GTMThreads->gt_array_size == 0)
 			newsize = GTM_MIN_THREADS;
@@ -159,7 +163,11 @@ GTM_ThreadRemove(GTM_ThreadInfo *thrinfo)
 	}
 
 	if (ii == GTMThreads->gt_array_size)
-		elog(ERROR, "Thread (%p) not found ", thrinfo);
+	{
+		elog(LOG, "Thread (%p) not found ", thrinfo);
+		GTM_RWLockRelease(&GTMThreads->gt_lock);
+		return -1;
+	}
 
 	GTMThreads->gt_threads[ii] = NULL;
 	GTMThreads->gt_thread_count--;
@@ -207,7 +215,11 @@ GTM_ThreadCreate(GTM_ConnectionInfo *conninfo,
 	 * starting the thread
 	 */
 	if (GTM_ThreadAdd(thrinfo) == -1)
-		elog(ERROR, "Error starting a new thread");
+	{
+		GTM_RWLockDestroy(&thrinfo->thr_lock);
+		pfree(thrinfo);
+		return NULL;
+	}
 
 	/*
 	 * Set up memory contextes before actually starting the threads
@@ -254,10 +266,21 @@ GTM_ThreadCreate(GTM_ConnectionInfo *conninfo,
 	 * Return the thrinfo structure to the caller
 	 */
 	if ((err = pthread_create(&thrinfo->thr_id, NULL, GTM_ThreadMainWrapper,
-							 thrinfo)))
-		ereport(ERROR,
+							  thrinfo)))
+	{
+		ereport(LOG,
 				(err,
-				 errmsg("Failed to create a new thread: error %d", err)));
+				 errmsg("Failed to create a new thread: error %s", strerror(err))));
+
+		MemoryContextDelete(thrinfo->thr_error_context);
+		MemoryContextDelete(thrinfo->thr_thread_context);
+
+		GTM_RWLockDestroy(&thrinfo->thr_lock);
+
+		GTM_ThreadRemove(thrinfo);
+
+		return NULL;
+	}
 
 	/*
 	 * Ensure that the resources are released when the thread exits. (We used
@@ -372,6 +395,8 @@ GTM_ThreadCleanup(void *argp)
 	MemoryContextDelete(thrinfo->thr_thread_context);
 	thrinfo->thr_thread_context = NULL;
 
+	GTM_RWLockDestroy(&thrinfo->thr_lock);
+
 	/*
 	 * TODO Now cleanup the thrinfo structure itself and remove it from the global
 	 * array.
diff --git a/src/gtm/main/main.c b/src/gtm/main/main.c
index 371ebcd7d5..73ed02662a 100644
--- a/src/gtm/main/main.c
+++ b/src/gtm/main/main.c
@@ -958,7 +958,7 @@ ServerLoop(void)
 			{
 				ereport(LOG,
 						(EACCES,
-						 errmsg("select() failed in postmaster: %m")));
+						 errmsg("select() failed in main thread: %m")));
 				return STATUS_ERROR;
 			}
 		}
@@ -989,8 +989,6 @@ ServerLoop(void)
 
 						if (GTMAddConnection(port, standby) != STATUS_OK)
 						{
-							elog(ERROR, "Too many connections");
-
 							gtm_standby_disconnect_from_standby(standby);
 
 							StreamClose(port->sock);
@@ -1439,16 +1437,7 @@ GTMAddConnection(Port *port, GTM_Conn *standby)
 {
 	GTM_ConnectionInfo *conninfo = NULL;
 
-	conninfo = (GTM_ConnectionInfo *)palloc(sizeof (GTM_ConnectionInfo));
-	memset(conninfo, 0, sizeof(GTM_ConnectionInfo));
-
-	if (conninfo == NULL)
-	{
-		ereport(ERROR,
-				(ENOMEM,
-				 	errmsg("Out of memory")));
-		return STATUS_ERROR;
-	}
+	conninfo = (GTM_ConnectionInfo *)palloc0(sizeof (GTM_ConnectionInfo));
 
 	elog(DEBUG3, "Started new connection");
 	conninfo->con_port = port;
@@ -1463,10 +1452,7 @@ GTMAddConnection(Port *port, GTM_Conn *standby)
 	 * XXX Start the thread
 	 */
 	if (GTM_ThreadCreate(conninfo, GTM_ThreadMain) == NULL)
-	{
-		elog(ERROR, "failed to create a new thread");
 		return STATUS_ERROR;
-	}
 
 	return STATUS_OK;
 }
diff --git a/src/gtm/proxy/proxy_main.c b/src/gtm/proxy/proxy_main.c
index a9f141a5a1..b03441e9ce 100644
--- a/src/gtm/proxy/proxy_main.c
+++ b/src/gtm/proxy/proxy_main.c
@@ -916,7 +916,7 @@ ConnCreate(int serverFd)
 		ereport(LOG,
 				(ENOMEM,
 				 errmsg("out of memory")));
-		exit(1);
+		return NULL;
 	}
 
 	if (StreamConnection(serverFd, port) != STATUS_OK)
@@ -924,7 +924,8 @@ ConnCreate(int serverFd)
 		if (port->sock >= 0)
 			StreamClose(port->sock);
 		ConnFree(port);
-		port = NULL;
+
+		return NULL;
 	}
 
 	port->conn_id = InvalidGTMProxyConnID;
@@ -1027,7 +1028,7 @@ ServerLoop(void)
 			{
 				ereport(DEBUG1,
 						(EACCES,
-						 errmsg("select() failed in postmaster: %m")));
+						 errmsg("select() failed in main thread: %m")));
 				return STATUS_ERROR;
 			}
 		}
@@ -1053,7 +1054,6 @@ ServerLoop(void)
 					{
 						if (GTMProxyAddConnection(port) != STATUS_OK)
 						{
-							elog(ERROR, "Too many connections");
 							StreamClose(port->sock);
 							ConnFree(port);
 						}
@@ -1553,7 +1553,7 @@ GTMProxyAddConnection(Port *port)
 
 	if (conninfo == NULL)
 	{
-		ereport(ERROR,
+		ereport(LOG,
 				(ENOMEM,
 					errmsg("Out of memory")));
 		return STATUS_ERROR;
@@ -1565,7 +1565,8 @@ GTMProxyAddConnection(Port *port)
 	/*
 	 * Add the conninfo struct to the next worker thread in round-robin manner
 	 */
-	GTMProxy_ThreadAddConnection(conninfo);
+	if (!GTMProxy_ThreadAddConnection(conninfo))
+		return STATUS_ERROR;
 
 	return STATUS_OK;
 }
diff --git a/src/gtm/proxy/proxy_thread.c b/src/gtm/proxy/proxy_thread.c
index 81fffc4a9f..4f91a98c68 100644
--- a/src/gtm/proxy/proxy_thread.c
+++ b/src/gtm/proxy/proxy_thread.c
@@ -374,7 +374,8 @@ GTMProxy_ThreadAddConnection(GTMProxy_ConnectionInfo *conninfo)
 	if (thrinfo->thr_conn_count >= GTM_PROXY_MAX_CONNECTIONS)
 	{
 		GTM_MutexLockRelease(&thrinfo->thr_lock);
-		elog(ERROR, "Too many connections");
+		elog(LOG, "Too many connections");
+		return NULL;
 	}
 
 	connIndx = -1;
@@ -393,7 +394,8 @@ GTMProxy_ThreadAddConnection(GTMProxy_ConnectionInfo *conninfo)
 	if (connIndx == -1)
 	{
 		GTM_MutexLockRelease(&thrinfo->thr_lock);
-		elog(ERROR, "Too many connections - could not find a free slot");
+		elog(LOG, "Too many connections - could not find a free slot");
+		return NULL;
 	}
 
 	/*