This commit adds error handling in gtm_proxy so that it now check

GTM communication error and handle the error so that:

If GTM connectin retry is specified by configuration parameters
gtm_connect_retry_idle, gtm_connect_retry_count and
gtm_connect_retry_interval, gtm_proxy will retry to establish
connection to GTM, while waiting for reconnect request from
gtm_ctl.

If retry failed, then it will wait for reconnect request from
gtm_ctl as specified by err_wait_idle, err_wait_cound and
err_wait_interval.

GTM error should be detected, gtm-standby should be promoted
to ACT and gtm_proxy should should be reconnected to the promoted
gtm within these timers.

14 files changed, 535 insertions, 166 deletions

diff --git a/doc-xc/src/sgml/ref/gtm_proxy.sgmlin b/doc-xc/src/sgml/ref/gtm_proxy.sgmlin
index d8ee7f6327..570f0184be 100644
--- a/doc-xc/src/sgml/ref/gtm_proxy.sgmlin
+++ b/doc-xc/src/sgml/ref/gtm_proxy.sgmlin
@@ -48,11 +48,11 @@ PostgreSQL documentation
    You must provide gtm configuration
    file <filename>gtm_proxy.conf</filename> placed at gtm working directory
    as specified by <literal>-D</literal> command line option.  The
-   configuration file specifies gtm running envinment and resources.
+   configuration file specifies gtm running environment and resources.
   </para>
 
   <para>
-   Some of the parameters specified in the control file can be overriden by 
+   Some of the parameters specified in the control file can be overridden by 
    command line options.
   </para>
 
@@ -93,6 +93,98 @@ PostgreSQL documentation
 -->
   <variablelist>
 
+   <varlistentry id="gtm-proxy-opt-err-wait-count" xreflabel="gtm_proxy_opt_err_wait_count">
+    <term><varname>err_wait_count</varname> (<type>integer</type>)</term>
+    <indexterm>
+     <primary><varname>err_wait_count</varname> configuration parameter</primary>
+    </indexterm>
+    <listitem>
+     <para>
+      Specifies how many times GTM-Proxy detects reconnect command from <application>gtm_ctl</application>
+      when communication error with GTM is detected.
+      Default value is zero.
+      Refer to <varname>err_wait_idle</varname> and <varname>err_wait_interval</varname>.
+     </para>
+    </listitem>
+   </varlistentry>
+
+   <varlistentry id="gtm-proxy-opt-err-wait-idle" xreflabel="gtm_proxy_opt_err_wait_count">
+    <term><varname>err_wait_idle</varname> (<type>integer</type>)</term>
+    <indexterm>
+     <primary><varname>err_wait_idle</varname> configuration parameter</primary>
+    </indexterm>
+    <listitem>
+     <para>
+      Specifies how long in second GTM-Proxy waits before it begin to detect reconnect
+      from <application>gtm_ctl</application>
+      when communication error with GTM is detected.
+      Default value is zero.
+      Refer to <varname>err_wait_count</varname> and <varname>err_wait_interval</varname>.
+     </para>
+    </listitem>
+   </varlistentry>
+
+   <varlistentry id="gtm-proxy-opt-err-wait-interval" xreflabel="gtm_proxy_opt_err_wait_interval">
+    <term><varname>err_wait_interval</varname> (<type>integer</type>)</term>
+    <indexterm>
+     <primary><varname>err_wait_interval</varname> configuration parameter</primary>
+    </indexterm>
+    <listitem>
+     <para>
+      Specifies how long in second GTM-Proxy waits between each detection of reconnect
+      from <application>gtm_ctl</application>
+      when communication error with GTM is detected.
+      Default value is zero.
+      Refer to <varname>err_wait_count</varname> and <varname>err_wait_idle</varname>.
+     </para>
+    </listitem>
+   </varlistentry>
+
+   <varlistentry id="gtm-proxy-opt-gtm-connect-retry-count" xreflabel="gtm_proxy_opt_gtm_connect_retry_count">
+    <term><varname>gtm_connect_retry_count</varname> (<type>integer</type>)</term>
+    <indexterm>
+     <primary><varname>gtm_connect_retry_count</varname> configuration parameter</primary>
+    </indexterm>
+    <listitem>
+     <para>
+      Specifies how many times GTM-Proxy retries to connect to GTM
+      when communication error with GTM is detected.
+      Default value is zero.
+      Refer to <varname>gtm_connect_retry_idle</varname> and <varname>gtm_connect_retry_interval</varname>.
+     </para>
+    </listitem>
+   </varlistentry>
+
+   <varlistentry id="gtm-proxy-opt-gtm-connect-retry-idle" xreflabel="gtm_proxy_opt_gtm_connect_retry_idle">
+    <term><varname>gtm_connect_retry_idle</varname> (<type>integer</type>)</term>
+    <indexterm>
+     <primary><varname>gtm_connect_retry_idle</varname> configuration parameter</primary>
+    </indexterm>
+    <listitem>
+     <para>
+      Specifies how long in second GTM-Proxy waits before it retries to connect to GTM when
+      communication error with GTM is detected.
+      Default value is zero.
+      Refer to <varname>gtm_connect_retry_count</varname> and <varname>gtm_connect_retry_interval</varname>.
+     </para>
+    </listitem>
+   </varlistentry>
+
+   <varlistentry id="gtm-proxy-opt-gtm-connect-retry-interval" xreflabel="gtm_proxy_opt_gtm_connect_retry_interval">
+    <term><varname>gtm_connect_retry_interval</varname> (<type>integer</type>)</term>
+    <indexterm>
+     <primary><varname>gtm_connect_retry_interval</varname> configuration parameter</primary>
+    </indexterm>
+    <listitem>
+     <para>
+      Specifies how log in second GTM-Proxy waits between each retry to connect to GTM when
+      communication error with GTM is detected.
+      Default value is zero.
+      Refer to <varname>gtm_connect_retry_count</varname> and <varname>gtm_connect_retry_idle</varname>.
+     </para>
+    </listitem>
+   </varlistentry>
+
    <varlistentry id="gtm-proxy-opt-gtm-host" xreflabel="gtm_proxy_opt_gtm_host">
     <term><varname>gtm_host</varname> (<type>string</type>)</term>
     <indexterm>
diff --git a/src/gtm/client/fe-misc.c b/src/gtm/client/fe-misc.c
index 930c8ec41e..98dbe2190c 100644
--- a/src/gtm/client/fe-misc.c
+++ b/src/gtm/client/fe-misc.c
@@ -582,8 +582,11 @@ gtmpqReadData(GTM_Conn *conn)
 retry3:
 	nread = recv(conn->sock, conn->inBuffer + conn->inEnd,
 						  conn->inBufSize - conn->inEnd, 0);
+	conn->last_call = GTM_LastCall_RECV;
 	if (nread < 0)
 	{
+		conn->last_errno = SOCK_ERRNO;
+
 		if (SOCK_ERRNO == EINTR)
 			goto retry3;
 		/* Some systems return EAGAIN/EWOULDBLOCK for no data */
@@ -604,6 +607,9 @@ retry3:
 				   "could not receive data from server:\n");
 		return -1;
 	}
+	else
+		conn->last_errno = 0;
+
 	if (nread > 0)
 	{
 		conn->inEnd += nread;
@@ -668,8 +674,10 @@ retry3:
 retry4:
 	nread = recv(conn->sock, conn->inBuffer + conn->inEnd,
 						  conn->inBufSize - conn->inEnd, 0);
+	conn->last_call = GTM_LastCall_RECV;
 	if (nread < 0)
 	{
+		conn->last_errno = SOCK_ERRNO;
 		if (SOCK_ERRNO == EINTR)
 			goto retry4;
 		/* Some systems return EAGAIN/EWOULDBLOCK for no data */
@@ -690,6 +698,8 @@ retry4:
 				   "could not receive data from server: \n");
 		return -1;
 	}
+	else
+		conn->last_errno = 0;
 	if (nread > 0)
 	{
 		conn->inEnd += nread;
@@ -741,9 +751,11 @@ gtmpqSendSome(GTM_Conn *conn, int len)
 		int			sent;
 
 		sent = send(conn->sock, ptr, len, 0);
+		conn->last_call = GTM_LastCall_SEND;
 
 		if (sent < 0)
 		{
+			conn->last_errno = SOCK_ERRNO;
 			/*
 			 * Anything except EAGAIN/EWOULDBLOCK/EINTR is trouble. If it's
 			 * EPIPE or ECONNRESET, assume we've lost the backend connection
@@ -795,6 +807,7 @@ gtmpqSendSome(GTM_Conn *conn, int len)
 			ptr += sent;
 			len -= sent;
 			remaining -= sent;
+			conn->last_errno = 0;
 		}
 
 		if (len > 0)
diff --git a/src/gtm/libpq/pqcomm.c b/src/gtm/libpq/pqcomm.c
index 9647b8488c..d2a6e9b6b5 100644
--- a/src/gtm/libpq/pqcomm.c
+++ b/src/gtm/libpq/pqcomm.c
@@ -509,9 +509,11 @@ pq_recvbuf(Port *myport)
 
 		r = recv(myport->sock, myport->PqRecvBuffer + myport->PqRecvLength,
 						PQ_BUFFER_SIZE - myport->PqRecvLength, 0);
+		myport->last_call = GTM_LastCall_RECV;
 
 		if (r < 0)
 		{
+			myport->last_errno = errno;
 			if (errno == EINTR)
 				continue;		/* Ok if interrupted */
 
@@ -525,6 +527,8 @@ pq_recvbuf(Port *myport)
 					 errmsg("could not receive data from client: %m")));
 			return EOF;
 		}
+		else
+			myport->last_errno = 0;
 		if (r == 0)
 		{
 			/*
@@ -820,9 +824,11 @@ internal_flush(Port *myport)
 		int			r;
 
 		r = send(myport->sock, bufptr, bufend - bufptr, 0);
+		myport->last_call = GTM_LastCall_SEND;
 
 		if (r <= 0)
 		{
+			myport->last_errno = errno;
 			if (errno == EINTR)
 				continue;		/* Ok if we were interrupted */
 
@@ -850,6 +856,8 @@ internal_flush(Port *myport)
 			myport->PqSendPointer = 0;
 			return EOF;
 		}
+		else
+			myport->last_errno = 0;
 
 		last_reported_send_errno = 0;	/* reset after any successful send */
 		bufptr += r;
diff --git a/src/gtm/main/gtm.conf.sample b/src/gtm/main/gtm.conf.sample
index 4121849eef..d0e3fdb629 100644
--- a/src/gtm/main/gtm.conf.sample
+++ b/src/gtm/main/gtm.conf.sample
@@ -53,6 +53,6 @@
 #keepalives_count = 0			# Keepalives_count internal parameter.
 #log_file = 'gtm.log'			# Log file name
 #log_min_messages = WARNING		# log_min_messages.  Default WARNING.
-				  	# Valid value: DEBUG, DEBUG5, DEBUG4, DEBU3,
+				  	# Valid value: DEBUG, DEBUG5, DEBUG4, DEBUG3,
 					# DEBUG2, DEBUG1, INFO, NOTICE, WARNING,
 					# ERROR, LOG, FATAL, PANIC
diff --git a/src/gtm/proxy/gtm_proxy.conf.sample b/src/gtm/proxy/gtm_proxy.conf.sample
index 97eefd0c3d..3b275a43e6 100644
--- a/src/gtm/proxy/gtm_proxy.conf.sample
+++ b/src/gtm/proxy/gtm_proxy.conf.sample
@@ -27,30 +27,48 @@
 #------------------------------------------------------------------------------
 # GENERAL PARAMETERS
 #------------------------------------------------------------------------------
-#nodename = ''				# Specifies the node name.
-					# (changes requires restart)
+#nodename = ''					# Specifies the node name.
+								# (changes requires restart)
 #listen_addresses = '*'			# Listen addresses of this GTM.
-					# (changes requires restart)
-#port = 6666				# Port number of this GTM.
-					# (changes requires restart)
+								# (changes requires restart)
+#port = 6666					# Port number of this GTM.
+								# (changes requires restart)
 
 #------------------------------------------------------------------------------
 # GTM PROXY PARAMETERS
 #------------------------------------------------------------------------------
-#worker_threads = 1			# Number of the worker thread of this
-					# GTM proxy
-					# (changes requires restart)
+#worker_threads = 1				# Number of the worker thread of this
+								# GTM proxy
+								# (changes requires restart)
 
 #------------------------------------------------------------------------------
 # GTM CONNECTION PARAMETERS
 #------------------------------------------------------------------------------
 # Those parameters are used to connect to a GTM server
-#gtm_host = '' 				# Listen address of the active GTM.
-					# (changes requires restart)
-#gtm_port = 				# Port number of the active GTM.
-					# (changes requires restart)
+#gtm_host = '' 					# Listen address of the active GTM.
+								# (changes requires restart)
+#gtm_port = 					# Port number of the active GTM.
+								# (changes requires restart)
 
 #------------------------------------------------------------------------------
+# Behavior at GTM communication error
+#------------------------------------------------------------------------------
+#gtm_connect_retry_idle = 0		# How long (in secs) to wait before GTM proxy
+			   					# retries to connect to GTM.
+#gtm_connect_retry_count = 0	# How many times GTM proxy retries to connect
+								# to GTM.
+#gtm_connect_retry_interval = 0	# How long (in secs) to wait until the next
+								# retry to connect to GTM.
+#err_wait_idle = 0				# How long (in secs) GTM proxy should wait
+			   	 				# before it detects reconnect from gtm_ctl.
+#err_wait_count = 0				# How many times GTM should detect reconnect
+								# from gtm_ctl.
+#err_wait_interval = 0			# How long (in secs) GTM proxy should wait
+								# until next detection of reconnect from
+								# gtm_ctl.
+#
+#
+#------------------------------------------------------------------------------
 # Other options
 #------------------------------------------------------------------------------
 #keepalives_idle = 0			# Keepalives_idle parameter.
@@ -58,7 +76,7 @@
 #keepalives_count = 0			# Keepalives_count internal parameter.
 #log_file = 'gtm_proxy.log'		# Log file name
 #log_min_messages = WARNING		# log_min_messages.  Default WARNING.
-				  	# Valid value: DEBUG, DEBUG5, DEBUG4, DEBU3,
-					# DEBUG2, DEBUG1, INFO, NOTICE, WARNING,
-					# ERROR, LOG, FATAL, PANIC.
+							  	# Valid value: DEBUG, DEBUG5, DEBUG4, DEBUG3,
+								# DEBUG2, DEBUG1, INFO, NOTICE, WARNING,
+								# ERROR, LOG, FATAL, PANIC.
 
diff --git a/src/gtm/proxy/gtm_proxy_opt.c b/src/gtm/proxy/gtm_proxy_opt.c
index 8048f39896..6fdd470b2f 100644
--- a/src/gtm/proxy/gtm_proxy_opt.c
+++ b/src/gtm/proxy/gtm_proxy_opt.c
@@ -47,14 +47,18 @@ extern int log_min_messages;
 extern int keepalives_idle;
 extern int keepalives_count;
 extern int keepalives_interval;
-extern bool GTMErrorWaitOpt;
 extern char *GTMServerHost;
 extern int GTMProxyPortNumber;
+extern bool IsGTMConnectRetryRequired;
+extern int GTMConnectRetryIdle;
+extern int GTMConnectRetryCount;
+extern int GTMConnectRetryInterval;
 extern int GTMServerPortNumber;
 extern int GTMServerKeepalivesIdle;
 extern int GTMServerKeepalivesInterval;
 extern int GTMServerKeepalivesCount;
-extern int GTMErrorWaitSecs;
+extern int GTMErrorWaitIdle;
+extern int GTMErrorWaitInterval;
 extern int GTMErrorWaitCount;
 extern int GTMProxyWorkerThreads;
 extern char *GTMProxyDataDir;
@@ -146,15 +150,6 @@ Config_Type_Names();
 
 struct config_bool ConfigureNamesBool[] =
 {
-	{
-		{GTM_OPTNAME_ERR_WAIT_OPT, GTMC_SIGHUP,
-		 gettext_noop("If GTM_Proxy waits for reconnect when GTM communication error is encountered."),
-		 NULL,
-		 0
-		},
-		&GTMErrorWaitOpt,
-		false, false, NULL
-	},
 	/* End-of-list marker */
 	{
 		{NULL, 0, NULL, NULL, 0}, NULL, false, false, NULL
@@ -165,7 +160,8 @@ struct config_bool ConfigureNamesBool[] =
 struct config_int ConfigureNamesInt[] =
 {
 	{
-		{GTM_OPTNAME_PORT, GTMC_STARTUP,
+		{
+			GTM_OPTNAME_PORT, GTMC_STARTUP,
 			gettext_noop("Listen Port of GTM_Proxy server."),
 			NULL,
 			0
@@ -175,7 +171,8 @@ struct config_int ConfigureNamesInt[] =
 		0, NULL
 	},
 	{
-		{GTM_OPTNAME_GTM_PORT, GTMC_SIGHUP,
+		{
+			GTM_OPTNAME_GTM_PORT, GTMC_SIGHUP,
 			gettext_noop("GTM server port number."),
 			NULL,
 			0
@@ -185,7 +182,42 @@ struct config_int ConfigureNamesInt[] =
 	    0, NULL
 	},
 	{
-		{GTM_OPTNAME_KEEPALIVES_IDLE, GTMC_STARTUP,
+		{
+			GTM_OPTNAME_CONNECT_RETRY_COUNT, GTMC_SIGHUP,
+		 	gettext_noop("Retry count to try to reconnect to GTM."),
+		 	NULL,
+		 	0
+		},
+		&GTMConnectRetryCount,
+		0, 0, INT_MAX,
+		0, NULL
+	},
+	{
+		{
+			GTM_OPTNAME_CONNECT_RETRY_IDLE, GTMC_SIGHUP,
+		 	gettext_noop("Idle time in second before GTM standby retries "
+						 "connection to GTM."),
+		 	NULL,
+		 	GTMOPT_UNIT_TIME
+		},
+		&GTMConnectRetryIdle,
+		0, 0, INT_MAX,
+		0, NULL
+	},
+	{
+		{
+			GTM_OPTNAME_CONNECT_RETRY_INTERVAL, GTMC_SIGHUP,
+			gettext_noop("Interval in second to detect reconnect command."),
+			NULL,
+			GTMOPT_UNIT_TIME
+		},
+		&GTMConnectRetryInterval,
+		0, 0, INT_MAX,
+		0, NULL
+	},
+	{
+		{
+			GTM_OPTNAME_KEEPALIVES_IDLE, GTMC_STARTUP,
 			gettext_noop("Sets \"keepalives_idle\" option for the connection to GTM."),
 		    NULL,
 			GTMOPT_UNIT_TIME
@@ -195,7 +227,8 @@ struct config_int ConfigureNamesInt[] =
 		0, NULL
 	},
 	{
-		{GTM_OPTNAME_KEEPALIVES_INTERVAL, GTMC_STARTUP,
+		{
+			GTM_OPTNAME_KEEPALIVES_INTERVAL, GTMC_STARTUP,
 			gettext_noop("Sets \"keepalives_interval\" option fo the connetion to GTM."),
 		 	NULL,
 			GTMOPT_UNIT_TIME
@@ -205,7 +238,8 @@ struct config_int ConfigureNamesInt[] =
 		0, NULL
 	},
 	{
-		{GTM_OPTNAME_KEEPALIVES_COUNT, GTMC_STARTUP,
+		{
+			GTM_OPTNAME_KEEPALIVES_COUNT, GTMC_STARTUP,
 			gettext_noop("Sets \"keepalives_count\" option to the connection to GTM."),
 			NULL, 
 			0
@@ -215,19 +249,36 @@ struct config_int ConfigureNamesInt[] =
 		0, NULL
 	},
 	{
-		{GTM_OPTNAME_ERR_WAIT_INTERVAL, GTMC_SIGHUP,
-		 gettext_noop("Wait interval to wait for reconnect."),
-		 gettext_noop("This parameter determines GTM Proxy behavior when GTM communication error is encountered."),
-		 0
+		{
+			GTM_OPTNAME_ERR_WAIT_IDLE, GTMC_SIGHUP,
+			gettext_noop("Time duration after connection to GTM failed and "
+						 "wait for reconnect command begins."),
+			gettext_noop("This parameter determines GTM Proxy behavior "
+						 "when GTM communication error is encountered."),
+			0
 		},
-		&GTMErrorWaitSecs,
+		&GTMErrorWaitIdle,
 		0, 0, INT_MAX,
 		0, NULL
 	},
 	{
-		{GTM_OPTNAME_ERR_WAIT_COUNT, GTMC_SIGHUP,
-		 gettext_noop("Number of err_wait_interval to wait for reconnect."),
-		 gettext_noop("This parameter determines GTM Prox behavior when GTM communication error is encountered."),
+		{
+			GTM_OPTNAME_ERR_WAIT_INTERVAL, GTMC_SIGHUP,
+			gettext_noop("Wait interval to wait for reconnect."),
+			gettext_noop("This parameter determines GTM Proxy behavior "
+						 "when GTM communication error is encountered."),
+			0
+		},
+		&GTMErrorWaitInterval,
+		0, 0, INT_MAX,
+		0, NULL
+	},
+	{
+		{
+			GTM_OPTNAME_ERR_WAIT_COUNT, GTMC_SIGHUP,
+			gettext_noop("Number of err_wait_interval to wait for reconnect."),
+			gettext_noop("This parameter determines GTM Prox behavior "
+						 "when GTM communication error is encountered."),
 		 0
 		},
 		&GTMErrorWaitCount,
@@ -235,10 +286,11 @@ struct config_int ConfigureNamesInt[] =
 		0, NULL
 	},
 	{
-		{GTM_OPTNAME_WORKER_THREADS, GTMC_STARTUP,
-		 gettext_noop("Number of worker thread."),
-		 NULL,
-		 0
+		{
+			GTM_OPTNAME_WORKER_THREADS, GTMC_STARTUP,
+			gettext_noop("Number of worker thread."),
+			NULL,
+			0
 		},
 		&GTMProxyWorkerThreads,
 		GTM_PROXY_DEFAULT_WORKERS, 1, INT_MAX,
@@ -262,7 +314,8 @@ struct config_real ConfigureNamesReal[] =
 struct config_string ConfigureNamesString[] =
 {
 	{
-		{GTM_OPTNAME_DATA_DIR, GTMC_STARTUP,
+		{
+			GTM_OPTNAME_DATA_DIR, GTMC_STARTUP,
 			gettext_noop("Work directory."),
 			NULL,
 			0
@@ -274,7 +327,8 @@ struct config_string ConfigureNamesString[] =
 	},
 
 	{
-		{GTM_OPTNAME_CONFIG_FILE, GTMC_SIGHUP,
+		{
+			GTM_OPTNAME_CONFIG_FILE, GTMC_SIGHUP,
 		 	gettext_noop("Configuration file name."),
 		 	NULL,
 		 	0
@@ -286,7 +340,8 @@ struct config_string ConfigureNamesString[] =
 	},
 
 	{
-		{GTM_OPTNAME_LISTEN_ADDRESSES, GTMC_STARTUP,
+		{
+			GTM_OPTNAME_LISTEN_ADDRESSES, GTMC_STARTUP,
 			gettext_noop("Listen address."),
 			NULL,
 			0
@@ -297,7 +352,8 @@ struct config_string ConfigureNamesString[] =
 	},
 
 	{
-		{GTM_OPTNAME_NODENAME, GTMC_STARTUP,
+		{
+			GTM_OPTNAME_NODENAME, GTMC_STARTUP,
 		 	gettext_noop("My node name."),
 			NULL,
 		 	0,
@@ -308,7 +364,8 @@ struct config_string ConfigureNamesString[] =
 	},
 
 	{
-		{GTM_OPTNAME_GTM_HOST, GTMC_SIGHUP,
+		{
+			GTM_OPTNAME_GTM_HOST, GTMC_SIGHUP,
 			gettext_noop("Address of target GTM ACT."),
 			NULL,
 			0
@@ -319,7 +376,8 @@ struct config_string ConfigureNamesString[] =
 	},
 
 	{
-		{GTM_OPTNAME_LOG_FILE, GTMC_SIGHUP,
+		{
+			GTM_OPTNAME_LOG_FILE, GTMC_SIGHUP,
 			gettext_noop("Log file name."),
 			NULL,
 			0
@@ -330,7 +388,8 @@ struct config_string ConfigureNamesString[] =
 	},
 
 	{
-		{GTM_OPTNAME_ERROR_REPORTER, GTMC_SIGHUP,
+		{
+			GTM_OPTNAME_ERROR_REPORTER, GTMC_SIGHUP,
 			gettext_noop("Command to report various errors."),
 			NULL,
 			0
@@ -341,7 +400,8 @@ struct config_string ConfigureNamesString[] =
 	},
 
 	{
-		{GTM_OPTNAME_STATUS_READER, GTMC_SIGHUP,
+		{
+			GTM_OPTNAME_STATUS_READER, GTMC_SIGHUP,
 			gettext_noop("Command to get status of global XC node status."),
 			gettext_noop("Runs when configuration file is read by SIGHUP"),
 			0
@@ -361,7 +421,8 @@ struct config_string ConfigureNamesString[] =
 struct config_enum ConfigureNamesEnum[] =
 {
 	{
-		{GTM_OPTNAME_LOG_MIN_MESSAGES, GTMC_SIGHUP,
+		{
+			GTM_OPTNAME_LOG_MIN_MESSAGES, GTMC_SIGHUP,
 			gettext_noop("Minimum message level to write to the log file."),
 			NULL,
 		 	0
diff --git a/src/gtm/proxy/proxy_main.c b/src/gtm/proxy/proxy_main.c
index 1586fd7d13..8317d3dc61 100644
--- a/src/gtm/proxy/proxy_main.c
+++ b/src/gtm/proxy/proxy_main.c
@@ -66,13 +66,18 @@ char		*GTMProxyConfigFileName;
 char		*GTMConfigFileName;
 
 /* GTM communication error handling options */
-bool		GTMErrorWaitOpt = FALSE;		/* Wait and assume XCM if TRUE */
-int			GTMErrorWaitSecs = 0;			/* Duration of each wait */
+int			GTMErrorWaitIdle = 0;
+int			GTMErrorWaitInterval = 0;		/* Duration of each wait */
 int			GTMErrorWaitCount = 0;			/* How many durations to wait */
 
 char		*GTMServerHost;
 int			GTMServerPortNumber;
 
+/* GTM connection retry info */
+int			GTMConnectRetryIdle = 0;
+int			GTMConnectRetryCount = 0;
+int			GTMConnectRetryInterval = 0;
+
 /*
  * Keepalives setup for the connection with GTM server
  */
@@ -136,6 +141,8 @@ static void GTMProxy_ProxyPGXCNodeCommand(GTMProxy_ConnectionInfo *conninfo,
 
 static void ProcessCommand(GTMProxy_ConnectionInfo *conninfo,
 		GTM_Conn *gtm_conn, StringInfo input_message);
+static GTM_Conn *HandleGTMError(GTM_Conn *gtm_conn);
+static GTM_Conn *HandlePostCommand(GTMProxy_ConnectionInfo *conninfo, GTM_Conn *gtm_conn);
 static void ProcessPGXCNodeCommand(GTMProxy_ConnectionInfo *conninfo,
 		GTM_Conn *gtm_conn, GTM_MessageType mtype, StringInfo message);
 static void ProcessTransactionCommand(GTMProxy_ConnectionInfo *conninfo,
@@ -168,7 +175,7 @@ static void RegisterProxy(bool is_reconnect);
 static void UnregisterProxy(void);
 static GTM_Conn *ConnectGTM(void);
 static void ReleaseCmdBackup(GTMProxy_CommandInfo *cmdinfo);
-static void workerThreadReconnectToGTMstandby(void);
+static void workerThreadReconnectToGTM(void);
 
 /*
  * One-time initialization. It's called immediately after the main process
@@ -202,6 +209,8 @@ MainThreadInit()
 
 	memset((char *)thrinfo, 0, sizeof(GTMProxy_ThreadInfo));
 
+	memset((char *)thrinfo, 0, sizeof(GTMProxy_ThreadInfo));
+
 	if (SetMyThreadInfo(thrinfo))
 	{
 		fprintf(stderr, "SetMyThreadInfo failed: %d", errno);
@@ -290,6 +299,7 @@ read_token(char *line, char **next)
 			return(tok);
 		}
 		if (*next_token == ' ' || *next_token == '\t')
+
 		{
 			*next_token = 0;
 			*next = next_token + 1;
@@ -692,55 +702,55 @@ main(int argc, char *argv[])
 	 */
 	if (listen_addresses)
 	{
-		SetConfigOption("listen_addresses", listen_addresses, GTMC_STARTUP, GTMC_S_OVERRIDE);
+		SetConfigOption(GTM_OPTNAME_LISTEN_ADDRESSES, listen_addresses, GTMC_STARTUP, GTMC_S_OVERRIDE);
 		free(listen_addresses);
 		listen_addresses = NULL;
 	}
 	if (node_name)
 	{
-		SetConfigOption("nodename", node_name, GTMC_STARTUP, GTMC_S_OVERRIDE);
+		SetConfigOption(GTM_OPTNAME_NODENAME, node_name, GTMC_STARTUP, GTMC_S_OVERRIDE);
 		free(node_name);
 		node_name = NULL;
 	}
 	if (proxy_port_number)
 	{
-		SetConfigOption("port", proxy_port_number, GTMC_STARTUP, GTMC_S_OVERRIDE);
+		SetConfigOption(GTM_OPTNAME_PORT, proxy_port_number, GTMC_STARTUP, GTMC_S_OVERRIDE);
 		free(proxy_port_number);
 		proxy_port_number = NULL;
 	}
 	if (proxy_worker_threads)
 	{
-		SetConfigOption("worker_threads", proxy_worker_threads, GTMC_STARTUP, GTMC_S_OVERRIDE);
+		SetConfigOption(GTM_OPTNAME_WORKER_THREADS, proxy_worker_threads, GTMC_STARTUP, GTMC_S_OVERRIDE);
 		free(proxy_worker_threads);
 		proxy_worker_threads = NULL;
 	}
 	if (log_file)
 	{
-		SetConfigOption("log_file", log_file, GTMC_STARTUP, GTMC_S_OVERRIDE);
+		SetConfigOption(GTM_OPTNAME_LOG_FILE, log_file, GTMC_STARTUP, GTMC_S_OVERRIDE);
 		free(log_file);
 		log_file = NULL;
 	}
 	if (gtm_host)
 	{
-		SetConfigOption("gtm_host", gtm_host, GTMC_STARTUP, GTMC_S_OVERRIDE);
+		SetConfigOption(GTM_OPTNAME_GTM_HOST, gtm_host, GTMC_STARTUP, GTMC_S_OVERRIDE);
 		free(gtm_host);
 		gtm_host = NULL;
 	}
 	if (gtm_port)
 	{
-		SetConfigOption("gtm_port", gtm_port, GTMC_STARTUP, GTMC_S_OVERRIDE);
+		SetConfigOption(GTM_OPTNAME_GTM_PORT, gtm_port, GTMC_STARTUP, GTMC_S_OVERRIDE);
 		free(gtm_port);
 		gtm_port = NULL;
 	}
 	if (gtm_err_wait_secs)
 	{
-		SetConfigOption("err_wait_interval", gtm_err_wait_secs, GTMC_STARTUP, GTMC_S_OVERRIDE);
+		SetConfigOption(GTM_OPTNAME_ERR_WAIT_INTERVAL, gtm_err_wait_secs, GTMC_STARTUP, GTMC_S_OVERRIDE);
 		free(gtm_err_wait_secs);
 		gtm_err_wait_secs = NULL;
 	}
 	if (gtm_err_wait_count)
 	{
-		SetConfigOption("err_wait_count", gtm_err_wait_count, GTMC_STARTUP, GTMC_S_OVERRIDE);
+		SetConfigOption(GTM_OPTNAME_ERR_WAIT_COUNT, gtm_err_wait_count, GTMC_STARTUP, GTMC_S_OVERRIDE);
 		free(gtm_err_wait_count);
 		gtm_err_wait_count = NULL;
 	}
@@ -765,23 +775,6 @@ main(int argc, char *argv[])
 	}
 
 	/*
-	 * Validate GTM communication error handling option
-	 */
-	if (GTMErrorWaitSecs > 0 && GTMErrorWaitCount > 0)
-	{
-		if (GTMErrorWaitOpt == false)
-		{
-			GTMErrorWaitSecs = 0;
-			GTMErrorWaitCount = 0;
-		}
-	}
-	else
-	{
-		GTMErrorWaitOpt = false;
-		GTMErrorWaitSecs = 0;
-		GTMErrorWaitCount = 0;
-	}
-	/*
 	 * GTM accepts no non-option switch arguments.
 	 */
 	if (optind < argc)
@@ -1124,8 +1117,8 @@ GTMProxy_ThreadMain(void *argp)
 	/*
 	 * Set GTM communication error handling options.
 	 */
-	thrinfo->thr_gtm_conn->gtmErrorWaitOpt = GTMErrorWaitOpt;
-	thrinfo->thr_gtm_conn->gtmErrorWaitSecs = GTMErrorWaitSecs;
+	thrinfo->thr_gtm_conn->gtmErrorWaitIdle = GTMErrorWaitIdle;
+	thrinfo->thr_gtm_conn->gtmErrorWaitInterval = GTMErrorWaitInterval;
 	thrinfo->thr_gtm_conn->gtmErrorWaitCount = GTMErrorWaitCount;
 
 	thrinfo->reconnect_issued = FALSE;
@@ -1260,7 +1253,7 @@ GTMProxy_ThreadMain(void *argp)
 					else
 					{
 						/* SIGUSR2 here */
-						workerThreadReconnectToGTMstandby();
+						workerThreadReconnectToGTM();
 					}
 				}
 
@@ -1340,7 +1333,7 @@ setjmp_again:
 			 * SIGUSR2 is detected and jumped here
 			 * Reconnection phase
 			 */
-			workerThreadReconnectToGTMstandby();
+			workerThreadReconnectToGTM();
 
 			/*
 			 * Correction of pending works.
@@ -1403,6 +1396,7 @@ setjmp_again:
 					case 'C':
 						ProcessCommand(thrinfo->thr_conn, thrinfo->thr_gtm_conn,
 								&input_message);
+						HandlePostCommand(thrinfo->thr_conn, thrinfo->thr_gtm_conn);
 						break;
 
 					case 'X':
@@ -1472,7 +1466,24 @@ setjmp_again:
 			{
 				Enable_Longjmp();
 				if ((res = GTMPQgetResult(thrinfo->thr_gtm_conn)) == NULL)
+				{
+					/*
+					 * Here's another place to check GTM communication error.
+					 * In this case, backup of each command will be taken care of
+					 * by ProcessResponse() so if socket read/write error is recorded,
+					 * disconnect GTM connection, retry connection and then if it faile,
+					 * wait for reconnect from gtm_ctl.
+					 */
+					if ((thrinfo->thr_gtm_conn->last_errno != 0) || (thrinfo->thr_gtm_conn->status == CONNECTION_BAD))
+					{
+						/*
+						 * Please note that error handling can end up with longjmp() and
+						 * may not return here.
+						 */
+						HandleGTMError(thrinfo->thr_gtm_conn);
+					}
 					elog(ERROR, "GTMPQgetResult failed");
+				}
 				Disable_Longjmp();
 			}
 
@@ -1585,6 +1596,153 @@ ProcessCommand(GTMProxy_ConnectionInfo *conninfo, GTM_Conn *gtm_conn,
 	}
 
 	conninfo->con_pending_msg = mtype;
+
+}
+
+/*
+ * This funciton mainly takes care of GTM communcation error.
+ *
+ * Communication error was stored in last_errno chaned from GTMProxy_ConnectionInfo.
+ * Note that it is set to zero if the last send/receive/read/write succeeds.
+ *
+ * If error is detected, then it tries to connect to GTM again, if it is
+ * specified by configuration parameters.
+ *
+ * Relevant configuration parameters are: gtm_connect_retry_idle, gtm_connect_retry_count
+ * and gtm_connect_retry_interval.
+ *
+ * If it is not successfull or configuration parameter does not specify it,
+ * then, according to another confugration parameters, it waits "reconnect"
+ * command from gtm_proxy.
+ *
+ * Relevant configuration parameters are: err_wait_idle, err_wait_count, and
+ * err_wait_interval.
+ */
+
+static GTM_Conn *
+HandleGTMError(GTM_Conn *gtm_conn)
+{
+	GTM_Conn *new_gtm_conn;
+	int		 ii;
+
+	Assert(gtm_conn && gtm_conn->last_errno != 0);
+
+	if (GTMConnectRetryCount > 0)
+	{
+		int ii;
+		char gtm_connect_string[1024];
+
+		elog(NOTICE, 
+			 "GTM communication error was detected.  Retrying connection. idle: %d, count = %d, interval = %d.",
+			 GTMConnectRetryIdle, GTMConnectRetryCount, GTMConnectRetryInterval);
+		GTMPQfinish(gtm_conn);
+		sprintf(gtm_connect_string, "host=%s port=%d node_name=%s remote_type=%d",
+				GTMServerHost, GTMServerPortNumber, GTMProxyNodeName, GTM_NODE_GTM_PROXY);
+		/* Wait and retry reconnect */
+		if (GTMConnectRetryIdle > 0) {
+			elog(DEBUG1, "Waiting %d secs.", GTMConnectRetryIdle);
+			pg_usleep((long)GTMConnectRetryIdle * 1000000L);
+		}
+		/* GTM connection retry */
+		for (ii = 0; ii < GTMConnectRetryCount; ii++)
+		{
+			/* 
+			 * Connect retry 
+			 * Because this proxy has been registered to current
+			 * GTM, we don't re-register it.
+			 *
+			 * Please note that GTM-Proxy accepts "reconnect" from gtm_ctl
+			 * even while it is retrying to connect to GTM.
+			 */
+			elog(DEBUG1, "Try to reconnect to GTM, count %d.", ii);
+			Disable_Longjmp();
+			new_gtm_conn = PQconnectGTM(gtm_connect_string);
+			Enable_Longjmp();
+			if (new_gtm_conn != NULL) {
+				elog(NOTICE, "GTM connection retry succeeded, count %d.", ii);
+				break;
+			}
+			/* Wait if not successful */
+			elog(DEBUG1, "Reconnect failed.  Sleeping %d secs.", GTMConnectRetryInterval);
+			Disable_Longjmp();
+			pg_usleep((long)GTMConnectRetryInterval * 1000000L);
+			Enable_Longjmp();
+		}
+		if (new_gtm_conn != NULL)
+		{
+			GetMyThreadInfo->thr_gtm_conn = new_gtm_conn;
+			return(new_gtm_conn);
+		}
+		elog(NOTICE, "GTM connection retry failed.");
+	}
+	if (GTMErrorWaitIdle == 0 && GTMErrorWaitInterval == 0 && GTMErrorWaitCount == 0)
+	{
+		/*
+		 * GTM communication error detected, retry failed
+		 * but cannot wait for reconnect.
+		 */
+		elog(FATAL, 
+			 "No action specified to wait for reconnect.");
+		exit(1);		/* Just in case */
+	}
+	/*
+	 * Now wait for reconnect, SIGUSR2.
+	 *
+	 * All the controls are done by Disable_Logjmp() and Enable_Longjmp().
+	 * No longjump occurs after Disable_longjmp() and before Enable_longjmp().
+	 * Longjmp occurs inside Enable_longjmp(), only when SIGUSR2 was handled
+	 * correctly.
+	 *
+	 * For details, see gtm_proxy.h.
+	 */
+	elog(NOTICE, 
+		 "Waiting for reconnect action from gtm_ctl. idie: %d, count: %d, interval:%d", 
+		 GTMErrorWaitIdle, GTMErrorWaitCount, GTMErrorWaitInterval);
+	Disable_Longjmp();
+	elog(DEBUG1, "Witing %d secs.", GTMErrorWaitIdle);
+	pg_usleep((long)GTMErrorWaitIdle * 1000000L);
+	Enable_Longjmp();
+
+	for (ii = 0; ii < GTMErrorWaitCount; ii++)
+	{
+		Disable_Longjmp();
+		elog(DEBUG1, "Waiting %d secs, count %d.", GTMErrorWaitInterval, ii);
+		pg_usleep((long)GTMErrorWaitInterval * 1000000L);
+		Enable_Longjmp();
+	}
+	/*
+	 * No reconnect received.
+	 */
+	elog(FATAL,
+		 "No reconnect command recdeived frm gtm_ctl.");
+	exit(1);		/* Just in case */
+}
+
+static GTM_Conn *
+HandlePostCommand(GTMProxy_ConnectionInfo *conninfo, GTM_Conn *gtm_conn)
+{
+	int		connIdx = conninfo->con_id;
+
+	Assert(conninfo && gtm_conn);
+	/*
+	 * Check if the response was handled without error.
+	 * In this case, use last_errno to detect the error
+	 * because system call error is only one case to detect GTM error.
+	 */
+	if (gtm_conn->last_errno != 0)
+	{
+		return(HandleGTMError(gtm_conn));
+	}
+	else
+	{
+		/*
+		 * Command handled without error.  Clear the backup.
+		 */
+		resetStringInfo(&GetMyThreadInfo->thr_inBufData[connIdx]);
+		GetMyThreadInfo->thr_any_backup[connIdx] = FALSE;
+		return(gtm_conn);
+	}
+
 }
 
 static void
@@ -1907,11 +2065,11 @@ ReadCommand(GTMProxy_ConnectionInfo *conninfo, StringInfo inBuf)
 
 		copyStringInfo(&(GetMyThreadInfo->thr_inBufData[connIdx]), inBuf);
 
-		/* The next line should be added when we add the code to clear backup
-		 * when the response is processed. */
-#if 0
+		/* 
+		 * The next line is added because we added the code to clear backup
+		 * when the response is processed. 
+		 */
 		GetMyThreadInfo->thr_any_backup[connIdx] = TRUE;
-#endif
 	}
 	else
 	{
@@ -3157,8 +3315,9 @@ static void ReleaseCmdBackup(GTMProxy_CommandInfo *cmdinfo)
 	resetStringInfo(&(GetMyThreadInfo->thr_inBufData[connIdx]));
 }
 
+
 static void
-workerThreadReconnectToGTMstandby(void)
+workerThreadReconnectToGTM(void)
 {
 	char gtm_connect_string[1024];
 
@@ -3171,9 +3330,10 @@ workerThreadReconnectToGTMstandby(void)
 	PG_SETMASK(&UnBlockSig);
 
 	/* Disconnect the current connection and re-connect to the new GTM */
-	GTMPQfinish(GetMyThreadInfo->thr_gtm_conn);
+	if (GetMyThreadInfo->thr_gtm_conn)
+		GTMPQfinish(GetMyThreadInfo->thr_gtm_conn);
 	sprintf(gtm_connect_string, "host=%s port=%d node_name=%s remote_type=%d",
-			NewGTMServerHost, NewGTMServerPortNumber, GTMProxyNodeName, GTM_NODE_GTM_PROXY);
+			GTMServerHost, GTMServerPortNumber, GTMProxyNodeName, GTM_NODE_GTM_PROXY);
 	elog(LOG, "Worker thread connecting to %s", gtm_connect_string);
 	GetMyThreadInfo->thr_gtm_conn = PQconnectGTM(gtm_connect_string);
 
@@ -3182,8 +3342,8 @@ workerThreadReconnectToGTMstandby(void)
 	elog(LOG, "Worker thread connection done.");
 
 	/* Set GTM communication error handling option */
-	GetMyThreadInfo->thr_gtm_conn->gtmErrorWaitOpt = GTMErrorWaitOpt;
-	GetMyThreadInfo->thr_gtm_conn->gtmErrorWaitSecs = GTMErrorWaitSecs;
+	GetMyThreadInfo->thr_gtm_conn->gtmErrorWaitIdle = GTMErrorWaitIdle;
+	GetMyThreadInfo->thr_gtm_conn->gtmErrorWaitInterval = GTMErrorWaitInterval;
 	GetMyThreadInfo->thr_gtm_conn->gtmErrorWaitCount = GTMErrorWaitCount;
 
 	/* Initialize the command processing */
diff --git a/src/include/gtm/gtm_c.h b/src/include/gtm/gtm_c.h
index a9a5a6a95b..370e5d7559 100644
--- a/src/include/gtm/gtm_c.h
+++ b/src/include/gtm/gtm_c.h
@@ -126,6 +126,15 @@ typedef struct GTM_StartupPacket {
 	bool					sp_ispostmaster;
 } GTM_StartupPacket;
 
+typedef enum GTM_PortLastCall
+{
+	GTM_LastCall_NONE = 0,
+	GTM_LastCall_SEND,
+	GTM_LastCall_RECV,
+	GTM_LastCall_READ,
+	GTM_LastCall_WRITE
+} GTM_PortLastCall;
+
 #define InvalidGlobalTransactionId		((GlobalTransactionId) 0)
 
 /*
diff --git a/src/include/gtm/gtm_client.h b/src/include/gtm/gtm_client.h
index 9d3c5b06b1..971277816e 100644
--- a/src/include/gtm/gtm_client.h
+++ b/src/include/gtm/gtm_client.h
@@ -23,63 +23,63 @@
 
 typedef union GTM_ResultData
 {
-	GTM_TransactionHandle			grd_txnhandle;		/* TXN_BEGIN */
+	GTM_TransactionHandle		grd_txnhandle;	/* TXN_BEGIN */
 
 	struct
 	{
 		GlobalTransactionId		gxid;
 		GTM_Timestamp			timestamp;
-	} grd_gxid_tp;							/* TXN_BEGIN_GETGXID */
+	} grd_gxid_tp;								/* TXN_BEGIN_GETGXID */
 
 	GlobalTransactionId			grd_gxid;		/* TXN_PREPARE
-									 * TXN_START_PREPARED
-									 * TXN_COMMIT
-									 * TXN_COMMIT_PREPARED
-									 * TXN_ROLLBACK
-									 */
+												 * TXN_START_PREPARED
+												 * TXN_COMMIT
+												 * TXN_COMMIT_PREPARED
+												 * TXN_ROLLBACK
+												 */
 	
 	GlobalTransactionId			grd_next_gxid;
 
 	struct
 	{
-		GTM_TransactionHandle		txnhandle;
+		GTM_TransactionHandle	txnhandle;
 		GlobalTransactionId		gxid;
-	} grd_txn;							/* TXN_GET_GXID */
+	} grd_txn;									/* TXN_GET_GXID */
 
 	GTM_SequenceKeyData			grd_seqkey;		/* SEQUENCE_INIT
-									 * SEQUENCE_RESET
-									 * SEQUENCE_CLOSE */
+												 * SEQUENCE_RESET
+												 * SEQUENCE_CLOSE */
 	struct
 	{
 		GTM_SequenceKeyData		seqkey;
 		GTM_Sequence			seqval;
-	} grd_seq;							/* SEQUENCE_GET_CURRENT
-									   SEQUENCE_GET_NEXT */
+	} grd_seq;									/* SEQUENCE_GET_CURRENT
+												 * SEQUENCE_GET_NEXT */
 	struct
 	{
 		int				seq_count;
 		GTM_SeqInfo			**seq;
-	} grd_seq_list;							/* SEQUENCE_GET_LIST */
+	} grd_seq_list;								/* SEQUENCE_GET_LIST */
 
 	struct
 	{
-		int				txn_count; 		/* TXN_BEGIN_GETGXID_MULTI */
+		int				txn_count; 				/* TXN_BEGIN_GETGXID_MULTI */
 		GlobalTransactionId		start_gxid;
 		GTM_Timestamp			timestamp;
 	} grd_txn_get_multi;
 
 	struct
 	{
-		int				txn_count;		/* TXN_COMMIT_MULTI */
+		int				txn_count;				/* TXN_COMMIT_MULTI */
 		int				status[GTM_MAX_GLOBAL_TRANSACTIONS];
 	} grd_txn_rc_multi;
 
 	struct
 	{
-		GTM_TransactionHandle		txnhandle;		/* SNAPSHOT_GXID_GET */
+		GTM_TransactionHandle	txnhandle;		/* SNAPSHOT_GXID_GET */
 		GlobalTransactionId		gxid;			/* SNAPSHOT_GET */
-		int				txn_count;		/* SNAPSHOT_GET_MULTI */
-		int				status[GTM_MAX_GLOBAL_TRANSACTIONS];
+		int						txn_count;		/* SNAPSHOT_GET_MULTI */
+		int						status[GTM_MAX_GLOBAL_TRANSACTIONS];
 	} grd_txn_snap_multi;
 
 	struct
@@ -88,7 +88,7 @@ typedef union GTM_ResultData
 		GlobalTransactionId		prepared_gxid;
 		int				nodelen;
 		char			*nodestring;
-	} grd_txn_get_gid_data;						/* TXN_GET_GID_DATA_RESULT */
+	} grd_txn_get_gid_data;					/* TXN_GET_GID_DATA_RESULT */
 
 	struct
 	{
@@ -98,7 +98,7 @@ typedef union GTM_ResultData
 
 	struct
 	{
-		GTM_PGXCNodeType		type;			/* NODE_REGISTER */
+		GTM_PGXCNodeType	type;			/* NODE_REGISTER */
 		size_t				len;
 		char				*node_name;		/* NODE_UNREGISTER */
 	} grd_node;
@@ -129,8 +129,8 @@ typedef union GTM_ResultData
 typedef struct GTM_Result
 {
 	GTM_ResultType		gr_type;
-	int			gr_msglen;
-	int			gr_status;
+	int					gr_msglen;
+	int					gr_status;
 	GTM_ProxyMsgHeader	gr_proxyhdr;
 	GTM_ResultData		gr_resdata;
 	/*
@@ -138,13 +138,13 @@ typedef struct GTM_Result
 	 * of the xip array. If these items are pushed inside the union, they may
 	 * get overwritten by other members in the union
 	 */
-	int			gr_xip_size;
+	int					gr_xip_size;
 	GTM_SnapshotData	gr_snapshot;
 
 	/*
 	 * Similarly, keep the buffer for proxying data outside the union
 	 */
-	char			*gr_proxy_data;
+	char		*gr_proxy_data;
 	int			gr_proxy_datalen;
 } GTM_Result;
 
diff --git a/src/include/gtm/gtm_opt.h b/src/include/gtm/gtm_opt.h
index 704950e5ea..fd8a3101d8 100644
--- a/src/include/gtm/gtm_opt.h
+++ b/src/include/gtm/gtm_opt.h
@@ -334,8 +334,11 @@ const char *const config_type_names[] =\
 #define GTM_OPTNAME_DATA_DIR			"data_dir"
 #define GTM_OPTNAME_ERR_WAIT_COUNT		"err_wait_count"
 #define GTM_OPTNAME_ERR_WAIT_INTERVAL	"err_wait_interval"
-#define GTM_OPTNAME_ERR_WAIT_OPT		"err_wait_opt"
+#define GTM_OPTNAME_ERR_WAIT_IDLE		"err_wait_idle"
 #define GTM_OPTNAME_ERROR_REPORTER		"error_reporter"
+#define GTM_OPTNAME_CONNECT_RETRY_COUNT "gtm_connect_retry_count"
+#define GTM_OPTNAME_CONNECT_RETRY_IDLE  "gtm_connect_retry_idle"
+#define GTM_OPTNAME_CONNECT_RETRY_INTERVAL "gtm_connect_retry_interval"
 #define GTM_OPTNAME_GTM_HOST			"gtm_host"
 #define GTM_OPTNAME_GTM_PORT			"gtm_port"
 #define GTM_OPTNAME_KEEPALIVES_IDLE		"keepalives_idle"
diff --git a/src/include/gtm/gtm_proxy.h b/src/include/gtm/gtm_proxy.h
index b0d30c03b6..85fc0bbdc5 100644
--- a/src/include/gtm/gtm_proxy.h
+++ b/src/include/gtm/gtm_proxy.h
@@ -85,7 +85,9 @@ typedef struct GTMProxy_ThreadInfo
 	int				thr_criticalsec_count;
 
 	GTMProxy_ThreadStatus	thr_status;
-	GTMProxy_ConnectionInfo	*thr_conn;	/* Current active */
+	GTMProxy_ConnectionInfo	*thr_conn;		/* Current set of connections from clients */
+	uint32					thr_conn_count;	/* number of connections served by this thread */
+
 
 	/* 
 	 * The structure member type/sequence upto this point must match the
@@ -104,9 +106,6 @@ typedef struct GTMProxy_ThreadInfo
 	 */
 	int32					thr_seqno;
 
-	/* number of connections served by this thread */
-	uint32					thr_conn_count;
-
 	/* connection array */
 	GTMProxy_ConnectionInfo	*thr_all_conns[GTM_PROXY_MAX_CONNECTIONS];
 	struct pollfd			thr_poll_fds[GTM_PROXY_MAX_CONNECTIONS];
@@ -119,7 +118,7 @@ typedef struct GTMProxy_ThreadInfo
 	gtm_List 					*thr_processed_commands;
 	gtm_List 					*thr_pending_commands[MSG_TYPE_COUNT];
 
-	GTM_Conn				*thr_gtm_conn;
+	GTM_Conn				*thr_gtm_conn;		/* Connection to GTM */
 
 	/* Reconnect Info */
 	int						can_accept_SIGUSR2;
diff --git a/src/include/gtm/libpq-be.h b/src/include/gtm/libpq-be.h
index 1fcdff1cd0..4432e706c3 100644
--- a/src/include/gtm/libpq-be.h
+++ b/src/include/gtm/libpq-be.h
@@ -40,23 +40,25 @@
 
 typedef struct Port
 {
-	int			sock;			/* File descriptor */
-	SockAddr		laddr;			/* local addr (postmaster) */
-	SockAddr		raddr;			/* remote addr (client) */
-	char			*remote_host;		/* name (or ip addr) of remote host */
-	char			*remote_port;		/* text rep of remote port */
+	int			sock;				/* File descriptor */
+	SockAddr	laddr;				/* local addr (postmaster) */
+	SockAddr	raddr;				/* remote addr (client) */
+	char		*remote_host;		/* name (or ip addr) of remote host */
+	char		*remote_port;		/* text rep of remote port */
+	GTM_PortLastCall last_call;		/* Last syscall to this port */
+	int			last_errno;			/* Last errno. zero if the last call succeeds */
 
-	GTMProxy_ConnID		conn_id;		/* RequestID of this command */
+	GTMProxy_ConnID	conn_id;		/* RequestID of this command */
 
-	GTM_PGXCNodeType	remote_type;		/* Type of remote connection */
-	char			*node_name;
-	bool			is_postmaster;		/* Is remote a node postmaster? */
+	GTM_PGXCNodeType	remote_type;	/* Type of remote connection */
+	char		*node_name;
+	bool		is_postmaster;		/* Is remote a node postmaster? */
 #define PQ_BUFFER_SIZE 8192
 
-	char			PqSendBuffer[PQ_BUFFER_SIZE];
+	char		PqSendBuffer[PQ_BUFFER_SIZE];
 	int			PqSendPointer;		/* Next index to store a byte in PqSendBuffer */
 
-	char 			PqRecvBuffer[PQ_BUFFER_SIZE];
+	char 		PqRecvBuffer[PQ_BUFFER_SIZE];
 	int			PqRecvPointer;		/* Next index to read a byte from PqRecvBuffer */
 	int			PqRecvLength;		/* End of data available in PqRecvBuffer */
 
@@ -78,7 +80,7 @@ typedef struct Port
 	 * GTM communication error handling.  See libpq-int.h for details.
 	 */
 	int			connErr_WaitOpt;
-	int			connErr_WaitSecs;
+	int			connErr_WaitInterval;
 	int			connErr_WaitCount;
 } Port;
 
diff --git a/src/include/gtm/libpq-fe.h b/src/include/gtm/libpq-fe.h
index 31274698c8..175f06502d 100644
--- a/src/include/gtm/libpq-fe.h
+++ b/src/include/gtm/libpq-fe.h
@@ -54,15 +54,15 @@ typedef enum
 	 * The existence of these should never be relied upon - they should only
 	 * be used for user feedback or similar purposes.
 	 */
-	CONNECTION_STARTED,			/* Waiting for connection to be made.  */
-	CONNECTION_MADE,			/* Connection OK; waiting to send.	   */
-	CONNECTION_AWAITING_RESPONSE,		/* Waiting for a response from the
-										 * postmaster.		  */
-	CONNECTION_AUTH_OK,			/* Received authentication; waiting for
-								 * backend startup. */
-	CONNECTION_SETENV,			/* Negotiating environment. */
-	CONNECTION_SSL_STARTUP,		/* Negotiating SSL. */
-	CONNECTION_NEEDED			/* Internal state: connect() needed */
+	CONNECTION_STARTED,				/* Waiting for connection to be made.  */
+	CONNECTION_MADE,				/* Connection OK; waiting to send.	   */
+	CONNECTION_AWAITING_RESPONSE,	/* Waiting for a response from the
+									 * postmaster.		  */
+	CONNECTION_AUTH_OK,				/* Received authentication; waiting for
+									 * backend startup. */
+	CONNECTION_SETENV,				/* Negotiating environment. */
+	CONNECTION_SSL_STARTUP,			/* Negotiating SSL. */
+	CONNECTION_NEEDED				/* Internal state: connect() needed */
 } ConnStatusType;
 
 typedef enum
diff --git a/src/include/gtm/libpq-int.h b/src/include/gtm/libpq-int.h
index 7b9f4d6c69..0b4bba83c6 100644
--- a/src/include/gtm/libpq-int.h
+++ b/src/include/gtm/libpq-int.h
@@ -36,16 +36,16 @@
 struct gtm_conn
 {
 	/* Saved values of connection options */
-	char		*pghost;		/* the machine on which the server is running */
+	char		*pghost;			/* the machine on which the server is running */
 	char		*pghostaddr;		/* the IPv4 address of the machine on which
-					 	* the server is running, in IPv4
-					 	* numbers-and-dots notation. Takes precedence
-					 	* over above. */
-	char		*pgport;		/* the server's communication port */
+									 * the server is running, in IPv4
+									 * numbers-and-dots notation. Takes precedence
+									 * over above. */
+	char		*pgport;			/* the server's communication port */
 	char		*connect_timeout;	/* connection timeout (numeric string) */
 	char		*gc_node_name;		/* PGXC Node Name */
-	int		remote_type;		/* is this a connection to/from a proxy ? */
-	int		is_postmaster;		/* is this connection to/from a postmaster instance */
+	int			remote_type;		/* is this a connection to/from a proxy ? */
+	int			is_postmaster;		/* is this connection to/from a postmaster instance */
 
 	/* Optional file to write trace info to */
 	FILE		*Pfdebug;
@@ -54,30 +54,34 @@ struct gtm_conn
 	ConnStatusType	status;
 
 	/* Connection data */
-	int		sock;			/* Unix FD for socket, -1 if not connected */
+	int			sock;			/* Unix FD for socket, -1 if not connected */
 	SockAddr	laddr;			/* Local address */
 	SockAddr	raddr;			/* Remote address */
 
+	/* Error info for GTM communication */
+	GTM_PortLastCall	last_call;	/* Last syscall to this sock. */
+	int					last_errno;	/* Last errno.  zero if the last call succeeds. */
+
 	/* Transient state needed while establishing connection */
-	struct addrinfo	*addrlist;		/* list of possible backend addresses */
-	struct addrinfo	*addr_cur;		/* the one currently being tried */
+	struct addrinfo	*addrlist;	/* list of possible backend addresses */
+	struct addrinfo	*addr_cur;	/* the one currently being tried */
 	int		addrlist_family;	/* needed to know how to free addrlist */
 
 	/* Buffer for data received from backend and not yet processed */
-	char		*inBuffer;		/* currently allocated buffer */
+	char	*inBuffer;		/* currently allocated buffer */
 	int		inBufSize;		/* allocated size of buffer */
 	int		inStart;		/* offset to first unconsumed data in buffer */
 	int		inCursor;		/* next byte to tentatively consume */
 	int		inEnd;			/* offset to first position after avail data */
 
 	/* Buffer for data not yet sent to backend */
-	char		*outBuffer;		/* currently allocated buffer */
+	char	*outBuffer;		/* currently allocated buffer */
 	int		outBufSize;		/* allocated size of buffer */
 	int		outCount;		/* number of chars waiting in buffer */
 
 	/* State for constructing messages in outBuffer */
-	int		outMsgStart;		/* offset to msg start (length word); if -1,
-						 * msg has no length word */
+	int		outMsgStart;	/* offset to msg start (length word); if -1,
+							 * msg has no length word */
 	int		outMsgEnd;		/* offset to msg end (so far) */
 
 	/* Buffer for current error message */
@@ -87,9 +91,9 @@ struct gtm_conn
 	PQExpBufferData	workBuffer; /* expansible string */
 
 	/* Options to handle GTM communication error */
-	int		gtmErrorWaitOpt;	/* If true, wait reconnect signal. */
-	int		gtmErrorWaitSecs;	/* Duration of the wait time in second */
-	int		gtmErrorWaitCount;	/* How many durations to wait */
+	int		gtmErrorWaitIdle;		/* If true, wait reconnect signal. */
+	int		gtmErrorWaitInterval;	/* Duration of the wait time in second */
+	int		gtmErrorWaitCount;		/* How many durations to wait */
 
 	/* Pointer to the result of last operation */
 	GTM_Result	*result;