Add a user configurable parameter to control the number of rows fetched from

the remote side during RemoteSubplan execution.

This allows us to experiment with different sizes more easily. Playing with the
fetch size also exposed couple of problems fixed in this same commit.

1. We were incorrectly forgetting a connection response combiner while
suspending a portal, leading to errors later when we try to buffer the results
because the connection must be used for other queries.

2. The remote cursor name was not getting set properly, thus datanodes
complaining about non-existent cursors.

3 files changed, 21 insertions, 5 deletions

diff --git a/src/backend/pgxc/pool/execRemote.c b/src/backend/pgxc/pool/execRemote.c
index 9cf3541ac7..f4b2c7fe2a 100644
--- a/src/backend/pgxc/pool/execRemote.c
+++ b/src/backend/pgxc/pool/execRemote.c
@@ -68,6 +68,9 @@
  */
 #define END_QUERY_TIMEOUT	1000
 
+/* Declarations used by guc.c */
+int PGXLRemoteFetchSize;
+
 typedef struct
 {
 	xact_callback function;
@@ -1346,7 +1349,7 @@ FetchTuple(ResponseCombiner *combiner)
 				return NULL;
 			}
 
-			if (pgxc_node_send_execute(conn, combiner->cursor, 1000) != 0)
+			if (pgxc_node_send_execute(conn, combiner->cursor, PGXLRemoteFetchSize) != 0)
 			{
 				ereport(ERROR,
 						(errcode(ERRCODE_INTERNAL_ERROR),
@@ -1394,7 +1397,7 @@ FetchTuple(ResponseCombiner *combiner)
 			 */
 			if (combiner->merge_sort || combiner->probing_primary)
 			{
-				if (pgxc_node_send_execute(conn, combiner->cursor, 1000) != 0)
+				if (pgxc_node_send_execute(conn, combiner->cursor, PGXLRemoteFetchSize) != 0)
 					ereport(ERROR,
 							(errcode(ERRCODE_INTERNAL_ERROR),
 							 errmsg("Failed to send execute cursor '%s' to node %u", combiner->cursor, conn->nodeoid)));
@@ -1413,7 +1416,7 @@ FetchTuple(ResponseCombiner *combiner)
 			 * Tell the node to fetch data in background, next loop when we 
 			 * pgxc_node_receive, data is already there, so we can run faster
 			 * */
-			if (pgxc_node_send_execute(conn, combiner->cursor, 1000) != 0)
+			if (pgxc_node_send_execute(conn, combiner->cursor, PGXLRemoteFetchSize) != 0)
 			{
 				ereport(ERROR,
 						(errcode(ERRCODE_INTERNAL_ERROR),
@@ -1692,7 +1695,6 @@ handle_response(PGXCNodeHandle *conn, ResponseCombiner *combiner)
 			case 's':			/* PortalSuspended */
 				/* No activity is expected on the connection until next query */
 				PGXCNodeSetConnectionState(conn, DN_CONNECTION_STATE_IDLE);
-				conn->combiner = NULL;
 				return RESPONSE_SUSPENDED;
 			case '1': /* ParseComplete */
 			case '2': /* BindComplete */
@@ -5819,7 +5821,7 @@ primary_mode_phase_two:
 
 		if (plan->cursor)
 		{
-			fetch = 1000;
+			fetch = PGXLRemoteFetchSize;
 			if (plan->unique)
 				snprintf(cursor, NAMEDATALEN, "%s_%d", plan->cursor, plan->unique);
 			else
@@ -5993,6 +5995,7 @@ primary_mode_phase_two:
 				}
 				else
 				{
+					combiner->cursor = pstrdup(cursor);
 					combiner->cursor_count = combiner->conn_count;
 					combiner->cursor_connections = (PGXCNodeHandle **) palloc(
 								combiner->conn_count * sizeof(PGXCNodeHandle *));
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index d1d95b9106..3d4493fcdd 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -3099,6 +3099,17 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"pgxl_remote_fetch_size", PGC_USERSET, UNGROUPED,
+			gettext_noop("Number of maximum tuples to fetch in one remote iteration"),
+			NULL,
+			0
+		},
+		&PGXLRemoteFetchSize,
+		1000, 1, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 #endif
 #endif /* PGXC */
 
diff --git a/src/include/pgxc/execRemote.h b/src/include/pgxc/execRemote.h
index 45ad90738d..928e55d627 100644
--- a/src/include/pgxc/execRemote.h
+++ b/src/include/pgxc/execRemote.h
@@ -223,6 +223,8 @@ typedef struct RemoteStmt
 	List	   *distributionRestrict;
 } RemoteStmt;
 
+extern int PGXLRemoteFetchSize;
+
 typedef void (*xact_callback) (bool isCommit, void *args);
 
 /* Copy command just involves Datanodes */