Param handling re-work (Fix for bug ID: 3522907)

Moved the parameter handling part from the plan phase to the
execute phase. See bug ID 3522907 for details.

14 files changed, 374 insertions, 243 deletions

diff --git a/src/backend/commands/prepare.c b/src/backend/commands/prepare.c
index 219608b571..41b00ba1f0 100644
--- a/src/backend/commands/prepare.c
+++ b/src/backend/commands/prepare.c
@@ -503,7 +503,7 @@ SetRemoteStatementName(Plan *plan, const char *stmt_name, int num_params,
 		char name[NAMEDATALEN];
 
 		/* Nothing to do if parameters are already set for this query */
-		if (remotequery->remote_num_params != 0)
+		if (remotequery->rq_num_params != 0)
 			return 0;
 
 		if (stmt_name)
@@ -546,8 +546,8 @@ SetRemoteStatementName(Plan *plan, const char *stmt_name, int num_params,
 					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
 					 errmsg("Passing parameters in PREPARE statement is not supported")));
 
-		remotequery->remote_num_params = num_params;
-		remotequery->remote_param_types = param_types;
+		remotequery->rq_num_params = num_params;
+		remotequery->rq_param_types = param_types;
 	}
 	else if (IsA(plan, ModifyTable))
 	{
diff --git a/src/backend/executor/execTuples.c b/src/backend/executor/execTuples.c
index 314ee2e4fd..ad7c569f93 100644
--- a/src/backend/executor/execTuples.c
+++ b/src/backend/executor/execTuples.c
@@ -655,89 +655,6 @@ ExecCopySlotMinimalTuple(TupleTableSlot *slot)
 								   slot->tts_isnull);
 }
 
-#ifdef PGXC
-/* --------------------------------
- *		ExecCopySlotDatarow
- *			Obtain a copy of a slot's data row.  The copy is
- *			palloc'd in the current memory context.
- * 			Pointer to the datarow is returned as a var parameter, function
- * 			returns the length of the data row
- *			The slot itself is undisturbed
- * --------------------------------
- */
-int
-ExecCopySlotDatarow(TupleTableSlot *slot, char **datarow)
-{
-	Assert(datarow);
-
-	if (slot->tts_dataRow)
-	{
-		/* if we already have datarow make a copy */
-		*datarow = (char *)palloc(slot->tts_dataLen);
-		memcpy(*datarow, slot->tts_dataRow, slot->tts_dataLen);
-		return slot->tts_dataLen;
-	}
-	else
-	{
-		TupleDesc	 	tdesc = slot->tts_tupleDescriptor;
-		StringInfoData	buf;
-		uint16 			n16;
-		int 			i;
-
-		initStringInfo(&buf);
-		/* Number of parameter values */
-		n16 = htons(tdesc->natts);
-		appendBinaryStringInfo(&buf, (char *) &n16, 2);
-
-		/* ensure we have all values */
-		slot_getallattrs(slot);
-		for (i = 0; i < tdesc->natts; i++)
-		{
-			uint32 n32;
-
-			if (slot->tts_isnull[i])
-			{
-				n32 = htonl(-1);
-				appendBinaryStringInfo(&buf, (char *) &n32, 4);
-			}
-			else
-			{
-				Form_pg_attribute attr = tdesc->attrs[i];
-				Oid		typOutput;
-				bool	typIsVarlena;
-				Datum	pval;
-				char   *pstring;
-				int		len;
-
-				/* Get info needed to output the value */
-				getTypeOutputInfo(attr->atttypid, &typOutput, &typIsVarlena);
-				/*
-				 * If we have a toasted datum, forcibly detoast it here to avoid
-				 * memory leakage inside the type's output routine.
-				 */
-				if (typIsVarlena)
-					pval = PointerGetDatum(PG_DETOAST_DATUM(slot->tts_values[i]));
-				else
-					pval = slot->tts_values[i];
-
-				/* Convert Datum to string */
-				pstring = OidOutputFunctionCall(typOutput, pval);
-
-				/* copy data to the buffer */
-				len = strlen(pstring);
-				n32 = htonl(len);
-				appendBinaryStringInfo(&buf, (char *) &n32, 4);
-				appendBinaryStringInfo(&buf, pstring, len);
-			}
-		}
-		/* copy data to the buffer */
-		*datarow = palloc(buf.len);
-		memcpy(*datarow, buf.data, buf.len);
-		pfree(buf.data);
-		return buf.len;
-	}
-}
-#endif
 
 /* --------------------------------
  *		ExecFetchSlotTuple
diff --git a/src/backend/executor/functions.c b/src/backend/executor/functions.c
index 6af20e10dd..e6b57539b4 100644
--- a/src/backend/executor/functions.c
+++ b/src/backend/executor/functions.c
@@ -491,24 +491,6 @@ init_execution_state(List *queryTree_list,
 					   errmsg("%s is not allowed in a non-volatile function",
 							  CreateCommandTag(stmt))));
 
-#ifdef PGXC
-			if (IS_PGXC_COORDINATOR && !IsConnFromCoord())
-			{
-				if (queryTree->commandType != CMD_UTILITY)
-				{
-					/*
-					* The parameterised queries in RemoteQuery nodes will be prepared
-					* on the Datanode, and need parameter types for the same. Set the
-					* parameter types and their number in all RemoteQuery nodes in the
-					* plan
-					*/
-					SetRemoteStatementName(((PlannedStmt *)stmt)->planTree, NULL,
-											fcache->pinfo->nargs,
-											fcache->pinfo->argtypes, 0);
-				}
-			}
-#endif /* PGXC */
-
 			/* OK, build the execution_state for this query */
 			newes = (execution_state *) palloc(sizeof(execution_state));
 			if (preves)
diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c
index fc3abb5724..7eeb5b9af8 100644
--- a/src/backend/nodes/copyfuncs.c
+++ b/src/backend/nodes/copyfuncs.c
@@ -1027,12 +1027,12 @@ _copyRemoteQuery(const RemoteQuery *from)
 	COPY_SCALAR_FIELD(force_autocommit);
 	COPY_STRING_FIELD(statement);
 	COPY_STRING_FIELD(cursor);
-	COPY_SCALAR_FIELD(remote_num_params);
-	if (from->remote_param_types)
-		COPY_POINTER_FIELD(remote_param_types,
-			sizeof(from->remote_param_types[0]) * from->remote_num_params);
+	COPY_SCALAR_FIELD(rq_num_params);
+	if (from->rq_param_types)
+		COPY_POINTER_FIELD(rq_param_types,
+			sizeof(from->rq_param_types[0]) * from->rq_num_params);
 	else
-		newnode->remote_param_types = NULL;
+		newnode->rq_param_types = NULL;
 	COPY_SCALAR_FIELD(exec_type);
 	COPY_SCALAR_FIELD(is_temp);
 	COPY_SCALAR_FIELD(rq_finalise_aggs);
@@ -1043,6 +1043,7 @@ _copyRemoteQuery(const RemoteQuery *from)
 	COPY_NODE_FIELD(query_var_tlist);
 	COPY_SCALAR_FIELD(has_row_marks);
 	COPY_SCALAR_FIELD(has_ins_child_sel_parent);
+	COPY_SCALAR_FIELD(rq_params_internal);
 
 	return newnode;
 }
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 624a0e5014..080047c8e7 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -475,11 +475,11 @@ _outRemoteQuery(StringInfo str, const RemoteQuery *node)
 	WRITE_BOOL_FIELD(force_autocommit);
 	WRITE_STRING_FIELD(statement);
 	WRITE_STRING_FIELD(cursor);
-	WRITE_INT_FIELD(remote_num_params);
+	WRITE_INT_FIELD(rq_num_params);
 
-	appendStringInfo(str, " :remote_param_types");
-	for (i = 0; i < node->remote_num_params; i++)
-		appendStringInfo(str, " %d", node->remote_param_types[i]);
+	appendStringInfo(str, " :rq_param_types");
+	for (i = 0; i < node->rq_num_params; i++)
+		appendStringInfo(str, " %d", node->rq_param_types[i]);
 
 	WRITE_ENUM_FIELD(exec_type, RemoteQueryExecType);
 	WRITE_BOOL_FIELD(is_temp);
@@ -490,6 +490,7 @@ _outRemoteQuery(StringInfo str, const RemoteQuery *node)
 	WRITE_NODE_FIELD(coord_var_tlist);
 	WRITE_NODE_FIELD(query_var_tlist);
 	WRITE_BOOL_FIELD(has_ins_child_sel_parent);
+	WRITE_BOOL_FIELD(rq_params_internal);
 }
 
 static void
diff --git a/src/backend/optimizer/plan/pgxcplan.c b/src/backend/optimizer/plan/pgxcplan.c
index e818da4d4f..7bb24fdf0a 100644
--- a/src/backend/optimizer/plan/pgxcplan.c
+++ b/src/backend/optimizer/plan/pgxcplan.c
@@ -62,7 +62,6 @@ static PlannedStmt *pgxc_handle_exec_direct(Query *query, int cursorOptions,
 static RemoteQuery *pgxc_FQS_create_remote_plan(Query *query,
 												ExecNodes *exec_nodes,
 												bool is_exec_direct);
-static void pgxc_set_remote_parameters(PlannedStmt *plan, ParamListInfo boundParams);
 static bool pgxc_locate_grouping_columns(PlannerInfo *root, List *tlist,
 											AttrNumber *grpColIdx);
 static List *pgxc_process_grouping_targetlist(List *local_tlist,
@@ -1198,15 +1197,10 @@ create_remotedml_plan(PlannerInfo *root, Plan *topplan, CmdType cmdtyp)
 		Index			resultRelationIndex = lfirst_int(rel);
 		RangeTblEntry	*res_rel;
 		RelationLocInfo	*rel_loc_info;
-		Oid				*param_types;		/* Types of query parameters */
 		RemoteQuery		*fstep;
 		RangeTblEntry	*dummy_rte;			/* RTE for the remote query node */
 		Plan			*sourceDataPlan;	/* plan producing source data */
-		List			*sourceTargetList = NULL;
-		int				tot_prepparams;
 		char			*relname;
-		int				i;
-		ListCell		*elt;
 
 		relcount++;
 
@@ -1223,33 +1217,14 @@ create_remotedml_plan(PlannerInfo *root, Plan *topplan, CmdType cmdtyp)
 		if (rel_loc_info == NULL)
 			continue;
 
-		/* Get the plan that is supposed to supply source data to this plan */
-		sourceDataPlan = list_nth(mt->plans, relcount);
-		/*
-		 * Get the target list of the plan that is supposed to
-		 * supply source data to this plan
-		 */
-		sourceTargetList = sourceDataPlan->targetlist;
-		tot_prepparams = list_length(sourceTargetList);
-
-		/* Allocate the array for parameter types */
-		param_types = (Oid *) palloc0(sizeof (Oid) * tot_prepparams);
+		fstep = make_remotequery(NIL, NIL, resultRelationIndex);
 
 		/*
-		* The parameter types of all the supplied parameters
-		* will be the same as the types of the target list entries
-		* in the source data plan
-		*/
-		i = 0;
-		foreach(elt, sourceTargetList)
-		{
-			TargetEntry *tle = lfirst(elt);
-
-			/* Set up the parameter type */
-			param_types[i++] = exprType((Node *) tle->expr);
-		}
+		 * DML planning generates its own parameters that refer to the source
+		 * data plan.
+		 */
+		fstep->rq_params_internal = true;
 
-		fstep = make_remotequery(NIL, NIL, resultRelationIndex);
 		fstep->is_temp = IsTempTable(res_rel->relid);
 		fstep->read_only = false;
 
@@ -1258,8 +1233,11 @@ create_remotedml_plan(PlannerInfo *root, Plan *topplan, CmdType cmdtyp)
 									list_nth(mt->returningLists, relcount),
 									resultRelationIndex);
 
+		/* Get the plan that is supposed to supply source data to this plan */
+		sourceDataPlan = list_nth(mt->plans, relcount);
+
 		pgxc_build_dml_statement(root, cmdtyp, resultRelationIndex, fstep,
-									sourceTargetList);
+									sourceDataPlan->targetlist);
 
 		fstep->combine_type = get_plan_combine_type(cmdtyp,
 													rel_loc_info->locatorType);
@@ -1283,8 +1261,6 @@ create_remotedml_plan(PlannerInfo *root, Plan *topplan, CmdType cmdtyp)
 			fstep->exec_nodes->en_expr = pgxc_set_en_expr(res_rel->relid,
 														resultRelationIndex);
 
-		SetRemoteStatementName((Plan *) fstep, NULL, tot_prepparams,
-											param_types, 0);
 		dummy_rte = make_dummy_remote_rte(relname,
 										makeAlias("REMOTE_DML_QUERY", NIL));
 		root->parse->rtable = lappend(root->parse->rtable, dummy_rte);
@@ -2213,7 +2189,6 @@ pgxc_planner(Query *query, int cursorOptions, ParamListInfo boundParams)
 
 	/* we need Coordinator for evaluation, invoke standard planner */
 	result = standard_planner(query, cursorOptions, boundParams);
-	pgxc_set_remote_parameters(result, boundParams);
 	return result;
 }
 
@@ -2269,9 +2244,6 @@ pgxc_handle_exec_direct(Query *query, int cursorOptions,
 		}
 	}
 
-	/* Set existing remote parameters */
-	pgxc_set_remote_parameters(result, boundParams);
-
 	return result;
 }
 /*
@@ -2396,9 +2368,6 @@ pgxc_FQS_planner(Query *query, int cursorOptions, ParamListInfo boundParams)
 	if (query->utilityStmt && IsA(query->utilityStmt, DeclareCursorStmt))
 		fetch_ctid_of(result->planTree, query);
 
-	/* Set existing remote parameters */
-	pgxc_set_remote_parameters(result, boundParams);
-
 	return result;
 }
 
@@ -2643,57 +2612,6 @@ pgxc_query_contains_utility(List *queries)
 
 
 /*
- * pgxc_set_remote_parameters
- *
- * Set the list of remote parameters for remote plan
- */
-static void
-pgxc_set_remote_parameters(PlannedStmt *plan, ParamListInfo boundParams)
-{
-	Oid *param_types;
-	int cntParam, i;
-
-	/* Leave if no plan */
-	if (!plan)
-		return;
-
-	/* Leave if no parameters */
-	if (!boundParams)
-		return;
-
-	/*
-	 * Count the number of remote parameters available
-	 * We need to take into account all the parameters
-	 * that are prior to the latest available. This insures
-	 * that remote node will not complain about an incorrect
-	 * number of parameter. In case parameters with no types
-	 * are taken into account, they are considered as NULL entries.
-	 */
-	cntParam = 0;
-	for (i = 0; i < boundParams->numParams; i++)
-	{
-		if (OidIsValid(boundParams->params[i].ptype))
-			cntParam = i + 1;
-	}
-
-	/* If there are no parameters available, simply leave */
-	if (cntParam == 0)
-		return;
-
-	param_types = (Oid *) palloc(sizeof(Oid) * cntParam);
-
-	/* Then fill the array of types */
-	for (i = 0; i < cntParam; i++)
-		param_types[i] = boundParams->params[i].ptype;
-
-	/* Finally save the parameters in plan */
-	SetRemoteStatementName(plan->planTree, NULL,
-						   cntParam, param_types, 0);
-
-	return;
-}
-
-/*
  * create_remotesort_plan
  * Check if we can push down the ORDER BY clause to the datanode/s thereby
  * getting partially sorted results.
diff --git a/src/backend/pgxc/pool/execRemote.c b/src/backend/pgxc/pool/execRemote.c
index 15eccd6fda..f19eb0498f 100644
--- a/src/backend/pgxc/pool/execRemote.c
+++ b/src/backend/pgxc/pool/execRemote.c
@@ -172,6 +172,7 @@ static void pgxc_node_report_error(RemoteQueryState *combiner);
 static TupleTableSlot *getrow_for_tapesort(RemoteQueryState *combiner,
 											TupleTableSlot *scanslot);
 static bool IsReturningDMLOnReplicatedTable(RemoteQuery *rq);
+static void SetDataRowForIntParams(TupleTableSlot *slot, RemoteQueryState *rq_state);
 
 /*
  * Create a structure to store parameters needed to combine responses from
@@ -2554,12 +2555,7 @@ ExecInitRemoteQuery(RemoteQuery *node, EState *estate, int eflags)
 	 * If there are parameters supplied, get them into a form to be sent to the
 	 * Datanodes with bind message. We should not have had done this before.
 	 */
-	if (estate->es_param_list_info)
-	{
-		Assert(!remotestate->paramval_data);
-		remotestate->paramval_len = ParamListToDataRow(estate->es_param_list_info,
-												&remotestate->paramval_data);
-	}
+	SetDataRowForExtParams(estate->es_param_list_info, remotestate);
 
 	/*
 	 * Initialize result tuple type and projection info.
@@ -2780,7 +2776,7 @@ pgxc_start_command_on_connection(PGXCNodeHandle *connection,
 
 	if (snapshot && pgxc_node_send_snapshot(connection, snapshot))
 		return false;
-	if (step->statement || step->cursor || step->remote_param_types)
+	if (step->statement || step->cursor || remotestate->rqs_num_params)
 	{
 		/* need to use Extended Query Protocol */
 		int	fetch = 0;
@@ -2808,8 +2804,8 @@ pgxc_start_command_on_connection(PGXCNodeHandle *connection,
 							prepared ? NULL : step->sql_statement,
 							step->statement,
 							step->cursor,
-							step->remote_num_params,
-							step->remote_param_types,
+							remotestate->rqs_num_params,
+							remotestate->rqs_param_types,
 							remotestate->paramval_len,
 							remotestate->paramval_data,
 							send_desc,
@@ -3445,6 +3441,15 @@ ExecEndRemoteQuery(RemoteQueryState *node)
 		node->paramval_len = 0;
 	}
 
+	/* Free the param types if they are newly allocated */
+	if (node->rqs_param_types &&
+	    node->rqs_param_types != ((RemoteQuery*)node->ss.ps.plan)->rq_param_types)
+	{
+		pfree(node->rqs_param_types);
+		node->rqs_param_types = NULL;
+		node->rqs_num_params = 0;
+	}
+
 	if (node->ss.ss_currentRelation)
 		ExecCloseScanRelation(node->ss.ss_currentRelation);
 
@@ -3506,29 +3511,47 @@ close_node_cursors(PGXCNodeHandle **connections, int conn_count, char *cursor)
 /*
  * Encode parameter values to format of DataRow message (the same format is
  * used in Bind) to prepare for sending down to Datanodes.
- * The buffer to store encoded value is palloc'ed and returned as the result
- * parameter. Function returns size of the result
+ * The data row is copied to RemoteQueryState.paramval_data.
  */
-int
-ParamListToDataRow(ParamListInfo params, char** result)
+void
+SetDataRowForExtParams(ParamListInfo paraminfo, RemoteQueryState *rq_state)
 {
 	StringInfoData buf;
 	uint16 n16;
 	int i;
 	int real_num_params = 0;
+	RemoteQuery *node = (RemoteQuery*) rq_state->ss.ps.plan;
+
+	/* If there are no parameters, there is no data to BIND. */
+	if (!paraminfo)
+		return;
+
+	/*
+	 * If this query has been generated internally as a part of two-step DML
+	 * statement, it uses only the internal parameters for input values taken
+	 * from the source data, and it never uses external parameters. So even if
+	 * parameters were being set externally, they won't be present in this
+	 * statement (they might be present in the source data query). In such
+	 * case where parameters refer to the values returned by SELECT query, the
+	 * parameter data and parameter types would be set in SetDataRowForIntParams().
+	 */
+	if (node->rq_params_internal)
+		return;
+
+	Assert(!rq_state->paramval_data);
 
 	/*
 	 * It is necessary to fetch parameters
 	 * before looking at the output value.
 	 */
-	for (i = 0; i < params->numParams; i++)
+	for (i = 0; i < paraminfo->numParams; i++)
 	{
 		ParamExternData *param;
 
-		param = &params->params[i];
+		param = &paraminfo->params[i];
 
-		if (!OidIsValid(param->ptype) && params->paramFetch != NULL)
-			(*params->paramFetch) (params, i + 1);
+		if (!OidIsValid(param->ptype) && paraminfo->paramFetch != NULL)
+			(*paraminfo->paramFetch) (paraminfo, i + 1);
 
 		/*
 		 * This is the last parameter found as useful, so we need
@@ -3547,8 +3570,9 @@ ParamListToDataRow(ParamListInfo params, char** result)
 	 */
 	if (real_num_params == 0)
 	{
-		*result = NULL;
-		return 0;
+		rq_state->paramval_data = NULL;
+		rq_state->paramval_len = 0;
+		return;
 	}
 
 	initStringInfo(&buf);
@@ -3560,7 +3584,7 @@ ParamListToDataRow(ParamListInfo params, char** result)
 	/* Parameter values */
 	for (i = 0; i < real_num_params; i++)
 	{
-		ParamExternData *param = &params->params[i];
+		ParamExternData *param = &paraminfo->params[i];
 		uint32 n32;
 
 		/*
@@ -3603,11 +3627,35 @@ ParamListToDataRow(ParamListInfo params, char** result)
 		}
 	}
 
-	/* Take data from the buffer */
-	*result = palloc(buf.len);
-	memcpy(*result, buf.data, buf.len);
-	pfree(buf.data);
-	return buf.len;
+
+	/*
+	 * If parameter types are not already set, infer them from
+	 * the paraminfo.
+	 */
+	if (node->rq_num_params > 0)
+	{
+		/*
+		 * Use the already known param types for BIND. Parameter types
+		 * can be already known when the same plan is executed multiple
+		 * times.
+		 */
+		if (node->rq_num_params != real_num_params)
+			elog(ERROR, "Number of user-supplied parameters do not match "
+						"the number of remote parameters");
+		rq_state->rqs_num_params = node->rq_num_params;
+		rq_state->rqs_param_types = node->rq_param_types;
+	}
+	else
+	{
+		rq_state->rqs_num_params = real_num_params;
+		rq_state->rqs_param_types = (Oid *) palloc(sizeof(Oid) * real_num_params);
+		for (i = 0; i < real_num_params; i++)
+			rq_state->rqs_param_types[i] = paraminfo->params[i].ptype;
+	}
+
+	/* Assign the newly allocated data row to paramval */
+	rq_state->paramval_data = buf.data;
+	rq_state->paramval_len = buf.len;
 }
 
 
@@ -4112,8 +4160,7 @@ ExecProcNodeDMLInXC(RemoteQueryState *resultRemoteRel,
 	 * Use data row returned by the previous step as parameter for
 	 * the DML to be executed in this step.
 	 */
-	resultRemoteRel->paramval_len = ExecCopySlotDatarow(previousStepSlot,
-											&resultRemoteRel->paramval_data);
+	SetDataRowForIntParams(previousStepSlot, resultRemoteRel);
 
 	/*
 	 * do_query calls get_exec_connections to determine target nodes
@@ -4958,3 +5005,111 @@ getrow_for_tapesort(RemoteQueryState *combiner, TupleTableSlot *scanslot)
 			(errcode(ERRCODE_INTERNAL_ERROR),
 			 errmsg("Did not get response complete message for the connection.")));
 }
+
+
+/* --------------------------------
+ * SetDataRowForIntParams: Form a BIND data row for internal parameters.
+ * This function is called when the data for the parameters of remote
+ * statement resides in some plan slot of an internally generated remote
+ * statement rather than from some extern params supplied by the caller of the
+ * query. Currently DML is the only case where we generate a query with
+ * internal parameters.
+ * The parameter data is constructed from the slot data, and stored in
+ * RemoteQueryState.paramval_data.
+ * At the same time, remote parameter types are inferred from the slot
+ * tuple descriptor, and stored in RemoteQueryState.rqs_param_types.
+ * On subsequent calls, these param types are re-used.
+ * The slot itself is undisturbed.
+ * --------------------------------
+ */
+static void
+SetDataRowForIntParams(TupleTableSlot *slot, RemoteQueryState *rq_state)
+{
+	TupleDesc	tdesc = slot->tts_tupleDescriptor;
+	int			att_index;
+
+	Assert(tdesc != NULL);
+
+	/*
+	 * Infer param types from the tuple desc. But we have to do it only the
+	 * first time: the interal parameters remain the same while processing all
+	 * the source data rows because the data slot tupdesc never changes.
+	 * Even though we can determine the internal param types during planning, we
+	 * want to do it here: we don't want to set the param types and param data
+	 * at two different places. Doing them together here helps us to make sure
+	 * that the order of param types are in line with the order of the param
+	 * data.
+	 */
+	if (rq_state->rqs_num_params == 0)
+	{
+		rq_state->rqs_num_params = tdesc->natts;
+		rq_state->rqs_param_types =
+			(Oid *) palloc(sizeof(Oid) * rq_state->rqs_num_params);
+		for (att_index = 0; att_index < rq_state->rqs_num_params; att_index++)
+			rq_state->rqs_param_types[att_index] = tdesc->attrs[att_index]->atttypid;
+	}
+
+	/* if we already have datarow make a copy */
+	if (slot->tts_dataRow)
+	{
+		rq_state->paramval_data = (char *)palloc(slot->tts_dataLen);
+		memcpy(rq_state->paramval_data, slot->tts_dataRow, slot->tts_dataLen);
+		rq_state->paramval_len = slot->tts_dataLen;
+	}
+	else
+	{
+		StringInfoData	buf;
+		uint16			n16;
+
+		initStringInfo(&buf);
+		/* Number of parameter values */
+		n16 = htons(tdesc->natts);
+		appendBinaryStringInfo(&buf, (char *) &n16, 2);
+
+		/* ensure we have all values */
+		slot_getallattrs(slot);
+		for (att_index = 0; att_index < tdesc->natts; att_index++)
+		{
+			uint32 n32;
+
+			if (slot->tts_isnull[att_index])
+			{
+				n32 = htonl(-1);
+				appendBinaryStringInfo(&buf, (char *) &n32, 4);
+			}
+			else
+			{
+				Form_pg_attribute attr = tdesc->attrs[att_index];
+				Oid		typOutput;
+				bool	typIsVarlena;
+				Datum	pval;
+				char   *pstring;
+				int		len;
+
+				/* Get info needed to output the value */
+				getTypeOutputInfo(attr->atttypid, &typOutput, &typIsVarlena);
+				/*
+				 * If we have a toasted datum, forcibly detoast it here to avoid
+				 * memory leakage inside the type's output routine.
+				 */
+				if (typIsVarlena)
+					pval = PointerGetDatum(PG_DETOAST_DATUM(slot->tts_values[att_index]));
+				else
+					pval = slot->tts_values[att_index];
+
+				/* Convert Datum to string */
+				pstring = OidOutputFunctionCall(typOutput, pval);
+
+				/* copy data to the buffer */
+				len = strlen(pstring);
+				n32 = htonl(len);
+				appendBinaryStringInfo(&buf, (char *) &n32, 4);
+				appendBinaryStringInfo(&buf, pstring, len);
+			}
+		}
+
+		/* Assign the newly allocated data row to paramval */
+		rq_state->paramval_data = buf.data;
+		rq_state->paramval_len = buf.len;
+	}
+}
diff --git a/src/include/executor/tuptable.h b/src/include/executor/tuptable.h
index ed15896653..6ea58632fd 100644
--- a/src/include/executor/tuptable.h
+++ b/src/include/executor/tuptable.h
@@ -171,9 +171,6 @@ extern TupleTableSlot *ExecStoreVirtualTuple(TupleTableSlot *slot);
 extern TupleTableSlot *ExecStoreAllNullTuple(TupleTableSlot *slot);
 extern HeapTuple ExecCopySlotTuple(TupleTableSlot *slot);
 extern MinimalTuple ExecCopySlotMinimalTuple(TupleTableSlot *slot);
-#ifdef PGXC
-extern int ExecCopySlotDatarow(TupleTableSlot *slot, char **datarow);
-#endif
 extern HeapTuple ExecFetchSlotTuple(TupleTableSlot *slot);
 extern MinimalTuple ExecFetchSlotMinimalTuple(TupleTableSlot *slot);
 extern Datum ExecFetchSlotTupleDatum(TupleTableSlot *slot);
diff --git a/src/include/optimizer/pgxcplan.h b/src/include/optimizer/pgxcplan.h
index ecbf8d6eab..eb923b3494 100644
--- a/src/include/optimizer/pgxcplan.h
+++ b/src/include/optimizer/pgxcplan.h
@@ -88,11 +88,12 @@ typedef struct
 	bool			force_autocommit;	/* some commands like VACUUM require autocommit mode */
 	char			*statement;		/* if specified use it as a PreparedStatement name on Datanodes */
 	char			*cursor;		/* if specified use it as a Portal name on Datanodes */
-	int			remote_num_params;		/* number of parameters specified for Prepared remote statement */
-	Oid			*remote_param_types;		/* parameter types, this pointer is shared
-							 * across all the RemoteQuery nodes in the
-							 * plan. So, don't change this once set.
-							 */
+	int				rq_num_params;		/* number of parameters present in remote statement */
+	Oid				*rq_param_types;	/* parameter types for the remote statement */
+	bool			rq_params_internal;	/* Do the query params refer to the source data
+										 * plan as against user-supplied params ?
+										 */
+
 	RemoteQueryExecType	exec_type;
 	bool			is_temp;		/* determine if this remote node is based
 									 * on a temporary objects (no 2PC) */
diff --git a/src/include/pgxc/execRemote.h b/src/include/pgxc/execRemote.h
index c89be6052a..169be003b4 100644
--- a/src/include/pgxc/execRemote.h
+++ b/src/include/pgxc/execRemote.h
@@ -121,6 +121,8 @@ typedef struct RemoteQueryState
 	/* Support for parameters */
 	char	   *paramval_data;		/* parameter data, format is like in BIND */
 	int			paramval_len;		/* length of parameter values data */
+	Oid		   *rqs_param_types;	/* Types of the remote params */
+	int			rqs_num_params;
 
 	int			eflags;			/* capability flags to pass to tuplestore */
 	bool		eof_underlying; /* reached end of underlying plan? */
@@ -171,7 +173,7 @@ extern void BufferConnection(PGXCNodeHandle *conn);
 
 extern void ExecRemoteQueryReScan(RemoteQueryState *node, ExprContext *exprCtxt);
 
-extern int ParamListToDataRow(ParamListInfo params, char** result);
+extern void SetDataRowForExtParams(ParamListInfo params, RemoteQueryState *rq_state);
 
 extern void ExecCloseRemoteStatement(const char *stmt_name, List *nodelist);
 extern void PreCommit_Remote(char *prepareGID, bool preparedLocalNode);
diff --git a/src/test/regress/expected/plpgsql_1.out b/src/test/regress/expected/plpgsql_1.out
index 7cae70eb8a..a959ec26f3 100644
--- a/src/test/regress/expected/plpgsql_1.out
+++ b/src/test/regress/expected/plpgsql_1.out
@@ -1548,9 +1548,28 @@ update PSlot set slotlink = 'HS.base.hub1.1' where slotname = 'PS.base.b2';
 --
 -- PGXCTODO: This is failing due to issue 3522907, complicated SELECT queries in plpgsql functions
 select * from PField_v1 where pfname = 'PF0_1' order by slotname;
-ERROR:  there is no parameter $6
-CONTEXT:  SQL statement "select          * from WSlot where slotname = rec.backlink"
-PL/pgSQL function pslot_backlink_view(character) line 31 at SQL statement
+ pfname |       slotname       |          backside          |      patch       
+--------+----------------------+----------------------------+------------------
+ PF0_1  | PS.base.a1           | WS.001.1a in room 001 -> - | PS.base.ta1 -> -
+ PF0_1  | PS.base.a2           | WS.001.1b in room 001 -> - | -
+ PF0_1  | PS.base.a3           | WS.001.2a in room 001 -> - | PS.base.ta2 -> -
+ PF0_1  | PS.base.a4           | -                          | -
+ PF0_1  | PS.base.a5           | -                          | -
+ PF0_1  | PS.base.a6           | -                          | -
+ PF0_1  | PS.base.b1           | WS.002.1a in room 002 -> - | PS.base.ta5 -> -
+ PF0_1  | PS.base.b2           | WS.002.1b in room 002 -> - | 
+ PF0_1  | PS.base.b3           | WS.002.2a in room 002 -> - | PS.base.tb2 -> -
+ PF0_1  | PS.base.b4           | WS.002.2b in room 002 -> - | -
+ PF0_1  | PS.base.b5           | WS.002.3a in room 002 -> - | -
+ PF0_1  | PS.base.b6           | WS.002.3b in room 002 -> - | -
+ PF0_1  | PS.base.c1           | WS.003.1a in room 003 -> - | -
+ PF0_1  | PS.base.c2           | WS.003.1b in room 003 -> - | -
+ PF0_1  | PS.base.c3           | WS.003.2a in room 003 -> - | -
+ PF0_1  | PS.base.c4           | WS.003.2b in room 003 -> - | -
+ PF0_1  | PS.base.c5           | WS.003.3a in room 003 -> - | -
+ PF0_1  | PS.base.c6           | WS.003.3b in room 003 -> - | -
+(18 rows)
+
 select * from PField_v1 where pfname = 'PF0_2' order by slotname;
  pfname | slotname | backside | patch 
 --------+----------+----------+-------
diff --git a/src/test/regress/expected/xc_params.out b/src/test/regress/expected/xc_params.out
new file mode 100644
index 0000000000..86f1aef6f8
--- /dev/null
+++ b/src/test/regress/expected/xc_params.out
@@ -0,0 +1,71 @@
+--
+-- xc_params
+--
+-- Tests usage of non-simple variables (specifically record types without
+-- %rowtype descriptor) in SQL statements inside a plpgsql function. 
+--
+--
+create table prm_emp(empno int, empname varchar);
+create function prm_func() returns varchar as
+$$
+declare emprow record;
+begin
+	insert into prm_emp values (1, 'abcd');
+
+	-- Initialize emprow
+	select into emprow * from prm_emp;
+
+	emprow.empname = 'xyz';
+	emprow.empno = emprow.empno + 1;
+
+	-- Test parameter handling in insert.
+	insert into prm_emp values (emprow.empno, emprow.empname);
+	-- At the same time, add some rows to be deleted for testing DELETEs
+	emprow.empno = emprow.empno + 1;
+	insert into prm_emp values (emprow.empno, 'tobedeleted1');
+	emprow.empno = emprow.empno + 1;
+	insert into prm_emp values (emprow.empno, 'tobedeleted2');
+
+	-- Test parameter handling for non-shippable UPDATE statement.
+	emprow.empno = 2;
+	emprow.empname = ' changed row 2';
+	-- This will update the 2nd row
+	update prm_emp set empname = empname || emprow.empname
+					where   (empname || now()) like '%xyz%';
+
+	emprow.empname = 'abcd';
+	-- This will update 1st row
+	update prm_emp set empname = emprow.empname || ' changed row 1'
+	                where empname = emprow.empname;
+
+	-- Test DELETEs. The final rows should not contain these rows.
+	-- Test shippable DELETEs.
+	delete from prm_emp where empname = 'tobedeleted1';
+	-- Test non-shippable DELETEs. Again, now() is used only to make non-shippable.
+	delete from prm_emp where empname = 'tobedeleted2' and (emprow.empname || now()) is not NULL;
+
+	-- Test SELECTs
+	emprow.empno = -1;
+	emprow.empname = '%xyz%';
+	select into emprow * from prm_emp where empname || now() like emprow.empname;
+
+	-- Return the SELECTed emp row.
+	return emprow.empno::text ||  ' | ' || emprow.empname;
+end
+$$ language plpgsql;
+select prm_func();
+       prm_func        
+-----------------------
+ 2 | xyz changed row 2
+(1 row)
+
+-- Show the final rows
+select * from prm_emp order by empno;
+ empno |      empname       
+-------+--------------------
+     1 | abcd changed row 1
+     2 | xyz changed row 2
+(2 rows)
+
+drop table prm_emp;
+drop function prm_func();
diff --git a/src/test/regress/serial_schedule b/src/test/regress/serial_schedule
index 1bc011fbaf..81cbfc4e4e 100644
--- a/src/test/regress/serial_schedule
+++ b/src/test/regress/serial_schedule
@@ -153,3 +153,4 @@ test: xc_notrans_block
 test: xc_limit
 test: xc_sort
 test: xc_returning
+test: xc_params
diff --git a/src/test/regress/sql/xc_params.sql b/src/test/regress/sql/xc_params.sql
new file mode 100644
index 0000000000..3cceb29193
--- /dev/null
+++ b/src/test/regress/sql/xc_params.sql
@@ -0,0 +1,66 @@
+--
+-- xc_params
+--
+-- Tests usage of non-simple variables (specifically record types without
+-- %rowtype descriptor) in SQL statements inside a plpgsql function. 
+--
+--
+
+create table prm_emp(empno int, empname varchar);
+
+
+create function prm_func() returns varchar as
+$$
+declare emprow record;
+begin
+	insert into prm_emp values (1, 'abcd');
+
+	-- Initialize emprow
+	select into emprow * from prm_emp;
+
+	emprow.empname = 'xyz';
+	emprow.empno = emprow.empno + 1;
+
+	-- Test parameter handling in insert.
+	insert into prm_emp values (emprow.empno, emprow.empname);
+	-- At the same time, add some rows to be deleted for testing DELETEs
+	emprow.empno = emprow.empno + 1;
+	insert into prm_emp values (emprow.empno, 'tobedeleted1');
+	emprow.empno = emprow.empno + 1;
+	insert into prm_emp values (emprow.empno, 'tobedeleted2');
+
+	-- Test parameter handling for non-shippable UPDATE statement.
+	emprow.empno = 2;
+	emprow.empname = ' changed row 2';
+	-- This will update the 2nd row
+	update prm_emp set empname = empname || emprow.empname
+					where   (empname || now()) like '%xyz%';
+
+	emprow.empname = 'abcd';
+	-- This will update 1st row
+	update prm_emp set empname = emprow.empname || ' changed row 1'
+	                where empname = emprow.empname;
+
+	-- Test DELETEs. The final rows should not contain these rows.
+	-- Test shippable DELETEs.
+	delete from prm_emp where empname = 'tobedeleted1';
+	-- Test non-shippable DELETEs. Again, now() is used only to make non-shippable.
+	delete from prm_emp where empname = 'tobedeleted2' and (emprow.empname || now()) is not NULL;
+
+	-- Test SELECTs
+	emprow.empno = -1;
+	emprow.empname = '%xyz%';
+	select into emprow * from prm_emp where empname || now() like emprow.empname;
+
+	-- Return the SELECTed emp row.
+	return emprow.empno::text ||  ' | ' || emprow.empname;
+end
+$$ language plpgsql;
+
+select prm_func();
+
+-- Show the final rows
+select * from prm_emp order by empno;
+
+drop table prm_emp;
+drop function prm_func();