diff options
author | Pavan Deolasee | 2015-07-20 11:54:49 +0000 |
---|---|---|
committer | Pavan Deolasee | 2015-07-20 11:54:49 +0000 |
commit | df4c23f01b1a64b310fd572b87bc8000c225b432 (patch) | |
tree | 50d36eb7a7f9538fd95e4ed136c6e52f20e1c927 | |
parent | 13b3e6d58331cac6c3556fb6d273bcbde5d4d872 (diff) |
Add a facility to track waited-for XIDs for a transaction
A transaction may wait for one or more transactions to finish before proceeding
with its operation. For example, an UPDATEing transaction may wait for other
transaction if it has already updated/deleted the tuple and decide the next
action based on other transaction's outcome as well as its own isolation level.
Sometimes it may happen a transaction is marked as committed on a datanode, but
GTM has not yet received a message to this effect. We have seen that this can
lead to breakage in MVCC properties when more than one tuple version may
satisfy MVCC checks. For specifically, when a transaction which is already
committed on the datanode is still seen as in-progress, but a later transaction
which again updated the same tuple is seen as committed as per a snapshot
obtained from GTM. Such snapshots can see both, most old and most recent
versions of a tuple as visible.
This patch adds an ability to track XIDs on which a transaction may have waited
and later sends that list to GTM. GTM must not commit a transaction unless all
such transactions on which it has waited for are also finished. Till such time,
GTM will send back STATUS_DELAYED response to the client. The client must retry
commit until its done. We believe the window is extremely small and its a
corner case. So such retries should not add much overhead to the system.
-rw-r--r-- | src/backend/access/transam/gtm.c | 17 | ||||
-rw-r--r-- | src/backend/access/transam/xact.c | 207 | ||||
-rw-r--r-- | src/backend/pgxc/pool/execRemote.c | 42 | ||||
-rw-r--r-- | src/backend/storage/lmgr/lmgr.c | 4 | ||||
-rw-r--r-- | src/gtm/client/fe-protocol.c | 7 | ||||
-rw-r--r-- | src/gtm/client/gtm_client.c | 111 | ||||
-rw-r--r-- | src/gtm/main/gtm_txn.c | 113 | ||||
-rw-r--r-- | src/gtm/proxy/proxy_main.c | 25 | ||||
-rw-r--r-- | src/gtm/test2/test_txn.c | 8 | ||||
-rw-r--r-- | src/include/access/gtm.h | 7 | ||||
-rw-r--r-- | src/include/c.h | 1 | ||||
-rw-r--r-- | src/include/gtm/gtm_client.h | 29 | ||||
-rw-r--r-- | src/include/gtm/gtm_txn.h | 8 | ||||
-rw-r--r-- | src/include/pgxc/execRemote.h | 1 |
14 files changed, 512 insertions, 68 deletions
diff --git a/src/backend/access/transam/gtm.c b/src/backend/access/transam/gtm.c index 347dc480d1..2a8e78547a 100644 --- a/src/backend/access/transam/gtm.c +++ b/src/backend/access/transam/gtm.c @@ -187,7 +187,8 @@ BeginTranAutovacuumGTM(void) } int -CommitTranGTM(GlobalTransactionId gxid) +CommitTranGTM(GlobalTransactionId gxid, int waited_xid_count, + GlobalTransactionId *waited_xids) { int ret; @@ -198,7 +199,7 @@ CommitTranGTM(GlobalTransactionId gxid) ret = -1; if (conn) #endif - ret = commit_transaction(conn, gxid); + ret = commit_transaction(conn, gxid, waited_xid_count, waited_xids); /* * If something went wrong (timeout), try and reset GTM connection. @@ -211,7 +212,7 @@ CommitTranGTM(GlobalTransactionId gxid) InitGTM(); #ifdef XCP if (conn) - ret = commit_transaction(conn, gxid); + ret = commit_transaction(conn, gxid, waited_xid_count, waited_xids); #endif } @@ -228,7 +229,9 @@ CommitTranGTM(GlobalTransactionId gxid) * and for COMMIT PREPARED. */ int -CommitPreparedTranGTM(GlobalTransactionId gxid, GlobalTransactionId prepared_gxid) +CommitPreparedTranGTM(GlobalTransactionId gxid, + GlobalTransactionId prepared_gxid, int waited_xid_count, + GlobalTransactionId *waited_xids) { int ret = 0; @@ -239,7 +242,8 @@ CommitPreparedTranGTM(GlobalTransactionId gxid, GlobalTransactionId prepared_gxi ret = -1; if (conn) #endif - ret = commit_prepared_transaction(conn, gxid, prepared_gxid); + ret = commit_prepared_transaction(conn, gxid, prepared_gxid, + waited_xid_count, waited_xids); /* * If something went wrong (timeout), try and reset GTM connection. @@ -253,7 +257,8 @@ CommitPreparedTranGTM(GlobalTransactionId gxid, GlobalTransactionId prepared_gxi InitGTM(); #ifdef XCP if (conn) - ret = commit_prepared_transaction(conn, gxid, prepared_gxid); + ret = commit_prepared_transaction(conn, gxid, prepared_gxid, + waited_xid_count, waited_xids); #endif } currentGxid = InvalidGlobalTransactionId; diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c index 30e36576cb..c72dd98d92 100644 --- a/src/backend/access/transam/xact.c +++ b/src/backend/access/transam/xact.c @@ -73,6 +73,9 @@ #include "storage/procarray.h" #include "storage/sinvaladt.h" #include "storage/smgr.h" +#ifdef XCP +#include "tcop/tcopprot.h" +#endif #include "utils/builtins.h" #include "utils/catcache.h" #include "utils/combocid.h" @@ -223,6 +226,10 @@ typedef struct TransactionStateData bool didLogXid; /* has xid been included in WAL record? */ int parallelModeLevel; /* Enter/ExitParallelMode counter */ struct TransactionStateData *parent; /* back link to parent */ +#ifdef XCP + int waitedForXidsCount; /* count of xids we waited to finish */ + TransactionId *waitedForXids; /* xids we waited to finish */ +#endif } TransactionStateData; typedef TransactionStateData *TransactionState; @@ -426,6 +433,14 @@ static void AtSubCommit_Memory(void); static void AtSubStart_Memory(void); static void AtSubStart_ResourceOwner(void); +#ifdef XCP +static void AtSubCommit_WaitedXids(void); +static void AtSubAbort_WaitedXids(void); +static void AtEOXact_WaitedXids(void); +static void TransactionRecordXidWait_Internal(TransactionState s, + TransactionId xid); +#endif + static void ShowTransactionState(const char *str); static void ShowTransactionStateRec(TransactionState state); static const char *BlockStateAsString(TBlockState blockState); @@ -2498,6 +2513,10 @@ CommitTransaction(void) } #endif +#ifdef XCP + AtEOXact_WaitedXids(); +#endif + /* Prevent cancel/die interrupt while cleaning up */ HOLD_INTERRUPTS(); @@ -2712,11 +2731,15 @@ AtEOXact_GlobalTxn(bool commit) if (GlobalTransactionIdIsValid(s->auxilliaryTransactionId) && GlobalTransactionIdIsValid(s->topGlobalTransansactionId)) CommitPreparedTranGTM(s->topGlobalTransansactionId, - s->auxilliaryTransactionId); + s->auxilliaryTransactionId, + s->waitedForXidsCount, + s->waitedForXids); else if (GlobalTransactionIdIsValid(s->topGlobalTransansactionId)) - CommitTranGTM(s->topGlobalTransansactionId); + CommitTranGTM(s->topGlobalTransansactionId, + s->waitedForXidsCount, + s->waitedForXids); else if (GlobalTransactionIdIsValid(s->auxilliaryTransactionId)) - CommitTranGTM(s->auxilliaryTransactionId); + CommitTranGTM(s->auxilliaryTransactionId, 0, NULL); } else { @@ -2740,7 +2763,7 @@ AtEOXact_GlobalTxn(bool commit) { IsXidFromGTM = false; if (commit) - CommitTranGTM(s->topGlobalTransansactionId); + CommitTranGTM(s->topGlobalTransansactionId, 0, NULL); else RollbackTranGTM(s->topGlobalTransansactionId); @@ -2774,6 +2797,14 @@ AtEOXact_GlobalTxn(bool commit) s->topGlobalTransansactionId = InvalidGlobalTransactionId; s->auxilliaryTransactionId = InvalidGlobalTransactionId; + if (IS_PGXC_LOCAL_COORDINATOR) + { + if (s->waitedForXids) + pfree(s->waitedForXids); + } + s->waitedForXids = NULL; + s->waitedForXidsCount = 0; + SetNextTransactionId(InvalidTransactionId); } @@ -2994,6 +3025,10 @@ PrepareTransaction(void) AtPrepare_MultiXact(); AtPrepare_RelationMap(); +#ifdef XCP + AtEOXact_WaitedXids(); +#endif + /* * Here is where we really truly prepare. * @@ -3138,6 +3173,16 @@ PrepareTransaction(void) s->topGlobalTransansactionId = InvalidGlobalTransactionId; ForgetTransactionLocalNode(); } +#ifdef XCP + if (IS_PGXC_LOCAL_COORDINATOR) + { + if (s->waitedForXids) + pfree(s->waitedForXids); + } + s->waitedForXids = NULL; + s->waitedForXidsCount = 0; +#endif + SetNextTransactionId(InvalidTransactionId); /* @@ -3421,6 +3466,16 @@ CleanupTransaction(void) XactTopTransactionId = InvalidTransactionId; nParallelCurrentXids = 0; +#ifdef XCP + if (IS_PGXC_LOCAL_COORDINATOR) + { + if (s->waitedForXids) + pfree(s->waitedForXids); + } + s->waitedForXids = NULL; + s->waitedForXidsCount = 0; +#endif + /* * done with abort processing, set current transaction state back to * default @@ -5369,6 +5424,9 @@ CommitSubTransaction(void) AtEOSubXact_HashTables(true, s->nestingLevel); AtEOSubXact_PgStat(true, s->nestingLevel); AtSubCommit_Snapshot(s->nestingLevel); +#ifdef XCP + AtSubCommit_WaitedXids(); +#endif /* * We need to restore the upper transaction's read-only state, in case the @@ -5516,6 +5574,9 @@ AbortSubTransaction(void) AtEOSubXact_HashTables(false, s->nestingLevel); AtEOSubXact_PgStat(false, s->nestingLevel); AtSubAbort_Snapshot(s->nestingLevel); +#ifdef XCP + AtSubAbort_WaitedXids(); +#endif } /* @@ -6666,4 +6727,142 @@ IsPGXCNodeXactDatanodeDirect(void) #endif !IsConnFromCoord(); } + +#ifdef XCP +static void +TransactionRecordXidWait_Internal(TransactionState s, TransactionId xid) +{ + int i; + + if (s->waitedForXids == NULL) + { + /* + * XIDs recorded on the local coordinator, which are mostly collected + * from various remote nodes, must survive across transaction + * boundaries since they are sent to the GTM after local transaction is + * committed. So we track them in the TopMemoryContext and make extra + * efforts to free that memory later. Note that we do this only when we + * are running in the top-level transaction. For subtranctions, they + * will be copied to the parent transaction at the commit time. So at + * subtranction level, they can be tracked in a transaction-local + * memory without any problem + */ + if (IS_PGXC_LOCAL_COORDINATOR && (s->parent == NULL)) + s->waitedForXids = (TransactionId *) + MemoryContextAlloc(TopMemoryContext, sizeof (TransactionId) * + MaxConnections); + else + s->waitedForXids = (TransactionId *) + MemoryContextAlloc(CurTransactionContext, sizeof (TransactionId) * + MaxConnections); + + s->waitedForXidsCount = 0; + } + + elog(DEBUG2, "TransactionRecordXidWait_Internal - recording %d", xid); + + for (i = 0; i < s->waitedForXidsCount; i++) + { + if (TransactionIdEquals(xid, s->waitedForXids[i])) + { + elog(DEBUG2, "TransactionRecordXidWait_Internal - xid %d already recorded", xid); + return; + } + } + + /* + * We track maximum MaxConnections xids. In case of overflow, we forget the + * earliest recorded xid. That should be enough for all practical purposes + * since what we are guarding against is a very small window where a + * transaction which is already committed on a datanode has not yet got an + * opportunity to send its status to the GTM. Such transactions can only be + * running on a different coordinator session. So tracking MaxConnections + * worth xids seems like more than enough + */ + if (s->waitedForXidsCount == MaxConnections) + { + memmove(&s->waitedForXids[0], + &s->waitedForXids[1], + (s->waitedForXidsCount - 1) * sizeof (TransactionId)); + s->waitedForXids[s->waitedForXidsCount - 1] = xid; + } + else + s->waitedForXids[s->waitedForXidsCount++] = xid; + +} + +void +TransactionRecordXidWait(TransactionId xid) +{ + TransactionRecordXidWait_Internal(CurrentTransactionState, xid); +} + +static void +AtSubCommit_WaitedXids() +{ + TransactionState s = CurrentTransactionState; + int i; + + Assert(s->parent != NULL); + + /* + * Move the recorded XIDs to the parent structure + */ + for (i = 0; i < s->waitedForXidsCount; i++) + TransactionRecordXidWait_Internal(s->parent, s->waitedForXids[i]); + + /* And we can safely free them now */ + if (s->waitedForXids != NULL) + pfree(s->waitedForXids); + s->waitedForXids = NULL; + s->waitedForXidsCount = 0; +} + +static void +AtSubAbort_WaitedXids() +{ + TransactionState s = CurrentTransactionState; + + if (s->waitedForXids != NULL) + pfree(s->waitedForXids); + s->waitedForXids = NULL; + s->waitedForXidsCount = 0; + +} + +static void +AtEOXact_WaitedXids(void) +{ + TransactionState s = CurrentTransactionState; + TransactionId *sendXids; + + /* + * Report the set of XIDs this transaction (and its committed + * subtransactions) had waited-for to the coordinator. The coordinator will + * then forward the list to the GTM who ensures that the logical ordering + * between these transactions and this transaction is correctly followed. + */ + if (whereToSendOutput == DestRemote && !IS_PGXC_LOCAL_COORDINATOR) + { + if (s->waitedForXidsCount > 0) + { + int i; + + /* + * Convert the XIDs in network order and send to the client + */ + sendXids = (TransactionId *) palloc (sizeof (TransactionId) * + s->waitedForXidsCount); + for (i = 0; i < s->waitedForXidsCount; i++) + sendXids[i] = htonl(s->waitedForXids[i]); + + pq_putmessage('W', + (const char *)sendXids, + s->waitedForXidsCount * sizeof (TransactionId)); + pfree(sendXids); + } + } + +} +#endif #endif diff --git a/src/backend/pgxc/pool/execRemote.c b/src/backend/pgxc/pool/execRemote.c index d6c9e7176e..0bb7df6cdc 100644 --- a/src/backend/pgxc/pool/execRemote.c +++ b/src/backend/pgxc/pool/execRemote.c @@ -1193,6 +1193,31 @@ HandleDatanodeCommandId(RemoteQueryState *combiner, char *msg_body, size_t len) } /* + * Record waited-for XIDs received from the remote nodes into the transaction + * state + */ +static void +HandleWaitXids(char *msg_body, size_t len) +{ + int xid_count; + uint32 n32; + int cur; + int i; + + /* Get the xid count */ + xid_count = len / sizeof (TransactionId); + + cur = 0; + for (i = 0; i < xid_count; i++) + { + Assert(cur < len); + memcpy(&n32, &msg_body[cur], sizeof (TransactionId)); + cur = cur + sizeof (TransactionId); + TransactionRecordXidWait(ntohl(n32)); + } +} + +/* * Examine the specified combiner state and determine if command was completed * successfully */ @@ -2207,6 +2232,9 @@ pgxc_node_receive_responses(const int conn_count, PGXCNodeHandle ** connections, case RESPONSE_ERROR: /* no handling needed, just wait for ReadyForQuery */ break; + + case RESPONSE_WAITXIDS: + break; #endif default: /* Inconsistent responses */ @@ -2370,6 +2398,11 @@ handle_response(PGXCNodeHandle *conn, ResponseCombiner *combiner) return RESPONSE_BARRIER_OK; case 'I': /* EmptyQuery */ return RESPONSE_COMPLETE; +#ifdef XCP + case 'W': + HandleWaitXids(msg, msg_len); + return RESPONSE_WAITXIDS; +#endif default: /* sync lost? */ elog(WARNING, "Received unsupported message type: %c", msg_type); @@ -7098,7 +7131,7 @@ FinishRemotePreparedTransaction(char *prepareGID, bool commit) if (commit) { pgxc_node_remote_commit(); - CommitPreparedTranGTM(prepare_gxid, gxid); + CommitPreparedTranGTM(prepare_gxid, gxid, 0, NULL); } else { @@ -7133,7 +7166,12 @@ FinishRemotePreparedTransaction(char *prepareGID, bool commit) if (commit) { - CommitPreparedTranGTM(prepare_gxid, gxid); + /* + * XXX For explicit 2PC, there will be enough delay for any + * waited-committed transactions to send a final COMMIT message to the + * GTM. + */ + CommitPreparedTranGTM(prepare_gxid, gxid, 0, NULL); } else { diff --git a/src/backend/storage/lmgr/lmgr.c b/src/backend/storage/lmgr/lmgr.c index c052949749..763f1cfdce 100644 --- a/src/backend/storage/lmgr/lmgr.c +++ b/src/backend/storage/lmgr/lmgr.c @@ -542,6 +542,10 @@ XactLockTableWait(TransactionId xid, Relation rel, ItemPointer ctid, error_context_stack = &callback; } +#ifdef XCP + TransactionRecordXidWait(xid); +#endif + for (;;) { Assert(TransactionIdIsValid(xid)); diff --git a/src/gtm/client/fe-protocol.c b/src/gtm/client/fe-protocol.c index d18c531eff..3e43036140 100644 --- a/src/gtm/client/fe-protocol.c +++ b/src/gtm/client/fe-protocol.c @@ -396,6 +396,7 @@ gtmpqParseSuccess(GTM_Conn *conn, GTM_Result *result) case TXN_BEGIN_GETGXID_AUTOVACUUM_RESULT: case TXN_PREPARE_RESULT: case TXN_START_PREPARED_RESULT: + case TXN_ROLLBACK_RESULT: if (gtmpqGetnchar((char *)&result->gr_resdata.grd_gxid, sizeof (GlobalTransactionId), conn)) result->gr_status = GTM_RESULT_ERROR; @@ -403,10 +404,12 @@ gtmpqParseSuccess(GTM_Conn *conn, GTM_Result *result) case TXN_COMMIT_RESULT: case TXN_COMMIT_PREPARED_RESULT: - case TXN_ROLLBACK_RESULT: - if (gtmpqGetnchar((char *)&result->gr_resdata.grd_gxid, + if (gtmpqGetnchar((char *)&result->gr_resdata.grd_eof_txn.gxid, sizeof (GlobalTransactionId), conn)) result->gr_status = GTM_RESULT_ERROR; + if (gtmpqGetnchar((char *)&result->gr_resdata.grd_eof_txn.status, + sizeof (int), conn)) + result->gr_status = GTM_RESULT_ERROR; break; case TXN_GET_GXID_RESULT: diff --git a/src/gtm/client/gtm_client.c b/src/gtm/client/gtm_client.c index 4f77731a4c..8808735c6c 100644 --- a/src/gtm/client/gtm_client.c +++ b/src/gtm/client/gtm_client.c @@ -48,6 +48,8 @@ void GTM_FreeResult(GTM_Result *result, GTM_PGXCNodeType remote_type); static GTM_Result *makeEmptyResultIfIsNull(GTM_Result *oldres); static int commit_prepared_transaction_internal(GTM_Conn *conn, GlobalTransactionId gxid, GlobalTransactionId prepared_gxid, + int waited_xid_count, + GlobalTransactionId *waited_xids, bool is_backup); static int start_prepared_transaction_internal(GTM_Conn *conn, GlobalTransactionId gxid, char *gid, char *nodestring, bool is_backup); @@ -70,7 +72,10 @@ static GTM_Sequence get_next_internal(GTM_Conn *conn, GTM_SequenceKey key, bool static int set_val_internal(GTM_Conn *conn, GTM_SequenceKey key, GTM_Sequence nextval, bool iscalled, bool is_backup); #endif static int reset_sequence_internal(GTM_Conn *conn, GTM_SequenceKey key, bool is_backup); -static int commit_transaction_internal(GTM_Conn *conn, GlobalTransactionId gxid, bool is_backup); +static int commit_transaction_internal(GTM_Conn *conn, GlobalTransactionId gxid, + int waited_xid_count, + GlobalTransactionId *waited_xids, + bool is_backup); static int close_sequence_internal(GTM_Conn *conn, GTM_SequenceKey key, bool is_backup); static int rename_sequence_internal(GTM_Conn *conn, GTM_SequenceKey key, GTM_SequenceKey newkey, bool is_backup); static int alter_sequence_internal(GTM_Conn *conn, GTM_SequenceKey key, GTM_Sequence increment, @@ -596,29 +601,52 @@ send_failed: int bkup_commit_transaction(GTM_Conn *conn, GlobalTransactionId gxid) { - return commit_transaction_internal(conn, gxid, true); + return commit_transaction_internal(conn, gxid, 0, NULL, true); } int -commit_transaction(GTM_Conn *conn, GlobalTransactionId gxid) +commit_transaction(GTM_Conn *conn, GlobalTransactionId gxid, + int waited_xid_count, GlobalTransactionId *waited_xids) { - return commit_transaction_internal(conn, gxid, false); + if (waited_xid_count == 0) + { + int txn_count_out; + int status_out; + int status; + status = commit_transaction_multi(conn, 1, &gxid, &txn_count_out, + &status_out); + Assert(txn_count_out == 1); + return status; + } + else + return commit_transaction_internal(conn, gxid, waited_xid_count, + waited_xids, false); } static int -commit_transaction_internal(GTM_Conn *conn, GlobalTransactionId gxid, bool is_backup) +commit_transaction_internal(GTM_Conn *conn, GlobalTransactionId gxid, + int waited_xid_count, GlobalTransactionId *waited_xids, + bool is_backup) { GTM_Result *res = NULL; time_t finish_time; +retry: /* Start the message. */ if (gtmpqPutMsgStart('C', true, conn) || gtmpqPutInt(is_backup ? MSG_BKUP_TXN_COMMIT : MSG_TXN_COMMIT, sizeof (GTM_MessageType), conn) || - gtmpqPutnchar((char *)&gxid, sizeof (GlobalTransactionId), conn)) + gtmpqPutnchar((char *)&gxid, sizeof (GlobalTransactionId), conn) || + gtmpqPutInt(waited_xid_count, sizeof (int), conn)) goto send_failed; + if (waited_xid_count > 0) + { + if (gtmpqPutnchar((char *) waited_xids, waited_xid_count * sizeof (GlobalTransactionId), conn)) + goto send_failed; + } + /* Finish the message. */ if (gtmpqPutMsgEnd(conn)) goto send_failed; @@ -641,6 +669,33 @@ commit_transaction_internal(GTM_Conn *conn, GlobalTransactionId gxid, bool is_ba { Assert(res->gr_type == TXN_COMMIT_RESULT); Assert(res->gr_resdata.grd_gxid == gxid); + + if (waited_xid_count > 0) + { + if (res->gr_resdata.grd_eof_txn.status == STATUS_DELAYED) + { + /* + * GTM may decide to delay the transaction commit if one or + * more of the XIDs we had waited to finish for hasn't yet + * made to the GTM. While this window is very small, we + * need to guard against that to ensure that a transaction + * which is already seen as committed by datanodes is not + * reported as in-progress by GTM. Also, we don't wait at + * the GTM for other transactions to finish because that + * could potentially lead to deadlocks. So instead just + * sleep for a while (1ms right now) and retry the + * operation. + * + * Since the transactions we are waiting for are in fact + * already committed and hence we don't see a reason why we + * might end up in an inifinite loop. Nevertheless, it + * might make sense to flash a warning and proceed after + * certain number of retries + */ + pg_usleep(1000); + goto retry; + } + } } return res->gr_status; @@ -655,30 +710,49 @@ send_failed: } int -commit_prepared_transaction(GTM_Conn *conn, GlobalTransactionId gxid, GlobalTransactionId prepared_gxid) +commit_prepared_transaction(GTM_Conn *conn, + GlobalTransactionId gxid, + GlobalTransactionId prepared_gxid, + int waited_xid_count, + GlobalTransactionId *waited_xids) { - return commit_prepared_transaction_internal(conn, gxid, prepared_gxid, false); + return commit_prepared_transaction_internal(conn, gxid, prepared_gxid, + waited_xid_count, waited_xids, false); } int bkup_commit_prepared_transaction(GTM_Conn *conn, GlobalTransactionId gxid, GlobalTransactionId prepared_gxid) { - return commit_prepared_transaction_internal(conn, gxid, prepared_gxid, true); + return commit_prepared_transaction_internal(conn, gxid, prepared_gxid, 0, + NULL, true); } static int -commit_prepared_transaction_internal(GTM_Conn *conn, GlobalTransactionId gxid, GlobalTransactionId prepared_gxid, bool is_backup) +commit_prepared_transaction_internal(GTM_Conn *conn, + GlobalTransactionId gxid, + GlobalTransactionId prepared_gxid, + int waited_xid_count, + GlobalTransactionId *waited_xids, + bool is_backup) { GTM_Result *res = NULL; time_t finish_time; +retry: /* Start the message */ if (gtmpqPutMsgStart('C', true, conn) || gtmpqPutInt(is_backup ? MSG_BKUP_TXN_COMMIT_PREPARED : MSG_TXN_COMMIT_PREPARED, sizeof (GTM_MessageType), conn) || gtmpqPutnchar((char *)&gxid, sizeof (GlobalTransactionId), conn) || - gtmpqPutnchar((char *)&prepared_gxid, sizeof (GlobalTransactionId), conn)) + gtmpqPutnchar((char *)&prepared_gxid, sizeof (GlobalTransactionId), conn) || + gtmpqPutInt(waited_xid_count, sizeof (int), conn)) goto send_failed; + if (waited_xid_count > 0) + { + if (gtmpqPutnchar((char *) waited_xids, waited_xid_count * sizeof (GlobalTransactionId), conn)) + goto send_failed; + } + /* Finish the message */ if (gtmpqPutMsgEnd(conn)) goto send_failed; @@ -701,6 +775,15 @@ commit_prepared_transaction_internal(GTM_Conn *conn, GlobalTransactionId gxid, G { Assert(res->gr_type == TXN_COMMIT_PREPARED_RESULT); Assert(res->gr_resdata.grd_gxid == gxid); + if (waited_xid_count > 0) + { + if (res->gr_resdata.grd_eof_txn.status == STATUS_DELAYED) + { + /* See comments in commit_transaction_internal() */ + pg_usleep(1000); + goto retry; + } + } } return res->gr_status; @@ -2022,7 +2105,8 @@ send_failed: } int -bkup_commit_transaction_multi(GTM_Conn *conn, int txn_count, GlobalTransactionId *gxid) +bkup_commit_transaction_multi(GTM_Conn *conn, int txn_count, + GlobalTransactionId *gxid) { int ii; @@ -2035,8 +2119,7 @@ bkup_commit_transaction_multi(GTM_Conn *conn, int txn_count, GlobalTransactionId for (ii = 0; ii < txn_count; ii++) { - if (gtmpqPutnchar((char *)&gxid[ii], - sizeof (GlobalTransactionId), conn)) + if (gtmpqPutnchar((char *)&gxid[ii], sizeof (GlobalTransactionId), conn)) goto send_failed; } diff --git a/src/gtm/main/gtm_txn.c b/src/gtm/main/gtm_txn.c index 8f09d7c128..cd55018e22 100644 --- a/src/gtm/main/gtm_txn.c +++ b/src/gtm/main/gtm_txn.c @@ -149,8 +149,8 @@ GlobalTransactionIdGetStatus(GlobalTransactionId transactionId) /* * Given the GXID, find the corresponding transaction handle. */ -GTM_TransactionHandle -GTM_GXIDToHandle(GlobalTransactionId gxid) +static GTM_TransactionHandle +GTM_GXIDToHandle_Internal(GlobalTransactionId gxid, bool warn) { gtm_ListCell *elem = NULL; GTM_TransactionInfo *gtm_txninfo = NULL; @@ -174,13 +174,26 @@ GTM_GXIDToHandle(GlobalTransactionId gxid) return gtm_txninfo->gti_handle; else { - ereport(WARNING, + if (warn) + ereport(WARNING, (ERANGE, errmsg("No transaction handle for gxid: %d", gxid))); return InvalidTransactionHandle; } } +GTM_TransactionHandle +GTM_GXIDToHandle(GlobalTransactionId gxid) +{ + return GTM_GXIDToHandle_Internal(gxid, true); +} + +bool +GTM_IsGXIDInProgress(GlobalTransactionId gxid) +{ + return (GTM_GXIDToHandle_Internal(gxid, false) != + InvalidTransactionHandle); +} /* * Given the GID (for a prepared transaction), find the corresponding * transaction handle. @@ -979,20 +992,27 @@ int GTM_CommitTransactionGXID(GlobalTransactionId gxid) { GTM_TransactionHandle txn = GTM_GXIDToHandle(gxid); - return GTM_CommitTransaction(txn); + return GTM_CommitTransaction(txn, 0, NULL); } /* * Commit multiple transactions in one go */ int -GTM_CommitTransactionMulti(GTM_TransactionHandle txn[], int txn_count, int status[]) +GTM_CommitTransactionMulti(GTM_TransactionHandle txn[], int txn_count, + int waited_xid_count, GlobalTransactionId *waited_xids, + int status[]) { GTM_TransactionInfo *gtm_txninfo[txn_count]; + GTM_TransactionInfo *remove_txninfo[txn_count]; + int remove_count = 0; int ii; for (ii = 0; ii < txn_count; ii++) { + int jj; + bool waited_xid_running = false; + gtm_txninfo[ii] = GTM_HandleToTransactionInfo(txn[ii]); if (gtm_txninfo[ii] == NULL) @@ -1000,6 +1020,28 @@ GTM_CommitTransactionMulti(GTM_TransactionHandle txn[], int txn_count, int statu status[ii] = STATUS_ERROR; continue; } + + /* + * If any of the waited_xids is still running, we must delay commit for + * this transaction till all such waited_xids are finished + */ + for (jj = 0; jj < waited_xid_count; jj++) + { + if (GTM_IsGXIDInProgress(waited_xids[jj])) + { + elog(DEBUG1, "Xact %d not yet finished, xact %d will be delayed", + waited_xids[jj], gtm_txninfo[ii]->gti_gxid); + waited_xid_running = true; + break; + } + } + + if (waited_xid_running) + { + status[ii] = STATUS_DELAYED; + continue; + } + /* * Mark the transaction as being aborted */ @@ -1007,11 +1049,13 @@ GTM_CommitTransactionMulti(GTM_TransactionHandle txn[], int txn_count, int statu gtm_txninfo[ii]->gti_state = GTM_TXN_COMMIT_IN_PROGRESS; GTM_RWLockRelease(>m_txninfo[ii]->gti_lock); status[ii] = STATUS_OK; + + remove_txninfo[remove_count++] = gtm_txninfo[ii]; } - GTM_RemoveTransInfoMulti(gtm_txninfo, txn_count); + GTM_RemoveTransInfoMulti(remove_txninfo, remove_count); - return txn_count; + return remove_count; } /* @@ -1041,10 +1085,11 @@ GTM_PrepareTransaction(GTM_TransactionHandle txn) * Commit a transaction */ int -GTM_CommitTransaction(GTM_TransactionHandle txn) +GTM_CommitTransaction(GTM_TransactionHandle txn, int waited_xid_count, + GlobalTransactionId *waited_xids) { int status; - GTM_CommitTransactionMulti(&txn, 1, &status); + GTM_CommitTransactionMulti(&txn, 1, waited_xid_count, waited_xids, &status); return status; } @@ -1714,6 +1759,9 @@ ProcessCommitTransactionCommand(Port *myport, StringInfo message, bool is_backup GlobalTransactionId gxid; MemoryContext oldContext; int status = STATUS_OK; + int waited_xid_count; + GlobalTransactionId *waited_xids; + const char *data = pq_getmsgbytes(message, sizeof (gxid)); if (data == NULL) @@ -1723,6 +1771,13 @@ ProcessCommitTransactionCommand(Port *myport, StringInfo message, bool is_backup memcpy(&gxid, data, sizeof (gxid)); txn = GTM_GXIDToHandle(gxid); + waited_xid_count = pq_getmsgint(message, sizeof (int)); + if (waited_xid_count > 0) + { + waited_xids = pq_getmsgbytes(message, + waited_xid_count * sizeof (GlobalTransactionId)); + } + pq_getmsgend(message); oldContext = MemoryContextSwitchTo(TopMemoryContext); @@ -1730,13 +1785,17 @@ ProcessCommitTransactionCommand(Port *myport, StringInfo message, bool is_backup /* * Commit the transaction */ - status = GTM_CommitTransaction(txn); + status = GTM_CommitTransaction(txn, waited_xid_count, waited_xids); MemoryContextSwitchTo(oldContext); if(!is_backup) { - if (GetMyThreadInfo->thr_conn->standby) + /* + * If the transaction is successfully committed on the GTM master then + * send a backup message to the GTM slave to redo the action locally + */ + if ((GetMyThreadInfo->thr_conn->standby) && (status == STATUS_OK)) { /* * Backup first @@ -1769,7 +1828,7 @@ ProcessCommitTransactionCommand(Port *myport, StringInfo message, bool is_backup pq_sendbytes(&buf, (char *)&proxyhdr, sizeof (GTM_ProxyMsgHeader)); } pq_sendbytes(&buf, (char *)&gxid, sizeof(gxid)); - pq_sendint(&buf, status, sizeof(status)); + pq_sendbytes(&buf, (char *)&status, sizeof(status)); pq_endmessage(myport, &buf); if (myport->remote_type != GTM_NODE_GTM_PROXY) @@ -1801,6 +1860,8 @@ ProcessCommitPreparedTransactionCommand(Port *myport, StringInfo message, bool i MemoryContext oldContext; int status[txn_count]; int ii; + int waited_xid_count; + GlobalTransactionId *waited_xids; for (ii = 0; ii < txn_count; ii++) { @@ -1814,6 +1875,13 @@ ProcessCommitPreparedTransactionCommand(Port *myport, StringInfo message, bool i elog(DEBUG1, "ProcessCommitTransactionCommandMulti: gxid(%u), handle(%u)", gxid[ii], txn[ii]); } + waited_xid_count = pq_getmsgint(message, sizeof (int)); + if (waited_xid_count > 0) + { + waited_xids = pq_getmsgbytes(message, + waited_xid_count * sizeof (GlobalTransactionId)); + } + pq_getmsgend(message); oldContext = MemoryContextSwitchTo(TopMemoryContext); @@ -1823,13 +1891,26 @@ ProcessCommitPreparedTransactionCommand(Port *myport, StringInfo message, bool i /* * Commit the prepared transaction. */ - GTM_CommitTransactionMulti(txn, txn_count, status); + GTM_CommitTransactionMulti(txn, txn_count, waited_xid_count, + waited_xids, status); MemoryContextSwitchTo(oldContext); if (!is_backup) { - if (GetMyThreadInfo->thr_conn->standby) + /* + * If we successfully committed the transaction on the GTM master, then + * also send a backup message to the GTM slave to redo the action + * locally + * + * GTM_CommitTransactionMulti() above is used to only commit the main + * and the auxiliary GXID. Since we either commit or delay both of + * these GXIDs together, its enough to just test for one of the GXIDs. + * If the transaction commit is delayed, the backup message will be + * sent when the GTM master receives COMMIT message again and + * successfully commits the transaction + */ + if ((GetMyThreadInfo->thr_conn->standby) && (status[0] == STATUS_OK)) { /* Backup first */ int _rc; @@ -1862,7 +1943,7 @@ ProcessCommitPreparedTransactionCommand(Port *myport, StringInfo message, bool i pq_sendbytes(&buf, (char *)&proxyhdr, sizeof (GTM_ProxyMsgHeader)); } pq_sendbytes(&buf, (char *)&gxid[0], sizeof(GlobalTransactionId)); - pq_sendint(&buf, status[0], 4); + pq_sendbytes(&buf, (char *)&status[0], 4); pq_endmessage(myport, &buf); if (myport->remote_type != GTM_NODE_GTM_PROXY) @@ -2193,7 +2274,7 @@ ProcessCommitTransactionCommandMulti(Port *myport, StringInfo message, bool is_b /* * Commit the transaction */ - GTM_CommitTransactionMulti(txn, txn_count, status); + GTM_CommitTransactionMulti(txn, txn_count, 0, NULL, status); MemoryContextSwitchTo(oldContext); diff --git a/src/gtm/proxy/proxy_main.c b/src/gtm/proxy/proxy_main.c index 0534ee4a31..33b2266f0a 100644 --- a/src/gtm/proxy/proxy_main.c +++ b/src/gtm/proxy/proxy_main.c @@ -1596,6 +1596,7 @@ ProcessCommand(GTMProxy_ConnectionInfo *conninfo, GTM_Conn *gtm_conn, case MSG_SEQUENCE_RENAME: case MSG_SEQUENCE_ALTER: case MSG_BARRIER: + case MSG_TXN_COMMIT: #ifdef XCP case MSG_REGISTER_SESSION: #endif @@ -1610,7 +1611,7 @@ ProcessCommand(GTMProxy_ConnectionInfo *conninfo, GTM_Conn *gtm_conn, case MSG_TXN_BEGIN: case MSG_TXN_BEGIN_GETGXID: - case MSG_TXN_COMMIT: + case MSG_TXN_COMMIT_MULTI: case MSG_TXN_ROLLBACK: case MSG_TXN_GET_GXID: ProcessTransactionCommand(conninfo, gtm_conn, mtype, input_message); @@ -1655,7 +1656,6 @@ ProcessCommand(GTMProxy_ConnectionInfo *conninfo, GTM_Conn *gtm_conn, static GTM_Conn * HandleGTMError(GTM_Conn *gtm_conn) { - Assert(gtm_conn && gtm_conn->last_errno != 0); elog(NOTICE, "GTM communication error was detected. Retrying connection, interval = %d.", @@ -1758,6 +1758,7 @@ IsProxiedMessage(GTM_MessageType mtype) case MSG_SEQUENCE_RENAME: case MSG_SEQUENCE_ALTER: case MSG_SNAPSHOT_GET: + case MSG_TXN_COMMIT: return true; default: @@ -1823,7 +1824,7 @@ ProcessResponse(GTMProxy_ThreadInfo *thrinfo, GTMProxy_CommandInfo *cmdinfo, ReleaseCmdBackup(cmdinfo); break; - case MSG_TXN_COMMIT: + case MSG_TXN_COMMIT_MULTI: if (res->gr_type != TXN_COMMIT_MULTI_RESULT) { ReleaseCmdBackup(cmdinfo); @@ -1842,9 +1843,13 @@ ProcessResponse(GTMProxy_ThreadInfo *thrinfo, GTMProxy_CommandInfo *cmdinfo, if (res->gr_resdata.grd_txn_rc_multi.status[cmdinfo->ci_res_index] == STATUS_OK) { + int txn_count = 1; + int status = STATUS_OK; + pq_beginmessage(&buf, 'S'); - pq_sendint(&buf, TXN_COMMIT_RESULT, 4); - pq_sendbytes(&buf, (char *)&cmdinfo->ci_data.cd_rc.gxid, sizeof (GlobalTransactionId)); + pq_sendint(&buf, TXN_COMMIT_MULTI_RESULT, 4); + pq_sendbytes(&buf, &txn_count, sizeof (int)); + pq_sendbytes(&buf, &status, sizeof (int)); pq_endmessage(cmdinfo->ci_conn->con_port, &buf); pq_flush(cmdinfo->ci_conn->con_port); } @@ -1958,6 +1963,7 @@ ProcessResponse(GTMProxy_ThreadInfo *thrinfo, GTMProxy_CommandInfo *cmdinfo, case MSG_SEQUENCE_RENAME: case MSG_SEQUENCE_ALTER: case MSG_SNAPSHOT_GET: + case MSG_TXN_COMMIT: Assert(IsProxiedMessage(cmdinfo->ci_mtype)); if ((res->gr_proxyhdr.ph_conid == InvalidGTMProxyConnID) || (res->gr_proxyhdr.ph_conid >= GTM_PROXY_MAX_CONNECTIONS) || @@ -2296,7 +2302,12 @@ ProcessTransactionCommand(GTMProxy_ConnectionInfo *conninfo, GTM_Conn *gtm_conn, GTMProxy_CommandPending(conninfo, mtype, cmd_data); break; - case MSG_TXN_COMMIT: + case MSG_TXN_COMMIT_MULTI: + { + int txn_count = pq_getmsgint(message, sizeof (int)); + Assert (txn_count == 1); + } + /* fall through */ case MSG_TXN_ROLLBACK: { const char *data = pq_getmsgbytes(message, @@ -2694,7 +2705,7 @@ GTMProxy_ProcessPendingCommands(GTMProxy_ThreadInfo *thrinfo) thrinfo->thr_pending_commands[ii] = gtm_NIL; break; - case MSG_TXN_COMMIT: + case MSG_TXN_COMMIT_MULTI: if (gtmpqPutInt(MSG_TXN_COMMIT_MULTI, sizeof (GTM_MessageType), gtm_conn) || gtmpqPutInt(gtm_list_length(thrinfo->thr_pending_commands[ii]), sizeof(int), gtm_conn)) elog(ERROR, "Error sending data"); diff --git a/src/gtm/test2/test_txn.c b/src/gtm/test2/test_txn.c index c29b733aa2..55097a1fba 100644 --- a/src/gtm/test2/test_txn.c +++ b/src/gtm/test2/test_txn.c @@ -65,7 +65,7 @@ test_txn_02() rc = prepare_transaction(conn, gxid); _ASSERT( rc>=0 ); - rc = commit_transaction(conn, gxid); + rc = commit_transaction(conn, gxid, 0, NULL); _ASSERT( rc>=0 ); TEARDOWN(); @@ -82,7 +82,7 @@ test_txn_03() gxid = begin_transaction_autovacuum(conn, GTM_ISOLATION_SERIALIZABLE); _ASSERT( gxid != InvalidGlobalTransactionId ); - rc = commit_transaction(conn, gxid); + rc = commit_transaction(conn, gxid, 0, NULL); _ASSERT( rc>=0 ); TEARDOWN(); @@ -116,7 +116,7 @@ test_txn_11() printf("test_txn_11: prepared_gxid=%d\n", prepared_gxid); _ASSERT( rc>=0 ); - rc = commit_prepared_transaction(conn, gxid, prepared_gxid); + rc = commit_prepared_transaction(conn, gxid, prepared_gxid, 0, NULL); _ASSERT( rc>=0 ); TEARDOWN(); @@ -159,7 +159,7 @@ test_txn_53() SETUP(); - rc = commit_transaction(conn, InvalidGlobalTransactionId); + rc = commit_transaction(conn, InvalidGlobalTransactionId, 0, NULL); _ASSERT( rc==-1 ); TEARDOWN(); diff --git a/src/include/access/gtm.h b/src/include/access/gtm.h index b606976276..9a376d3bdc 100644 --- a/src/include/access/gtm.h +++ b/src/include/access/gtm.h @@ -27,7 +27,8 @@ extern void InitGTM(void); extern void CloseGTM(void); extern GlobalTransactionId BeginTranGTM(GTM_Timestamp *timestamp); extern GlobalTransactionId BeginTranAutovacuumGTM(void); -extern int CommitTranGTM(GlobalTransactionId gxid); +extern int CommitTranGTM(GlobalTransactionId gxid, int waited_xid_count, + GlobalTransactionId *waited_xids); extern int RollbackTranGTM(GlobalTransactionId gxid); extern int StartPreparedTranGTM(GlobalTransactionId gxid, char *gid, @@ -38,7 +39,9 @@ extern int GetGIDDataGTM(char *gid, GlobalTransactionId *prepared_gxid, char **nodestring); extern int CommitPreparedTranGTM(GlobalTransactionId gxid, - GlobalTransactionId prepared_gxid); + GlobalTransactionId prepared_gxid, + int waited_xid_count, + GlobalTransactionId *waited_xids); extern GTM_Snapshot GetSnapshotGTM(GlobalTransactionId gxid, bool canbe_grouped); diff --git a/src/include/c.h b/src/include/c.h index 56aebbe026..450248a001 100644 --- a/src/include/c.h +++ b/src/include/c.h @@ -963,6 +963,7 @@ typedef NameData *Name; #define STATUS_FOUND (1) #define STATUS_WAITING (2) #define STATUS_NOT_FOUND (3) +#define STATUS_DELAYED (4) /* diff --git a/src/include/gtm/gtm_client.h b/src/include/gtm/gtm_client.h index e154e4979c..7a6c61d811 100644 --- a/src/include/gtm/gtm_client.h +++ b/src/include/gtm/gtm_client.h @@ -36,12 +36,17 @@ typedef union GTM_ResultData GTM_Timestamp timestamp; } grd_gxid_tp; /* TXN_BEGIN_GETGXID */ - GlobalTransactionId grd_gxid; /* TXN_PREPARE - * TXN_START_PREPARED - * TXN_COMMIT - * TXN_COMMIT_PREPARED - * TXN_ROLLBACK - */ + GlobalTransactionId grd_gxid; /* TXN_PREPARE + * TXN_START_PREPARED + * TXN_ROLLBACK + */ + struct { + GlobalTransactionId gxid; + /* TXN_COMMIT + * TXN_COMMIT_PREPARED + */ + int status; + } grd_eof_txn; GlobalTransactionId grd_next_gxid; @@ -185,9 +190,14 @@ GlobalTransactionId begin_transaction_autovacuum(GTM_Conn *conn, GTM_IsolationLe int bkup_begin_transaction_autovacuum(GTM_Conn *conn, GlobalTransactionId gxid, GTM_IsolationLevel isolevel, uint32 client_id); -int commit_transaction(GTM_Conn *conn, GlobalTransactionId gxid); +int commit_transaction(GTM_Conn *conn, GlobalTransactionId gxid, + int waited_xid_count, + GlobalTransactionId *waited_xids); int bkup_commit_transaction(GTM_Conn *conn, GlobalTransactionId gxid); -int commit_prepared_transaction(GTM_Conn *conn, GlobalTransactionId gxid, GlobalTransactionId prepared_gxid); +int commit_prepared_transaction(GTM_Conn *conn, GlobalTransactionId gxid, + GlobalTransactionId prepared_gxid, + int waited_xid_count, + GlobalTransactionId *waited_xids); int bkup_commit_prepared_transaction(GTM_Conn *conn, GlobalTransactionId gxid, GlobalTransactionId prepared_gxid); int abort_transaction(GTM_Conn *conn, GlobalTransactionId gxid); int bkup_abort_transaction(GTM_Conn *conn, GlobalTransactionId gxid); @@ -218,7 +228,8 @@ int commit_transaction_multi(GTM_Conn *conn, int txn_count, GlobalTransactionId *gxid, int *txn_count_out, int *status_out); int -bkup_commit_transaction_multi(GTM_Conn *conn, int txn_count, GlobalTransactionId *gxid); +bkup_commit_transaction_multi(GTM_Conn *conn, int txn_count, + GlobalTransactionId *gxid); int abort_transaction_multi(GTM_Conn *conn, int txn_count, GlobalTransactionId *gxid, int *txn_count_out, int *status_out); diff --git a/src/include/gtm/gtm_txn.h b/src/include/gtm/gtm_txn.h index 23e2382ad6..d220e81df1 100644 --- a/src/include/gtm/gtm_txn.h +++ b/src/include/gtm/gtm_txn.h @@ -184,6 +184,7 @@ extern GTM_Transactions GTMTransactions; GTM_TransactionInfo *GTM_HandleToTransactionInfo(GTM_TransactionHandle handle); GTM_TransactionHandle GTM_GXIDToHandle(GlobalTransactionId gxid); GTM_TransactionHandle GTM_GIDToHandle(char *gid); +bool GTM_IsGXIDInProgress(GlobalTransactionId gxid); /* Transaction Control */ void GTM_InitTxnManager(void); @@ -197,8 +198,11 @@ int GTM_BeginTransactionMulti(GTM_IsolationLevel isolevel[], int GTM_RollbackTransaction(GTM_TransactionHandle txn); int GTM_RollbackTransactionMulti(GTM_TransactionHandle txn[], int txn_count, int status[]); int GTM_RollbackTransactionGXID(GlobalTransactionId gxid); -int GTM_CommitTransaction(GTM_TransactionHandle txn); -int GTM_CommitTransactionMulti(GTM_TransactionHandle txn[], int txn_count, int status[]); +int GTM_CommitTransaction(GTM_TransactionHandle txn, + int waited_xid_count, GlobalTransactionId *waited_xids); +int GTM_CommitTransactionMulti(GTM_TransactionHandle txn[], int txn_count, + int waited_xid_count, GlobalTransactionId *waited_xids, + int status[]); int GTM_CommitTransactionGXID(GlobalTransactionId gxid); int GTM_PrepareTransaction(GTM_TransactionHandle txn); int GTM_StartPreparedTransaction(GTM_TransactionHandle txn, diff --git a/src/include/pgxc/execRemote.h b/src/include/pgxc/execRemote.h index 3f4d19e858..87fc6fdedc 100644 --- a/src/include/pgxc/execRemote.h +++ b/src/include/pgxc/execRemote.h @@ -50,6 +50,7 @@ extern bool EnforceTwoPhaseCommit; #ifdef XCP #define RESPONSE_ERROR 6 #define RESPONSE_READY 10 +#define RESPONSE_WAITXIDS 11 #endif typedef enum |