Add debug facility to GTM_RWLock

This can be turned on by #define GTM_LOCK_DEBUG and should be useful for
deadlock detections and other such causes

4 files changed, 185 insertions, 16 deletions

diff --git a/src/gtm/common/gtm_lock.c b/src/gtm/common/gtm_lock.c
index 9cc368aafd..53ccd0a463 100644
--- a/src/gtm/common/gtm_lock.c
+++ b/src/gtm/common/gtm_lock.c
@@ -29,15 +29,101 @@ bool
 GTM_RWLockAcquire(GTM_RWLock *lock, GTM_LockMode mode)
 {
 	int status = EINVAL;
+#ifdef GTM_LOCK_DEBUG
+	int indx;
+	int ii;
+#endif
 
 	switch (mode)
 	{
 		case GTM_LOCKMODE_WRITE:
+#ifdef GTM_LOCK_DEBUG
+			pthread_mutex_lock(&lock->lk_debug_mutex);
+			for (ii = 0; ii < lock->rd_holders_count; ii++)
+			{
+				if (pthread_equal(lock->rd_holders[ii], pthread_self()))
+					elog(WARNING, "Thread %p already owns a read-lock and may deadlock",
+							(void *) pthread_self());
+			}
+			if (pthread_equal(lock->wr_owner, pthread_self()))
+				elog(WARNING, "Thread %p already owns a write-lock and may deadlock",
+						(void *) pthread_self());
+			indx = lock->wr_waiters_count;
+			if (indx < GTM_LOCK_DEBUG_MAX_READ_TRACKERS)
+				lock->wr_waiters[lock->wr_waiters_count++] = pthread_self();
+			else
+				indx = -1;
+			pthread_mutex_unlock(&lock->lk_debug_mutex);
+#endif
 			status = pthread_rwlock_wrlock(&lock->lk_lock);
+#ifdef GTM_LOCK_DEBUG
+			if (!status)
+			{
+				pthread_mutex_lock(&lock->lk_debug_mutex);
+				lock->wr_granted = true;
+				lock->wr_owner = pthread_self();
+				lock->rd_holders_count = 0;
+				lock->rd_holders_overflow = false;
+				if (indx != -1)
+				{
+					lock->wr_waiters[indx] = 0;
+					lock->wr_waiters_count--;
+				}
+				pthread_mutex_unlock(&lock->lk_debug_mutex);
+			}
+			else
+				elog(ERROR, "pthread_rwlock_wrlock returned %d", status);
+#endif
 			break;
 
 		case GTM_LOCKMODE_READ:
+#ifdef GTM_LOCK_DEBUG
+			pthread_mutex_lock(&lock->lk_debug_mutex);
+			if (lock->wr_waiters_count > 0)
+			{
+				for (ii = 0; ii < lock->rd_holders_count; ii++)
+				{
+					if (pthread_equal(lock->rd_holders[ii], pthread_self()))
+						elog(WARNING, "Thread %p already owns a read-lock and "
+								"there are blocked writers - this may deadlock",
+									(void *) pthread_self());
+				}
+			}
+			if (pthread_equal(lock->wr_owner, pthread_self()))
+				elog(WARNING, "Thread %p already owns a write-lock and may deadlock",
+						(void *) pthread_self());
+			indx = lock->rd_waiters_count;
+			if (indx < GTM_LOCK_DEBUG_MAX_READ_TRACKERS)
+				lock->rd_waiters[lock->rd_waiters_count++] = pthread_self();
+			else
+				indx = -1;
+			pthread_mutex_unlock(&lock->lk_debug_mutex);
+#endif
+			/* Now acquire the lock */
 			status = pthread_rwlock_rdlock(&lock->lk_lock);
+
+#ifdef GTM_LOCK_DEBUG
+			if (!status)
+			{
+				pthread_mutex_lock(&lock->lk_debug_mutex);
+				lock->wr_granted = false;
+				if (lock->rd_holders_count == GTM_LOCK_DEBUG_MAX_READ_TRACKERS)
+					lock->rd_holders_overflow = true;
+				else
+				{
+					lock->rd_holders[lock->rd_holders_count++] = pthread_self();
+					lock->rd_holders_overflow = false;
+					if (indx != -1)
+					{
+						lock->rd_waiters[indx] = 0;
+						lock->rd_waiters_count--;
+					}
+				}
+				pthread_mutex_unlock(&lock->lk_debug_mutex);
+			}
+			else
+				elog(ERROR, "pthread_rwlock_rdlock returned %d", status);
+#endif
 			break;
 
 		default:
@@ -56,6 +142,42 @@ GTM_RWLockRelease(GTM_RWLock *lock)
 {
 	int status;
 	status = pthread_rwlock_unlock(&lock->lk_lock);
+#ifdef GTM_LOCK_DEBUG
+	if (status)
+		elog(PANIC, "pthread_rwlock_unlock returned %d", status);
+	else
+	{
+		pthread_mutex_lock(&lock->lk_debug_mutex);
+		if (lock->wr_granted)
+		{
+			Assert(pthread_equal(lock->wr_owner, pthread_self()));
+			lock->wr_granted = false;
+			lock->wr_owner = 0;
+		}
+		else
+		{
+			int ii;
+			bool found = false;
+			for (ii = 0; ii < lock->rd_holders_count; ii++)
+			{
+				if (pthread_equal(lock->rd_holders[ii], pthread_self()))
+				{
+					found = true;
+					lock->rd_holders[ii] =
+						lock->rd_holders[lock->rd_holders_count - 1];
+					lock->rd_holders_count--;
+					lock->rd_holders[lock->rd_holders_count] = 0;
+					break;
+				}
+			}
+
+			if (!found && !lock->rd_holders_overflow)
+				elog(PANIC, "Thread %p does not own a read-lock",
+						(void *)pthread_self());
+		}
+		pthread_mutex_unlock(&lock->lk_debug_mutex);
+	}
+#endif
 	return status ? false : true;
 }
 
@@ -65,6 +187,10 @@ GTM_RWLockRelease(GTM_RWLock *lock)
 int
 GTM_RWLockInit(GTM_RWLock *lock)
 {
+#ifdef GTM_LOCK_DEBUG
+	memset(lock, 0, sizeof (GTM_RWLock));
+	pthread_mutex_init(&lock->lk_debug_mutex, NULL);
+#endif
 	return pthread_rwlock_init(&lock->lk_lock, NULL);
 }
 
@@ -92,10 +218,38 @@ GTM_RWLockConditionalAcquire(GTM_RWLock *lock, GTM_LockMode mode)
 	{
 		case GTM_LOCKMODE_WRITE:
 			status = pthread_rwlock_trywrlock(&lock->lk_lock);
+#ifdef GTM_LOCK_DEBUG
+			if (!status)
+			{
+				pthread_mutex_lock(&lock->lk_debug_mutex);
+				lock->wr_granted = true;
+				lock->wr_owner = pthread_self();
+				lock->rd_holders_count = 0;
+				lock->rd_holders_overflow = false;
+				pthread_mutex_unlock(&lock->lk_debug_mutex);
+			}
+#endif
 			break;
 
 		case GTM_LOCKMODE_READ:
 			status = pthread_rwlock_tryrdlock(&lock->lk_lock);
+#ifdef GTM_LOCK_DEBUG
+			if (!status)
+			{
+				pthread_mutex_lock(&lock->lk_debug_mutex);
+				if (lock->rd_holders_count == GTM_LOCK_DEBUG_MAX_READ_TRACKERS)
+				{
+					elog(WARNING, "Too many threads waiting for a read-lock");
+					lock->rd_holders_overflow = true;
+				}
+				else
+				{
+					lock->rd_holders[lock->rd_holders_count++] = pthread_self();
+					lock->rd_holders_overflow = false;
+				}
+				pthread_mutex_unlock(&lock->lk_debug_mutex);
+			}
+#endif
 			break;
 
 		default:
diff --git a/src/gtm/main/gtm_standby.c b/src/gtm/main/gtm_standby.c
index 0455455323..45b545b107 100644
--- a/src/gtm/main/gtm_standby.c
+++ b/src/gtm/main/gtm_standby.c
@@ -47,9 +47,6 @@ gtm_standby_start_startup(void)
 	}
 	elog(LOG, "Connection established to the GTM active.");
 
-	/* Initialize standby lock */
-	Recovery_InitStandbyLock();
-
 	return 1;
 }
 
diff --git a/src/gtm/main/main.c b/src/gtm/main/main.c
index 0f3d89f08c..d122591f32 100644
--- a/src/gtm/main/main.c
+++ b/src/gtm/main/main.c
@@ -213,6 +213,9 @@ InitGTMProcess()
 static void
 BaseInit()
 {
+	/* Initialize standby lock before doing anything else */
+	Recovery_InitStandbyLock();
+
 	checkDataDir();
 	SetDataDir();
 	ChangeToDataDir();
@@ -547,18 +550,6 @@ main(int argc, char *argv[])
 		free(dest_port);
 		dest_port = NULL;
 	}
-	/*
-	 * Check options for the standby mode.
-	 */
-	if (Recovery_IsStandby())
-	{
-		if (active_addr == NULL || active_port < 1)
-		{
-			help(argv[0]);
-			exit(1);
-		}
-		Recovery_StandbySetConnInfo(active_addr, active_port);
-	}
 
 	if (GTMDataDir == NULL)
 	{
@@ -603,6 +594,20 @@ main(int argc, char *argv[])
 	BaseInit();
 
 	/*
+	 * Check options for the standby mode. Do it after StandbyLock has been
+	 * initialised in BaseInit()
+	 */
+	if (Recovery_IsStandby())
+	{
+		if (active_addr == NULL || active_port < 1)
+		{
+			help(argv[0]);
+			exit(1);
+		}
+		Recovery_StandbySetConnInfo(active_addr, active_port);
+	}
+
+	/*
 	 * Establish a connection between the active and standby.
 	 */
 	if (Recovery_IsStandby())
diff --git a/src/include/gtm/gtm_lock.h b/src/include/gtm/gtm_lock.h
index c7f79dd959..26e8faee49 100644
--- a/src/include/gtm/gtm_lock.h
+++ b/src/include/gtm/gtm_lock.h
@@ -16,10 +16,23 @@
 #define GTM_LOCK_H
 
 #include <pthread.h>
-
 typedef struct GTM_RWLock
 {
 	pthread_rwlock_t lk_lock;
+#ifdef GTM_LOCK_DEBUG
+#define GTM_LOCK_DEBUG_MAX_READ_TRACKERS	1024
+	pthread_mutex_t	lk_debug_mutex;
+	int				wr_waiters_count;
+	pthread_t		wr_waiters[GTM_LOCK_DEBUG_MAX_READ_TRACKERS];
+	bool			wr_granted;
+	pthread_t		wr_owner;
+	int				rd_holders_count;
+	bool			rd_holders_overflow;
+	pthread_t		rd_holders[GTM_LOCK_DEBUG_MAX_READ_TRACKERS];
+	int				rd_waiters_count;
+	bool			rd_waiters_overflow;
+	pthread_t		rd_waiters[GTM_LOCK_DEBUG_MAX_READ_TRACKERS];
+#endif
 } GTM_RWLock;
 
 typedef struct GTM_MutexLock