path: root/src/gtm/proxy/proxy_thread.c

/*-------------------------------------------------------------------------
 *
 * proxy_thread.c
 *
 * Portions Copyright (c) 1996-2009, PostgreSQL Global Development Group
 * Portions Copyright (c) 1994, Regents of the University of California
 * Portions Copyright (c) 2010-2012 Postgres-XC Development Group
 *
 *
 * IDENTIFICATION
 *	  $PostgreSQL$
 *
 *-------------------------------------------------------------------------
 */
#include <pthread.h>
#include "gtm/gtm_proxy.h"
#include "gtm/memutils.h"
#include "gtm/libpq.h"

static void *GTMProxy_ThreadMainWrapper(void *argp);
static void GTMProxy_ThreadCleanup(void *argp);

GTMProxy_Threads	GTMProxyThreadsData;
GTMProxy_Threads *GTMProxyThreads = &GTMProxyThreadsData;

#define GTM_PROXY_MIN_THREADS 32			/* Provision for minimum threads */
#define GTM_PROXY_MAX_THREADS 1024		/* Max threads allowed in the GTMProxy */
#define GTMProxyThreadsFull	(GTMProxyThreads->gt_thread_count == GTMProxyThreads->gt_array_size)

extern int GTMProxyWorkerThreads;
extern GTMProxy_ThreadInfo **Proxy_ThreadInfo;

/*
 * Add the given thrinfo structure to the global array, expanding it if
 * necessary
 */
int
GTMProxy_ThreadAdd(GTMProxy_ThreadInfo *thrinfo)
{
	int ii;

	GTM_RWLockAcquire(&GTMProxyThreads->gt_lock, GTM_LOCKMODE_WRITE);

	if (GTMProxyThreadsFull)
	{
		GTMProxy_ThreadInfo **threads;
		uint32 newsize;

		/*
		 * TODO Optimize lock management by not holding any locks during memory
		 * allocation
		 */
		if (GTMProxyThreads->gt_array_size == GTM_PROXY_MAX_THREADS)
		{
			GTM_RWLockRelease(&GTMProxyThreads->gt_lock);
			elog(ERROR, "Too many threads active");
		}

		if (GTMProxyThreads->gt_array_size == 0)
			newsize = GTM_PROXY_MIN_THREADS;
		else
		{
			/*
			 * We ran out of the array size. Just double the size, bound by the
			 * upper limit
			 */
			newsize = GTMProxyThreads->gt_array_size * 2;
		}

		/* Can't have more than GTM_PROXY_MAX_THREADS */
		if (newsize > GTM_PROXY_MAX_THREADS)
			newsize = GTM_PROXY_MAX_THREADS;

		if (GTMProxyThreads->gt_threads == NULL)
			threads = (GTMProxy_ThreadInfo **)palloc0(sizeof (GTMProxy_ThreadInfo *) * newsize);
		else
		{
			void *old_ptr = GTMProxyThreads->gt_threads;
			threads = (GTMProxy_ThreadInfo **)palloc0(sizeof (GTMProxy_ThreadInfo *) * newsize);
			memcpy(threads, old_ptr,
					GTMProxyThreads->gt_array_size * sizeof (GTMProxy_ThreadInfo *));
			pfree(old_ptr);
		}

		GTMProxyThreads->gt_threads = threads;
		GTMProxyThreads->gt_array_size = newsize;
	}

	/*
	 * Now that we have free entries in the array, find a free slot and add the
	 * thrinfo pointer to it.
	 *
	 * TODO Optimize this later by tracking few free slots and reusing them.
	 * The free slots can be updated when a thread exits and reused when a new
	 * thread is added to the pool.
	 */
	for (ii = 0; ii < GTMProxyThreads->gt_array_size; ii++)
	{
		if (GTMProxyThreads->gt_threads[ii] == NULL)
		{
			GTMProxyThreads->gt_threads[ii] = thrinfo;
			GTMProxyThreads->gt_thread_count++;
			break;
		}
	}
	GTM_RWLockRelease(&GTMProxyThreads->gt_lock);

	/*
	 * Track the slot information in the thrinfo. This is useful to quickly
	 * find the slot given the thrinfo structure.
	 */
	thrinfo->thr_localid = ii;
	return ii;
}

int
GTMProxy_ThreadRemove(GTMProxy_ThreadInfo *thrinfo)
{
	/*
	 * XXX To be implemeneted
	 */
	return 0;
}

/*
 * Create a new thread and assign the given connection to it.
 *
 * This function is responsible for setting up the various memory contextes for
 * the thread as well as registering this thread with the Thread Manager.
 *
 * Upon successful creation, the thread will start running the given
 * "startroutine". The thread information is returned to the calling process.
 */
GTMProxy_ThreadInfo *
GTMProxy_ThreadCreate(void *(* startroutine)(void *), int idx)
{
	GTMProxy_ThreadInfo *thrinfo;
	int err;

	thrinfo = (GTMProxy_ThreadInfo *)malloc(sizeof (GTMProxy_ThreadInfo));
	memset(thrinfo, 0, sizeof (GTMProxy_ThreadInfo));

	GTM_MutexLockInit(&thrinfo->thr_lock);
	GTM_CVInit(&thrinfo->thr_cv);

	/*
	 * Initialize communication area with SIGUSR2 signal handler (reconnect)
	 */
	Proxy_ThreadInfo[idx] = thrinfo;

	/*
	 * The thread status is set to GTM_PROXY_THREAD_STARTING and will be changed by
	 * the thread itself when it actually starts executing
	 */
	thrinfo->thr_status = GTM_PROXY_THREAD_STARTING;

	/*
	 * Install the ThreadInfo structure in the global array. We do this before
	 * starting the thread
	 */
	if (GTMProxy_ThreadAdd(thrinfo) == -1)
		elog(ERROR, "Error starting a new thread");

	/*
	 * Set up memory contextes before actually starting the threads
	 *
	 * The TopThreadContext is a child of TopMemoryContext and it will last as
	 * long as the main process or this thread lives
	 *
	 * Thread context is not shared between other threads
	 */
	thrinfo->thr_thread_context = AllocSetContextCreate(TopMemoryContext,
														"TopMemoryContext",
														ALLOCSET_DEFAULT_MINSIZE,
														ALLOCSET_DEFAULT_INITSIZE,
														ALLOCSET_DEFAULT_MAXSIZE,
														false);

	/*
	 * Since the thread is not yes started, TopMemoryContext still points to
	 * the context of the calling thread
	 */
	thrinfo->thr_parent_context = TopMemoryContext;

	/*
	 * Each thread gets its own ErrorContext and its a child of ErrorContext of
	 * the main process
	 *
	 * This is a thread-specific context and is not shared between other
	 * threads
	 */
	thrinfo->thr_error_context = AllocSetContextCreate(ErrorContext,
													   "ErrorContext",
													   8 * 1024,
													   8 * 1024,
													   8 * 1024,
													   false);

	thrinfo->thr_startroutine = startroutine;

	/*
	 * Now start the thread. The thread will start executing the given
	 * "startroutine". The thrinfo structure is also passed to the thread. Any
	 * additional parameters should be passed via the thrinfo strcuture.
	 *
	 * Return the thrinfo structure to the caller
	 */
	if ((err = pthread_create(&thrinfo->thr_id, NULL, GTMProxy_ThreadMainWrapper,
							 thrinfo)))
		ereport(ERROR,
				(err,
				 errmsg("Failed to create a new thread: error %d", err)));

	/*
	 * Prepare to reclaim resources used by the thread once it exits. (We used
	 * to do this inside GTMProxy_ThreadMainWrapper, but its not clear if
	 * thrinfo->thr_id will be set by the time the routine starts executing.
	 * Its safer to do that here instead
	 */
	pthread_detach(thrinfo->thr_id);

	return thrinfo;
}

/*
 * Exit the current thread
 */
void
GTMProxy_ThreadExit(void)
{
	/* XXX To be implemented */
}

int
GTMProxy_ThreadJoin(GTMProxy_ThreadInfo *thrinfo)
{
	int error;
	void *data;

	error = pthread_join(thrinfo->thr_id, &data);

	return error;
}

/*
 * Get thread information for the given thread, identified by the
 * thread_id
 */
GTMProxy_ThreadInfo *
GTMProxy_GetThreadInfo(GTM_ThreadID thrid)
{

	return NULL;
}

/*
 * Cleanup routine for the thread
 */
static void
GTMProxy_ThreadCleanup(void *argp)
{
	GTMProxy_ThreadInfo *thrinfo = (GTMProxy_ThreadInfo *)argp;

	elog(DEBUG1, "Cleaning up thread state");

	/* Release any currently held mutex and rwlocks */
	GTM_MutexLockReleaseAll();
	GTM_RWLockReleaseAll();

	/*
	 * TODO Close the open connection.
	 */
	StreamClose(thrinfo->thr_conn->con_port->sock);

	/*
	 * Switch to the memory context of the main process so that we can free up
	 * our memory contextes easily.
	 *
	 * XXX We don't setup cleanup handlers for the main process. So this
	 * routine would never be called for the main process/thread
	 */
	MemoryContextSwitchTo(thrinfo->thr_parent_context);

	MemoryContextDelete(thrinfo->thr_message_context);
	thrinfo->thr_message_context = NULL;

	MemoryContextDelete(thrinfo->thr_error_context);
	thrinfo->thr_error_context = NULL;

	MemoryContextDelete(thrinfo->thr_thread_context);
	thrinfo->thr_thread_context = NULL;

	/*
	 * TODO Now cleanup the thrinfo structure itself and remove it from the global
	 * array.
	 */


	/*
	 * Reset the thread-specific information. This should be done only after we
	 * are sure that memory contextes are not required
	 *
	 * Note: elog calls need memory contextes, so no elog calls beyond this
	 * point.
	 */
	SetMyThreadInfo(NULL);

	return;
}

/*
 * A wrapper around the start routine of the thread. This helps us doing any
 * initialization and setting up cleanup handlers before the main routine is
 * started
 */
void *
GTMProxy_ThreadMainWrapper(void *argp)
{
	GTMProxy_ThreadInfo *thrinfo = (GTMProxy_ThreadInfo *)argp;

	pthread_detach(thrinfo->thr_id);

	SetMyThreadInfo(thrinfo);
	MemoryContextSwitchTo(TopMemoryContext);

	pthread_cleanup_push(GTMProxy_ThreadCleanup, thrinfo);
	thrinfo->thr_startroutine(thrinfo);
	pthread_cleanup_pop(1);

	return thrinfo;
}

/*
 * Add the given connection info structure to a thread which is selected by a
 * round-robin manner. The caller is responsible for only accepting the
 * connection. Other things including the authentication is done by the worker
 * thread when it finds a new entry in the connection list.
 *
 * Return the reference to the GTMProxy_ThreadInfo structure of the thread
 * which will be serving this connection
 */
GTMProxy_ThreadInfo *
GTMProxy_ThreadAddConnection(GTMProxy_ConnectionInfo *conninfo)
{
	GTMProxy_ThreadInfo *thrinfo = NULL;
	GTMProxy_ConnID connIndx, ii;

	/*
	 * Get the next thread in the queue
	 */
	GTM_RWLockAcquire(&GTMProxyThreads->gt_lock, GTM_LOCKMODE_WRITE);

	/*
	 * Always start with thread 1 because thread 0 is the main thread
	 */
	if (GTMProxyThreads->gt_next_worker == 0)
		GTMProxyThreads->gt_next_worker = 1;

	thrinfo = GTMProxyThreads->gt_threads[GTMProxyThreads->gt_next_worker];

	/*
	 * Set the next worker thread before releasing the lock
	 */
	GTMProxyThreads->gt_next_worker++;
	if (GTMProxyThreads->gt_next_worker == GTMProxyThreads->gt_thread_count)
	   GTMProxyThreads->gt_next_worker = 1;

	GTM_RWLockRelease(&GTMProxyThreads->gt_lock);

	/*
	 * Lock the threadninfo structure to safely add the new connection to the
	 * thread structure. The thread will see the connection when it queries the
	 * socket descriptor in the next cycle
	 */
	GTM_MutexLockAcquire(&thrinfo->thr_lock);

	if (thrinfo->thr_conn_count >= GTM_PROXY_MAX_CONNECTIONS)
	{
		GTM_MutexLockRelease(&thrinfo->thr_lock);
		elog(LOG, "Too many connections");
		return NULL;
	}

	connIndx = -1;
	for (ii = 0; ii < GTM_PROXY_MAX_CONNECTIONS; ii++)
	{
		if (thrinfo->thr_all_conns[ii] == NULL)
		{
			/*
			 * Great, found a free slot to track the connection
			 */
			connIndx = ii;
			break;
		}
	}

	if (connIndx == -1)
	{
		GTM_MutexLockRelease(&thrinfo->thr_lock);
		elog(LOG, "Too many connections - could not find a free slot");
		return NULL;
	}

	/*
	 * Save the array slotid in the conninfo structure. We send this to the GTM
	 * server as an identifier which the GTM server sends us back in the
	 * response. We use that information to route the response back to the
	 * approrpiate connection.
	 *
	 * Note that the reason to use the array slotid in the messages to/from GTM
	 * is to ensure that the corresponding connection can be quickly found
	 * while proxying responses back to the client.
	 */
	conninfo->con_id = connIndx;
	thrinfo->thr_all_conns[connIndx] = conninfo;

	/*
	 * We also maintain a map of currently used array slots in a separate data
	 * structure. This allows us to quickly iterate through all open
	 * connections servred by a thread. So while iterating through all open
	 * connections, the correct mechanism would be something as follow:
	 *
	 * for (ii = 0; ii < thrinfo->thr_conn_count; ii++)
	 * {
	 * 		int connIndx = thrinfo->thr_conn_map[ii];
	 * 	 	GTMProxy_ConnectionInfo *conninfo = * 	 	thrinfo->thr_all_conns[connIndx];	
	 * 	 	.....
	 * }
	 */  
	thrinfo->thr_conn_map[thrinfo->thr_conn_count] = connIndx;
	thrinfo->thr_conn_count++;

	/*
	 * Now increment the seqno since a new connection is added to the array.
	 * Before we do the next poll(), the fd array will be forced to be
	 * reconstructed.
	 */
   	thrinfo->thr_seqno++;

	/*
	 * Signal the worker thread if its waiting for connections to be added to
	 * its Q
	 *
	 * XXX May be we can first check the condition that this is the first
	 * connection in the array and also use signal instead of a bcast since
	 * only one thread is waiting on the cv.
	 */
	GTM_CVBcast(&thrinfo->thr_cv);
	GTM_MutexLockRelease(&thrinfo->thr_lock);

	return thrinfo;
}

/*
 * Remove the connection from the array and compact the array
 */
int
GTMProxy_ThreadRemoveConnection(GTMProxy_ThreadInfo *thrinfo, GTMProxy_ConnectionInfo *conninfo)
{
	int ii;
	int connIndx;

	/*
	 * Lock the threadninfo structure to safely remove the connection from the
	 * thread structure.
	 */
	GTM_MutexLockAcquire(&thrinfo->thr_lock);

	connIndx = -1;
	for (ii = 0; ii < GTM_PROXY_MAX_CONNECTIONS; ii++)
	{
		if (thrinfo->thr_all_conns[ii] == conninfo)
		{
			connIndx = ii;
			break;
		}
	}

	if (connIndx == -1)
	{
		GTM_MutexLockRelease(&thrinfo->thr_lock);
		elog(ERROR, "No such connection");
	}

	/*
	 * Reset command backup info
	 */
	thrinfo->thr_any_backup[connIndx] = FALSE;
	thrinfo->thr_qtype[connIndx] = 0;
	resetStringInfo(&(thrinfo->thr_inBufData[connIndx]));
	thrinfo->thr_all_conns[connIndx] = NULL;

	/*
	 * Now also removed the entry from thr_conn_map
	 */
	for (ii = 0; ii < thrinfo->thr_conn_count; ii++)
	{
		if (thrinfo->thr_conn_map[ii] == connIndx)
			break;
	}

	if (ii >= thrinfo->thr_conn_count)
	{
		GTM_MutexLockRelease(&thrinfo->thr_lock);
		elog(FATAL, "Failed to find connection mapping to %d", connIndx);
	}

	/*
	 * If this is the last entry in the array ? If not, then copy the last
	 * entry in this slot and mark the last slot an empty
	 */
	if ((ii + 1) < thrinfo->thr_conn_count)
	{
		/* Copy the last entry in this slot */
		thrinfo->thr_conn_map[ii] = thrinfo->thr_conn_map[thrinfo->thr_conn_count - 1];

		/* Mark the last slot free */
		thrinfo->thr_conn_map[thrinfo->thr_conn_count - 1] = -1;
	}
	else
	{
		/* This is the last entry in the array. Just mark it free */
		thrinfo->thr_conn_map[ii] = -1;
	}

	thrinfo->thr_conn_count--;

	/*
	 * Increment the seqno to ensure that the next time before we poll, the fd
	 * array is reconstructed.
	 */
	thrinfo->thr_seqno++;
	GTM_MutexLockRelease(&thrinfo->thr_lock);

	return 0;
}