diff options
author | Marko Kreen | 2012-07-17 12:30:04 +0000 |
---|---|---|
committer | Marko Kreen | 2012-07-18 09:35:21 +0000 |
commit | c686e22f6bf0b2d032cd25db7bbfeeafc359bd7c (patch) | |
tree | 2657fdbafc528bc27ec854a3b285ac1e7828eb18 | |
parent | 8dc8b6f5138f2c78b630b82c4308b3a08a99cc29 (diff) |
simple_local_consumer: function caller based on LocalConsumer
Consumer that runs a SQL for each event.
-rw-r--r-- | debian/skytools3.install | 1 | ||||
-rwxr-xr-x | scripts/simple_local_consumer.py | 66 | ||||
-rwxr-xr-x | setup_skytools.py | 1 |
3 files changed, 68 insertions, 0 deletions
diff --git a/debian/skytools3.install b/debian/skytools3.install index b7012149..7c6145d5 100644 --- a/debian/skytools3.install +++ b/debian/skytools3.install @@ -4,6 +4,7 @@ usr/bin/scriptmgr3 usr/bin/queue_mover3 usr/bin/queue_splitter3 usr/bin/simple_consumer3 +usr/bin/simple_local_consumer3 debian/skytools.ini /etc usr/lib/python*/site-packages/skytools-3.0/londiste usr/share/skytools3 diff --git a/scripts/simple_local_consumer.py b/scripts/simple_local_consumer.py new file mode 100755 index 00000000..6e3eb601 --- /dev/null +++ b/scripts/simple_local_consumer.py @@ -0,0 +1,66 @@ +#!/usr/bin/python + +"""Consumer that simply calls SQL query for each event. + +It tracks completed batches in local file. + +Config:: + # source database + src_db = + + # destination database + dst_db = + + # query to call + dst_query = select * from somefunc(%%(pgq.ev_data)s); +""" + + +import sys + +import pkgloader +pkgloader.require('skytools', '3.0') + +import pgq +import skytools + +class SimpleLocalConsumer(pgq.LocalConsumer): + __doc__ = __doc__ + + def reload(self): + super(SimpleLocalConsumer, self).reload() + self.dst_query = self.cf.get("dst_query") + + def process_local_event(self, db, batch_id, ev): + curs = self.get_database('dst_db', autocommit = 1).cursor() + + if ev.ev_type[:2] not in ('I:', 'U:', 'D:'): + return + + if ev.ev_data is None: + payload = {} + else: + payload = skytools.db_urldecode(ev.ev_data) + + payload['pgq.tick_id'] = self.batch_info['cur_tick_id'] + payload['pgq.ev_id'] = ev.ev_id + payload['pgq.ev_time'] = ev.ev_time + payload['pgq.ev_type'] = ev.ev_type + payload['pgq.ev_data'] = ev.ev_data + payload['pgq.ev_extra1'] = ev.ev_extra1 + payload['pgq.ev_extra2'] = ev.ev_extra2 + payload['pgq.ev_extra3'] = ev.ev_extra3 + payload['pgq.ev_extra4'] = ev.ev_extra4 + + self.log.debug(self.dst_query, payload) + curs.execute(self.dst_query, payload) + if curs.statusmessage[:6] == 'SELECT': + res = curs.fetchall() + self.log.debug(res) + else: + self.log.debug(curs.statusmessage) + +if __name__ == '__main__': + script = SimpleLocalConsumer("simple_local_consumer3", "src_db", sys.argv[1:]) + script.start() + diff --git a/setup_skytools.py b/setup_skytools.py index 0e4a65c8..5440068d 100755 --- a/setup_skytools.py +++ b/setup_skytools.py @@ -38,6 +38,7 @@ sfx_scripts = [ 'scripts/queue_splitter.py', 'scripts/queue_mover.py', 'scripts/simple_consumer.py', + 'scripts/simple_local_consumer.py', 'scripts/skytools_upgrade.py', ] # those do not need suffix (no conflict with 2.1) |