summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMarko Kreen2012-07-17 12:30:04 +0000
committerMarko Kreen2012-07-18 09:35:21 +0000
commitc686e22f6bf0b2d032cd25db7bbfeeafc359bd7c (patch)
tree2657fdbafc528bc27ec854a3b285ac1e7828eb18
parent8dc8b6f5138f2c78b630b82c4308b3a08a99cc29 (diff)
simple_local_consumer: function caller based on LocalConsumer
Consumer that runs a SQL for each event.
-rw-r--r--debian/skytools3.install1
-rwxr-xr-xscripts/simple_local_consumer.py66
-rwxr-xr-xsetup_skytools.py1
3 files changed, 68 insertions, 0 deletions
diff --git a/debian/skytools3.install b/debian/skytools3.install
index b7012149..7c6145d5 100644
--- a/debian/skytools3.install
+++ b/debian/skytools3.install
@@ -4,6 +4,7 @@ usr/bin/scriptmgr3
usr/bin/queue_mover3
usr/bin/queue_splitter3
usr/bin/simple_consumer3
+usr/bin/simple_local_consumer3
debian/skytools.ini /etc
usr/lib/python*/site-packages/skytools-3.0/londiste
usr/share/skytools3
diff --git a/scripts/simple_local_consumer.py b/scripts/simple_local_consumer.py
new file mode 100755
index 00000000..6e3eb601
--- /dev/null
+++ b/scripts/simple_local_consumer.py
@@ -0,0 +1,66 @@
+#!/usr/bin/python
+
+"""Consumer that simply calls SQL query for each event.
+
+It tracks completed batches in local file.
+
+Config::
+ # source database
+ src_db =
+
+ # destination database
+ dst_db =
+
+ # query to call
+ dst_query = select * from somefunc(%%(pgq.ev_data)s);
+"""
+
+
+import sys
+
+import pkgloader
+pkgloader.require('skytools', '3.0')
+
+import pgq
+import skytools
+
+class SimpleLocalConsumer(pgq.LocalConsumer):
+ __doc__ = __doc__
+
+ def reload(self):
+ super(SimpleLocalConsumer, self).reload()
+ self.dst_query = self.cf.get("dst_query")
+
+ def process_local_event(self, db, batch_id, ev):
+ curs = self.get_database('dst_db', autocommit = 1).cursor()
+
+ if ev.ev_type[:2] not in ('I:', 'U:', 'D:'):
+ return
+
+ if ev.ev_data is None:
+ payload = {}
+ else:
+ payload = skytools.db_urldecode(ev.ev_data)
+
+ payload['pgq.tick_id'] = self.batch_info['cur_tick_id']
+ payload['pgq.ev_id'] = ev.ev_id
+ payload['pgq.ev_time'] = ev.ev_time
+ payload['pgq.ev_type'] = ev.ev_type
+ payload['pgq.ev_data'] = ev.ev_data
+ payload['pgq.ev_extra1'] = ev.ev_extra1
+ payload['pgq.ev_extra2'] = ev.ev_extra2
+ payload['pgq.ev_extra3'] = ev.ev_extra3
+ payload['pgq.ev_extra4'] = ev.ev_extra4
+
+ self.log.debug(self.dst_query, payload)
+ curs.execute(self.dst_query, payload)
+ if curs.statusmessage[:6] == 'SELECT':
+ res = curs.fetchall()
+ self.log.debug(res)
+ else:
+ self.log.debug(curs.statusmessage)
+
+if __name__ == '__main__':
+ script = SimpleLocalConsumer("simple_local_consumer3", "src_db", sys.argv[1:])
+ script.start()
+
diff --git a/setup_skytools.py b/setup_skytools.py
index 0e4a65c8..5440068d 100755
--- a/setup_skytools.py
+++ b/setup_skytools.py
@@ -38,6 +38,7 @@ sfx_scripts = [
'scripts/queue_splitter.py',
'scripts/queue_mover.py',
'scripts/simple_consumer.py',
+ 'scripts/simple_local_consumer.py',
'scripts/skytools_upgrade.py',
]
# those do not need suffix (no conflict with 2.1)