1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
|
#! /usr/bin/env python
"""simple serial consumer for skytools3
it consumes events from a predefined queue and feeds them to a sql statement
Config template::
[simple_serial_consumer]
job_name = descriptive_name_for_job
src_db = dbname=sourcedb_test
dst_db = dbname=destdb port=1234 host=dbhost.com username=guest password=secret
pgq_queue_name = source_queue
logfile = ~/log/%(job_name)s.log
pidfile = ~/pid/%(job_name)s.pid
dst_query = select 1
use_skylog = 0
"""
"""Config example::
Create a queue named "echo_queue" in a database (like "testdb")
Register consumer "echo" to this queue
Start the echo consumer with config file shown below
(You may want to use -v to see, what will happen)
From some other window, insert something into the queue:
select pgq.insert_event('echo_queue','type','hello=world');
Enjoy the ride :)
If dst_query is set to "select 1" then echo consumer becomes a sink consumer
[simple_serial_consumer]
job_name = echo
src_db = dbname=testdb
dst_db = dbname=testdb
pgq_queue_name = echo_queue
logfile = ~/log/%(job_name)s.log
pidfile = ~/pid/%(job_name)s.pid
dst_query =
select *
from pgq.insert_event('echo_queue', %%(pgq.ev_type)s, %%(pgq.ev_data)s)
"""
import sys, pgq, skytools
skytools.sane_config = 1
class SimpleSerialConsumer(pgq.SerialConsumer):
doc_string = __doc__
def __init__(self, args):
pgq.SerialConsumer.__init__(self,"simple_serial_consumer","src_db","dst_db", args)
self.dst_query = self.cf.get("dst_query")
def process_remote_batch(self, db, batch_id, event_list, dst_db):
curs = dst_db.cursor()
for ev in event_list:
payload = skytools.db_urldecode(ev.data)
if payload is None:
payload = {}
payload['pgq.ev_type'] = ev.type
payload['pgq.ev_data'] = ev.data
payload['pgq.ev_id'] = ev.id
payload['pgq.ev_time'] = ev.time
payload['pgq.ev_extra1'] = ev.extra1
payload['pgq.ev_extra2'] = ev.extra2
payload['pgq.ev_extra3'] = ev.extra3
payload['pgq.ev_extra4'] = ev.extra4
self.log.debug(self.dst_query % payload)
curs.execute(self.dst_query, payload)
try:
res = curs.fetchone()
self.log.debug(res)
except:
pass
if __name__ == '__main__':
script = SimpleSerialConsumer(sys.argv[1:])
script.start()
|
