diff options
author | Marko Kreen | 2012-01-31 08:58:39 +0000 |
---|---|---|
committer | Marko Kreen | 2012-01-31 08:58:39 +0000 |
commit | 7c1861e7bbfde69da0904b68d38720786329af41 (patch) | |
tree | 712d7bb7af9263c0f1983cb7c1a3d59e0e089853 | |
parent | da4a0bc388a8ff45cee1f3481812cbfd6066c2e5 (diff) |
--sync-watermark: limit watermark to specific nodes
This will allow specified nodes to sync global watermark
with each other, without sending it upwards.
This isolates root node from lag on downstream nodes.
-rwxr-xr-x | python/londiste.py | 2 | ||||
-rw-r--r-- | python/pgq/cascade/admin.py | 13 | ||||
-rw-r--r-- | python/pgq/cascade/nodeinfo.py | 12 | ||||
-rw-r--r-- | python/pgq/cascade/worker.py | 97 | ||||
-rw-r--r-- | sql/pgq_node/expected/pgq_node_test.out | 48 | ||||
-rw-r--r-- | sql/pgq_node/functions/pgq_node.get_node_info.sql | 11 | ||||
-rw-r--r-- | sql/pgq_node/functions/pgq_node.set_node_attrs.sql | 35 | ||||
-rw-r--r-- | sql/pgq_node/functions/pgq_node.upgrade_schema.sql | 10 | ||||
-rw-r--r-- | sql/pgq_node/sql/pgq_node_test.sql | 2 | ||||
-rw-r--r-- | sql/pgq_node/structure/functions.sql | 1 | ||||
-rw-r--r-- | sql/pgq_node/structure/tables.sql | 2 | ||||
-rwxr-xr-x | tests/londiste/regen.sh | 6 |
12 files changed, 204 insertions, 35 deletions
diff --git a/python/londiste.py b/python/londiste.py index 5ce717b9..b7bc99e0 100755 --- a/python/londiste.py +++ b/python/londiste.py @@ -97,6 +97,8 @@ class Londiste(skytools.DBScript): help = "takeover: old node was root") g.add_option("--dead-branch", action = 'store_true', help = "takeover: old node was branch") + g.add_option("--sync-watermark", + help = "create-branch: list of node names to sync wm with") p.add_option_group(g) g = optparse.OptionGroup(p, "repair queue position") g.add_option("--rewind", action = "store_true", diff --git a/python/pgq/cascade/admin.py b/python/pgq/cascade/admin.py index 48f4e43a..964af081 100644 --- a/python/pgq/cascade/admin.py +++ b/python/pgq/cascade/admin.py @@ -94,6 +94,8 @@ class CascadeAdmin(skytools.AdminScript): help = "tag some node as dead") g.add_option("--dead-branch", action="store_true", help = "tag some node as dead") + g.add_option("--sync-watermark", + help = "list of node names to sync with") p.add_option_group(g) return p @@ -147,6 +149,7 @@ class CascadeAdmin(skytools.AdminScript): return self.log.info("Initializing node") + node_attrs = {} worker_name = self.options.worker if not worker_name: @@ -155,6 +158,11 @@ class CascadeAdmin(skytools.AdminScript): if combined_queue and node_type != 'leaf': raise Exception('--merge can be used only for leafs') + if self.options.sync_watermark: + if node_type != 'branch': + raise UsageError('--sync-watermark can be used only for branch nodes') + node_attrs['sync_watermark'] = self.options.sync_watermark + # register member if node_type == 'root': global_watermark = None @@ -221,6 +229,11 @@ class CascadeAdmin(skytools.AdminScript): self.extra_init(node_type, db, provider_db) + if node_attrs: + s_attrs = skytools.db_urlencode(node_attrs) + self.exec_cmd(db, "select * from pgq_node.set_node_attrs(%s, %s)", + [self.queue_name, s_attrs]) + self.log.info("Done") def extra_init(self, node_type, node_db, provider_db): diff --git a/python/pgq/cascade/nodeinfo.py b/python/pgq/cascade/nodeinfo.py index 48dea4b2..726b311e 100644 --- a/python/pgq/cascade/nodeinfo.py +++ b/python/pgq/cascade/nodeinfo.py @@ -6,6 +6,7 @@ __all__ = ['MemberInfo', 'NodeInfo', 'QueueInfo'] import datetime +import skytools # node types ROOT = 'root' @@ -49,6 +50,7 @@ class NodeInfo: combined_queue = None combined_type = None last_tick = None + node_attrs = {} def __init__(self, queue_name, row, main_worker = True, node_name = None): self.queue_name = queue_name @@ -83,6 +85,12 @@ class NodeInfo: self.combined_type = row['combined_type'] self.last_tick = row['worker_last_tick'] + self.node_attrs = {} + if 'node_attrs' in row: + a = row['node_attrs'] + if a: + self.node_attrs = skytools.db_urldecode(a) + def __get_target_queue(self): qname = None if self.type == LEAF: @@ -129,6 +137,10 @@ class NodeInfo: txt += ", NOT UPTODATE" lst.append(txt) + for k, v in self.node_attrs.items(): + txt = "Attr: %s=%s" % (k, v) + lst.append(txt) + for cname, row in self.cascaded_consumer_map.items(): err = row['cur_error'] if err: diff --git a/python/pgq/cascade/worker.py b/python/pgq/cascade/worker.py index 1d3f8325..a721eaa0 100644 --- a/python/pgq/cascade/worker.py +++ b/python/pgq/cascade/worker.py @@ -4,10 +4,11 @@ CascadedConsumer that also maintains node. """ -import sys, time +import sys, time, skytools from pgq.cascade.consumer import CascadedConsumer from pgq.producer import bulk_insert_events +from pgq.event import Event __all__ = ['CascadedWorker'] @@ -32,10 +33,20 @@ class WorkerState: filtered_copy = 0 # ok process_global_wm = 0 # ok + sync_watermark = 0 # ? + wm_sync_nodes = [] + def __init__(self, queue_name, nst): self.node_type = nst['node_type'] self.node_name = nst['node_name'] self.local_watermark = nst['local_watermark'] + self.global_watermark = nst['global_watermark'] + + self.node_attrs = {} + attrs = nst.get('node_attrs', '') + if attrs: + self.node_attrs = skytools.db_urldecode(attrs) + ntype = nst['node_type'] ctype = nst['combined_type'] if ntype == 'root': @@ -49,7 +60,12 @@ class WorkerState: self.process_tick_event = 1 self.keep_event_ids = 1 self.create_tick = 1 - self.process_global_wm = 1 + if 'sync_watermark' in self.node_attrs: + slist = self.node_attrs['sync_watermark'] + self.sync_watermark = 1 + self.wm_sync_nodes = slist.split(',') + else: + self.process_global_wm = 1 elif ntype == 'leaf' and not ctype: self.process_batch = 1 self.process_events = 1 @@ -139,8 +155,6 @@ class CascadedWorker(CascadedConsumer): self.process_remote_event(src_curs, dst_curs, ev) if ev.ev_id > max_id: max_id = ev.ev_id - if st.local_wm_publish: - self.publish_local_wm(src_db) if max_id > self.cur_max_id: self.cur_max_id = max_id @@ -195,22 +209,72 @@ class CascadedWorker(CascadedConsumer): self.create_branch_tick(dst_db, cur_tick, tick_time) return True - def publish_local_wm(self, src_db): + def publish_local_wm(self, src_db, dst_db): """Send local watermark to provider. """ - if not self.main_worker: - return + t = time.time() if t - self.local_wm_publish_time < self.local_wm_publish_period: return st = self._worker_state - self.log.debug("Publishing local watermark: %d" % st.local_watermark) + wm = st.local_watermark + if st.sync_watermark: + # dont send local watermark upstream + wm = self.batch_info['prev_tick_id'] + + self.log.debug("Publishing local watermark: %d" % wm) src_curs = src_db.cursor() q = "select * from pgq_node.set_subscriber_watermark(%s, %s, %s)" - src_curs.execute(q, [self.pgq_queue_name, st.node_name, st.local_watermark]) + src_curs.execute(q, [self.pgq_queue_name, st.node_name, wm]) + + # if last part fails, dont repeat it immediately self.local_wm_publish_time = t + if st.sync_watermark: + # instead sync 'global-watermark' with specific nodes + dst_curs = dst_db.cursor() + nmap = self._get_node_map(dst_curs) + dst_db.commit() + + wm = st.local_watermark + for node in st.wm_sync_nodes: + if node == st.node_name: + continue + if node not in nmap: + # dont ignore missing nodes - cluster may be partially set up + self.log.warning('Unknown node in sync_watermark list: %s' % node) + return + n = nmap[node] + if n['dead']: + # ignore dead nodes + continue + wmdb = self.get_database('wmdb', connstr = n['node_location'], autocommit = 1) + wmcurs = wmdb.cursor() + q = 'select local_watermark from pgq_node.get_node_info(%s)' + wmcurs.execute(q, [self.queue_name]) + row = wmcurs.fetchone() + if not row: + # partially set up node? + self.log.warning('Node not working: %s' % node) + elif row['local_watermark'] < wm: + # keep lowest wm + wm = row['local_watermark'] + self.close_database('wmdb') + + # now we have lowest wm, store it + q = "select pgq_node.set_global_watermark(%s, %s)" + dst_curs.execute(q, [self.queue_name, wm]) + dst_db.commit() + + def _get_node_map(self, curs): + q = "select node_name, node_location, dead from pgq_node.get_queue_locations(%s)" + curs.execute(q, [self.queue_name]) + res = {} + for row in curs.fetchall(): + res[row['node_name']] = row + return res + def process_remote_event(self, src_curs, dst_curs, ev): """Handle cascading events. """ @@ -245,7 +309,10 @@ class CascadedWorker(CascadedConsumer): q = "select * from pgq_node.unregister_location(%s, %s)" dst_curs.execute(q, [self.pgq_queue_name, node]) elif t == "pgq.global-watermark": - if st.process_global_wm: + if st.sync_watermark: + tick_id = int(ev.ev_data) + self.log.info('Ignoring global watermark %s' % tick_id) + elif st.process_global_wm: tick_id = int(ev.ev_data) q = "select * from pgq_node.set_global_watermark(%s, %s)" dst_curs.execute(q, [self.pgq_queue_name, tick_id]) @@ -288,6 +355,8 @@ class CascadedWorker(CascadedConsumer): tick_id = self.batch_info['tick_id'] tick_time = self.batch_info['batch_end'] self.create_branch_tick(dst_db, tick_id, tick_time) + if st.local_wm_publish: + self.publish_local_wm(src_db, dst_db) def create_branch_tick(self, dst_db, tick_id, tick_time): q = "select pgq.ticker(%s, %s, %s, %s)" @@ -308,6 +377,14 @@ class CascadedWorker(CascadedConsumer): return if len(self.ev_buf) >= self.max_evbuf: self.flush_events(dst_curs) + + if ev.type == 'pgq.global-watermark': + st = self._worker_state + if st.sync_watermark: + # replace payload with synced global watermark + row = ev._event_row.copy() + row['ev_data'] = str(st.global_watermark) + ev = Event(self.queue_name, row) self.ev_buf.append(ev) def flush_events(self, dst_curs): diff --git a/sql/pgq_node/expected/pgq_node_test.out b/sql/pgq_node/expected/pgq_node_test.out index 1e69d3d9..1bd4ce55 100644 --- a/sql/pgq_node/expected/pgq_node_test.out +++ b/sql/pgq_node/expected/pgq_node_test.out @@ -4,6 +4,10 @@ 0 (1 row) + sub_consumer | sub_id | co_name +--------------+--------+--------- +(0 rows) + upgrade_schema ---------------- 0 @@ -159,10 +163,16 @@ select queue_name, consumer_name, last_tick from pgq.get_consumer_info(); aqueue | node2_worker | 1 (3 rows) +select * from pgq_node.set_node_attrs('aqueue', 'test=1'); + ret_code | ret_note +----------+------------------------- + 200 | Node attributes updated +(1 row) + select * from pgq_node.get_node_info('aqueue'); - ret_code | ret_note | node_type | node_name | global_watermark | local_watermark | provider_node | provider_location | combined_queue | combined_type | worker_name | worker_paused | worker_uptodate | worker_last_tick -----------+----------+-----------+-----------+------------------+-----------------+---------------+-------------------+----------------+---------------+--------------+---------------+-----------------+------------------ - 100 | Ok | root | node1 | 1 | 1 | node1 | dbname=node1 | | | node1_worker | f | f | 3 + ret_code | ret_note | node_type | node_name | global_watermark | local_watermark | provider_node | provider_location | combined_queue | combined_type | worker_name | worker_paused | worker_uptodate | worker_last_tick | node_attrs +----------+----------+-----------+-----------+------------------+-----------------+---------------+-------------------+----------------+---------------+--------------+---------------+-----------------+------------------+------------ + 100 | Ok | root | node1 | 1 | 1 | node1 | dbname=node1 | | | node1_worker | f | f | 3 | test=1 (1 row) select * from pgq_node.get_subscriber_info('aqueue'); @@ -218,28 +228,28 @@ select * from pgq_node.local_state; (4 rows) select * from pgq_node.node_info; - queue_name | node_type | node_name | worker_name | combined_queue -------------+-----------+-----------+--------------+---------------- - aqueue | root | node1 | node1_worker | - bqueue | branch | node2 | node2_worker | + queue_name | node_type | node_name | worker_name | combined_queue | node_attrs +------------+-----------+-----------+--------------+----------------+------------ + aqueue | root | node1 | node1_worker | | test=1 + bqueue | branch | node2 | node2_worker | | (2 rows) select * from pgq_node.get_node_info('aqueue'); - ret_code | ret_note | node_type | node_name | global_watermark | local_watermark | provider_node | provider_location | combined_queue | combined_type | worker_name | worker_paused | worker_uptodate | worker_last_tick -----------+----------+-----------+-----------+------------------+-----------------+---------------+-------------------+----------------+---------------+--------------+---------------+-----------------+------------------ - 100 | Ok | root | node1 | 1 | 1 | node1 | dbname=node1 | | | node1_worker | f | f | 3 + ret_code | ret_note | node_type | node_name | global_watermark | local_watermark | provider_node | provider_location | combined_queue | combined_type | worker_name | worker_paused | worker_uptodate | worker_last_tick | node_attrs +----------+----------+-----------+-----------+------------------+-----------------+---------------+-------------------+----------------+---------------+--------------+---------------+-----------------+------------------+------------ + 100 | Ok | root | node1 | 1 | 1 | node1 | dbname=node1 | | | node1_worker | f | f | 3 | test=1 (1 row) select * from pgq_node.get_node_info('bqueue'); - ret_code | ret_note | node_type | node_name | global_watermark | local_watermark | provider_node | provider_location | combined_queue | combined_type | worker_name | worker_paused | worker_uptodate | worker_last_tick -----------+----------+-----------+-----------+------------------+-----------------+---------------+-------------------+----------------+---------------+--------------+---------------+-----------------+------------------ - 100 | Ok | branch | node2 | 1 | 1 | node1 | dbname=node1 | | | node2_worker | f | f | 1 + ret_code | ret_note | node_type | node_name | global_watermark | local_watermark | provider_node | provider_location | combined_queue | combined_type | worker_name | worker_paused | worker_uptodate | worker_last_tick | node_attrs +----------+----------+-----------+-----------+------------------+-----------------+---------------+-------------------+----------------+---------------+--------------+---------------+-----------------+------------------+------------ + 100 | Ok | branch | node2 | 1 | 1 | node1 | dbname=node1 | | | node2_worker | f | f | 1 | (1 row) select * from pgq_node.get_node_info('cqueue'); - ret_code | ret_note | node_type | node_name | global_watermark | local_watermark | provider_node | provider_location | combined_queue | combined_type | worker_name | worker_paused | worker_uptodate | worker_last_tick -----------+-----------------------+-----------+-----------+------------------+-----------------+---------------+-------------------+----------------+---------------+-------------+---------------+-----------------+------------------ - 404 | Unknown queue: cqueue | | | | | | | | | | | | + ret_code | ret_note | node_type | node_name | global_watermark | local_watermark | provider_node | provider_location | combined_queue | combined_type | worker_name | worker_paused | worker_uptodate | worker_last_tick | node_attrs +----------+-----------------------+-----------+-----------+------------------+-----------------+---------------+-------------------+----------------+---------------+-------------+---------------+-----------------+------------------+------------ + 404 | Unknown queue: cqueue | | | | | | | | | | | | | (1 row) select * from pgq_node.get_worker_state('aqueue'); @@ -347,9 +357,9 @@ select * from pgq_node.get_consumer_state('bqueue', 'random_consumer2'); (1 row) select * from pgq_node.get_node_info('bqueue'); - ret_code | ret_note | node_type | node_name | global_watermark | local_watermark | provider_node | provider_location | combined_queue | combined_type | worker_name | worker_paused | worker_uptodate | worker_last_tick -----------+----------+-----------+-----------+------------------+-----------------+---------------+-------------------+----------------+---------------+--------------+---------------+-----------------+------------------ - 100 | Ok | branch | node2 | 1 | 1 | node1 | dbname=node1 | | | node2_worker | f | f | 1 + ret_code | ret_note | node_type | node_name | global_watermark | local_watermark | provider_node | provider_location | combined_queue | combined_type | worker_name | worker_paused | worker_uptodate | worker_last_tick | node_attrs +----------+----------+-----------+-----------+------------------+-----------------+---------------+-------------------+----------------+---------------+--------------+---------------+-----------------+------------------+------------ + 100 | Ok | branch | node2 | 1 | 1 | node1 | dbname=node1 | | | node2_worker | f | f | 1 | (1 row) set session_replication_role = 'replica'; diff --git a/sql/pgq_node/functions/pgq_node.get_node_info.sql b/sql/pgq_node/functions/pgq_node.get_node_info.sql index 5f00d693..a11dbfc3 100644 --- a/sql/pgq_node/functions/pgq_node.get_node_info.sql +++ b/sql/pgq_node/functions/pgq_node.get_node_info.sql @@ -1,4 +1,6 @@ +drop function if exists pgq_node.get_node_info(text); + create or replace function pgq_node.get_node_info( in i_queue_name text, @@ -17,7 +19,8 @@ create or replace function pgq_node.get_node_info( out worker_name text, out worker_paused bool, out worker_uptodate bool, - out worker_last_tick bigint + out worker_last_tick bigint, + out node_attrs text ) returns record as $$ -- ---------------------------------------------------------------------- -- Function: pgq_node.get_node_info(1) @@ -46,10 +49,12 @@ declare begin select 100, 'Ok', n.node_type, n.node_name, c.node_type, c.queue_name, w.provider_node, l.node_location, - n.worker_name, w.paused, w.uptodate, w.last_tick_id + n.worker_name, w.paused, w.uptodate, w.last_tick_id, + n.node_attrs into ret_code, ret_note, node_type, node_name, combined_type, combined_queue, provider_node, provider_location, - worker_name, worker_paused, worker_uptodate, worker_last_tick + worker_name, worker_paused, worker_uptodate, worker_last_tick, + node_attrs from pgq_node.node_info n left join pgq_node.node_info c on (c.queue_name = n.combined_queue) left join pgq_node.local_state w on (w.queue_name = n.queue_name and w.consumer_name = n.worker_name) diff --git a/sql/pgq_node/functions/pgq_node.set_node_attrs.sql b/sql/pgq_node/functions/pgq_node.set_node_attrs.sql new file mode 100644 index 00000000..fb015ed7 --- /dev/null +++ b/sql/pgq_node/functions/pgq_node.set_node_attrs.sql @@ -0,0 +1,35 @@ + +create or replace function pgq_node.set_node_attrs( + in i_queue_name text, + in i_node_attrs text, + out ret_code int4, + out ret_note text) +returns record as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq_node.create_attrs(2) +-- +-- Set node attributes. +-- +-- Parameters: +-- i_node_name - cascaded queue name +-- i_node_attrs - urlencoded node attrs +-- +-- Returns: +-- 200 - ok +-- 404 - node not found +-- ---------------------------------------------------------------------- +begin + update pgq_node.node_info + set node_attrs = i_node_attrs + where queue_name = i_queue_name; + if not found then + select 404, 'Node not found' into ret_code, ret_note; + return; + end if; + + select 200, 'Node attributes updated' + into ret_code, ret_note; + return; +end; +$$ language plpgsql security definer; + diff --git a/sql/pgq_node/functions/pgq_node.upgrade_schema.sql b/sql/pgq_node/functions/pgq_node.upgrade_schema.sql index 61130cdb..d9c1b6f4 100644 --- a/sql/pgq_node/functions/pgq_node.upgrade_schema.sql +++ b/sql/pgq_node/functions/pgq_node.upgrade_schema.sql @@ -5,6 +5,16 @@ returns int4 as $$ declare cnt int4 = 0; begin + -- node_info.node_attrs + perform 1 from information_schema.columns + where table_schema = 'pgq_node' + and table_name = 'node_info' + and column_name = 'node_attrs'; + if not found then + alter table pgq_node.node_info add column node_attrs text; + cnt := cnt + 1; + end if; + return cnt; end; $$ language plpgsql; diff --git a/sql/pgq_node/sql/pgq_node_test.sql b/sql/pgq_node/sql/pgq_node_test.sql index 9f012718..1b2e1b92 100644 --- a/sql/pgq_node/sql/pgq_node_test.sql +++ b/sql/pgq_node/sql/pgq_node_test.sql @@ -35,6 +35,8 @@ select * from pgq.ticker('aqueue'); select * from pgq_node.set_subscriber_watermark('aqueue', 'node2', 3); select queue_name, consumer_name, last_tick from pgq.get_consumer_info(); +select * from pgq_node.set_node_attrs('aqueue', 'test=1'); + select * from pgq_node.get_node_info('aqueue'); select * from pgq_node.get_subscriber_info('aqueue'); diff --git a/sql/pgq_node/structure/functions.sql b/sql/pgq_node/structure/functions.sql index 821a5656..6b3cb1ff 100644 --- a/sql/pgq_node/structure/functions.sql +++ b/sql/pgq_node/structure/functions.sql @@ -51,6 +51,7 @@ select pgq_node.upgrade_schema(); \i functions/pgq_node.demote_root.sql \i functions/pgq_node.promote_branch.sql +\i functions/pgq_node.set_node_attrs.sql -- Group: Provider side operations - worker \i functions/pgq_node.register_subscriber.sql diff --git a/sql/pgq_node/structure/tables.sql b/sql/pgq_node/structure/tables.sql index 73abc774..bd8d4e1d 100644 --- a/sql/pgq_node/structure/tables.sql +++ b/sql/pgq_node/structure/tables.sql @@ -50,6 +50,7 @@ create table pgq_node.node_location ( -- provider_node - provider node name -- worker_name - consumer name that maintains this node -- combined_queue - on 'leaf' the target combined set name +-- node_attrs - urlencoded fields for worker -- -- Node types: -- root - data + batches is generated here @@ -62,6 +63,7 @@ create table pgq_node.node_info ( node_name text not null, worker_name text, combined_queue text, + node_attrs text, foreign key (queue_name, node_name) references pgq_node.node_location, check (node_type in ('root', 'branch', 'leaf')), diff --git a/tests/londiste/regen.sh b/tests/londiste/regen.sh index a362b91b..46741f7e 100755 --- a/tests/londiste/regen.sh +++ b/tests/londiste/regen.sh @@ -71,15 +71,15 @@ msg "Install londiste3 and initialize nodes" run londiste3 $v conf/londiste_db1.ini create-root node1 'dbname=db1' run londiste3 $v conf/londiste_db2.ini create-branch node2 'dbname=db2' --provider='dbname=db1' run londiste3 $v conf/londiste_db3.ini create-branch node3 'dbname=db3' --provider='dbname=db1' -run londiste3 $v conf/londiste_db4.ini create-branch node4 'dbname=db4' --provider='dbname=db2' -run londiste3 $v conf/londiste_db5.ini create-branch node5 'dbname=db5' --provider='dbname=db3' +run londiste3 $v conf/londiste_db4.ini create-branch node4 'dbname=db4' --provider='dbname=db2' --sync-watermark=node4,node5 +run londiste3 $v conf/londiste_db5.ini create-branch node5 'dbname=db5' --provider='dbname=db3' --sync-watermark=node4,node5 msg "Run ticker" run pgqd $v -d conf/pgqd.ini run sleep 5 msg "See topology" -run londiste3 $v conf/londiste_db4.ini status +run londiste3 $v conf/londiste_db1.ini status msg "Run londiste3 daemon for each node" for db in $db_list; do |