diff options
author | martinko | 2013-02-27 09:44:55 +0000 |
---|---|---|
committer | martinko | 2013-02-27 09:44:55 +0000 |
commit | a49189ad06ae4811c1b9bb1e35a6442ce5faae37 (patch) | |
tree | 232badfac814380f9c413f08264d513cad593f06 | |
parent | 92484db1f617b4a5dcccb3234a43d53514786eb6 (diff) |
pgq/cascade: significant speed up of getting status on big cascades
e.g.: 1000 node cascade status went from minutes to seconds
-rw-r--r-- | python/pgq/cascade/admin.py | 79 | ||||
-rw-r--r-- | python/pgq/cascade/nodeinfo.py | 13 |
2 files changed, 64 insertions, 28 deletions
diff --git a/python/pgq/cascade/admin.py b/python/pgq/cascade/admin.py index 58fa16e3..c47ec02c 100644 --- a/python/pgq/cascade/admin.py +++ b/python/pgq/cascade/admin.py @@ -10,8 +10,14 @@ setadm.py INI pause NODE [CONS] """ -import sys, time, optparse, skytools, os.path - +import optparse +import os.path +import Queue +import sys +import threading +import time + +import skytools from skytools import UsageError, DBError from pgq.cascade.nodeinfo import * @@ -350,32 +356,59 @@ class CascadeAdmin(skytools.AdminScript): """Show set status.""" self.load_local_info() - for mname, minf in self.queue_info.member_map.iteritems(): - #inf = self.get_node_info(mname) - #self.queue_info.add_node(inf) - #continue + # prepare structs for workers + members = Queue.Queue() + for m in self.queue_info.member_map.itervalues(): + members.put(m) + nodes = Queue.Queue() - if not self.node_alive(mname): - node = NodeInfo(self.queue_name, None, node_name = mname) - self.queue_info.add_node(node) - continue + # launch workers and wait + n = min (members.qsize(), 1000) + for i in range(n): + t = threading.Thread (target = self._cmd_status_worker, args = (members, nodes)) + t.start() + members.join() + + while True: try: - db = self.get_database('look_db', connstr = minf.location, autocommit = 1) - curs = db.cursor() - curs.execute("select * from pgq_node.get_node_info(%s)", [self.queue_name]) - node = NodeInfo(self.queue_name, curs.fetchone()) - node.load_status(curs) - self.load_extra_status(curs, node) - self.queue_info.add_node(node) - except DBError, d: - msg = str(d).strip().split('\n', 1)[0] - print('Node %s failure: %s' % (mname, msg)) - node = NodeInfo(self.queue_name, None, node_name = mname) - self.queue_info.add_node(node) - self.close_database('look_db') + node = nodes.get_nowait() + except Queue.Empty: + break + self.queue_info.add_node(node) self.queue_info.print_tree() + def _cmd_status_worker (self, members, nodes): + # members in, nodes out, both thread-safe + while True: + try: + m = members.get_nowait() + except Queue.Empty: + break + node = self.load_node_status (m.name, m.location) + nodes.put(node) + members.task_done() + + def load_node_status (self, name, location): + """ Load node info & status """ + if not self.node_alive(name): + node = NodeInfo(self.queue_name, None, node_name = name) + return node + try: + conn = 'look_db.%s' % name + db = self.get_database(conn, connstr = location, autocommit = 1) + curs = db.cursor() + curs.execute("select * from pgq_node.get_node_info(%s)", [self.queue_name]) + node = NodeInfo(self.queue_name, curs.fetchone()) + node.load_status(curs) + self.load_extra_status(curs, node) + except DBError, d: + msg = str(d).strip().split('\n', 1)[0].strip() + print('Node %r failure: %s' % (name, msg)) + node = NodeInfo(self.queue_name, None, node_name = name) + self.close_database(conn) + return node + def cmd_node_status(self): """ Show status of a local node. diff --git a/python/pgq/cascade/nodeinfo.py b/python/pgq/cascade/nodeinfo.py index 726b311e..60f07583 100644 --- a/python/pgq/cascade/nodeinfo.py +++ b/python/pgq/cascade/nodeinfo.py @@ -150,7 +150,7 @@ class NodeInfo: err = err[:pos] lst.append("ERR: %s: %s" % (cname, err)) return lst - + def add_info_line(self, ln): self._info_lines.append(ln) @@ -166,6 +166,7 @@ class NodeInfo: for row in curs.fetchall(): cname = row['consumer_name'] self.consumer_map[cname] = row + q = "select current_timestamp - ticker_lag as tick_time,"\ " ticker_lag, current_timestamp as now "\ "from pgq.get_queue_info(%s)" @@ -180,7 +181,7 @@ class NodeInfo: class QueueInfo: """Info about cascaded queue. - + Slightly broken, as all info is per-node. """ @@ -192,8 +193,11 @@ class QueueInfo: self.add_node(self.local_node) for r in member_rows: - n = MemberInfo(r) - self.member_map[n.name] = n + m = MemberInfo(r) + self._add_member(m) + + def _add_member(self, member): + self.member_map[member.name] = member def get_member(self, name): return self.member_map.get(name) @@ -288,4 +292,3 @@ def _setpfx(pfx, sfx): def _node_key(n): return (n.levels, n.total_childs, n.name) - |