pgq/cascade: significant speed up of getting status on big cascades

e.g.: 1000 node cascade status went from minutes to seconds

2 files changed, 64 insertions, 28 deletions

diff --git a/python/pgq/cascade/admin.py b/python/pgq/cascade/admin.py
index 58fa16e3..c47ec02c 100644
--- a/python/pgq/cascade/admin.py
+++ b/python/pgq/cascade/admin.py
@@ -10,8 +10,14 @@ setadm.py INI pause NODE [CONS]
 
 """
 
-import sys, time, optparse, skytools, os.path
-
+import optparse
+import os.path
+import Queue
+import sys
+import threading
+import time
+
+import skytools
 from skytools import UsageError, DBError
 from pgq.cascade.nodeinfo import *
 
@@ -350,32 +356,59 @@ class CascadeAdmin(skytools.AdminScript):
         """Show set status."""
         self.load_local_info()
 
-        for mname, minf in self.queue_info.member_map.iteritems():
-            #inf = self.get_node_info(mname)
-            #self.queue_info.add_node(inf)
-            #continue
+        # prepare structs for workers
+        members = Queue.Queue()
+        for m in self.queue_info.member_map.itervalues():
+            members.put(m)
+        nodes = Queue.Queue()
 
-            if not self.node_alive(mname):
-                node = NodeInfo(self.queue_name, None, node_name = mname)
-                self.queue_info.add_node(node)
-                continue
+        # launch workers and wait
+        n = min (members.qsize(), 1000)
+        for i in range(n):
+            t = threading.Thread (target = self._cmd_status_worker, args = (members, nodes))
+            t.start()
+        members.join()
+
+        while True:
             try:
-                db = self.get_database('look_db', connstr = minf.location, autocommit = 1)
-                curs = db.cursor()
-                curs.execute("select * from pgq_node.get_node_info(%s)", [self.queue_name])
-                node = NodeInfo(self.queue_name, curs.fetchone())
-                node.load_status(curs)
-                self.load_extra_status(curs, node)
-                self.queue_info.add_node(node)
-            except DBError, d:
-                msg = str(d).strip().split('\n', 1)[0]
-                print('Node %s failure: %s' % (mname, msg))
-                node = NodeInfo(self.queue_name, None, node_name = mname)
-                self.queue_info.add_node(node)
-            self.close_database('look_db')
+                node = nodes.get_nowait()
+            except Queue.Empty:
+                break
+            self.queue_info.add_node(node)
 
         self.queue_info.print_tree()
 
+    def _cmd_status_worker (self, members, nodes):
+        # members in, nodes out, both thread-safe
+        while True:
+            try:
+                m = members.get_nowait()
+            except Queue.Empty:
+                break
+            node = self.load_node_status (m.name, m.location)
+            nodes.put(node)
+            members.task_done()
+
+    def load_node_status (self, name, location):
+        """ Load node info & status """
+        if not self.node_alive(name):
+            node = NodeInfo(self.queue_name, None, node_name = name)
+            return node
+        try:
+            conn = 'look_db.%s' % name
+            db = self.get_database(conn, connstr = location, autocommit = 1)
+            curs = db.cursor()
+            curs.execute("select * from pgq_node.get_node_info(%s)", [self.queue_name])
+            node = NodeInfo(self.queue_name, curs.fetchone())
+            node.load_status(curs)
+            self.load_extra_status(curs, node)
+        except DBError, d:
+            msg = str(d).strip().split('\n', 1)[0].strip()
+            print('Node %r failure: %s' % (name, msg))
+            node = NodeInfo(self.queue_name, None, node_name = name)
+        self.close_database(conn)
+        return node
+
     def cmd_node_status(self):
         """
         Show status of a local node.
diff --git a/python/pgq/cascade/nodeinfo.py b/python/pgq/cascade/nodeinfo.py
index 726b311e..60f07583 100644
--- a/python/pgq/cascade/nodeinfo.py
+++ b/python/pgq/cascade/nodeinfo.py
@@ -150,7 +150,7 @@ class NodeInfo:
                     err = err[:pos]
                 lst.append("ERR: %s: %s" % (cname, err))
         return lst
-    
+
     def add_info_line(self, ln):
         self._info_lines.append(ln)
 
@@ -166,6 +166,7 @@ class NodeInfo:
             for row in curs.fetchall():
                 cname = row['consumer_name']
                 self.consumer_map[cname] = row
+
             q = "select current_timestamp - ticker_lag as tick_time,"\
                 "  ticker_lag, current_timestamp as now "\
                 "from pgq.get_queue_info(%s)"
@@ -180,7 +181,7 @@ class NodeInfo:
 
 class QueueInfo:
     """Info about cascaded queue.
-    
+
     Slightly broken, as all info is per-node.
     """
 
@@ -192,8 +193,11 @@ class QueueInfo:
         self.add_node(self.local_node)
 
         for r in member_rows:
-            n = MemberInfo(r)
-            self.member_map[n.name] = n
+            m = MemberInfo(r)
+            self._add_member(m)
+
+    def _add_member(self, member):
+        self.member_map[member.name] = member
 
     def get_member(self, name):
         return self.member_map.get(name)
@@ -288,4 +292,3 @@ def _setpfx(pfx, sfx):
 
 def _node_key(n):
     return (n.levels, n.total_childs, n.name)
-