various tweaks to get regtest pass more predictibly

5 files changed, 71 insertions, 11 deletions

diff --git a/python/pgq/ticker.py b/python/pgq/ticker.py
index c218eaf1..8f726c78 100644
--- a/python/pgq/ticker.py
+++ b/python/pgq/ticker.py
@@ -143,10 +143,11 @@ class SmartTicker(skytools.DBScript):
     def work(self):
         db = self.get_database("db", autocommit = 1)
         cx = db.cursor()
+        queue_refresh = self.cf.getint('queue_refresh_period', 30)
 
         cur_time = time.time()
 
-        if cur_time >= self.refresh_time + 30:
+        if cur_time >= self.refresh_time + queue_refresh:
             self.refresh_queues(cx)
 
         if not self.seq_query:
diff --git a/python/skytools/scripting.py b/python/skytools/scripting.py
index cf976801..456fff69 100644
--- a/python/skytools/scripting.py
+++ b/python/skytools/scripting.py
@@ -396,13 +396,14 @@ class DBScript(object):
         self.stat_dict = {}
 
     def get_database(self, dbname, autocommit = 0, isolation_level = -1,
-                     cache = None, max_age = DEF_CONN_AGE):
+                     cache = None):
         """Load cached database connection.
         
         User must not store it permanently somewhere,
         as all connections will be invalidated on reset.
         """
 
+        max_age = self.cf.getint('connection_lifetime', DEF_CONN_AGE)
         if not cache:
             cache = dbname
         if cache in self.db_cache:
@@ -506,7 +507,11 @@ class DBScript(object):
                 return 1
 
     def work(self):
-        "Here should user's processing happen."
+        """Here should user's processing happen.
+
+        Return value is taken as boolean - if true, the next loop
+        starts immidiately.  If false, DBScript sleeps for a loop_delay.
+        """
         raise Exception("Nothing implemented?")
 
     def startup(self):
diff --git a/tests/londiste/conf/replic.ini b/tests/londiste/conf/replic.ini
index e34e4adf..22f1f01b 100644
--- a/tests/londiste/conf/replic.ini
+++ b/tests/londiste/conf/replic.ini
@@ -11,5 +11,7 @@ pgq_queue_name = londiste_replic
 pidfile = sys/pid.%(job_name)s
 logfile = sys/log.%(job_name)s
 
-loop_delay = 0.3
+loop_delay = 1
+
+connection_lifetime = 30
 
diff --git a/tests/londiste/conf/ticker.ini b/tests/londiste/conf/ticker.ini
index 5a35c85f..a3b96fe4 100644
--- a/tests/londiste/conf/ticker.ini
+++ b/tests/londiste/conf/ticker.ini
@@ -7,11 +7,15 @@ db = dbname=provider
 # how often to run maintenance [minutes]
 maint_delay_min = 1
 
-# how often to check fot activity [secs]
-loop_delay = 0.1
+# how often to check for activity [secs]
+loop_delay = 0.5
 
 logfile = sys/log.%(job_name)s
 pidfile = sys/pid.%(job_name)s
 
 use_skylog = 0
- 
+
+connection_lifetime = 21
+
+queue_refresh_period = 10
+
diff --git a/tests/londiste/run-tests.sh b/tests/londiste/run-tests.sh
index 2e6fa8a8..fd2e160d 100755
--- a/tests/londiste/run-tests.sh
+++ b/tests/londiste/run-tests.sh
@@ -1,14 +1,23 @@
 #! /bin/sh
 
-. ./env.sh
+. ../env.sh
+
+./gendb.sh
 
 script=londiste.py
 
+mwait () {
+  echo "Waiting $1 seconds..."
+  sleep $1
+  shift
+  echo "$@"
+}
+
 set -e
 
 $script  conf/replic.ini provider install
 
-psql -c "update pgq.queue set queue_ticker_idle_period = '3', queue_ticker_max_lag = '2'" provider
+psql -c "update pgq.queue set queue_ticker_idle_period = '15', queue_ticker_max_lag = '5'" provider
 
 pgqadm.py -d conf/ticker.ini ticker
 
@@ -18,7 +27,7 @@ $script  conf/replic.ini subscriber register
 $script  conf/replic.ini subscriber unregister
 
 $script -v -d conf/replic.ini replay
-$script -v -d conf/fwrite.ini replay
+#$script -v -d conf/fwrite.ini replay
 
 sleep 2
 
@@ -54,5 +63,44 @@ sleep 2
 $script  conf/replic.ini subscriber add data2
 sleep 2
 
-./testing.py conf/tester.ini
+echo "starting data gen script"
+
+./testing.py -d conf/tester.ini
+
+mwait 30 "vacuuming"
+psql -c "vacuum analyze" provider
+mwait 30 "vacuuming"
+psql -c "vacuum analyze" provider
+
+#echo "stopping script for a moment"
+#$script conf/replic.ini -s
+#mwait 15 "starting replica again"
+#$script conf/replic.ini replay -d -v
+
+mwait 90 "stopping tester skript"
+./testing.py -s conf/tester.ini
+
+#exit 0
+
+mwait 20 "comparing tables"
+$script conf/replic.ini compare --force -v
+$script conf/replic.ini repair --force -v
+
+mwait 10 "stopping replica"
+$script -v -s conf/replic.ini
+#$script -v -s conf/fwrite.ini
+
+mwait 10 "stopping ticker"
+pgqadm.py -s -v conf/ticker.ini 
+
+test -f sys/pid.replic.copy && {
+  echo "copy failed, still running"
+  kill `cat sys/pid.replic.copy`
+  sleep 3
+}
+
+echo "done?"
+ps aux|grep python
+
+