Merge branch 'master' of https://github.com/markokr/skytools into develop

15 files changed, 155 insertions, 71 deletions

diff --git a/NEWS b/NEWS
index 0fb04eca..603a396c 100644
--- a/NEWS
+++ b/NEWS
@@ -1,4 +1,56 @@
 
+2013-04-17  -  SkyTools 3.1.4  -  "Boldly Going Nowhere"
+
+  = Features =
+
+    * londiste create-node: Creation data from config.  The need to 2 connection
+      string in command line was major usability problem in create-* commands.
+      Now initial setup can be described in config.
+    * newgrants: Londiste can read from provider using non-privileged user.
+    * scripts/data_maintainer.py: Generic script for processing
+      large data sets in small batches.
+
+  = Minor features =
+
+    * londiste status: significant speed up of getting status on big cascades
+    * londiste: remote_extra_connstr config option that will be added
+      to remote node connect strings.  Needed when more than several cascaded
+      scripts are running, with different privileges in local and remote node.
+    * scriptmgr: User switching with sudo.  Script sections can
+      contain user= option that makes scriptmgr launch sudo to run script.
+    * londiste compare: Added --count-only switch
+    * BaseScript: skylog_locations config options
+    * skytools.Config: New default variables: config_dir, config_file.
+    * Parse & merge Postgres connect strings
+    * skytools_upgrade: add force option
+    * londiste: new drop_obsolete_partitions(2) function
+    * londiste: added dispatch handler arg retention_period
+
+  = Fixes =
+
+    * londiste: Sync SQL keywords with 9.3git.
+    * handers/dispatch.py: 'keep_latest' row_mode processes now deletes properly.
+    * configure.ac: Check for -lrt - needed when building against libevent.
+    * adminscripts: Make info commands not use pidfile.  Otherwise they will
+      not run if some writing admin command is running.
+    * Londiste compare: It's now compatible with 8.2.
+    * londiste.create_partition: Set owner to match parent table.
+    * londiste.create_trigger: Fixed skip-trigger check compatibility with postgres 9.x.
+    * londiste.create_trigger: Added check for after-trigger being overshadowed by skip-trigger.
+
+  = Cleanups =
+
+    * Refactor Consumer code.
+    * Remove: pgqadm and related code, its unmaintained
+    * Sweeping change to postpone log string formatting
+    * docs: copy-condition removed from londiste3 man page
+    * Various logging cleanups
+    * Londiste: fix trigger generation when extra params that are add-table specific are used (introduced in commit 364ade9)
+    * londiste: quote queue name in trigger args
+    * londiste: actually execute the ENABLE/DISABLE RULE query in londiste.create_partition
+    * londiste/handlers/dispatch: fixed issue with missing part_func description
+    * londiste.handler: disable shortcut for specifying handler args together with its name
+
 2012-12-21  -  SkyTools 3.1.3  -  "Chainsaw-wielding Toothfairy"
 
   = Features =
diff --git a/configure.ac b/configure.ac
index fbc36fe7..f3829de0 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,6 +1,6 @@
 dnl Process this file with autoconf to produce a configure script.
 
-AC_INIT(skytools, 3.1.3)
+AC_INIT(skytools, 3.1.4)
 AC_CONFIG_SRCDIR(python/londiste.py)
 AC_CONFIG_HEADER(lib/usual/config.h)
 AC_PREREQ([2.59])
diff --git a/debian/changelog b/debian/changelog
index 460ef96b..3025f329 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+skytools3 (3.1.4) experimental; urgency=low
+
+  * v3.1.4
+
+ -- Marko Kreen <[email protected]>  Wed, 17 Apr 2013 11:08:30 +0300
+
 skytools3 (3.1.3) experimental; urgency=low
 
   * v3.1.3
diff --git a/doc/TODO.txt b/doc/TODO.txt
index df78a053..31a49fa7 100644
--- a/doc/TODO.txt
+++ b/doc/TODO.txt
@@ -36,14 +36,6 @@ Low::
   . Replay rest of batches fully
   . Promote to root
 
-* Load public connect string for local node:
-  - from .ini (db=, public_db=?)
-  - with query sql
-
-* Load provider connect string:
-  - from ini (not good)
-  - describe connstr + query to look up node names?
-
 * tests for things that have not their own regtests
   or are not tested enough during other tests:
   - pgq.RemoteConsumer
@@ -69,11 +61,6 @@ Low::
   So separate counter for len(ev_data) needs to be added, that flushes
   if buffer would go over some specified amount of memory.
 
-* cascade status: parallel info gathering.  Sequential for-loop over
-  nodes can be slow if there are many nodes or some of them are
-  generally slow  (GP, bad network, etc).  Perhaps we can use
-  psycopg2 async mode for that.  Or threading.
-
 == Low Priority ==
 
 * dbscript: switch (-q) for silence for cron/init scripts.
diff --git a/doc/londiste3.txt b/doc/londiste3.txt
index 1cd3521a..c0954a58 100644
--- a/doc/londiste3.txt
+++ b/doc/londiste3.txt
@@ -75,36 +75,41 @@ partial replication and custom handlers for replication.
 
 == NODE INITIALIZATION COMMANDS ==
 
-Initialization commands are the only ones that requires a connection
-string argument. It is the connection string that Londiste3 will store,
-other nodes will use it to connect to the current node.
+Initialization commands will set up "public connect string" for
+current node.  It is a connect string that other nodes will
+use to connect to current node.  The local Londiste itself
+uses 'db' option from config file to connect to local node,
+which can have different user rights than scripts coming
+over public connect string.
 
-=== create-root <node> <connstr> ===
+Connect strings can be set in either command line or config file.
+Command line overrides config.  Setting them up in config
+might be more comfortable.  See `londiste3 --ini` for details.
+
+=== create-root <node> [<public_connstr_for_node>] ===
 
 Initializes a Master node.
 
 The <node> is the name of the node, it should be unique.
-The <connstr> argument is the connection string to the database on the
-Master node.
 
-=== create-branch <node> <connstr> --provider=<public_connstr> ===
+=== create-branch <node> [<public_connstr_for_node>] [--provider=<connstr_to_provider>] ===
 
 Initializes a Slave node which can be used as a reference for other
 nodes.
 
 The <node> is the name of the node, it should be unique.
-The <connstr> argument is the connection string to the database on the
-current node and <public_connstr> is the connection string to the
+The <public_connstr_for_node> argument is the connection string to the database
+on the current node and <connstr_to_provider> is the connection string to the
 provider database (it can be a root node or a branch node).
 
-=== create-leaf <node> <connstr> --provider=<public_connstr> ===
+=== create-leaf <node> [<public_connstr_for_node>] [--provider=<connstr_to_provider>] ===
 
-Initializes a Slave node which can not be used as a reference for other
+Initializes a slave node which can not be used as a reference for other
 nodes.
 
 The <node> is the name of the node, it should be unique.
-The <connstr> argument is the connection string to the database on the
-current node and <public_connstr> is the connection string to the
+The <public_connstr_for_node> argument is the connection string to the database
+on the current node and <connstr_to_provider> is the connection string to the
 provider database (it can be a root node or a branch node).
 
   --merge='qname'::
diff --git a/python/londiste.py b/python/londiste.py
index 6582abea..8860a313 100755
--- a/python/londiste.py
+++ b/python/londiste.py
@@ -148,6 +148,8 @@ class Londiste(skytools.DBScript):
                 help="don't merge tables from source queues", default=False)
         g.add_option("--max-parallel-copy", metavar = "NUM", type = "int",
                 help="max number of parallel copy processes")
+        g.add_option("--skip-non-existing", action="store_true",
+                help="add: skip object that does not exist")
         p.add_option_group(g)
 
         g = optparse.OptionGroup(p, "other options")
diff --git a/python/londiste/playback.py b/python/londiste/playback.py
index c262de53..8d0b2a94 100644
--- a/python/londiste/playback.py
+++ b/python/londiste/playback.py
@@ -201,10 +201,12 @@ class TableState(object):
         return self.max_parallel_copy and\
                     self.copy_pos >= self.max_parallel_copy
 
-    def interesting(self, ev, tick_id, copy_thread):
+    def interesting(self, ev, tick_id, copy_thread, copy_table_name):
         """Check if table wants this event."""
 
         if copy_thread:
+            if self.name != copy_table_name:
+                return False
             if self.state not in (TABLE_CATCHING_UP, TABLE_DO_SYNC):
                 return False
         else:
@@ -651,7 +653,7 @@ class Replicator(CascadedWorker):
     def handle_data_event(self, ev, dst_curs):
         """handle one truncate event"""
         t = self.get_table_by_name(ev.extra1)
-        if not t or not t.interesting(ev, self.cur_tick, self.copy_thread):
+        if not t or not t.interesting(ev, self.cur_tick, self.copy_thread, self.copy_table_name):
             self.stat_increase('ignored_events')
             return
 
@@ -667,7 +669,7 @@ class Replicator(CascadedWorker):
     def handle_truncate_event(self, ev, dst_curs):
         """handle one truncate event"""
         t = self.get_table_by_name(ev.extra1)
-        if not t or not t.interesting(ev, self.cur_tick, self.copy_thread):
+        if not t or not t.interesting(ev, self.cur_tick, self.copy_thread, self.copy_table_name):
             self.stat_increase('ignored_events')
             return
 
@@ -754,17 +756,6 @@ class Replicator(CascadedWorker):
 
         dst_curs.execute(buf)
 
-    def interesting(self, ev):
-        """See if event is interesting."""
-        if ev.type not in ('I', 'U', 'D', 'R'):
-            raise Exception('bug - bad event type in .interesting')
-        t = self.get_table_by_name(ev.extra1)
-        if not t:
-            return 0
-        if not t.interesting(ev, self.cur_tick, self.copy_thread):
-            return 0
-        return 1
-
     def add_set_table(self, dst_curs, tbl):
         """There was new table added to root, remember it."""
 
@@ -883,7 +874,8 @@ class Replicator(CascadedWorker):
         # pass same verbosity options as main script got
         if self.options.quiet:
             cmd.append('-q')
-        cmd += self.options.verbose * ['-v']
+        if self.options.verbose:
+            cmd += ['-v'] * self.options.verbose
 
         # let existing copy finish and clean its pidfile,
         # otherwise new copy will exit immediately.
diff --git a/python/londiste/setup.py b/python/londiste/setup.py
index 5a53d41a..20d3f5c0 100644
--- a/python/londiste/setup.py
+++ b/python/londiste/setup.py
@@ -82,6 +82,8 @@ class LondisteSetup(CascadeAdmin):
                 help="max number of parallel copy processes")
         p.add_option("--dest-table", metavar = "NAME",
                 help="add: name for actual table")
+        p.add_option("--skip-non-existing", action="store_true",
+                help="add: skip object that does not exist")
         return p
 
     def extra_init(self, node_type, node_db, provider_db):
@@ -163,8 +165,11 @@ class LondisteSetup(CascadeAdmin):
             for tbl in args:
                 tbl = skytools.fq_name(tbl)
                 if (tbl in src_tbls) and not src_tbls[tbl]['local']:
-                    self.log.error("Table %s does not exist on provider, need to switch to different provider", tbl)
-                    problems = True
+                    if self.options.skip_non_existing:
+                        self.log.warning("Table %s does not exist on provider", tbl)
+                    else:
+                        self.log.error("Table %s does not exist on provider, need to switch to different provider", tbl)
+                        problems = True
             if problems:
                 self.log.error("Problems, canceling operation")
                 sys.exit(1)
@@ -191,6 +196,7 @@ class LondisteSetup(CascadeAdmin):
         src_curs = src_db.cursor()
         dst_curs = dst_db.cursor()
         tbl_exists = skytools.exists_table(dst_curs, dest_table)
+        dst_db.commit()
 
         if dest_table == tbl:
             desc = tbl
@@ -218,6 +224,9 @@ class LondisteSetup(CascadeAdmin):
                 if src_dest_table != dest_table:
                     newname = dest_table
                 s.create(dst_curs, create_flags, log = self.log, new_table_name = newname)
+        elif not tbl_exists and self.options.skip_non_existing:
+            self.log.warning('Table %s does not exist on local node, skipping', desc)
+            return
 
         tgargs = self.build_tgargs()
 
@@ -402,8 +411,11 @@ class LondisteSetup(CascadeAdmin):
                 src_db.commit()
                 s.create(dst_curs, create_flags, log = self.log)
         elif not seq_exists:
-            self.log.warning('Sequence "%s" missing on subscriber, use --create if necessary', seq)
-            return
+            if self.options.skip_non_existing:
+                self.log.warning('Sequence "%s" missing on local node, skipping', seq)
+                return
+            else:
+                raise skytools.UsageError("Sequence %r missing on local node", seq)
 
         q = "select * from londiste.local_add_seq(%s, %s)"
         self.exec_cmd(dst_curs, q, [self.set_name, seq])
@@ -694,18 +706,19 @@ class LondisteSetup(CascadeAdmin):
         dst_curs = dst_db.cursor()
 
         partial = {}
-        done_pos = 1
         startup_info = 0
         while 1:
             dst_curs.execute(q, [self.queue_name])
             rows = dst_curs.fetchall()
             dst_db.commit()
 
+            total_count = 0
             cur_count = 0
             done_list = []
             for row in rows:
                 if not row['local']:
                     continue
+                total_count += 1
                 tbl = row['table_name']
                 if row['merge_state'] != 'ok':
                     partial[tbl] = 0
@@ -715,13 +728,14 @@ class LondisteSetup(CascadeAdmin):
                         partial[tbl] = 1
                         done_list.append(tbl)
 
+            done_count = total_count - cur_count
+
             if not startup_info:
-                self.log.info("%d table(s) to copy", len(partial))
+                self.log.info("%d/%d table(s) to copy", cur_count, total_count)
                 startup_info = 1
 
             for done in done_list:
-                self.log.info("%s: finished (%d/%d)", done, done_pos, len(partial))
-                done_pos += 1
+                self.log.info("%s: finished (%d/%d)", done, done_count, total_count)
 
             if cur_count == 0:
                 break
diff --git a/python/pgq/baseconsumer.py b/python/pgq/baseconsumer.py
index 9159106e..3ea1c6c6 100644
--- a/python/pgq/baseconsumer.py
+++ b/python/pgq/baseconsumer.py
@@ -285,6 +285,9 @@ class BaseConsumer(skytools.DBScript):
     def _launch_process_batch(self, db, batch_id, list):
         self.process_batch(db, batch_id, list)
 
+    def _make_event(self, queue_name, row):
+        return Event(queue_name, row)
+
     def _load_batch_events_old(self, curs, batch_id):
         """Fetch all events for this batch."""
 
@@ -298,7 +301,7 @@ class BaseConsumer(skytools.DBScript):
         # map them to python objects
         ev_list = []
         for r in rows:
-            ev = Event(self.queue_name, r)
+            ev = self._make_event(self.queue_name, r)
             ev_list.append(ev)
 
         return ev_list
diff --git a/python/pgq/cascade/nodeinfo.py b/python/pgq/cascade/nodeinfo.py
index 60f07583..e3c82555 100644
--- a/python/pgq/cascade/nodeinfo.py
+++ b/python/pgq/cascade/nodeinfo.py
@@ -240,7 +240,7 @@ class QueueInfo:
         for ln in datalines:
             print(self._DATAFMT % (_setpfx(pfx, '|'), ln))
         datalines = node.get_infolines()
-        print("%s%s" % (_setpfx(pfx, '+--'), node.get_title()))
+        print("%s%s" % (_setpfx(pfx, '+--: '), node.get_title()))
 
         for i, n in enumerate(node.child_list):
             sfx = ((i < len(node.child_list) - 1) and '  |' or '   ')
diff --git a/python/pgq/consumer.py b/python/pgq/consumer.py
index 294519d8..10cb0909 100644
--- a/python/pgq/consumer.py
+++ b/python/pgq/consumer.py
@@ -60,6 +60,9 @@ class Consumer(BaseConsumer):
 
     _batch_walker_class = RetriableBatchWalker
 
+    def _make_event(self, queue_name, row):
+        return RetriableWalkerEvent(self, queue_name, row)
+
     def _flush_retry(self, curs, batch_id, list):
         """Tag retry events."""
 
diff --git a/python/skytools/parsing.py b/python/skytools/parsing.py
index 318b1bf9..3d16a3a1 100644
--- a/python/skytools/parsing.py
+++ b/python/skytools/parsing.py
@@ -25,6 +25,8 @@ def parse_pgarray(array):
     >>> parse_pgarray("[0,3]={1,2,3}")
     ['1', '2', '3']
     """
+    if array is None:
+        return None
     if not array or array[0] not in ("{", "[") or array[-1] != '}':
         raise Exception("bad array format: must be surrounded with {}")
     res = []
diff --git a/python/skytools/querybuilder.py b/python/skytools/querybuilder.py
index c2eead2d..9930daab 100755
--- a/python/skytools/querybuilder.py
+++ b/python/skytools/querybuilder.py
@@ -319,8 +319,11 @@ class PLPyQuery:
                 arg_list = [arg_dict.get(k) for k in self.arg_map]
             return plpy.execute(self.plan, arg_list)
         except KeyError:
-            plpy.error("Missing argument: QUERY: %s  ARGS: %s VALUES: %s" % (
-                repr(self.sql), repr(self.arg_map), repr(arg_dict)))
+            need = set(self.arg_map)
+            got = set(arg_dict.keys())
+            missing = list(need.difference(got))
+            plpy.error("Missing arguments: [%s]  QUERY: %s" % (
+                ','.join(missing), repr(self.sql)))
 
     def __repr__(self):
         return 'PLPyQuery<%s>' % self.sql
@@ -341,7 +344,7 @@ def plpy_exec(gd, sql, args, all_keys_required = True):
     >>> res = plpy_exec(GD, "select {arg1}, {arg2:int4}, {arg1}", {'arg1': '3', 'arg2': '4'})
     DBG: plpy.execute(('PLAN', 'select $1, $2, $3', ['text', 'int4', 'text']), ['3', '4', '3'])
     >>> res = plpy_exec(GD, "select {arg1}, {arg2:int4}, {arg1}", {'arg1': '3'})
-    DBG: plpy.error("Missing argument: QUERY: 'select {arg1}, {arg2:int4}, {arg1}'  ARGS: ['arg1', 'arg2', 'arg1'] VALUES: {'arg1': '3'}")
+    DBG: plpy.error("Missing arguments: [arg2]  QUERY: 'select {arg1}, {arg2:int4}, {arg1}'")
     >>> res = plpy_exec(GD, "select {arg1}, {arg2:int4}, {arg1}", {'arg1': '3'}, False)
     DBG: plpy.execute(('PLAN', 'select $1, $2, $3', ['text', 'int4', 'text']), ['3', None, '3'])
     """
diff --git a/python/skytools/timeutil.py b/python/skytools/timeutil.py
index a29e050c..2ea63082 100644
--- a/python/skytools/timeutil.py
+++ b/python/skytools/timeutil.py
@@ -134,18 +134,24 @@ def datetime_to_timestamp(dt, local_time=True):
 
     Returns seconds since epoch as float.
 
-    >>> datetime_to_timestamp(parse_iso_timestamp("2005-06-01 15:00:59.33 +02"))
-    1117630859.33
-    >>> datetime_to_timestamp(datetime.fromtimestamp(1117630859.33, UTC))
-    1117630859.33
-    >>> datetime_to_timestamp(datetime.fromtimestamp(1117630859.33))
-    1117630859.33
+    >>> datetime_to_timestamp(parse_iso_timestamp("2005-06-01 15:00:59.5 +02"))
+    1117630859.5
+    >>> datetime_to_timestamp(datetime.fromtimestamp(1117630859.5, UTC))
+    1117630859.5
+    >>> datetime_to_timestamp(datetime.fromtimestamp(1117630859.5))
+    1117630859.5
     >>> now = datetime.utcnow()
     >>> now2 = datetime.utcfromtimestamp(datetime_to_timestamp(now, False))
+    >>> abs(now2.microsecond - now.microsecond) < 100
+    True
+    >>> now2 = now2.replace(microsecond = now.microsecond)
     >>> now == now2
     True
     >>> now = datetime.now()
     >>> now2 = datetime.fromtimestamp(datetime_to_timestamp(now))
+    >>> abs(now2.microsecond - now.microsecond) < 100
+    True
+    >>> now2 = now2.replace(microsecond = now.microsecond)
     >>> now == now2
     True
     """
diff --git a/scripts/grantfu.py b/scripts/grantfu.py
index 5d89d559..74739d80 100755
--- a/scripts/grantfu.py
+++ b/scripts/grantfu.py
@@ -85,16 +85,10 @@ class PConf(SafeConfigParser):
         return res
 
 class GrantFu:
-    def __init__(self, cf_file, revoke):
+    def __init__(self, cf, revoke):
+        self.cf = cf
         self.revoke = revoke
 
-        # load config
-        self.cf = PConf()
-        self.cf.read(cf_file)
-        if not self.cf.has_section("GrantFu"):
-            print "Incorrect config file, GrantFu sction missing"
-            sys.exit(1)
-
         # avoid putting grantfu vars into defaults, thus into every section
         self.group_list = []
         self.user_list = []
@@ -317,11 +311,26 @@ def main():
     if len(args) != 1:
         usage(1)
 
+    # load config
+    cf = PConf()
+    cf.read(args[0])
+    if not cf.has_section("GrantFu"):
+        print "Incorrect config file, GrantFu sction missing"
+        sys.exit(1)
+
     if tx:
         print "begin;\n"
 
-    g = GrantFu(args[0], revoke)
-    g.process()
+    # revokes and default grants
+    if revoke & (R_NEW | R_DEFS):
+        g = GrantFu(cf, revoke | R_ONLY)
+        g.process()
+        revoke = revoke & R_ONLY
+
+    # grants
+    if revoke & R_ONLY == 0:
+        g = GrantFu(cf, revoke & G_DEFS)
+        g.process()
 
     if tx:
         print "\ncommit;\n"