summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authormartinko2013-06-20 19:49:32 +0000
committermartinko2013-06-20 19:49:32 +0000
commita1242c473d04ed2dc69d82fb371f64808dfced79 (patch)
treeebd0c314b168efbaac9e5564e650908b7703a1c8
parentf8cc1e8c0834fc4ddee1e6a5694fe522f97224ce (diff)
parent53151b6eb4413e2af98ce3bf1e73008ad2ff3710 (diff)
Merge branch 'master' of https://github.com/markokr/skytools into develop
-rw-r--r--NEWS52
-rw-r--r--configure.ac2
-rw-r--r--debian/changelog6
-rw-r--r--doc/TODO.txt13
-rw-r--r--doc/londiste3.txt31
-rwxr-xr-xpython/londiste.py2
-rw-r--r--python/londiste/playback.py22
-rw-r--r--python/londiste/setup.py30
-rw-r--r--python/pgq/baseconsumer.py5
-rw-r--r--python/pgq/cascade/nodeinfo.py2
-rw-r--r--python/pgq/consumer.py3
-rw-r--r--python/skytools/parsing.py2
-rwxr-xr-xpython/skytools/querybuilder.py9
-rw-r--r--python/skytools/timeutil.py18
-rwxr-xr-xscripts/grantfu.py29
15 files changed, 155 insertions, 71 deletions
diff --git a/NEWS b/NEWS
index 0fb04eca..603a396c 100644
--- a/NEWS
+++ b/NEWS
@@ -1,4 +1,56 @@
+2013-04-17 - SkyTools 3.1.4 - "Boldly Going Nowhere"
+
+ = Features =
+
+ * londiste create-node: Creation data from config. The need to 2 connection
+ string in command line was major usability problem in create-* commands.
+ Now initial setup can be described in config.
+ * newgrants: Londiste can read from provider using non-privileged user.
+ * scripts/data_maintainer.py: Generic script for processing
+ large data sets in small batches.
+
+ = Minor features =
+
+ * londiste status: significant speed up of getting status on big cascades
+ * londiste: remote_extra_connstr config option that will be added
+ to remote node connect strings. Needed when more than several cascaded
+ scripts are running, with different privileges in local and remote node.
+ * scriptmgr: User switching with sudo. Script sections can
+ contain user= option that makes scriptmgr launch sudo to run script.
+ * londiste compare: Added --count-only switch
+ * BaseScript: skylog_locations config options
+ * skytools.Config: New default variables: config_dir, config_file.
+ * Parse & merge Postgres connect strings
+ * skytools_upgrade: add force option
+ * londiste: new drop_obsolete_partitions(2) function
+ * londiste: added dispatch handler arg retention_period
+
+ = Fixes =
+
+ * londiste: Sync SQL keywords with 9.3git.
+ * handers/dispatch.py: 'keep_latest' row_mode processes now deletes properly.
+ * configure.ac: Check for -lrt - needed when building against libevent.
+ * adminscripts: Make info commands not use pidfile. Otherwise they will
+ not run if some writing admin command is running.
+ * Londiste compare: It's now compatible with 8.2.
+ * londiste.create_partition: Set owner to match parent table.
+ * londiste.create_trigger: Fixed skip-trigger check compatibility with postgres 9.x.
+ * londiste.create_trigger: Added check for after-trigger being overshadowed by skip-trigger.
+
+ = Cleanups =
+
+ * Refactor Consumer code.
+ * Remove: pgqadm and related code, its unmaintained
+ * Sweeping change to postpone log string formatting
+ * docs: copy-condition removed from londiste3 man page
+ * Various logging cleanups
+ * Londiste: fix trigger generation when extra params that are add-table specific are used (introduced in commit 364ade9)
+ * londiste: quote queue name in trigger args
+ * londiste: actually execute the ENABLE/DISABLE RULE query in londiste.create_partition
+ * londiste/handlers/dispatch: fixed issue with missing part_func description
+ * londiste.handler: disable shortcut for specifying handler args together with its name
+
2012-12-21 - SkyTools 3.1.3 - "Chainsaw-wielding Toothfairy"
= Features =
diff --git a/configure.ac b/configure.ac
index fbc36fe7..f3829de0 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,6 +1,6 @@
dnl Process this file with autoconf to produce a configure script.
-AC_INIT(skytools, 3.1.3)
+AC_INIT(skytools, 3.1.4)
AC_CONFIG_SRCDIR(python/londiste.py)
AC_CONFIG_HEADER(lib/usual/config.h)
AC_PREREQ([2.59])
diff --git a/debian/changelog b/debian/changelog
index 460ef96b..3025f329 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+skytools3 (3.1.4) experimental; urgency=low
+
+ * v3.1.4
+
+ -- Marko Kreen <[email protected]> Wed, 17 Apr 2013 11:08:30 +0300
+
skytools3 (3.1.3) experimental; urgency=low
* v3.1.3
diff --git a/doc/TODO.txt b/doc/TODO.txt
index df78a053..31a49fa7 100644
--- a/doc/TODO.txt
+++ b/doc/TODO.txt
@@ -36,14 +36,6 @@ Low::
. Replay rest of batches fully
. Promote to root
-* Load public connect string for local node:
- - from .ini (db=, public_db=?)
- - with query sql
-
-* Load provider connect string:
- - from ini (not good)
- - describe connstr + query to look up node names?
-
* tests for things that have not their own regtests
or are not tested enough during other tests:
- pgq.RemoteConsumer
@@ -69,11 +61,6 @@ Low::
So separate counter for len(ev_data) needs to be added, that flushes
if buffer would go over some specified amount of memory.
-* cascade status: parallel info gathering. Sequential for-loop over
- nodes can be slow if there are many nodes or some of them are
- generally slow (GP, bad network, etc). Perhaps we can use
- psycopg2 async mode for that. Or threading.
-
== Low Priority ==
* dbscript: switch (-q) for silence for cron/init scripts.
diff --git a/doc/londiste3.txt b/doc/londiste3.txt
index 1cd3521a..c0954a58 100644
--- a/doc/londiste3.txt
+++ b/doc/londiste3.txt
@@ -75,36 +75,41 @@ partial replication and custom handlers for replication.
== NODE INITIALIZATION COMMANDS ==
-Initialization commands are the only ones that requires a connection
-string argument. It is the connection string that Londiste3 will store,
-other nodes will use it to connect to the current node.
+Initialization commands will set up "public connect string" for
+current node. It is a connect string that other nodes will
+use to connect to current node. The local Londiste itself
+uses 'db' option from config file to connect to local node,
+which can have different user rights than scripts coming
+over public connect string.
-=== create-root <node> <connstr> ===
+Connect strings can be set in either command line or config file.
+Command line overrides config. Setting them up in config
+might be more comfortable. See `londiste3 --ini` for details.
+
+=== create-root <node> [<public_connstr_for_node>] ===
Initializes a Master node.
The <node> is the name of the node, it should be unique.
-The <connstr> argument is the connection string to the database on the
-Master node.
-=== create-branch <node> <connstr> --provider=<public_connstr> ===
+=== create-branch <node> [<public_connstr_for_node>] [--provider=<connstr_to_provider>] ===
Initializes a Slave node which can be used as a reference for other
nodes.
The <node> is the name of the node, it should be unique.
-The <connstr> argument is the connection string to the database on the
-current node and <public_connstr> is the connection string to the
+The <public_connstr_for_node> argument is the connection string to the database
+on the current node and <connstr_to_provider> is the connection string to the
provider database (it can be a root node or a branch node).
-=== create-leaf <node> <connstr> --provider=<public_connstr> ===
+=== create-leaf <node> [<public_connstr_for_node>] [--provider=<connstr_to_provider>] ===
-Initializes a Slave node which can not be used as a reference for other
+Initializes a slave node which can not be used as a reference for other
nodes.
The <node> is the name of the node, it should be unique.
-The <connstr> argument is the connection string to the database on the
-current node and <public_connstr> is the connection string to the
+The <public_connstr_for_node> argument is the connection string to the database
+on the current node and <connstr_to_provider> is the connection string to the
provider database (it can be a root node or a branch node).
--merge='qname'::
diff --git a/python/londiste.py b/python/londiste.py
index 6582abea..8860a313 100755
--- a/python/londiste.py
+++ b/python/londiste.py
@@ -148,6 +148,8 @@ class Londiste(skytools.DBScript):
help="don't merge tables from source queues", default=False)
g.add_option("--max-parallel-copy", metavar = "NUM", type = "int",
help="max number of parallel copy processes")
+ g.add_option("--skip-non-existing", action="store_true",
+ help="add: skip object that does not exist")
p.add_option_group(g)
g = optparse.OptionGroup(p, "other options")
diff --git a/python/londiste/playback.py b/python/londiste/playback.py
index c262de53..8d0b2a94 100644
--- a/python/londiste/playback.py
+++ b/python/londiste/playback.py
@@ -201,10 +201,12 @@ class TableState(object):
return self.max_parallel_copy and\
self.copy_pos >= self.max_parallel_copy
- def interesting(self, ev, tick_id, copy_thread):
+ def interesting(self, ev, tick_id, copy_thread, copy_table_name):
"""Check if table wants this event."""
if copy_thread:
+ if self.name != copy_table_name:
+ return False
if self.state not in (TABLE_CATCHING_UP, TABLE_DO_SYNC):
return False
else:
@@ -651,7 +653,7 @@ class Replicator(CascadedWorker):
def handle_data_event(self, ev, dst_curs):
"""handle one truncate event"""
t = self.get_table_by_name(ev.extra1)
- if not t or not t.interesting(ev, self.cur_tick, self.copy_thread):
+ if not t or not t.interesting(ev, self.cur_tick, self.copy_thread, self.copy_table_name):
self.stat_increase('ignored_events')
return
@@ -667,7 +669,7 @@ class Replicator(CascadedWorker):
def handle_truncate_event(self, ev, dst_curs):
"""handle one truncate event"""
t = self.get_table_by_name(ev.extra1)
- if not t or not t.interesting(ev, self.cur_tick, self.copy_thread):
+ if not t or not t.interesting(ev, self.cur_tick, self.copy_thread, self.copy_table_name):
self.stat_increase('ignored_events')
return
@@ -754,17 +756,6 @@ class Replicator(CascadedWorker):
dst_curs.execute(buf)
- def interesting(self, ev):
- """See if event is interesting."""
- if ev.type not in ('I', 'U', 'D', 'R'):
- raise Exception('bug - bad event type in .interesting')
- t = self.get_table_by_name(ev.extra1)
- if not t:
- return 0
- if not t.interesting(ev, self.cur_tick, self.copy_thread):
- return 0
- return 1
-
def add_set_table(self, dst_curs, tbl):
"""There was new table added to root, remember it."""
@@ -883,7 +874,8 @@ class Replicator(CascadedWorker):
# pass same verbosity options as main script got
if self.options.quiet:
cmd.append('-q')
- cmd += self.options.verbose * ['-v']
+ if self.options.verbose:
+ cmd += ['-v'] * self.options.verbose
# let existing copy finish and clean its pidfile,
# otherwise new copy will exit immediately.
diff --git a/python/londiste/setup.py b/python/londiste/setup.py
index 5a53d41a..20d3f5c0 100644
--- a/python/londiste/setup.py
+++ b/python/londiste/setup.py
@@ -82,6 +82,8 @@ class LondisteSetup(CascadeAdmin):
help="max number of parallel copy processes")
p.add_option("--dest-table", metavar = "NAME",
help="add: name for actual table")
+ p.add_option("--skip-non-existing", action="store_true",
+ help="add: skip object that does not exist")
return p
def extra_init(self, node_type, node_db, provider_db):
@@ -163,8 +165,11 @@ class LondisteSetup(CascadeAdmin):
for tbl in args:
tbl = skytools.fq_name(tbl)
if (tbl in src_tbls) and not src_tbls[tbl]['local']:
- self.log.error("Table %s does not exist on provider, need to switch to different provider", tbl)
- problems = True
+ if self.options.skip_non_existing:
+ self.log.warning("Table %s does not exist on provider", tbl)
+ else:
+ self.log.error("Table %s does not exist on provider, need to switch to different provider", tbl)
+ problems = True
if problems:
self.log.error("Problems, canceling operation")
sys.exit(1)
@@ -191,6 +196,7 @@ class LondisteSetup(CascadeAdmin):
src_curs = src_db.cursor()
dst_curs = dst_db.cursor()
tbl_exists = skytools.exists_table(dst_curs, dest_table)
+ dst_db.commit()
if dest_table == tbl:
desc = tbl
@@ -218,6 +224,9 @@ class LondisteSetup(CascadeAdmin):
if src_dest_table != dest_table:
newname = dest_table
s.create(dst_curs, create_flags, log = self.log, new_table_name = newname)
+ elif not tbl_exists and self.options.skip_non_existing:
+ self.log.warning('Table %s does not exist on local node, skipping', desc)
+ return
tgargs = self.build_tgargs()
@@ -402,8 +411,11 @@ class LondisteSetup(CascadeAdmin):
src_db.commit()
s.create(dst_curs, create_flags, log = self.log)
elif not seq_exists:
- self.log.warning('Sequence "%s" missing on subscriber, use --create if necessary', seq)
- return
+ if self.options.skip_non_existing:
+ self.log.warning('Sequence "%s" missing on local node, skipping', seq)
+ return
+ else:
+ raise skytools.UsageError("Sequence %r missing on local node", seq)
q = "select * from londiste.local_add_seq(%s, %s)"
self.exec_cmd(dst_curs, q, [self.set_name, seq])
@@ -694,18 +706,19 @@ class LondisteSetup(CascadeAdmin):
dst_curs = dst_db.cursor()
partial = {}
- done_pos = 1
startup_info = 0
while 1:
dst_curs.execute(q, [self.queue_name])
rows = dst_curs.fetchall()
dst_db.commit()
+ total_count = 0
cur_count = 0
done_list = []
for row in rows:
if not row['local']:
continue
+ total_count += 1
tbl = row['table_name']
if row['merge_state'] != 'ok':
partial[tbl] = 0
@@ -715,13 +728,14 @@ class LondisteSetup(CascadeAdmin):
partial[tbl] = 1
done_list.append(tbl)
+ done_count = total_count - cur_count
+
if not startup_info:
- self.log.info("%d table(s) to copy", len(partial))
+ self.log.info("%d/%d table(s) to copy", cur_count, total_count)
startup_info = 1
for done in done_list:
- self.log.info("%s: finished (%d/%d)", done, done_pos, len(partial))
- done_pos += 1
+ self.log.info("%s: finished (%d/%d)", done, done_count, total_count)
if cur_count == 0:
break
diff --git a/python/pgq/baseconsumer.py b/python/pgq/baseconsumer.py
index 9159106e..3ea1c6c6 100644
--- a/python/pgq/baseconsumer.py
+++ b/python/pgq/baseconsumer.py
@@ -285,6 +285,9 @@ class BaseConsumer(skytools.DBScript):
def _launch_process_batch(self, db, batch_id, list):
self.process_batch(db, batch_id, list)
+ def _make_event(self, queue_name, row):
+ return Event(queue_name, row)
+
def _load_batch_events_old(self, curs, batch_id):
"""Fetch all events for this batch."""
@@ -298,7 +301,7 @@ class BaseConsumer(skytools.DBScript):
# map them to python objects
ev_list = []
for r in rows:
- ev = Event(self.queue_name, r)
+ ev = self._make_event(self.queue_name, r)
ev_list.append(ev)
return ev_list
diff --git a/python/pgq/cascade/nodeinfo.py b/python/pgq/cascade/nodeinfo.py
index 60f07583..e3c82555 100644
--- a/python/pgq/cascade/nodeinfo.py
+++ b/python/pgq/cascade/nodeinfo.py
@@ -240,7 +240,7 @@ class QueueInfo:
for ln in datalines:
print(self._DATAFMT % (_setpfx(pfx, '|'), ln))
datalines = node.get_infolines()
- print("%s%s" % (_setpfx(pfx, '+--'), node.get_title()))
+ print("%s%s" % (_setpfx(pfx, '+--: '), node.get_title()))
for i, n in enumerate(node.child_list):
sfx = ((i < len(node.child_list) - 1) and ' |' or ' ')
diff --git a/python/pgq/consumer.py b/python/pgq/consumer.py
index 294519d8..10cb0909 100644
--- a/python/pgq/consumer.py
+++ b/python/pgq/consumer.py
@@ -60,6 +60,9 @@ class Consumer(BaseConsumer):
_batch_walker_class = RetriableBatchWalker
+ def _make_event(self, queue_name, row):
+ return RetriableWalkerEvent(self, queue_name, row)
+
def _flush_retry(self, curs, batch_id, list):
"""Tag retry events."""
diff --git a/python/skytools/parsing.py b/python/skytools/parsing.py
index 318b1bf9..3d16a3a1 100644
--- a/python/skytools/parsing.py
+++ b/python/skytools/parsing.py
@@ -25,6 +25,8 @@ def parse_pgarray(array):
>>> parse_pgarray("[0,3]={1,2,3}")
['1', '2', '3']
"""
+ if array is None:
+ return None
if not array or array[0] not in ("{", "[") or array[-1] != '}':
raise Exception("bad array format: must be surrounded with {}")
res = []
diff --git a/python/skytools/querybuilder.py b/python/skytools/querybuilder.py
index c2eead2d..9930daab 100755
--- a/python/skytools/querybuilder.py
+++ b/python/skytools/querybuilder.py
@@ -319,8 +319,11 @@ class PLPyQuery:
arg_list = [arg_dict.get(k) for k in self.arg_map]
return plpy.execute(self.plan, arg_list)
except KeyError:
- plpy.error("Missing argument: QUERY: %s ARGS: %s VALUES: %s" % (
- repr(self.sql), repr(self.arg_map), repr(arg_dict)))
+ need = set(self.arg_map)
+ got = set(arg_dict.keys())
+ missing = list(need.difference(got))
+ plpy.error("Missing arguments: [%s] QUERY: %s" % (
+ ','.join(missing), repr(self.sql)))
def __repr__(self):
return 'PLPyQuery<%s>' % self.sql
@@ -341,7 +344,7 @@ def plpy_exec(gd, sql, args, all_keys_required = True):
>>> res = plpy_exec(GD, "select {arg1}, {arg2:int4}, {arg1}", {'arg1': '3', 'arg2': '4'})
DBG: plpy.execute(('PLAN', 'select $1, $2, $3', ['text', 'int4', 'text']), ['3', '4', '3'])
>>> res = plpy_exec(GD, "select {arg1}, {arg2:int4}, {arg1}", {'arg1': '3'})
- DBG: plpy.error("Missing argument: QUERY: 'select {arg1}, {arg2:int4}, {arg1}' ARGS: ['arg1', 'arg2', 'arg1'] VALUES: {'arg1': '3'}")
+ DBG: plpy.error("Missing arguments: [arg2] QUERY: 'select {arg1}, {arg2:int4}, {arg1}'")
>>> res = plpy_exec(GD, "select {arg1}, {arg2:int4}, {arg1}", {'arg1': '3'}, False)
DBG: plpy.execute(('PLAN', 'select $1, $2, $3', ['text', 'int4', 'text']), ['3', None, '3'])
"""
diff --git a/python/skytools/timeutil.py b/python/skytools/timeutil.py
index a29e050c..2ea63082 100644
--- a/python/skytools/timeutil.py
+++ b/python/skytools/timeutil.py
@@ -134,18 +134,24 @@ def datetime_to_timestamp(dt, local_time=True):
Returns seconds since epoch as float.
- >>> datetime_to_timestamp(parse_iso_timestamp("2005-06-01 15:00:59.33 +02"))
- 1117630859.33
- >>> datetime_to_timestamp(datetime.fromtimestamp(1117630859.33, UTC))
- 1117630859.33
- >>> datetime_to_timestamp(datetime.fromtimestamp(1117630859.33))
- 1117630859.33
+ >>> datetime_to_timestamp(parse_iso_timestamp("2005-06-01 15:00:59.5 +02"))
+ 1117630859.5
+ >>> datetime_to_timestamp(datetime.fromtimestamp(1117630859.5, UTC))
+ 1117630859.5
+ >>> datetime_to_timestamp(datetime.fromtimestamp(1117630859.5))
+ 1117630859.5
>>> now = datetime.utcnow()
>>> now2 = datetime.utcfromtimestamp(datetime_to_timestamp(now, False))
+ >>> abs(now2.microsecond - now.microsecond) < 100
+ True
+ >>> now2 = now2.replace(microsecond = now.microsecond)
>>> now == now2
True
>>> now = datetime.now()
>>> now2 = datetime.fromtimestamp(datetime_to_timestamp(now))
+ >>> abs(now2.microsecond - now.microsecond) < 100
+ True
+ >>> now2 = now2.replace(microsecond = now.microsecond)
>>> now == now2
True
"""
diff --git a/scripts/grantfu.py b/scripts/grantfu.py
index 5d89d559..74739d80 100755
--- a/scripts/grantfu.py
+++ b/scripts/grantfu.py
@@ -85,16 +85,10 @@ class PConf(SafeConfigParser):
return res
class GrantFu:
- def __init__(self, cf_file, revoke):
+ def __init__(self, cf, revoke):
+ self.cf = cf
self.revoke = revoke
- # load config
- self.cf = PConf()
- self.cf.read(cf_file)
- if not self.cf.has_section("GrantFu"):
- print "Incorrect config file, GrantFu sction missing"
- sys.exit(1)
-
# avoid putting grantfu vars into defaults, thus into every section
self.group_list = []
self.user_list = []
@@ -317,11 +311,26 @@ def main():
if len(args) != 1:
usage(1)
+ # load config
+ cf = PConf()
+ cf.read(args[0])
+ if not cf.has_section("GrantFu"):
+ print "Incorrect config file, GrantFu sction missing"
+ sys.exit(1)
+
if tx:
print "begin;\n"
- g = GrantFu(args[0], revoke)
- g.process()
+ # revokes and default grants
+ if revoke & (R_NEW | R_DEFS):
+ g = GrantFu(cf, revoke | R_ONLY)
+ g.process()
+ revoke = revoke & R_ONLY
+
+ # grants
+ if revoke & R_ONLY == 0:
+ g = GrantFu(cf, revoke & G_DEFS)
+ g.process()
if tx:
print "\ncommit;\n"