diff options
author | martinko | 2013-06-20 19:49:32 +0000 |
---|---|---|
committer | martinko | 2013-06-20 19:49:32 +0000 |
commit | a1242c473d04ed2dc69d82fb371f64808dfced79 (patch) | |
tree | ebd0c314b168efbaac9e5564e650908b7703a1c8 | |
parent | f8cc1e8c0834fc4ddee1e6a5694fe522f97224ce (diff) | |
parent | 53151b6eb4413e2af98ce3bf1e73008ad2ff3710 (diff) |
Merge branch 'master' of https://github.com/markokr/skytools into develop
-rw-r--r-- | NEWS | 52 | ||||
-rw-r--r-- | configure.ac | 2 | ||||
-rw-r--r-- | debian/changelog | 6 | ||||
-rw-r--r-- | doc/TODO.txt | 13 | ||||
-rw-r--r-- | doc/londiste3.txt | 31 | ||||
-rwxr-xr-x | python/londiste.py | 2 | ||||
-rw-r--r-- | python/londiste/playback.py | 22 | ||||
-rw-r--r-- | python/londiste/setup.py | 30 | ||||
-rw-r--r-- | python/pgq/baseconsumer.py | 5 | ||||
-rw-r--r-- | python/pgq/cascade/nodeinfo.py | 2 | ||||
-rw-r--r-- | python/pgq/consumer.py | 3 | ||||
-rw-r--r-- | python/skytools/parsing.py | 2 | ||||
-rwxr-xr-x | python/skytools/querybuilder.py | 9 | ||||
-rw-r--r-- | python/skytools/timeutil.py | 18 | ||||
-rwxr-xr-x | scripts/grantfu.py | 29 |
15 files changed, 155 insertions, 71 deletions
@@ -1,4 +1,56 @@ +2013-04-17 - SkyTools 3.1.4 - "Boldly Going Nowhere" + + = Features = + + * londiste create-node: Creation data from config. The need to 2 connection + string in command line was major usability problem in create-* commands. + Now initial setup can be described in config. + * newgrants: Londiste can read from provider using non-privileged user. + * scripts/data_maintainer.py: Generic script for processing + large data sets in small batches. + + = Minor features = + + * londiste status: significant speed up of getting status on big cascades + * londiste: remote_extra_connstr config option that will be added + to remote node connect strings. Needed when more than several cascaded + scripts are running, with different privileges in local and remote node. + * scriptmgr: User switching with sudo. Script sections can + contain user= option that makes scriptmgr launch sudo to run script. + * londiste compare: Added --count-only switch + * BaseScript: skylog_locations config options + * skytools.Config: New default variables: config_dir, config_file. + * Parse & merge Postgres connect strings + * skytools_upgrade: add force option + * londiste: new drop_obsolete_partitions(2) function + * londiste: added dispatch handler arg retention_period + + = Fixes = + + * londiste: Sync SQL keywords with 9.3git. + * handers/dispatch.py: 'keep_latest' row_mode processes now deletes properly. + * configure.ac: Check for -lrt - needed when building against libevent. + * adminscripts: Make info commands not use pidfile. Otherwise they will + not run if some writing admin command is running. + * Londiste compare: It's now compatible with 8.2. + * londiste.create_partition: Set owner to match parent table. + * londiste.create_trigger: Fixed skip-trigger check compatibility with postgres 9.x. + * londiste.create_trigger: Added check for after-trigger being overshadowed by skip-trigger. + + = Cleanups = + + * Refactor Consumer code. + * Remove: pgqadm and related code, its unmaintained + * Sweeping change to postpone log string formatting + * docs: copy-condition removed from londiste3 man page + * Various logging cleanups + * Londiste: fix trigger generation when extra params that are add-table specific are used (introduced in commit 364ade9) + * londiste: quote queue name in trigger args + * londiste: actually execute the ENABLE/DISABLE RULE query in londiste.create_partition + * londiste/handlers/dispatch: fixed issue with missing part_func description + * londiste.handler: disable shortcut for specifying handler args together with its name + 2012-12-21 - SkyTools 3.1.3 - "Chainsaw-wielding Toothfairy" = Features = diff --git a/configure.ac b/configure.ac index fbc36fe7..f3829de0 100644 --- a/configure.ac +++ b/configure.ac @@ -1,6 +1,6 @@ dnl Process this file with autoconf to produce a configure script. -AC_INIT(skytools, 3.1.3) +AC_INIT(skytools, 3.1.4) AC_CONFIG_SRCDIR(python/londiste.py) AC_CONFIG_HEADER(lib/usual/config.h) AC_PREREQ([2.59]) diff --git a/debian/changelog b/debian/changelog index 460ef96b..3025f329 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +skytools3 (3.1.4) experimental; urgency=low + + * v3.1.4 + + -- Marko Kreen <[email protected]> Wed, 17 Apr 2013 11:08:30 +0300 + skytools3 (3.1.3) experimental; urgency=low * v3.1.3 diff --git a/doc/TODO.txt b/doc/TODO.txt index df78a053..31a49fa7 100644 --- a/doc/TODO.txt +++ b/doc/TODO.txt @@ -36,14 +36,6 @@ Low:: . Replay rest of batches fully . Promote to root -* Load public connect string for local node: - - from .ini (db=, public_db=?) - - with query sql - -* Load provider connect string: - - from ini (not good) - - describe connstr + query to look up node names? - * tests for things that have not their own regtests or are not tested enough during other tests: - pgq.RemoteConsumer @@ -69,11 +61,6 @@ Low:: So separate counter for len(ev_data) needs to be added, that flushes if buffer would go over some specified amount of memory. -* cascade status: parallel info gathering. Sequential for-loop over - nodes can be slow if there are many nodes or some of them are - generally slow (GP, bad network, etc). Perhaps we can use - psycopg2 async mode for that. Or threading. - == Low Priority == * dbscript: switch (-q) for silence for cron/init scripts. diff --git a/doc/londiste3.txt b/doc/londiste3.txt index 1cd3521a..c0954a58 100644 --- a/doc/londiste3.txt +++ b/doc/londiste3.txt @@ -75,36 +75,41 @@ partial replication and custom handlers for replication. == NODE INITIALIZATION COMMANDS == -Initialization commands are the only ones that requires a connection -string argument. It is the connection string that Londiste3 will store, -other nodes will use it to connect to the current node. +Initialization commands will set up "public connect string" for +current node. It is a connect string that other nodes will +use to connect to current node. The local Londiste itself +uses 'db' option from config file to connect to local node, +which can have different user rights than scripts coming +over public connect string. -=== create-root <node> <connstr> === +Connect strings can be set in either command line or config file. +Command line overrides config. Setting them up in config +might be more comfortable. See `londiste3 --ini` for details. + +=== create-root <node> [<public_connstr_for_node>] === Initializes a Master node. The <node> is the name of the node, it should be unique. -The <connstr> argument is the connection string to the database on the -Master node. -=== create-branch <node> <connstr> --provider=<public_connstr> === +=== create-branch <node> [<public_connstr_for_node>] [--provider=<connstr_to_provider>] === Initializes a Slave node which can be used as a reference for other nodes. The <node> is the name of the node, it should be unique. -The <connstr> argument is the connection string to the database on the -current node and <public_connstr> is the connection string to the +The <public_connstr_for_node> argument is the connection string to the database +on the current node and <connstr_to_provider> is the connection string to the provider database (it can be a root node or a branch node). -=== create-leaf <node> <connstr> --provider=<public_connstr> === +=== create-leaf <node> [<public_connstr_for_node>] [--provider=<connstr_to_provider>] === -Initializes a Slave node which can not be used as a reference for other +Initializes a slave node which can not be used as a reference for other nodes. The <node> is the name of the node, it should be unique. -The <connstr> argument is the connection string to the database on the -current node and <public_connstr> is the connection string to the +The <public_connstr_for_node> argument is the connection string to the database +on the current node and <connstr_to_provider> is the connection string to the provider database (it can be a root node or a branch node). --merge='qname':: diff --git a/python/londiste.py b/python/londiste.py index 6582abea..8860a313 100755 --- a/python/londiste.py +++ b/python/londiste.py @@ -148,6 +148,8 @@ class Londiste(skytools.DBScript): help="don't merge tables from source queues", default=False) g.add_option("--max-parallel-copy", metavar = "NUM", type = "int", help="max number of parallel copy processes") + g.add_option("--skip-non-existing", action="store_true", + help="add: skip object that does not exist") p.add_option_group(g) g = optparse.OptionGroup(p, "other options") diff --git a/python/londiste/playback.py b/python/londiste/playback.py index c262de53..8d0b2a94 100644 --- a/python/londiste/playback.py +++ b/python/londiste/playback.py @@ -201,10 +201,12 @@ class TableState(object): return self.max_parallel_copy and\ self.copy_pos >= self.max_parallel_copy - def interesting(self, ev, tick_id, copy_thread): + def interesting(self, ev, tick_id, copy_thread, copy_table_name): """Check if table wants this event.""" if copy_thread: + if self.name != copy_table_name: + return False if self.state not in (TABLE_CATCHING_UP, TABLE_DO_SYNC): return False else: @@ -651,7 +653,7 @@ class Replicator(CascadedWorker): def handle_data_event(self, ev, dst_curs): """handle one truncate event""" t = self.get_table_by_name(ev.extra1) - if not t or not t.interesting(ev, self.cur_tick, self.copy_thread): + if not t or not t.interesting(ev, self.cur_tick, self.copy_thread, self.copy_table_name): self.stat_increase('ignored_events') return @@ -667,7 +669,7 @@ class Replicator(CascadedWorker): def handle_truncate_event(self, ev, dst_curs): """handle one truncate event""" t = self.get_table_by_name(ev.extra1) - if not t or not t.interesting(ev, self.cur_tick, self.copy_thread): + if not t or not t.interesting(ev, self.cur_tick, self.copy_thread, self.copy_table_name): self.stat_increase('ignored_events') return @@ -754,17 +756,6 @@ class Replicator(CascadedWorker): dst_curs.execute(buf) - def interesting(self, ev): - """See if event is interesting.""" - if ev.type not in ('I', 'U', 'D', 'R'): - raise Exception('bug - bad event type in .interesting') - t = self.get_table_by_name(ev.extra1) - if not t: - return 0 - if not t.interesting(ev, self.cur_tick, self.copy_thread): - return 0 - return 1 - def add_set_table(self, dst_curs, tbl): """There was new table added to root, remember it.""" @@ -883,7 +874,8 @@ class Replicator(CascadedWorker): # pass same verbosity options as main script got if self.options.quiet: cmd.append('-q') - cmd += self.options.verbose * ['-v'] + if self.options.verbose: + cmd += ['-v'] * self.options.verbose # let existing copy finish and clean its pidfile, # otherwise new copy will exit immediately. diff --git a/python/londiste/setup.py b/python/londiste/setup.py index 5a53d41a..20d3f5c0 100644 --- a/python/londiste/setup.py +++ b/python/londiste/setup.py @@ -82,6 +82,8 @@ class LondisteSetup(CascadeAdmin): help="max number of parallel copy processes") p.add_option("--dest-table", metavar = "NAME", help="add: name for actual table") + p.add_option("--skip-non-existing", action="store_true", + help="add: skip object that does not exist") return p def extra_init(self, node_type, node_db, provider_db): @@ -163,8 +165,11 @@ class LondisteSetup(CascadeAdmin): for tbl in args: tbl = skytools.fq_name(tbl) if (tbl in src_tbls) and not src_tbls[tbl]['local']: - self.log.error("Table %s does not exist on provider, need to switch to different provider", tbl) - problems = True + if self.options.skip_non_existing: + self.log.warning("Table %s does not exist on provider", tbl) + else: + self.log.error("Table %s does not exist on provider, need to switch to different provider", tbl) + problems = True if problems: self.log.error("Problems, canceling operation") sys.exit(1) @@ -191,6 +196,7 @@ class LondisteSetup(CascadeAdmin): src_curs = src_db.cursor() dst_curs = dst_db.cursor() tbl_exists = skytools.exists_table(dst_curs, dest_table) + dst_db.commit() if dest_table == tbl: desc = tbl @@ -218,6 +224,9 @@ class LondisteSetup(CascadeAdmin): if src_dest_table != dest_table: newname = dest_table s.create(dst_curs, create_flags, log = self.log, new_table_name = newname) + elif not tbl_exists and self.options.skip_non_existing: + self.log.warning('Table %s does not exist on local node, skipping', desc) + return tgargs = self.build_tgargs() @@ -402,8 +411,11 @@ class LondisteSetup(CascadeAdmin): src_db.commit() s.create(dst_curs, create_flags, log = self.log) elif not seq_exists: - self.log.warning('Sequence "%s" missing on subscriber, use --create if necessary', seq) - return + if self.options.skip_non_existing: + self.log.warning('Sequence "%s" missing on local node, skipping', seq) + return + else: + raise skytools.UsageError("Sequence %r missing on local node", seq) q = "select * from londiste.local_add_seq(%s, %s)" self.exec_cmd(dst_curs, q, [self.set_name, seq]) @@ -694,18 +706,19 @@ class LondisteSetup(CascadeAdmin): dst_curs = dst_db.cursor() partial = {} - done_pos = 1 startup_info = 0 while 1: dst_curs.execute(q, [self.queue_name]) rows = dst_curs.fetchall() dst_db.commit() + total_count = 0 cur_count = 0 done_list = [] for row in rows: if not row['local']: continue + total_count += 1 tbl = row['table_name'] if row['merge_state'] != 'ok': partial[tbl] = 0 @@ -715,13 +728,14 @@ class LondisteSetup(CascadeAdmin): partial[tbl] = 1 done_list.append(tbl) + done_count = total_count - cur_count + if not startup_info: - self.log.info("%d table(s) to copy", len(partial)) + self.log.info("%d/%d table(s) to copy", cur_count, total_count) startup_info = 1 for done in done_list: - self.log.info("%s: finished (%d/%d)", done, done_pos, len(partial)) - done_pos += 1 + self.log.info("%s: finished (%d/%d)", done, done_count, total_count) if cur_count == 0: break diff --git a/python/pgq/baseconsumer.py b/python/pgq/baseconsumer.py index 9159106e..3ea1c6c6 100644 --- a/python/pgq/baseconsumer.py +++ b/python/pgq/baseconsumer.py @@ -285,6 +285,9 @@ class BaseConsumer(skytools.DBScript): def _launch_process_batch(self, db, batch_id, list): self.process_batch(db, batch_id, list) + def _make_event(self, queue_name, row): + return Event(queue_name, row) + def _load_batch_events_old(self, curs, batch_id): """Fetch all events for this batch.""" @@ -298,7 +301,7 @@ class BaseConsumer(skytools.DBScript): # map them to python objects ev_list = [] for r in rows: - ev = Event(self.queue_name, r) + ev = self._make_event(self.queue_name, r) ev_list.append(ev) return ev_list diff --git a/python/pgq/cascade/nodeinfo.py b/python/pgq/cascade/nodeinfo.py index 60f07583..e3c82555 100644 --- a/python/pgq/cascade/nodeinfo.py +++ b/python/pgq/cascade/nodeinfo.py @@ -240,7 +240,7 @@ class QueueInfo: for ln in datalines: print(self._DATAFMT % (_setpfx(pfx, '|'), ln)) datalines = node.get_infolines() - print("%s%s" % (_setpfx(pfx, '+--'), node.get_title())) + print("%s%s" % (_setpfx(pfx, '+--: '), node.get_title())) for i, n in enumerate(node.child_list): sfx = ((i < len(node.child_list) - 1) and ' |' or ' ') diff --git a/python/pgq/consumer.py b/python/pgq/consumer.py index 294519d8..10cb0909 100644 --- a/python/pgq/consumer.py +++ b/python/pgq/consumer.py @@ -60,6 +60,9 @@ class Consumer(BaseConsumer): _batch_walker_class = RetriableBatchWalker + def _make_event(self, queue_name, row): + return RetriableWalkerEvent(self, queue_name, row) + def _flush_retry(self, curs, batch_id, list): """Tag retry events.""" diff --git a/python/skytools/parsing.py b/python/skytools/parsing.py index 318b1bf9..3d16a3a1 100644 --- a/python/skytools/parsing.py +++ b/python/skytools/parsing.py @@ -25,6 +25,8 @@ def parse_pgarray(array): >>> parse_pgarray("[0,3]={1,2,3}") ['1', '2', '3'] """ + if array is None: + return None if not array or array[0] not in ("{", "[") or array[-1] != '}': raise Exception("bad array format: must be surrounded with {}") res = [] diff --git a/python/skytools/querybuilder.py b/python/skytools/querybuilder.py index c2eead2d..9930daab 100755 --- a/python/skytools/querybuilder.py +++ b/python/skytools/querybuilder.py @@ -319,8 +319,11 @@ class PLPyQuery: arg_list = [arg_dict.get(k) for k in self.arg_map] return plpy.execute(self.plan, arg_list) except KeyError: - plpy.error("Missing argument: QUERY: %s ARGS: %s VALUES: %s" % ( - repr(self.sql), repr(self.arg_map), repr(arg_dict))) + need = set(self.arg_map) + got = set(arg_dict.keys()) + missing = list(need.difference(got)) + plpy.error("Missing arguments: [%s] QUERY: %s" % ( + ','.join(missing), repr(self.sql))) def __repr__(self): return 'PLPyQuery<%s>' % self.sql @@ -341,7 +344,7 @@ def plpy_exec(gd, sql, args, all_keys_required = True): >>> res = plpy_exec(GD, "select {arg1}, {arg2:int4}, {arg1}", {'arg1': '3', 'arg2': '4'}) DBG: plpy.execute(('PLAN', 'select $1, $2, $3', ['text', 'int4', 'text']), ['3', '4', '3']) >>> res = plpy_exec(GD, "select {arg1}, {arg2:int4}, {arg1}", {'arg1': '3'}) - DBG: plpy.error("Missing argument: QUERY: 'select {arg1}, {arg2:int4}, {arg1}' ARGS: ['arg1', 'arg2', 'arg1'] VALUES: {'arg1': '3'}") + DBG: plpy.error("Missing arguments: [arg2] QUERY: 'select {arg1}, {arg2:int4}, {arg1}'") >>> res = plpy_exec(GD, "select {arg1}, {arg2:int4}, {arg1}", {'arg1': '3'}, False) DBG: plpy.execute(('PLAN', 'select $1, $2, $3', ['text', 'int4', 'text']), ['3', None, '3']) """ diff --git a/python/skytools/timeutil.py b/python/skytools/timeutil.py index a29e050c..2ea63082 100644 --- a/python/skytools/timeutil.py +++ b/python/skytools/timeutil.py @@ -134,18 +134,24 @@ def datetime_to_timestamp(dt, local_time=True): Returns seconds since epoch as float. - >>> datetime_to_timestamp(parse_iso_timestamp("2005-06-01 15:00:59.33 +02")) - 1117630859.33 - >>> datetime_to_timestamp(datetime.fromtimestamp(1117630859.33, UTC)) - 1117630859.33 - >>> datetime_to_timestamp(datetime.fromtimestamp(1117630859.33)) - 1117630859.33 + >>> datetime_to_timestamp(parse_iso_timestamp("2005-06-01 15:00:59.5 +02")) + 1117630859.5 + >>> datetime_to_timestamp(datetime.fromtimestamp(1117630859.5, UTC)) + 1117630859.5 + >>> datetime_to_timestamp(datetime.fromtimestamp(1117630859.5)) + 1117630859.5 >>> now = datetime.utcnow() >>> now2 = datetime.utcfromtimestamp(datetime_to_timestamp(now, False)) + >>> abs(now2.microsecond - now.microsecond) < 100 + True + >>> now2 = now2.replace(microsecond = now.microsecond) >>> now == now2 True >>> now = datetime.now() >>> now2 = datetime.fromtimestamp(datetime_to_timestamp(now)) + >>> abs(now2.microsecond - now.microsecond) < 100 + True + >>> now2 = now2.replace(microsecond = now.microsecond) >>> now == now2 True """ diff --git a/scripts/grantfu.py b/scripts/grantfu.py index 5d89d559..74739d80 100755 --- a/scripts/grantfu.py +++ b/scripts/grantfu.py @@ -85,16 +85,10 @@ class PConf(SafeConfigParser): return res class GrantFu: - def __init__(self, cf_file, revoke): + def __init__(self, cf, revoke): + self.cf = cf self.revoke = revoke - # load config - self.cf = PConf() - self.cf.read(cf_file) - if not self.cf.has_section("GrantFu"): - print "Incorrect config file, GrantFu sction missing" - sys.exit(1) - # avoid putting grantfu vars into defaults, thus into every section self.group_list = [] self.user_list = [] @@ -317,11 +311,26 @@ def main(): if len(args) != 1: usage(1) + # load config + cf = PConf() + cf.read(args[0]) + if not cf.has_section("GrantFu"): + print "Incorrect config file, GrantFu sction missing" + sys.exit(1) + if tx: print "begin;\n" - g = GrantFu(args[0], revoke) - g.process() + # revokes and default grants + if revoke & (R_NEW | R_DEFS): + g = GrantFu(cf, revoke | R_ONLY) + g.process() + revoke = revoke & R_ONLY + + # grants + if revoke & R_ONLY == 0: + g = GrantFu(cf, revoke & G_DEFS) + g.process() if tx: print "\ncommit;\n" |