diff options
-rw-r--r-- | NEWS | 12 | ||||
-rw-r--r-- | configure.ac | 2 | ||||
-rw-r--r-- | debian/changelog | 12 | ||||
-rw-r--r-- | python/londiste/handlers/dispatch.py | 19 | ||||
-rw-r--r-- | python/skytools/scripting.py | 5 | ||||
-rw-r--r-- | python/skytools/timeutil.py | 14 | ||||
-rwxr-xr-x | python/walmgr.py | 2 | ||||
-rwxr-xr-x | scripts/simple_local_consumer.py | 6 | ||||
-rwxr-xr-x | setup_skytools.py | 56 | ||||
-rw-r--r-- | sql/londiste/functions/londiste.drop_obsolete_partitions.sql | 34 | ||||
-rw-r--r-- | sql/londiste/functions/londiste.is_obsolete_partition.sql | 56 | ||||
-rw-r--r-- | sql/londiste/functions/londiste.list_obsolete_partitions.sql | 62 | ||||
-rw-r--r-- | sql/londiste/structure/grants.ini | 5 | ||||
-rw-r--r-- | sql/pgq/functions/pgq.register_consumer.sql | 5 | ||||
-rw-r--r-- | sql/pgq/functions/pgq.unregister_consumer.sql | 12 | ||||
-rw-r--r-- | sql/pgq_node/structure/tables.sql | 9 |
16 files changed, 236 insertions, 75 deletions
@@ -1,4 +1,16 @@ +2014-04-10 - SkyTools 3.2.2 + + = Fixes = + + * skytools.scripting: moved psycopg2 reference to actual script using it + * skytools.timeutil: fixed for Python versions less than 2.7 + + = Cleanups = + + * libusual: updated to the latest version (that is already 9 months old) + * setup*.py: fixes; updated to point to correct licence + 2014-03-31 - SkyTools 3.2 - "Hit any user to continue" = Features = diff --git a/configure.ac b/configure.ac index 458deb5a..eec6c639 100644 --- a/configure.ac +++ b/configure.ac @@ -1,6 +1,6 @@ dnl Process this file with autoconf to produce a configure script. -AC_INIT(skytools, 3.2) +AC_INIT(skytools, 3.2.2) AC_CONFIG_SRCDIR(python/londiste.py) AC_CONFIG_HEADER(lib/usual/config.h) AC_PREREQ([2.59]) diff --git a/debian/changelog b/debian/changelog index 3712fc0b..f0cb9d34 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,15 @@ +skytools3 (3.2.2) experimental; urgency=low + + * v3.2.2 + + -- martinko <[email protected]> Thu, 10 Apr 2014 14:00:00 +0200 + +skytools3 (3.2.1) experimental; urgency=low + + * v3.2.1 + + -- martinko <[email protected]> Wed, 09 Apr 2014 17:34:51 +0200 + skytools3 (3.2) experimental; urgency=low * v3.2 diff --git a/python/londiste/handlers/dispatch.py b/python/londiste/handlers/dispatch.py index 90d01bbb..68e17083 100644 --- a/python/londiste/handlers/dispatch.py +++ b/python/londiste/handlers/dispatch.py @@ -849,8 +849,13 @@ class Dispatcher (ShardHandler): else if part function present in db, call it else clone master table""" curs = self.dst_curs + if (self.conf.ignore_old_events and self.conf.retention_period and + self.is_obsolete_partition (dst, self.conf.retention_period, self.conf.period)): + self.ignored_tables.add(dst) + return if skytools.exists_table(curs, dst): return + dst = quote_fqident(dst) vals = {'dest': dst, 'part': dst, @@ -926,6 +931,20 @@ class Dispatcher (ShardHandler): self.log.info("Dropped tables: %s", ", ".join(res)) return res + def is_obsolete_partition (self, partition_table, retention_period, partition_period): + """ Test partition name of partition-by-date parent table. + """ + curs = self.dst_curs + func = "londiste.is_obsolete_partition" + args = [partition_table, retention_period, partition_period] + sql = "select " + func + " (%s, %s, %s)" + self.log.debug("func: %s, args: %s", func, args) + curs.execute(sql, args) + res = curs.fetchone()[0] + if res: + self.log.info("Ignored table: %s", partition_table) + return res + def get_copy_condition(self, src_curs, dst_curs): """ Prepare where condition for copy and replay filtering. """ diff --git a/python/skytools/scripting.py b/python/skytools/scripting.py index 68ddc4d3..23660ced 100644 --- a/python/skytools/scripting.py +++ b/python/skytools/scripting.py @@ -14,7 +14,6 @@ import signal import sys import time -import psycopg2 import skytools import skytools.skylog @@ -672,7 +671,7 @@ class BaseScript(object): In case of daemon, if will be called in same process as work(), unlike __init__(). """ - pass + self.log.info("Script finished, exiting") # define some aliases (short-cuts / backward compatibility cruft) stat_add = stat_put # Old, deprecated function. @@ -958,7 +957,7 @@ class DBScript(BaseScript): sql_retry_formula_a = self.cf.getint("sql_retry_formula_a", 1) sql_retry_formula_b = self.cf.getint("sql_retry_formula_b", 5) sql_retry_formula_cap = self.cf.getint("sql_retry_formula_cap", 60) - elist = exceptions or (psycopg2.OperationalError,) + elist = exceptions or tuple([]) stime = time.time() tried = 0 dbc = None diff --git a/python/skytools/timeutil.py b/python/skytools/timeutil.py index 2ea63082..c04756b8 100644 --- a/python/skytools/timeutil.py +++ b/python/skytools/timeutil.py @@ -16,6 +16,20 @@ from datetime import datetime, timedelta, tzinfo __all__ = ['parse_iso_timestamp', 'FixedOffsetTimezone', 'datetime_to_timestamp'] +try: + timedelta.total_seconds # new in 2.7 +except AttributeError: + def total_seconds(td): + return float (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6 + + import ctypes + _get_dict = ctypes.pythonapi._PyObject_GetDictPtr + _get_dict.restype = ctypes.POINTER(ctypes.py_object) + _get_dict.argtypes = [ctypes.py_object] + d = _get_dict(timedelta)[0] + d['total_seconds'] = total_seconds + + class FixedOffsetTimezone(tzinfo): """Fixed offset in minutes east from UTC.""" __slots__ = ('__offset', '__name') diff --git a/python/walmgr.py b/python/walmgr.py index 2918b067..8de02bd4 100755 --- a/python/walmgr.py +++ b/python/walmgr.py @@ -1821,7 +1821,7 @@ STOP TIME: %(stop_time)s self.log.info("%s: not found (ignored)", srcname) # remove PG_RECEIVEXLOG file if it's present - if os.path.isfile(prxlogfile): + if os.path.isfile(prxlogfile) and not srcname.endswith('.history'): os.remove(prxlogfile) sys.exit(1) diff --git a/scripts/simple_local_consumer.py b/scripts/simple_local_consumer.py index 50177097..e78ba0b4 100755 --- a/scripts/simple_local_consumer.py +++ b/scripts/simple_local_consumer.py @@ -22,12 +22,15 @@ Config:: import sys +import psycopg2 + import pkgloader pkgloader.require('skytools', '3.0') import pgq import skytools + class SimpleLocalConsumer(pgq.LocalConsumer): __doc__ = __doc__ @@ -57,7 +60,8 @@ class SimpleLocalConsumer(pgq.LocalConsumer): payload['pgq.ev_extra4'] = ev.ev_extra4 self.log.debug(self.dst_query, payload) - retries, curs = self.execute_with_retry('dst_db', self.dst_query, payload) + retries, curs = self.execute_with_retry('dst_db', self.dst_query, payload, + exceptions = (psycopg2.OperationalError,)) if curs.statusmessage[:6] == 'SELECT': res = curs.fetchall() self.log.debug(res) diff --git a/setup_skytools.py b/setup_skytools.py index 0d2158d4..ce52e250 100755 --- a/setup_skytools.py +++ b/setup_skytools.py @@ -20,7 +20,7 @@ from subprocess import Popen INSTALL_SCRIPTS = 1 INSTALL_SQL = 1 -# dont build C module on win32 as it's unlikely to have dev env +# don't build C module on win32 as it's unlikely to have dev env BUILD_C_MOD = 1 if sys.platform == 'win32': BUILD_C_MOD = 0 @@ -53,19 +53,19 @@ if not INSTALL_SCRIPTS: # sql files we want to access from python sql_files = [ - 'sql/pgq/pgq.sql', - 'sql/londiste/londiste.sql', - 'sql/pgq_node/pgq_node.sql', - 'sql/pgq_coop/pgq_coop.sql', - 'sql/pgq_ext/pgq_ext.sql', - - 'sql/pgq/pgq.upgrade.sql', - 'sql/pgq_node/pgq_node.upgrade.sql', - 'sql/londiste/londiste.upgrade.sql', - 'sql/pgq_coop/pgq_coop.upgrade.sql', - 'sql/pgq_ext/pgq_ext.upgrade.sql', - 'upgrade/final/pgq.upgrade_2.1_to_3.0.sql', - 'upgrade/final/londiste.upgrade_2.1_to_3.1.sql', + 'sql/pgq/pgq.sql', + 'sql/londiste/londiste.sql', + 'sql/pgq_node/pgq_node.sql', + 'sql/pgq_coop/pgq_coop.sql', + 'sql/pgq_ext/pgq_ext.sql', + + 'sql/pgq/pgq.upgrade.sql', + 'sql/pgq_node/pgq_node.upgrade.sql', + 'sql/londiste/londiste.upgrade.sql', + 'sql/pgq_coop/pgq_coop.upgrade.sql', + 'sql/pgq_ext/pgq_ext.upgrade.sql', + 'upgrade/final/pgq.upgrade_2.1_to_3.0.sql', + 'upgrade/final/londiste.upgrade_2.1_to_3.1.sql', ] # sql files for special occasions @@ -87,7 +87,7 @@ def getvar(name, default): pass return default -# dont rename scripts on win32 +# don't rename scripts on win32 if sys.platform == 'win32': DEF_SUFFIX = '.py' DEF_NOSUFFIX = '.py' @@ -100,7 +100,7 @@ DEF_SUFFIX = getvar('SUFFIX', DEF_SUFFIX) DEF_SKYLOG = getvar('SKYLOG', '0') != '0' DEF_SK3_SUBDIR = getvar('SK3_SUBDIR', '0') != '0' -# create sql files if they dont exist +# create sql files if they don't exist def make_sql(): for fn in sql_files: if not os.path.isfile(fn): @@ -155,8 +155,8 @@ class sk3_build_scripts(build_scripts): # wrap generic install command class sk3_install(install): user_options = install.user_options + [ - ('sk3-subdir', None, 'install modules into "skytools-3.0" subdir'), - ('skylog', None, 'use "skylog" logging by default'), + ('sk3-subdir', None, 'install modules into "skytools-3.0" subdir'), + ('skylog', None, 'use "skylog" logging by default'), ] boolean_options = ['sk3-subdir', 'skylog'] sk3_subdir = DEF_SK3_SUBDIR @@ -187,7 +187,7 @@ if BUILD_C_MOD: ext = [ Extension("skytools._cquoting", ['python/modules/cquoting.c']), Extension("skytools._chashtext", ['python/modules/hashtext.c']), - ] + ] c_modules.extend(ext) # run actual setup @@ -198,15 +198,17 @@ setup( maintainer = "Marko Kreen", maintainer_email = "[email protected]", url = "http://pgfoundry.org/projects/skytools/", + description = "SkyTools - tools for PostgreSQL", + platforms = "POSIX, MacOS, Windows", package_dir = {'': 'python'}, packages = ['skytools', 'londiste', 'londiste.handlers', 'pgq', 'pgq.cascade'], data_files = [ - ('share/doc/skytools3/conf', [ - 'python/conf/wal-master.ini', - 'python/conf/wal-slave.ini', + ('share/doc/skytools3/conf', [ + 'python/conf/wal-master.ini', + 'python/conf/wal-slave.ini', ]), - ('share/skytools3', sql_files), - #('share/skytools3/extra', extra_sql_files), + ('share/skytools3', sql_files), + #('share/skytools3/extra', extra_sql_files), ], ext_modules = c_modules, scripts = sfx_scripts + nosfx_scripts, @@ -215,4 +217,10 @@ setup( 'build_scripts': sk3_build_scripts, 'install': sk3_install, }, + long_description = """ +This is a package of tools developed at Skype for replication and failover. +It includes a generic queuing framework (PgQ), easy-to-use replication +implementation (Londiste), tool for managing WAL based standby servers, +utility library for Python scripts, selection of scripts for specific jobs. +""" ) diff --git a/sql/londiste/functions/londiste.drop_obsolete_partitions.sql b/sql/londiste/functions/londiste.drop_obsolete_partitions.sql index 4346724e..1718fa73 100644 --- a/sql/londiste/functions/londiste.drop_obsolete_partitions.sql +++ b/sql/londiste/functions/londiste.drop_obsolete_partitions.sql @@ -21,40 +21,10 @@ as $$ -- Names of partitions dropped ------------------------------------------------------------------------------- declare - _schema text not null := lower (split_part (i_parent_table, '.', 1)); - _table text not null := lower (split_part (i_parent_table, '.', 2)); - _part text; - _expr text; - _dfmt text; + _part text; begin - if i_partition_period in ('year', 'yearly') then - _expr := '_[0-9]{4}'; - _dfmt := '_YYYY'; - elsif i_partition_period in ('month', 'monthly') then - _expr := '_[0-9]{4}_[0-9]{2}'; - _dfmt := '_YYYY_MM'; - elsif i_partition_period in ('day', 'daily') then - _expr := '_[0-9]{4}_[0-9]{2}_[0-9]{2}'; - _dfmt := '_YYYY_MM_DD'; - elsif i_partition_period in ('hour', 'hourly') then - _expr := '_[0-9]{4}_[0-9]{2}_[0-9]{2}_[0-9]{2}'; - _dfmt := '_YYYY_MM_DD_HH24'; - else - raise exception 'not supported i_partition_period: %', i_partition_period; - end if; - - if length (_table) = 0 then - _table := _schema; - _schema := 'public'; - end if; - for _part in - select quote_ident (t.schemaname) ||'.'|| quote_ident (t.tablename) - from pg_catalog.pg_tables t - where t.schemaname = _schema - and t.tablename ~ ('^'|| _table || _expr ||'$') - and t.tablename < _table || to_char (now() - i_retention_period, _dfmt) - order by 1 + select londiste.list_obsolete_partitions (i_parent_table, i_retention_period, i_partition_period) loop execute 'drop table '|| _part; return next _part; diff --git a/sql/londiste/functions/londiste.is_obsolete_partition.sql b/sql/londiste/functions/londiste.is_obsolete_partition.sql new file mode 100644 index 00000000..d7baea9c --- /dev/null +++ b/sql/londiste/functions/londiste.is_obsolete_partition.sql @@ -0,0 +1,56 @@ + +create or replace function londiste.is_obsolete_partition +( + in i_partition_table text, + in i_retention_period interval, + in i_partition_period text +) + returns boolean +as $$ +------------------------------------------------------------------------------- +-- Function: londiste.is_obsolete_partition(3) +-- +-- Test partition name of partition-by-date parent table. +-- +-- Parameters: +-- i_partition_table Partition table name we want to check +-- i_retention_period How long to keep partitions around +-- i_partition_period One of: year, month, day, hour +-- +-- Returns: +-- True if partition is too old, false if it is not, +-- null if its name does not match expected pattern. +------------------------------------------------------------------------------- +declare + _expr text; + _dfmt text; + _base text; +begin + if i_partition_period in ('year', 'yearly') then + _expr := '_[0-9]{4}'; + _dfmt := '_YYYY'; + elsif i_partition_period in ('month', 'monthly') then + _expr := '_[0-9]{4}_[0-9]{2}'; + _dfmt := '_YYYY_MM'; + elsif i_partition_period in ('day', 'daily') then + _expr := '_[0-9]{4}_[0-9]{2}_[0-9]{2}'; + _dfmt := '_YYYY_MM_DD'; + elsif i_partition_period in ('hour', 'hourly') then + _expr := '_[0-9]{4}_[0-9]{2}_[0-9]{2}_[0-9]{2}'; + _dfmt := '_YYYY_MM_DD_HH24'; + else + raise exception 'not supported i_partition_period: %', i_partition_period; + end if; + + _expr = '^(.+)' || _expr || '$'; + _base = substring (i_partition_table from _expr); + + if _base is null then + return null; + elsif i_partition_table < _base || to_char (now() - i_retention_period, _dfmt) then + return true; + else + return false; + end if; +end; +$$ language plpgsql; diff --git a/sql/londiste/functions/londiste.list_obsolete_partitions.sql b/sql/londiste/functions/londiste.list_obsolete_partitions.sql new file mode 100644 index 00000000..76af1cba --- /dev/null +++ b/sql/londiste/functions/londiste.list_obsolete_partitions.sql @@ -0,0 +1,62 @@ + +create or replace function londiste.list_obsolete_partitions +( + in i_parent_table text, + in i_retention_period interval, + in i_partition_period text +) + returns setof text +as $$ +------------------------------------------------------------------------------- +-- Function: londiste.list_obsolete_partitions(3) +-- +-- List obsolete partitions of partition-by-date parent table. +-- +-- Parameters: +-- i_parent_table Master table from which partitions are inherited +-- i_retention_period How long to keep partitions around +-- i_partition_period One of: year, month, day, hour +-- +-- Returns: +-- Names of partitions to be dropped +------------------------------------------------------------------------------- +declare + _schema text not null := split_part (i_parent_table, '.', 1); + _table text not null := split_part (i_parent_table, '.', 2); + _part text; + _expr text; + _dfmt text; +begin + if i_partition_period in ('year', 'yearly') then + _expr := '_[0-9]{4}'; + _dfmt := '_YYYY'; + elsif i_partition_period in ('month', 'monthly') then + _expr := '_[0-9]{4}_[0-9]{2}'; + _dfmt := '_YYYY_MM'; + elsif i_partition_period in ('day', 'daily') then + _expr := '_[0-9]{4}_[0-9]{2}_[0-9]{2}'; + _dfmt := '_YYYY_MM_DD'; + elsif i_partition_period in ('hour', 'hourly') then + _expr := '_[0-9]{4}_[0-9]{2}_[0-9]{2}_[0-9]{2}'; + _dfmt := '_YYYY_MM_DD_HH24'; + else + raise exception 'not supported i_partition_period: %', i_partition_period; + end if; + + if length (_table) = 0 then + _table := _schema; + _schema := 'public'; + end if; + + for _part in + select quote_ident (t.schemaname) ||'.'|| quote_ident (t.tablename) + from pg_catalog.pg_tables t + where t.schemaname = _schema + and t.tablename ~ ('^'|| _table || _expr ||'$') + and t.tablename < _table || to_char (now() - i_retention_period, _dfmt) + order by 1 + loop + return next _part; + end loop; +end; +$$ language plpgsql; diff --git a/sql/londiste/structure/grants.ini b/sql/londiste/structure/grants.ini index ca2a3765..80d83bc3 100644 --- a/sql/londiste/structure/grants.ini +++ b/sql/londiste/structure/grants.ini @@ -29,12 +29,14 @@ londiste_writer = execute on.functions = %(londiste_local_fns)s, %(londiste_internal_fns)s londiste_writer = execute + [5.seqs] londiste_writer = usage on.sequences = londiste.table_info_nr_seq, londiste.seq_info_nr_seq + [6.maint] pgq_admin = execute on.functions = londiste.periodic_maintenance() @@ -96,7 +98,8 @@ londiste_local_fns = londiste.drop_table_triggers(text, text), londiste.table_info_trigger(), londiste.create_partition(text, text, text, text, timestamptz, text), + londiste.is_obsolete_partition (text, interval, text), + londiste.list_obsolete_partitions (text, interval, text), londiste.drop_obsolete_partitions (text, interval, text), londiste.create_trigger(text,text,text[],text,text) - diff --git a/sql/pgq/functions/pgq.register_consumer.sql b/sql/pgq/functions/pgq.register_consumer.sql index 55b38ead..9f42d9bc 100644 --- a/sql/pgq/functions/pgq.register_consumer.sql +++ b/sql/pgq/functions/pgq.register_consumer.sql @@ -55,7 +55,7 @@ returns integer as $$ declare tmp text; last_tick bigint; - x_queue_id integer; + x_queue_id integer; x_consumer_id integer; queue integer; sub record; @@ -68,7 +68,8 @@ begin -- get consumer and create if new select co_id into x_consumer_id from pgq.consumer - where co_name = x_consumer_name; + where co_name = x_consumer_name + for update; if not found then insert into pgq.consumer (co_name) values (x_consumer_name); x_consumer_id := currval('pgq.consumer_co_id_seq'); diff --git a/sql/pgq/functions/pgq.unregister_consumer.sql b/sql/pgq/functions/pgq.unregister_consumer.sql index d495c88e..eeb2524f 100644 --- a/sql/pgq/functions/pgq.unregister_consumer.sql +++ b/sql/pgq/functions/pgq.unregister_consumer.sql @@ -6,8 +6,8 @@ returns integer as $$ -- ---------------------------------------------------------------------- -- Function: pgq.unregister_consumer(2) -- --- Unsubscriber consumer from the queue. Also consumer's --- retry events are deleted. +-- Unsubscribe consumer from the queue. +-- Also consumer's retry events are deleted. -- -- Parameters: -- x_queue_name - Name of the queue @@ -36,7 +36,7 @@ begin and s.sub_consumer = c.co_id and q.queue_name = x_queue_name and c.co_name = x_consumer_name - for update of s; + for update of s, c; if not found then return 0; end if; @@ -51,7 +51,6 @@ begin delete from pgq.subscription where sub_id = x_sub_id and sub_consumer = _consumer_id; - return 1; else -- delete main consumer (including possible subconsumers) @@ -63,7 +62,10 @@ begin -- this will drop subconsumers too delete from pgq.subscription where sub_id = x_sub_id; - + + delete from pgq.consumer + where co_id = _consumer_id; + return _sub_id_cnt; end if; diff --git a/sql/pgq_node/structure/tables.sql b/sql/pgq_node/structure/tables.sql index 464a4454..94739610 100644 --- a/sql/pgq_node/structure/tables.sql +++ b/sql/pgq_node/structure/tables.sql @@ -11,14 +11,14 @@ -- Root node sends minimal tick_id that must be kept. -- -- pgq.tick-id - ev_data: tick_id, extra1: queue_name --- Partition node inserts it's tick-id into combined queue. +-- Partition node inserts its tick-id into combined queue. -- -- ---------------------------------------------------------------------- create schema pgq_node; -- ---------------------------------------------------------------------- --- Table: pgq_node.location +-- Table: pgq_node.node_location -- -- Static table that just lists all members in set. -- @@ -46,7 +46,6 @@ create table pgq_node.node_location ( -- queue_name - cascaded queue name -- node_type - local node type -- node_name - local node name --- provider_node - provider node name -- worker_name - consumer name that maintains this node -- combined_queue - on 'leaf' the target combined set name -- node_attrs - urlencoded fields for worker @@ -93,7 +92,6 @@ create table pgq_node.local_state ( provider_node text not null, last_tick_id bigint not null, cur_error text, - paused boolean not null default false, uptodate boolean not null default false, @@ -103,7 +101,7 @@ create table pgq_node.local_state ( ); -- ---------------------------------------------------------------------- --- Table: pgq_node.subscriber +-- Table: pgq_node.subscriber_info -- -- List of nodes that subscribe to local node. -- @@ -119,6 +117,7 @@ create table pgq_node.subscriber_info ( watermark_name text not null, primary key (queue_name, subscriber_node), + foreign key (queue_name) references pgq_node.node_info, foreign key (queue_name, subscriber_node) references pgq_node.node_location, foreign key (worker_name) references pgq.consumer (co_name), foreign key (watermark_name) references pgq.consumer (co_name) |