diff options
| author | Marko Kreen | 2007-03-13 11:52:09 +0000 |
|---|---|---|
| committer | Marko Kreen | 2007-03-13 11:52:09 +0000 |
| commit | 50abdba44a031ad40b1886f941479f203ca92039 (patch) | |
| tree | 873e72d78cd48917b2907c4c63abf185649ebb54 | |
final public releaseskytools_2_1
203 files changed, 15899 insertions, 0 deletions
diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 00000000..f00e3a50 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,4 @@ + +Marko Kreen <[email protected]> - main coder +Martin Pihlak <[email protected]> - walmgr + diff --git a/COPYRIGHT b/COPYRIGHT new file mode 100644 index 00000000..c20f0b8b --- /dev/null +++ b/COPYRIGHT @@ -0,0 +1,16 @@ +SkyTools - tool collection for PostgreSQL + +Copyright (c) 2007 Marko Kreen, Skype Technologies OÜ + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..60ecb1a9 --- /dev/null +++ b/Makefile @@ -0,0 +1,79 @@ + +-include config.mak + +PYTHON ?= python + +pyver = $(shell $(PYTHON) -V 2>&1 | sed 's/^[^ ]* \([0-9]*\.[0-9]*\).*/\1/') + +SUBDIRS = sql + +all: python-all modules-all + +modules-all: config.mak + make -C sql all + +python-all: config.mak + $(PYTHON) setup.py build + +clean: + make -C sql clean + make -C doc clean + $(PYTHON) setup.py clean + rm -rf build + find python -name '*.py[oc]' -print | xargs rm -f + +install: python-install modules-install + +installcheck: + make -C sql installcheck + +modules-install: config.mak + make -C sql install DESTDIR=$(DESTDIR) + test \! -d compat || make -C compat $@ DESTDIR=$(DESTDIR) + +python-install: config.mak + $(PYTHON) setup.py install --prefix=$(prefix) --root=$(DESTDIR)/ + test \! -d compat || make -C compat $@ DESTDIR=$(DESTDIR) + +distclean: clean + for dir in $(SUBDIRS); do make -C $$dir $@ || exit 1; done + make -C doc $@ + rm -rf source.list dist skytools-* + find python -name '*.pyc' | xargs rm -f + rm -rf dist build + rm -rf autom4te.cache config.log config.status config.mak + +deb80: + ./configure + sed -e s/PGVER/8.0/g -e s/PYVER/$(pyver)/g < debian/packages.in > debian/packages + yada rebuild + debuild -uc -us -b + +deb81: + ./configure + sed -e s/PGVER/8.1/g -e s/PYVER/$(pyver)/g < debian/packages.in > debian/packages + yada rebuild + debuild -uc -us -b + +deb82: + ./configure + sed -e s/PGVER/8.2/g -e s/PYVER/$(pyver)/g < debian/packages.in > debian/packages + yada rebuild + debuild -uc -us -b + +tgz: config.mak + $(PYTHON) setup.py sdist -t source.cfg -m source.list + +debclean: distclean + rm -rf debian/tmp-* debian/build* debian/control debian/packages-tmp* + rm -f debian/files debian/rules debian/sub* debian/packages + +boot: configure + +configure: configure.ac + autoconf + + +.PHONY: all clean distclean install deb debclean tgz +.PHONY: python-all python-clean python-install + @@ -0,0 +1,5 @@ + +2007-03-xx ver 2.1 "Radioactive Candy" + + * Final public release. + @@ -0,0 +1,47 @@ + +SkyTools - tools for PostgreSQL +=============================== + +This is a package of tools in use in Skype for replication and +failover. Also it includes a generic queuing mechanism PgQ and +utility library for Python scripts. + +It contains following tools: + +PgQ +--- + +This is the queue machanism we use. Consists of PL/pgsql, PL/python +and C code in database, with Python framework on top of it. It is +based on snapshot based event handling ideas from Slony-I, +written for general usage. + +Features: + + * There can be several queues in database. + * There can be several producers than can insert into any queue. + * There can be several consumers on one queue and all consumers + see all events. + + +Londiste +-------- + +Replication tool written in Python, using PgQ as event transport. + +Features: +- Tables can be added one-by-one into set. +- Initial COPY for one table does not block event replay + for other tables. +- Can compare tables on both sides. + + +walmgr +------ + +This script will setup WAL archiving, does initial backup and +runtime WAL archive and restore. + + + + diff --git a/config.mak.in b/config.mak.in new file mode 100644 index 00000000..233f2cdf --- /dev/null +++ b/config.mak.in @@ -0,0 +1,10 @@ + +prefix = @prefix@ + +override PYTHON = @PYTHON@ +override PG_CONFIG = @PG_CONFIG@ + +PGXS = $(shell $(PG_CONFIG) --pgxs) + +DESTDIR = / + diff --git a/configure.ac b/configure.ac new file mode 100644 index 00000000..4aaa879e --- /dev/null +++ b/configure.ac @@ -0,0 +1,24 @@ +dnl Process this file with autoconf to produce a configure script. + +AC_INIT(skytools, 2.1) +AC_CONFIG_SRCDIR(python/pgqadm.py) + +dnl Find Python interpreter +AC_ARG_WITH(python, [ --with-python=PYTHON name of the Python executable (default: python)], +[ AC_MSG_CHECKING(for python) + PYTHON=$withval + AC_MSG_RESULT($PYTHON)], +[ AC_PATH_PROGS(PYTHON, python) ]) +test -n "$PYTHON" || AC_MSG_ERROR([Cannot continue without Python]) + +dnl Find PostgreSQL pg_config +AC_ARG_WITH(pgconfig, [ --with-pgconfig=PG_CONFIG path to pg_config (default: pg_config)], +[ AC_MSG_CHECKING(for pg_config) + PG_CONFIG=$withval + AC_MSG_RESULT($PG_CONFIG)], +[ AC_PATH_PROGS(PG_CONFIG, pg_config) ]) +test -n "$PG_CONFIG" || AC_MSG_ERROR([Cannot continue without pg_config]) + +dnl Write result +AC_OUTPUT([config.mak]) + diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 00000000..44774483 --- /dev/null +++ b/debian/changelog @@ -0,0 +1,6 @@ +skytools (2.1) unstable; urgency=low + + * cleanup + + -- Marko Kreen <[email protected]> Fri, 02 Feb 2007 12:38:17 +0200 + diff --git a/debian/packages.in b/debian/packages.in new file mode 100644 index 00000000..a9b07847 --- /dev/null +++ b/debian/packages.in @@ -0,0 +1,44 @@ +## debian/packages for skytools + +Source: skytools +Section: contrib/misc +Priority: extra +Maintainer: Marko Kreen <[email protected]> +Standards-Version: 3.6.2 +Description: PostgreSQL +Copyright: BSD + Copyright 2006 Marko Kreen +Build: sh + PG_CONFIG=/usr/lib/postgresql/PGVER/bin/pg_config \ + ./configure --prefix=/usr + make DESTDIR=$ROOT +Clean: sh + make distclean || make clean || true +Build-Depends: python-dev, postgresql-server-dev-PGVER + +Package: skytools +Architecture: any +Depends: python-psycopg | pythonPYVER-psycopg, skytools-modules-8.1 | skytools-modules-8.0, [] +Description: Skype database tools - Python parts + . + londiste - replication + pgqadm - generic event queue + walmgr - failover server scripts +Install: sh + make python-install DESTDIR=$ROOT prefix=/usr + +Package: skytools-modules-PGVER +Architecture: any +Depends: postgresql-PGVER, [] +Conflicts: postgresql-extras-PGVER +Description: Extra modules for PostgreSQL + It includes various extra modules for PostgreSQL: + . + txid - 8-byte transaction id's + logtriga - Trigger function to log change in SQL format. + logutriga - Trigger function to log change in urlencoded format. + londiste - Database parts of replication engine. + pgq - Generic queue in database. +Install: sh + make modules-install DESTDIR=$ROOT + diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 00000000..d9d94a3d --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,41 @@ + +wiki = https://developer.skype.com/SkypeGarage/DbProjects/SkyTools + +web = [email protected]:/home/pgfoundry.org/groups/skytools/htdocs/ + +EPYARGS = --no-private -u "http://pgfoundry.org/projects/skytools/" \ + -n "Skytools" + +all: + +upload: + devupload.sh overview.txt $(wiki) + devupload.sh londiste.txt $(wiki)/LondisteUsage + devupload.sh pgq-sql.txt $(wiki)/PgQdocs + devupload.sh pgq-nodupes.txt $(wiki)/PgqNoDupes + devupload.sh walmgr.txt $(wiki)/WalMgr + devupload.sh pgq-admin.txt $(wiki)/PgqAdm + +PY_PKGS = skytools skytools.config skytools.dbstruct skytools.gzlog \ + skytools.quoting skytools.scripting skytools.sqltools \ + pgq pgq.consumer pgq.event pgq.maint pgq.producer pgq.status pgq.ticker \ + londiste londiste.compare londiste.file_read londiste.file_write \ + londiste.installer londiste.playback londiste.repair londiste.setup \ + londiste.syncer londiste.table_copy + +apidoc: + rm -rf api + mkdir -p api + cd ../python && epydoc3 -o ../doc/api --html --no-private $(PY_PKGS) + +apiupload: apidoc + cd ../sql/pgq && rm -rf docs/pgq && make dox && mv docs/html docs/pgq + rsync -rtlz api $(web) + rsync -rtlz ../sql/pgq/docs/pgq $(web) + +clean: + rm -rf api + +distclean: + rm -rf ../sql/pgq/docs/pgq api + diff --git a/doc/TODO.txt b/doc/TODO.txt new file mode 100644 index 00000000..efc8a54e --- /dev/null +++ b/doc/TODO.txt @@ -0,0 +1,44 @@ + +web: + - walmgr + - pgqadm + - todo + + +londiste link <qname> +londiste unlink <qname> + +Immidiate +========= + +* londiste swithcover support / deny triggers +* deb: /etc/skylog.ini should be conffile +* RemoteConsumer/SerialConsumer/pgq_ext sanity, too much duplication + +Near future +============ + +* londiste: create tables on subscriber +* skytools: switch for silence for cron scripts +* docs: londiste, pgq/python, pgq/sql, skytools +* txid: decide on renaming functions +* logtriga: way to switch off logging for some connection +* pgq: separately installable fkeys for all tables for testing +* logdb: hostname +* pgq_ext: solve event tracking +* contrib/*.sql loading from python - need to check db version +* ideas from SlonyI: + - force timestamps to ISO + - when buffering queries, check their size +* DBScript: failure to write pidfile should be logged (crontscripts) + +Just ideas +=========== + +* logtriga: use pgq.insert_event_directly? +* pgq/sql: rewrite pgq.insert_event in C or logtriga in plpython? +* skytools: config-less operation? +* skytools: config from database? +* skytools: partial sql parser for log processing +* pgqadm ticker logic into db, to make easier other implementations? + diff --git a/doc/londiste.txt b/doc/londiste.txt new file mode 100644 index 00000000..bdb26a52 --- /dev/null +++ b/doc/londiste.txt @@ -0,0 +1,64 @@ + + +== Config file == + +{{{ +[londiste] +job_name = test_to_subcriber + +# source database, where the queue resides +provider_db = dbname=provider port=6000 host=127.0.0.1 + +# destination database +subscriber_db = dbname=subscriber port=6000 host=127.0.0.1 + +# the queue where to listen on +pgq_queue_name = londiste.replika + +# where to log +logfile = ~/log/%(job_name)s.log + +# pidfile is used for avoiding duplicate processes +pidfile = ~/pid/%(job_name)s.pid + +}}} + +== Command line overview == + +{{{ +$ londiste.py --help +usage: londiste.py [options] INI CMD [subcmd args] + +commands: + provider install installs modules, creates queue + provider add TBL ... add table to queue + provider remove TBL ... remove table from queue + provider tables show all tables linked to queue + subscriber install installs schema + subscriber register attaches subscriber to queue (also done by replay) + subscriber unregister detach subscriber from queue + subscriber add TBL ... add table to subscriber + subscriber remove TBL ... remove table from subscriber + subscriber resync TBL ... do full copy again + subscriber tables list tables subscriber has attached to + subscriber missing list tables subscriber has not yet attached to + subscriber check compare table structure on both sides + subscriber fkeys print out fkey drop/create commands + replay replay events to subscriber + copy full copy of table, internal cmd + compare [TBL ...] compare table contents on both sides + repair [TBL ...] repair data on subscriber + +options: + -h, --help show this help message and exit + -q, --quiet make program silent + -v, --verbose make program verbose + -d, --daemon go background + --expect-sync no copy needed (for add command) + --force ignore some warnings + + control running process: + -r, --reload reload config (send SIGHUP) + -s, --stop stop program safely (send SIGINT) + -k, --kill kill program immidiately (send SIGTERM) +}}} diff --git a/doc/overview.txt b/doc/overview.txt new file mode 100644 index 00000000..98235564 --- /dev/null +++ b/doc/overview.txt @@ -0,0 +1,179 @@ +#pragma section-numbers 2 + += SkyTools = + +[[TableOfContents]] + +== Intro == + +This is package of tools we use at Skype to manage our cluster of [http://www.postgresql.org PostgreSQL] +servers. They are put together for our own convinience and also because they build on each other, +so managing them separately is pain. + +The code is hosted at [http://pgfoundry.org PgFoundry] site: + + http://pgfoundry.org/projects/skytools/ + +There are our [http://pgfoundry.org/frs/?group_id=1000206 downloads] and +[http://lists.pgfoundry.org/mailman/listinfo/skytools-users mailing list]. +Also [http://pgfoundry.org/scm/?group_id=1000206 CVS] +and [http://pgfoundry.org/tracker/?group_id=1000206 bugtracker]. + +== High-level tools == + +Those are script that are meant for end-user. +In our case that means database administrators. + +=== Londiste === + +Replication engine written in Python. It uses PgQ as transport mechanism. +Its main goals are robustness and easy usage. Thus its not as complete +and featureful as Slony-I. + +Docs: ./LondisteUsage + +''' Features ''' + + * Tables can be added one-by-one into set. + * Initial COPY for one table does not block event replay for other tables. + * Can compare tables on both sides. + * Easy installation. + +''' Missing features ''' + + * No support for sequences. Thus its not possible to use it for keeping + failover server up-to-date. We use WalMgr for that. + + * Does not understand cascaded replication, when one subscriber acts + as provider to another one and it dies, the last one loses sync with the first one. + In other words - it understands only pair of servers. + +''' Sample usage ''' +{{{ +$ londiste.py replic.ini provider install +$ londiste.py replic.ini subscriber install +$ londiste.py replic.ini replay -d +$ londiste.py replic.ini provider add users orders +$ londiste.py replic.ini subscriber add users +}}} + +=== PgQ === + +Generic queue implementation. Based on ideas from [http://www.slony1.info/ Slony-I] - +snapshot based event batching. + +''' Features ''' + + * Generic multi-consumer, multi-producer queue. + * There can be several consumers on one queue. + * It is guaranteed that each of them sees a event at least once. + But it's not guaranteed that it sees it only once. + * The goal is to provide a clean API as SQL functions. The frameworks + on top of that don't need to understand internal details. + +''' Technical design ''' + + * Events are batched using snapshots (like Slony-I). + * Consumers are poll-only, they don't need to do any administrative work. + * Queue administration is separate process from consumers. + * Tolerant of long transactions. + * Easy to monitor. + +''' Docs ''' + + * SQL API overview: ./PgQdocs + * SQL API detailed docs: http://skytools.projects.postgresql.org/pgq/ + * Administrative tool usage: ./PgqAdm + +=== WalMgr === + +Python script for hot failover. Tries to make setup +initial copy and later switch easy for admins. + + * Docs: ./WalMgr + +Sample: + +{{{ + [ .. prepare config .. ] + + master$ walmgr setup + master$ walmgr backup + slave$ walmgr restore + + [ .. main server down, switch failover server to normal mode: ] + + slave$ walmgr restore +}}} + +== Low-level tools == + +Those are building blocks for the PgQ and Londiste. +Useful for database developers. + +=== txid === + + Provides 8-byte transaction id-s for external usage. + +=== logtriga === + + Trigger function for table event logging in "partial SQL" format. + Based on Slony-I logtrigger. Used in londiste for replication. + +=== logutriga === + + Trigger function for table event logging in urlencoded format. + Written in PL/Python. For cases where data manipulation is necessary. + +== Developement frameworks == + +=== skytools - Python framework for database scripting === + +This collect various utilities for Python scripts for databases. + +''' Topics ''' + + * Daemonization + * Logging + * Configuration. + * Skeleton class for scripts. + * Quoting (SQL/COPY) + * COPY helpers. + * Database object lookup. + * Table structure detection. + +Documentation: http://skytools.projects.postgresql.org/api/ + +=== pgq - Python framework for PgQ consumers === + +This builds on scripting framework above. + +Documentation: http://skytools.projects.postgresql.org/api/ + +== Sample scripts == + +Those are specialized script that are based on skytools/pgq framework. +Can be considered examples, although they are used in production in Skype. + +=== Special data moving scripts === + +There are couple of scripts for situations where regular replication +does not fit. They all operate on `logutriga()` urlencoded queues. + + * `cube_dispatcher`: Multi-table partitioning on change date, with optional keep-all-row-versions mode. + * `table_dispatcher`: configurable partitioning for one table. + * `bulk_loader`: aggregates changes for slow databases. Instead of each change in separate statement, + does minimal amount of DELETE-s and then big COPY. + +|| Script || Supported operations || Number of tables || Partitioning || +|| table_dispatcher || INSERT || 1 || any || +|| cube_dispatcher || INSERT/UPDATE || any || change time || +|| bulk_loader || INSERT/UPDATE/DELETE || any || none || + +=== queue_mover === + +Simply copies all events from one queue to another. + +=== scriptmgr === + +Allows to start and stop several scripts together. diff --git a/doc/pgq-admin.txt b/doc/pgq-admin.txt new file mode 100644 index 00000000..1ea9fddb --- /dev/null +++ b/doc/pgq-admin.txt @@ -0,0 +1,42 @@ += PgqAdm = + +== Config == + +{{{ +[pgqadm] +job_name = pgqadm_somedb + +db = dbname=somedb + +# how often to run maintenance [minutes] +maint_delay_min = 5 + +# how often to check for activity [secs] +loop_delay = 0.1 + +logfile = ~/log/%(job_name)s.log +pidfile = ~/pid/%(job_name)s.pid + +}}} + +== Command line usage == + +{{{ +$ pgqadm.py --help +usage: pgqadm.py [options] INI CMD [subcmd args] + +commands: + ticker start ticking & maintenance process + status show overview of queue healt + +options: + -h, --help show this help message and exit + -q, --quiet make program silent + -v, --verbose make program verbose + -d, --daemon go background + + control running process: + -r, --reload reload config (send SIGHUP) + -s, --stop stop program safely (send SIGINT) + -k, --kill kill program immidiately (send SIGTERM) +}}} diff --git a/doc/pgq-nodupes.txt b/doc/pgq-nodupes.txt new file mode 100644 index 00000000..933cab56 --- /dev/null +++ b/doc/pgq-nodupes.txt @@ -0,0 +1,33 @@ += Avoiding duplicate events =
+
+It is pretty burdensome to check if event is already processed,
+especially on bulk data moving. Here's a way how this can be avoided.
+
+First, consumer must guarantee that it processes all events in one tx.
+
+Consumer itself can tag events for retry, but then it must be able to handle them later.
+
+ * If the PgQ queue and event data handling happen in same database,
+ the consumer must simply call pgq.finish_batch() inside the event-processing
+ transaction.
+
+ * If the event processing happens in different database, the consumer
+ must store the batch_id into destination database, inside the same
+ transaction as the event processing happens.
+
+ Only after committing it, consumer can call pgq.finish_batch() in queue database
+ and commit that.
+
+ As the batches come in sequence, there's no need to remember full log of batch_id's,
+ it's enough to keep the latest batch_id.
+
+ Then at the start of every batch, consumer can check if the batch_id already
+ exists in destination database, and if it does, then just tag batch done,
+ without processing.
+
+With this, there's no need for consumer to check for already processed
+events.
+
+NB: This assumes the event processing is transaction-able - failures
+will be rollbacked. If event processing includes communication with
+world outside database, eg. sending email, such handling won't work.
diff --git a/doc/pgq-sql.txt b/doc/pgq-sql.txt new file mode 100644 index 00000000..9414594e --- /dev/null +++ b/doc/pgq-sql.txt @@ -0,0 +1,191 @@ += PgQ - queue for PostgreSQL = + +== Queue creation == + +{{{ + pgq.create_queue(queue_name text) +}}} + +Initialize event queue. + +Returns 0 if event queue already exists, 1 otherwise. + +== Producer == + +{{{ + pgq.insert_event(queue_name text, ev_type, ev_data) + pgq.insert_event(queue_name text, ev_type, ev_data, extra1, extra2, extra3, extra4) +}}} + +Generate new event. This should be called inside main tx - thus +rollbacked with it if needed. + + +== Consumer == + +{{{ + pgq.register_consumer(queue_name text, consumer_id text) +}}} + +Attaches this consumer to particular event queue. + +Returns 0 if the consumer was already attached, 1 otherwise. + +{{{ + pgq.unregister_consumer(queue_name text, consumer_id text) +}}} + +Unregister and drop resources allocated to customer. + + +{{{ + pgq.next_batch(queue_name text, consumer_id text) +}}} + +Allocates next batch of events to consumer. + +Returns batch id (int8), to be used in processing functions. If no batches +are available, returns NULL. That means that the ticker has not cut them yet. +This is the appropriate moment for consumer to sleep. + +{{{ + pgq.get_batch_events(batch_id int8) +}}} + +`pgq.get_batch_events()` returns a set of events in this batch. + +There may be no events in the batch. This is normal. The batch must still be closed +with pgq.finish_batch(). + +Event fields: (ev_id int8, ev_time timestamptz, ev_txid int8, ev_retry int4, ev_type text, + ev_data text, ev_extra1, ev_extra2, ev_extra3, ev_extra4) + +{{{ + pgq.event_failed(batch_id int8, event_id int8, reason text) +}}} + +Tag event as 'failed' - it will be stored, but not further processing is done. + +{{{ + pgq.event_retry(batch_id int8, event_id int8, retry_seconds int4) +}}} + +Tag event for 'retry' - after x seconds the event will be re-inserted +into main queue. + +{{{ + pgq.finish_batch(batch_id int8) +}}} + +Tag batch as finished. Until this is not done, the consumer will get +same batch again. + +After calling finish_batch consumer cannot do any operations with events +of that batch. All operations must be done before. + +== Failed queue operation == + +Events tagged as failed just stay on their queue. Following +functions can be used to manage them. + +{{{ + pgq.failed_event_list(queue_name, consumer) + pgq.failed_event_list(queue_name, consumer, cnt, offset) + pgq.failed_event_count(queue_name, consumer) +}}} + +Get info about the queue. + +Event fields are same as for pgq.get_batch_events() + +{{{ + pgq.failed_event_delete(queue_name, consumer, event_id) + pgq.failed_event_retry(queue_name, consumer, event_id) +}}} + +Remove an event from queue, or retry it. + +== Info operations == + +{{{ + pgq.get_queue_info() +}}} + +Get list of queues. + +Result: () + +{{{ + pgq.get_consumer_info() + pgq.get_consumer_info(queue_name) + pgq.get_consumer_info(queue_name, consumer) +}}} + +Get list of active consumers. + +Result: () + +{{{ + pgq.get_batch_info(batch_id) +}}} + +Get info about batch. + +Result fields: () + +== Notes == + +Consumer '''must''' be able to process same event several times. + +== Example == + +First, create event queue: + +{{{ + select pgq.create_queue('LogEvent'); +}}} + +Then, producer side can do whenever it wishes: + +{{{ + select pgq.insert_event('LogEvent', 'data', 'DataFor123'); +}}} + +First step for consumer is to register: + +{{{ + select pgq.register_consumer('LogEvent', 'TestConsumer'); +}}} + +Then it can enter into consuming loop: + +{{{ + begin; + select pgq.next_batch('LogEvent', 'TestConsumer'); [into batch_id] + commit; +}}} + +That will reserve a batch of events for this consumer. + +To see the events in batch: + +{{{ + select * from pgq.get_batch_events(batch_id); +}}} + +That will give all events in batch. The processing does not need to be happen +all in one transaction, framework can split the work how it wants. + +If a events failed or needs to be tried again, framework can call: + +{{{ + select pgq.event_retry(batch_id, event_id, 60); + select pgq.event_failed(batch_id, event_id, 'Record deleted'); +}}} + +When all done, notify core about it: + +{{{ + select pgq.finish_batch(batch_id) +}}} + diff --git a/doc/walmgr.txt b/doc/walmgr.txt new file mode 100644 index 00000000..a05ea644 --- /dev/null +++ b/doc/walmgr.txt @@ -0,0 +1,82 @@ +#pragma section-numbers 2 + += WalMgr = + +[[TableOfContents]] + +== Step-by-step instructions == + +=== no-password ssh access from one to other === + + master$ test -f ~/.ssh/id_dsa.pub || ssh-keygen -t dsa + master$ scp .ssh/id_dsa.pub slave: + slave$ cat id_dsa.pub >> ~/.ssh/authorized_keys + +=== Configure paths === + + master$ edit master.ini + slave$ edit slave.ini + slave$ mkdir data.master logs.full logs.partial + +=== Start archival process === + + master$ ./walmgr.py setup + +=== Do full backup+restore === + + master$ ./walmgr.py backup + slave$ ./walmgr.py restore + + 'walmgr.py restore' moves data in place and starts postmaster, + that starts replaying logs as they appear. + +=== In-progress WAL segments can be backup by command: === + + master$ ./walmgr.py sync + +=== If need to stop replay on slave and boot into normal mode, do: === + + slave$ ./walmgr.py boot + +== Configuration == + +=== master.ini === + +{{{ +[wal-master] +logfile = master.log +use_skylog = 0 + +master_db = dbname=template1 +master_data = /var/lib/postgresql/8.0/main +master_config = /etc/postgresql/8.0/main/postgresql.conf + +slave = slave:/var/lib/postgresql/walshipping + +completed_wals = %(slave)s/logs.complete +partial_wals = %(slave)s/logs.partial +full_backup = %(slave)s/data.master + +# syncdaemon update frequency +loop_delay = 10.0 + +}}} + +=== slave.ini === + +{{{ +[wal-slave] +logfile = slave.log +use_skylog = 0 + +slave_data = /var/lib/postgresql/8.0/main +slave_stop_cmd = /etc/init.d/postgresql-8.0 stop +slave_start_cmd = /etc/init.d/postgresql-8.0 start + +slave = /var/lib/postgresql/walshipping +completed_wals = %(slave)s/logs.complete +partial_wals = %(slave)s/logs.partial +full_backup = %(slave)s/data.master + +keep_old_logs = 0 +}}} diff --git a/python/conf/londiste.ini b/python/conf/londiste.ini new file mode 100644 index 00000000..a1506a32 --- /dev/null +++ b/python/conf/londiste.ini @@ -0,0 +1,16 @@ + +[londiste] +job_name = test_to_subcriber + +provider_db = dbname=provider port=6000 host=127.0.0.1 +subscriber_db = dbname=subscriber port=6000 host=127.0.0.1 + +# it will be used as sql ident so no dots/spaces +pgq_queue_name = londiste.replika + +logfile = ~/log/%(job_name)s.log +pidfile = ~/pid/%(job_name)s.pid + +# both events and ticks will be copied there +#mirror_queue = replika_mirror + diff --git a/python/conf/pgqadm.ini b/python/conf/pgqadm.ini new file mode 100644 index 00000000..a2e92f6b --- /dev/null +++ b/python/conf/pgqadm.ini @@ -0,0 +1,18 @@ + +[pgqadm] + +job_name = pgqadm_somedb + +db = dbname=provider port=6000 host=127.0.0.1 + +# how often to run maintenance [minutes] +maint_delay_min = 5 + +# how often to check for activity [secs] +loop_delay = 0.1 + +logfile = ~/log/%(job_name)s.log +pidfile = ~/pid/%(job_name)s.pid + +use_skylog = 0 + diff --git a/python/conf/skylog.ini b/python/conf/skylog.ini new file mode 100644 index 00000000..150ef934 --- /dev/null +++ b/python/conf/skylog.ini @@ -0,0 +1,76 @@ +; notes: +; - 'args' is mandatory in [handler_*] sections +; - in lists there must not be spaces + +; +; top-level config +; + +; list of all loggers +[loggers] +keys=root +; root logger sees everything. there can be per-job configs by +; specifing loggers with job_name of the script + +; list of all handlers +[handlers] +;; seems logger module immidiately initalized all handlers, +;; whether they are actually used or not. so better +;; keep this list in sync with actual handler list +;keys=stderr,logdb,logsrv,logfile +keys=stderr + +; list of all formatters +[formatters] +keys=short,long,none + +; +; map specific loggers to specifig handlers +; +[logger_root] +level=DEBUG +;handlers=stderr,logdb,logsrv,logfile +handlers=stderr + +; +; configure formatters +; +[formatter_short] +format=%(asctime)s %(levelname)s %(message)s +datefmt=%H:%M + +[formatter_long] +format=%(asctime)s %(process)s %(levelname)s %(message)s + +[formatter_none] +format=%(message)s + +; +; configure handlers +; + +; file. args: stream +[handler_stderr] +class=StreamHandler +args=(sys.stderr,) +formatter=short + +; log into db. args: conn_string +[handler_logdb] +class=skylog.LogDBHandler +args=("host=127.0.0.1 port=5432 user=logger dbname=logdb",) +formatter=none +level=INFO + +; JSON messages over UDP. args: host, port +[handler_logsrv] +class=skylog.UdpLogServerHandler +args=('127.0.0.1', 6666) +formatter=none + +; rotating logfile. args: filename, maxsize, maxcount +[handler_logfile] +class=skylog.EasyRotatingFileHandler +args=('~/log/%(job_name)s.log', 100*1024*1024, 3) +formatter=long + diff --git a/python/conf/wal-master.ini b/python/conf/wal-master.ini new file mode 100644 index 00000000..5ae8cb2b --- /dev/null +++ b/python/conf/wal-master.ini @@ -0,0 +1,18 @@ +[wal-master] +logfile = master.log +use_skylog = 0 + +master_db = dbname=template1 +master_data = /var/lib/postgresql/8.0/main +master_config = /etc/postgresql/8.0/main/postgresql.conf + + +slave = slave:/var/lib/postgresql/walshipping + +completed_wals = %(slave)s/logs.complete +partial_wals = %(slave)s/logs.partial +full_backup = %(slave)s/data.master + +# syncdaemon update frequency +loop_delay = 10.0 + diff --git a/python/conf/wal-slave.ini b/python/conf/wal-slave.ini new file mode 100644 index 00000000..912bf756 --- /dev/null +++ b/python/conf/wal-slave.ini @@ -0,0 +1,15 @@ +[wal-slave] +logfile = slave.log +use_skylog = 0 + +slave_data = /var/lib/postgresql/8.0/main +slave_stop_cmd = /etc/init.d/postgresql-8.0 stop +slave_start_cmd = /etc/init.d/postgresql-8.0 start + +slave = /var/lib/postgresql/walshipping +completed_wals = %(slave)s/logs.complete +partial_wals = %(slave)s/logs.partial +full_backup = %(slave)s/data.master + +keep_old_logs = 0 + diff --git a/python/londiste.py b/python/londiste.py new file mode 100755 index 00000000..9e5684ea --- /dev/null +++ b/python/londiste.py @@ -0,0 +1,130 @@ +#! /usr/bin/env python + +"""Londiste launcher. +""" + +import sys, os, optparse, skytools + +# python 2.3 will try londiste.py first... +import sys, os.path +if os.path.exists(os.path.join(sys.path[0], 'londiste.py')) \ + and not os.path.exists(os.path.join(sys.path[0], 'londiste')): + del sys.path[0] + +from londiste import * + +command_usage = """ +%prog [options] INI CMD [subcmd args] + +commands: + provider install installs modules, creates queue + provider add TBL ... add table to queue + provider remove TBL ... remove table from queue + provider tables show all tables linked to queue + provider seqs show all sequences on provider + + subscriber install installs schema + subscriber add TBL ... add table to subscriber + subscriber remove TBL ... remove table from subscriber + subscriber tables list tables subscriber has attached to + subscriber seqs list sequences subscriber is interested + subscriber missing list tables subscriber has not yet attached to + subscriber link QUE create mirror queue + subscriber unlink dont mirror queue + + replay replay events to subscriber + + switchover switch the roles between provider & subscriber + compare [TBL ...] compare table contents on both sides + repair [TBL ...] repair data on subscriber + copy full copy of table, internal cmd + subscriber check compare table structure on both sides + subscriber fkeys print out fkey drop/create commands + subscriber resync TBL ... do full copy again + subscriber register attaches subscriber to queue (also done by replay) + subscriber unregister detach subscriber from queue +""" + +"""switchover: +goal is to launch a replay with reverse config + +- should link be required? (link should guarantee all tables/seqs same?) +- should link auto-add tables for subscriber + +1. lock all tables on provider, in order specified by 'nr' +2. wait until old replay is past the point +3. sync seq +4. replace queue triggers on provider with deny triggers +5. replace deny triggers on subscriber with queue triggers +6. sync pgq tick seqs? change pgq config? + +""" + +class Londiste(skytools.DBScript): + def __init__(self, args): + skytools.DBScript.__init__(self, 'londiste', args) + + if self.options.rewind or self.options.reset: + self.script = Replicator(args) + return + + if len(self.args) < 2: + print "need command" + sys.exit(1) + cmd = self.args[1] + + if cmd =="provider": + script = ProviderSetup(args) + elif cmd == "subscriber": + script = SubscriberSetup(args) + elif cmd == "replay": + method = self.cf.get('method', 'direct') + if method == 'direct': + script = Replicator(args) + elif method == 'file_write': + script = FileWrite(args) + elif method == 'file_write': + script = FileWrite(args) + else: + print "unknown method, quitting" + sys.exit(1) + elif cmd == "copy": + script = CopyTable(args) + elif cmd == "compare": + script = Comparator(args) + elif cmd == "repair": + script = Repairer(args) + elif cmd == "upgrade": + script = UpgradeV2(args) + else: + print "Unknown command '%s', use --help for help" % cmd + sys.exit(1) + + self.script = script + + def start(self): + self.script.start() + + def init_optparse(self, parser=None): + p = skytools.DBScript.init_optparse(self, parser) + p.set_usage(command_usage.strip()) + + g = optparse.OptionGroup(p, "expert options") + g.add_option("--force", action="store_true", + help = "add: ignore table differences, repair: ignore lag") + g.add_option("--expect-sync", action="store_true", dest="expect_sync", + help = "add: no copy needed", default=False) + g.add_option("--skip-truncate", action="store_true", dest="skip_truncate", + help = "copy: keep old data", default=False) + g.add_option("--rewind", action="store_true", + help = "replay: sync queue pos with subscriber") + g.add_option("--reset", action="store_true", + help = "replay: forget queue pos on subscriber") + p.add_option_group(g) + + return p + +if __name__ == '__main__': + script = Londiste(sys.argv[1:]) + script.start() + diff --git a/python/londiste/__init__.py b/python/londiste/__init__.py new file mode 100644 index 00000000..97d67433 --- /dev/null +++ b/python/londiste/__init__.py @@ -0,0 +1,12 @@ + +"""Replication on top of PgQ.""" + +from playback import * +from compare import * +from file_read import * +from file_write import * +from setup import * +from table_copy import * +from installer import * +from repair import * + diff --git a/python/londiste/compare.py b/python/londiste/compare.py new file mode 100644 index 00000000..0029665b --- /dev/null +++ b/python/londiste/compare.py @@ -0,0 +1,45 @@ +#! /usr/bin/env python + +"""Compares tables in replication set. + +Currently just does count(1) on both sides. +""" + +import sys, os, time, skytools + +__all__ = ['Comparator'] + +from syncer import Syncer + +class Comparator(Syncer): + def process_sync(self, tbl, src_db, dst_db): + """Actual comparision.""" + + src_curs = src_db.cursor() + dst_curs = dst_db.cursor() + + self.log.info('Counting %s' % tbl) + + q = "select count(1) from only _TABLE_" + q = self.cf.get('compare_sql', q) + q = q.replace('_TABLE_', tbl) + + self.log.debug("srcdb: " + q) + src_curs.execute(q) + src_row = src_curs.fetchone() + src_str = ", ".join(map(str, src_row)) + self.log.info("srcdb: res = %s" % src_str) + + self.log.debug("dstdb: " + q) + dst_curs.execute(q) + dst_row = dst_curs.fetchone() + dst_str = ", ".join(map(str, dst_row)) + self.log.info("dstdb: res = %s" % dst_str) + + if src_str != dst_str: + self.log.warning("%s: Results do not match!" % tbl) + +if __name__ == '__main__': + script = Comparator(sys.argv[1:]) + script.start() + diff --git a/python/londiste/file_read.py b/python/londiste/file_read.py new file mode 100644 index 00000000..2902bda5 --- /dev/null +++ b/python/londiste/file_read.py @@ -0,0 +1,52 @@ + +"""Reads events from file instead of db queue.""" + +import sys, os, re, skytools + +from playback import * +from table_copy import * + +__all__ = ['FileRead'] + +file_regex = r"^tick_0*([0-9]+)\.sql$" +file_rc = re.compile(file_regex) + + +class FileRead(CopyTable): + """Reads events from file instead of db queue. + + Incomplete implementation. + """ + + def __init__(self, args, log = None): + CopyTable.__init__(self, args, log, copy_thread = 0) + + def launch_copy(self, tbl): + # copy immidiately + self.do_copy(t) + + def work(self): + last_batch = self.get_last_batch(curs) + list = self.get_file_list() + + def get_list(self): + """Return list of (first_batch, full_filename) pairs.""" + + src_dir = self.cf.get('file_src') + list = os.listdir(src_dir) + list.sort() + res = [] + for fn in list: + m = file_rc.match(fn) + if not m: + self.log.debug("Ignoring file: %s" % fn) + continue + full = os.path.join(src_dir, fn) + batch_id = int(m.group(1)) + res.append((batch_id, full)) + return res + +if __name__ == '__main__': + script = Replicator(sys.argv[1:]) + script.start() + diff --git a/python/londiste/file_write.py b/python/londiste/file_write.py new file mode 100644 index 00000000..86e16aae --- /dev/null +++ b/python/londiste/file_write.py @@ -0,0 +1,67 @@ + +"""Writes events into file.""" + +import sys, os, skytools +from cStringIO import StringIO +from playback import * + +__all__ = ['FileWrite'] + +class FileWrite(Replicator): + """Writes events into file. + + Incomplete implementation. + """ + + last_successful_batch = None + + def load_state(self, batch_id): + # maybe check if batch exists on filesystem? + self.cur_tick = self.cur_batch_info['tick_id'] + self.prev_tick = self.cur_batch_info['prev_tick_id'] + return 1 + + def process_batch(self, db, batch_id, ev_list): + pass + + def save_state(self, do_commit): + # nothing to save + pass + + def sync_tables(self, dst_db): + # nothing to sync + return 1 + + def interesting(self, ev): + # wants all of them + return 1 + + def handle_data_event(self, ev): + fmt = self.sql_command[ev.type] + sql = fmt % (ev.ev_extra1, ev.data) + row = "%s -- txid:%d" % (sql, ev.txid) + self.sql_list.append(row) + ev.tag_done() + + def handle_system_event(self, ev): + row = "-- sysevent:%s txid:%d data:%s" % ( + ev.type, ev.txid, ev.data) + self.sql_list.append(row) + ev.tag_done() + + def flush_sql(self): + self.sql_list.insert(0, "-- tick:%d prev:%s" % ( + self.cur_tick, self.prev_tick)) + self.sql_list.append("-- end_tick:%d\n" % self.cur_tick) + # store result + dir = self.cf.get("file_dst") + fn = os.path.join(dir, "tick_%010d.sql" % self.cur_tick) + f = open(fn, "w") + buf = "\n".join(self.sql_list) + f.write(buf) + f.close() + +if __name__ == '__main__': + script = Replicator(sys.argv[1:]) + script.start() + diff --git a/python/londiste/installer.py b/python/londiste/installer.py new file mode 100644 index 00000000..6f190ab2 --- /dev/null +++ b/python/londiste/installer.py @@ -0,0 +1,26 @@ + +"""Functions to install londiste and its depentencies into database.""" + +import os, skytools + +__all__ = ['install_provider', 'install_subscriber'] + +provider_object_list = [ + skytools.DBFunction('logtriga', 0, sql_file = "logtriga.sql"), + skytools.DBFunction('get_current_snapshot', 0, sql_file = "txid.sql"), + skytools.DBSchema('pgq', sql_file = "pgq.sql"), + skytools.DBSchema('londiste', sql_file = "londiste.sql") +] + +subscriber_object_list = [ + skytools.DBSchema('londiste', sql_file = "londiste.sql") +] + +def install_provider(curs, log): + """Installs needed code into provider db.""" + skytools.db_install(curs, provider_object_list, log) + +def install_subscriber(curs, log): + """Installs needed code into subscriber db.""" + skytools.db_install(curs, subscriber_object_list, log) + diff --git a/python/londiste/playback.py b/python/londiste/playback.py new file mode 100644 index 00000000..2bcb1bc7 --- /dev/null +++ b/python/londiste/playback.py @@ -0,0 +1,558 @@ +#! /usr/bin/env python + +"""Basic replication core.""" + +import sys, os, time +import skytools, pgq + +__all__ = ['Replicator', 'TableState', + 'TABLE_MISSING', 'TABLE_IN_COPY', 'TABLE_CATCHING_UP', + 'TABLE_WANNA_SYNC', 'TABLE_DO_SYNC', 'TABLE_OK'] + +# state # owner - who is allowed to change +TABLE_MISSING = 0 # main +TABLE_IN_COPY = 1 # copy +TABLE_CATCHING_UP = 2 # copy +TABLE_WANNA_SYNC = 3 # main +TABLE_DO_SYNC = 4 # copy +TABLE_OK = 5 # setup + +SYNC_OK = 0 # continue with batch +SYNC_LOOP = 1 # sleep, try again +SYNC_EXIT = 2 # nothing to do, exit skript + +class Counter(object): + """Counts table statuses.""" + + missing = 0 + copy = 0 + catching_up = 0 + wanna_sync = 0 + do_sync = 0 + ok = 0 + + def __init__(self, tables): + """Counts and sanity checks.""" + for t in tables: + if t.state == TABLE_MISSING: + self.missing += 1 + elif t.state == TABLE_IN_COPY: + self.copy += 1 + elif t.state == TABLE_CATCHING_UP: + self.catching_up += 1 + elif t.state == TABLE_WANNA_SYNC: + self.wanna_sync += 1 + elif t.state == TABLE_DO_SYNC: + self.do_sync += 1 + elif t.state == TABLE_OK: + self.ok += 1 + # only one table is allowed to have in-progress copy + if self.copy + self.catching_up + self.wanna_sync + self.do_sync > 1: + raise Exception('Bad table state') + +class TableState(object): + """Keeps state about one table.""" + def __init__(self, name, log): + self.name = name + self.log = log + self.forget() + self.changed = 0 + + def forget(self): + self.state = TABLE_MISSING + self.str_snapshot = None + self.from_snapshot = None + self.sync_tick_id = None + self.ok_batch_count = 0 + self.last_tick = 0 + self.changed = 1 + + def change_snapshot(self, str_snapshot, tag_changed = 1): + if self.str_snapshot == str_snapshot: + return + self.log.debug("%s: change_snapshot to %s" % (self.name, str_snapshot)) + self.str_snapshot = str_snapshot + if str_snapshot: + self.from_snapshot = skytools.Snapshot(str_snapshot) + else: + self.from_snapshot = None + + if tag_changed: + self.ok_batch_count = 0 + self.last_tick = None + self.changed = 1 + + def change_state(self, state, tick_id = None): + if self.state == state and self.sync_tick_id == tick_id: + return + self.state = state + self.sync_tick_id = tick_id + self.changed = 1 + self.log.debug("%s: change_state to %s" % (self.name, + self.render_state())) + + def render_state(self): + """Make a string to be stored in db.""" + + if self.state == TABLE_MISSING: + return None + elif self.state == TABLE_IN_COPY: + return 'in-copy' + elif self.state == TABLE_CATCHING_UP: + return 'catching-up' + elif self.state == TABLE_WANNA_SYNC: + return 'wanna-sync:%d' % self.sync_tick_id + elif self.state == TABLE_DO_SYNC: + return 'do-sync:%d' % self.sync_tick_id + elif self.state == TABLE_OK: + return 'ok' + + def parse_state(self, merge_state): + """Read state from string.""" + + state = -1 + if merge_state == None: + state = TABLE_MISSING + elif merge_state == "in-copy": + state = TABLE_IN_COPY + elif merge_state == "catching-up": + state = TABLE_CATCHING_UP + elif merge_state == "ok": + state = TABLE_OK + elif merge_state == "?": + state = TABLE_OK + else: + tmp = merge_state.split(':') + if len(tmp) == 2: + self.sync_tick_id = int(tmp[1]) + if tmp[0] == 'wanna-sync': + state = TABLE_WANNA_SYNC + elif tmp[0] == 'do-sync': + state = TABLE_DO_SYNC + + if state < 0: + raise Exception("Bad table state: %s" % merge_state) + + return state + + def loaded_state(self, merge_state, str_snapshot): + self.log.debug("loaded_state: %s: %s / %s" % ( + self.name, merge_state, str_snapshot)) + self.change_snapshot(str_snapshot, 0) + self.state = self.parse_state(merge_state) + self.changed = 0 + if merge_state == "?": + self.changed = 1 + + def interesting(self, ev, tick_id, copy_thread): + """Check if table wants this event.""" + + if copy_thread: + if self.state not in (TABLE_CATCHING_UP, TABLE_DO_SYNC): + return False + else: + if self.state != TABLE_OK: + return False + + # if no snapshot tracking, then accept always + if not self.from_snapshot: + return True + + # uninteresting? + if self.from_snapshot.contains(ev.txid): + return False + + # after couple interesting batches there no need to check snapshot + # as there can be only one partially interesting batch + if tick_id != self.last_tick: + self.last_tick = tick_id + self.ok_batch_count += 1 + + # disable batch tracking + if self.ok_batch_count > 3: + self.change_snapshot(None) + return True + +class SeqCache(object): + def __init__(self): + self.seq_list = [] + self.val_cache = {} + + def set_seq_list(self, seq_list): + self.seq_list = seq_list + new_cache = {} + for seq in seq_list: + val = self.val_cache.get(seq) + if val: + new_cache[seq] = val + self.val_cache = new_cache + + def resync(self, src_curs, dst_curs): + if len(self.seq_list) == 0: + return + dat = ".last_value, ".join(self.seq_list) + dat += ".last_value" + q = "select %s from %s" % (dat, ",".join(self.seq_list)) + src_curs.execute(q) + row = src_curs.fetchone() + for i in range(len(self.seq_list)): + seq = self.seq_list[i] + cur = row[i] + old = self.val_cache.get(seq) + if old != cur: + q = "select setval(%s, %s)" + dst_curs.execute(q, [seq, cur]) + self.val_cache[seq] = cur + +class Replicator(pgq.SerialConsumer): + """Replication core.""" + + sql_command = { + 'I': "insert into %s %s;", + 'U': "update only %s set %s;", + 'D': "delete from only %s where %s;", + } + + # batch info + cur_tick = 0 + prev_tick = 0 + + def __init__(self, args): + pgq.SerialConsumer.__init__(self, 'londiste', 'provider_db', 'subscriber_db', args) + + # tick table in dst for SerialConsumer(). keep londiste stuff under one schema + self.dst_completed_table = "londiste.completed" + + self.table_list = [] + self.table_map = {} + + self.copy_thread = 0 + self.maint_time = 0 + self.seq_cache = SeqCache() + self.maint_delay = self.cf.getint('maint_delay', 600) + self.mirror_queue = self.cf.get('mirror_queue', '') + + def process_remote_batch(self, src_db, batch_id, ev_list, dst_db): + "All work for a batch. Entry point from SerialConsumer." + + # this part can play freely with transactions + + dst_curs = dst_db.cursor() + + self.cur_tick = self.cur_batch_info['tick_id'] + self.prev_tick = self.cur_batch_info['prev_tick_id'] + + self.load_table_state(dst_curs) + self.sync_tables(dst_db) + + # now the actual event processing happens. + # they must be done all in one tx in dst side + # and the transaction must be kept open so that + # the SerialConsumer can save last tick and commit. + + self.handle_seqs(dst_curs) + self.handle_events(dst_curs, ev_list) + self.save_table_state(dst_curs) + + def handle_seqs(self, dst_curs): + if self.copy_thread: + return + + q = "select * from londiste.subscriber_get_seq_list(%s)" + dst_curs.execute(q, [self.pgq_queue_name]) + seq_list = [] + for row in dst_curs.fetchall(): + seq_list.append(row[0]) + + self.seq_cache.set_seq_list(seq_list) + + src_curs = self.get_database('provider_db').cursor() + self.seq_cache.resync(src_curs, dst_curs) + + def sync_tables(self, dst_db): + """Table sync loop. + + Calls appropriate handles, which is expected to + return one of SYNC_* constants.""" + + self.log.debug('Sync tables') + while 1: + cnt = Counter(self.table_list) + if self.copy_thread: + res = self.sync_from_copy_thread(cnt, dst_db) + else: + res = self.sync_from_main_thread(cnt, dst_db) + + if res == SYNC_EXIT: + self.log.debug('Sync tables: exit') + self.detach() + sys.exit(0) + elif res == SYNC_OK: + return + elif res != SYNC_LOOP: + raise Exception('Program error') + + self.log.debug('Sync tables: sleeping') + time.sleep(3) + dst_db.commit() + self.load_table_state(dst_db.cursor()) + dst_db.commit() + + def sync_from_main_thread(self, cnt, dst_db): + "Main thread sync logic." + + # + # decide what to do - order is imortant + # + if cnt.do_sync: + # wait for copy thread to catch up + return SYNC_LOOP + elif cnt.wanna_sync: + # copy thread wants sync, if not behind, do it + t = self.get_table_by_state(TABLE_WANNA_SYNC) + if self.cur_tick >= t.sync_tick_id: + self.change_table_state(dst_db, t, TABLE_DO_SYNC, self.cur_tick) + return SYNC_LOOP + else: + return SYNC_OK + elif cnt.catching_up: + # active copy, dont worry + return SYNC_OK + elif cnt.copy: + # active copy, dont worry + return SYNC_OK + elif cnt.missing: + # seems there is no active copy thread, launch new + t = self.get_table_by_state(TABLE_MISSING) + self.change_table_state(dst_db, t, TABLE_IN_COPY) + + # the copy _may_ happen immidiately + self.launch_copy(t) + + # there cannot be interesting events in current batch + # but maybe there's several tables, lets do them in one go + return SYNC_LOOP + else: + # seems everything is in sync + return SYNC_OK + + def sync_from_copy_thread(self, cnt, dst_db): + "Copy thread sync logic." + + # + # decide what to do - order is imortant + # + if cnt.do_sync: + # main thread is waiting, catch up, then handle over + t = self.get_table_by_state(TABLE_DO_SYNC) + if self.cur_tick == t.sync_tick_id: + self.change_table_state(dst_db, t, TABLE_OK) + return SYNC_EXIT + elif self.cur_tick < t.sync_tick_id: + return SYNC_OK + else: + self.log.error("copy_sync: cur_tick=%d sync_tick=%d" % ( + self.cur_tick, t.sync_tick_id)) + raise Exception('Invalid table state') + elif cnt.wanna_sync: + # wait for main thread to react + return SYNC_LOOP + elif cnt.catching_up: + # is there more work? + if self.work_state: + return SYNC_OK + + # seems we have catched up + t = self.get_table_by_state(TABLE_CATCHING_UP) + self.change_table_state(dst_db, t, TABLE_WANNA_SYNC, self.cur_tick) + return SYNC_LOOP + elif cnt.copy: + # table is not copied yet, do it + t = self.get_table_by_state(TABLE_IN_COPY) + self.do_copy(t) + + # forget previous value + self.work_state = 1 + + return SYNC_LOOP + else: + # nothing to do + return SYNC_EXIT + + def handle_events(self, dst_curs, ev_list): + "Actual event processing happens here." + + ignored_events = 0 + self.sql_list = [] + mirror_list = [] + for ev in ev_list: + if not self.interesting(ev): + ignored_events += 1 + ev.tag_done() + continue + + if ev.type in ('I', 'U', 'D'): + self.handle_data_event(ev, dst_curs) + else: + self.handle_system_event(ev, dst_curs) + + if self.mirror_queue: + mirror_list.append(ev) + + # finalize table changes + self.flush_sql(dst_curs) + self.stat_add('ignored', ignored_events) + + # put events into mirror queue if requested + if self.mirror_queue: + self.fill_mirror_queue(mirror_list, dst_curs) + + def handle_data_event(self, ev, dst_curs): + fmt = self.sql_command[ev.type] + sql = fmt % (ev.extra1, ev.data) + self.sql_list.append(sql) + if len(self.sql_list) > 200: + self.flush_sql(dst_curs) + ev.tag_done() + + def flush_sql(self, dst_curs): + if len(self.sql_list) == 0: + return + + buf = "\n".join(self.sql_list) + self.sql_list = [] + + dst_curs.execute(buf) + + def interesting(self, ev): + if ev.type not in ('I', 'U', 'D'): + return 1 + t = self.get_table_by_name(ev.extra1) + if t: + return t.interesting(ev, self.cur_tick, self.copy_thread) + else: + return 0 + + def handle_system_event(self, ev, dst_curs): + "System event." + + if ev.type == "T": + self.log.info("got new table event: "+ev.data) + # check tables to be dropped + name_list = [] + for name in ev.data.split(','): + name_list.append(name.strip()) + + del_list = [] + for tbl in self.table_list: + if tbl.name in name_list: + continue + del_list.append(tbl) + + # separate loop to avoid changing while iterating + for tbl in del_list: + self.log.info("Removing table %s from set" % tbl.name) + self.remove_table(tbl, dst_curs) + + ev.tag_done() + else: + self.log.warning("Unknows op %s" % ev.type) + ev.tag_failed("Unknown operation") + + def remove_table(self, tbl, dst_curs): + del self.table_map[tbl.name] + self.table_list.remove(tbl) + q = "select londiste.subscriber_remove_table(%s, %s)" + dst_curs.execute(q, [self.pgq_queue_name, tbl.name]) + + def load_table_state(self, curs): + """Load table state from database. + + Todo: if all tables are OK, there is no need + to load state on every batch. + """ + + q = """select table_name, snapshot, merge_state + from londiste.subscriber_get_table_list(%s) + """ + curs.execute(q, [self.pgq_queue_name]) + + new_list = [] + new_map = {} + for row in curs.dictfetchall(): + t = self.get_table_by_name(row['table_name']) + if not t: + t = TableState(row['table_name'], self.log) + t.loaded_state(row['merge_state'], row['snapshot']) + new_list.append(t) + new_map[t.name] = t + + self.table_list = new_list + self.table_map = new_map + + def save_table_state(self, curs): + """Store changed table state in database.""" + + for t in self.table_list: + if not t.changed: + continue + merge_state = t.render_state() + self.log.info("storing state of %s: copy:%d new_state:%s" % ( + t.name, self.copy_thread, merge_state)) + q = "select londiste.subscriber_set_table_state(%s, %s, %s, %s)" + curs.execute(q, [self.pgq_queue_name, + t.name, t.str_snapshot, merge_state]) + t.changed = 0 + + def change_table_state(self, dst_db, tbl, state, tick_id = None): + tbl.change_state(state, tick_id) + self.save_table_state(dst_db.cursor()) + dst_db.commit() + + self.log.info("Table %s status changed to '%s'" % ( + tbl.name, tbl.render_state())) + + def get_table_by_state(self, state): + "get first table with specific state" + + for t in self.table_list: + if t.state == state: + return t + raise Exception('No table was found with state: %d' % state) + + def get_table_by_name(self, name): + if name.find('.') < 0: + name = "public.%s" % name + if name in self.table_map: + return self.table_map[name] + return None + + def fill_mirror_queue(self, ev_list, dst_curs): + # insert events + rows = [] + fields = ['ev_type', 'ev_data', 'ev_extra1'] + for ev in mirror_list: + rows.append((ev.type, ev.data, ev.extra1)) + pgq.bulk_insert_events(dst_curs, rows, fields, self.mirror_queue) + + # create tick + q = "select pgq.ticker(%s, %s)" + dst_curs.execute(q, [self.mirror_queue, self.cur_tick]) + + def launch_copy(self, tbl_stat): + self.log.info("Launching copy process") + script = sys.argv[0] + conf = self.cf.filename + if self.options.verbose: + cmd = "%s -d -v %s copy" + else: + cmd = "%s -d %s copy" + cmd = cmd % (script, conf) + self.log.debug("Launch args: "+repr(cmd)) + res = os.system(cmd) + self.log.debug("Launch result: "+repr(res)) + +if __name__ == '__main__': + script = Replicator(sys.argv[1:]) + script.start() + diff --git a/python/londiste/repair.py b/python/londiste/repair.py new file mode 100644 index 00000000..ec4bd404 --- /dev/null +++ b/python/londiste/repair.py @@ -0,0 +1,284 @@ + +"""Repair data on subscriber. + +Walks tables by primary key and searcher +missing inserts/updates/deletes. +""" + +import sys, os, time, psycopg, skytools + +from syncer import Syncer + +__all__ = ['Repairer'] + +def unescape(s): + return skytools.unescape_copy(s) + +def get_pkey_list(curs, tbl): + """Get list of pkey fields in right order.""" + + oid = skytools.get_table_oid(curs, tbl) + q = """SELECT k.attname FROM pg_index i, pg_attribute k + WHERE i.indrelid = %s AND k.attrelid = i.indexrelid + AND i.indisprimary AND k.attnum > 0 AND NOT k.attisdropped + ORDER BY k.attnum""" + curs.execute(q, [oid]) + list = [] + for row in curs.fetchall(): + list.append(row[0]) + return list + +def get_column_list(curs, tbl): + """Get list of columns in right order.""" + + oid = skytools.get_table_oid(curs, tbl) + q = """SELECT a.attname FROM pg_attribute a + WHERE a.attrelid = %s + AND a.attnum > 0 AND NOT a.attisdropped + ORDER BY a.attnum""" + curs.execute(q, [oid]) + list = [] + for row in curs.fetchall(): + list.append(row[0]) + return list + +class Repairer(Syncer): + """Walks tables in primary key order and checks if data matches.""" + + + def process_sync(self, tbl, src_db, dst_db): + """Actual comparision.""" + + src_curs = src_db.cursor() + dst_curs = dst_db.cursor() + + self.log.info('Checking %s' % tbl) + + self.common_fields = [] + self.pkey_list = [] + copy_tbl = self.gen_copy_tbl(tbl, src_curs, dst_curs) + + dump_src = tbl + ".src" + dump_dst = tbl + ".dst" + + self.log.info("Dumping src table: %s" % tbl) + self.dump_table(tbl, copy_tbl, src_curs, dump_src) + src_db.commit() + self.log.info("Dumping dst table: %s" % tbl) + self.dump_table(tbl, copy_tbl, dst_curs, dump_dst) + dst_db.commit() + + self.log.info("Sorting src table: %s" % tbl) + + s_in, s_out = os.popen4("sort --version") + s_ver = s_out.read() + del s_in, s_out + if s_ver.find("coreutils") > 0: + args = "-S 30%" + else: + args = "" + os.system("sort %s -T . -o %s.sorted %s" % (args, dump_src, dump_src)) + self.log.info("Sorting dst table: %s" % tbl) + os.system("sort %s -T . -o %s.sorted %s" % (args, dump_dst, dump_dst)) + + self.dump_compare(tbl, dump_src + ".sorted", dump_dst + ".sorted") + + os.unlink(dump_src) + os.unlink(dump_dst) + os.unlink(dump_src + ".sorted") + os.unlink(dump_dst + ".sorted") + + def gen_copy_tbl(self, tbl, src_curs, dst_curs): + self.pkey_list = get_pkey_list(src_curs, tbl) + dst_pkey = get_pkey_list(dst_curs, tbl) + if dst_pkey != self.pkey_list: + self.log.error('pkeys do not match') + sys.exit(1) + + src_cols = get_column_list(src_curs, tbl) + dst_cols = get_column_list(dst_curs, tbl) + field_list = [] + for f in self.pkey_list: + field_list.append(f) + for f in src_cols: + if f in self.pkey_list: + continue + if f in dst_cols: + field_list.append(f) + + self.common_fields = field_list + + tbl_expr = "%s (%s)" % (tbl, ",".join(field_list)) + + self.log.debug("using copy expr: %s" % tbl_expr) + + return tbl_expr + + def dump_table(self, tbl, copy_tbl, curs, fn): + f = open(fn, "w", 64*1024) + curs.copy_to(f, copy_tbl) + size = f.tell() + f.close() + self.log.info('Got %d bytes' % size) + + def get_row(self, ln): + t = ln[:-1].split('\t') + row = {} + for i in range(len(self.common_fields)): + row[self.common_fields[i]] = t[i] + return row + + def dump_compare(self, tbl, src_fn, dst_fn): + self.log.info("Comparing dumps: %s" % tbl) + self.cnt_insert = 0 + self.cnt_update = 0 + self.cnt_delete = 0 + self.total_src = 0 + self.total_dst = 0 + f1 = open(src_fn, "r", 64*1024) + f2 = open(dst_fn, "r", 64*1024) + src_ln = f1.readline() + dst_ln = f2.readline() + if src_ln: self.total_src += 1 + if dst_ln: self.total_dst += 1 + + fix = "fix.%s.sql" % tbl + if os.path.isfile(fix): + os.unlink(fix) + + while src_ln or dst_ln: + keep_src = keep_dst = 0 + if src_ln != dst_ln: + src_row = self.get_row(src_ln) + dst_row = self.get_row(dst_ln) + + cmp = self.cmp_keys(src_row, dst_row) + if cmp > 0: + # src > dst + self.got_missed_delete(tbl, dst_row) + keep_src = 1 + elif cmp < 0: + # src < dst + self.got_missed_insert(tbl, src_row) + keep_dst = 1 + else: + if self.cmp_data(src_row, dst_row) != 0: + self.got_missed_update(tbl, src_row, dst_row) + + if not keep_src: + src_ln = f1.readline() + if src_ln: self.total_src += 1 + if not keep_dst: + dst_ln = f2.readline() + if dst_ln: self.total_dst += 1 + + self.log.info("finished %s: src: %d rows, dst: %d rows,"\ + " missed: %d inserts, %d updates, %d deletes" % ( + tbl, self.total_src, self.total_dst, + self.cnt_insert, self.cnt_update, self.cnt_delete)) + + def got_missed_insert(self, tbl, src_row): + self.cnt_insert += 1 + fld_list = self.common_fields + val_list = [] + for f in fld_list: + v = unescape(src_row[f]) + val_list.append(skytools.quote_literal(v)) + q = "insert into %s (%s) values (%s);" % ( + tbl, ", ".join(fld_list), ", ".join(val_list)) + self.show_fix(tbl, q, 'insert') + + def got_missed_update(self, tbl, src_row, dst_row): + self.cnt_update += 1 + fld_list = self.common_fields + set_list = [] + whe_list = [] + for f in self.pkey_list: + self.addcmp(whe_list, f, unescape(src_row[f])) + for f in fld_list: + v1 = src_row[f] + v2 = dst_row[f] + if self.cmp_value(v1, v2) == 0: + continue + + self.addeq(set_list, f, unescape(v1)) + self.addcmp(whe_list, f, unescape(v2)) + + q = "update only %s set %s where %s;" % ( + tbl, ", ".join(set_list), " and ".join(whe_list)) + self.show_fix(tbl, q, 'update') + + def got_missed_delete(self, tbl, dst_row, pkey_list): + self.cnt_delete += 1 + whe_list = [] + for f in self.pkey_list: + self.addcmp(whe_list, f, unescape(dst_row[f])) + q = "delete from only %s where %s;" % (tbl, " and ".join(whe_list)) + self.show_fix(tbl, q, 'delete') + + def show_fix(self, tbl, q, desc): + #self.log.warning("missed %s: %s" % (desc, q)) + fn = "fix.%s.sql" % tbl + open(fn, "a").write("%s\n" % q) + + def addeq(self, list, f, v): + vq = skytools.quote_literal(v) + s = "%s = %s" % (f, vq) + list.append(s) + + def addcmp(self, list, f, v): + if v is None: + s = "%s is null" % f + else: + vq = skytools.quote_literal(v) + s = "%s = %s" % (f, vq) + list.append(s) + + def cmp_data(self, src_row, dst_row): + for k in self.common_fields: + v1 = src_row[k] + v2 = dst_row[k] + if self.cmp_value(v1, v2) != 0: + return -1 + return 0 + + def cmp_value(self, v1, v2): + if v1 == v2: + return 0 + + # try to work around tz vs. notz + z1 = len(v1) + z2 = len(v2) + if z1 == z2 + 3 and z2 >= 19 and v1[z2] == '+': + v1 = v1[:-3] + if v1 == v2: + return 0 + elif z1 + 3 == z2 and z1 >= 19 and v2[z1] == '+': + v2 = v2[:-3] + if v1 == v2: + return 0 + + return -1 + + def cmp_keys(self, src_row, dst_row): + """Compare primary keys of the rows. + + Returns 1 if src > dst, -1 if src < dst and 0 if src == dst""" + + # None means table is done. tag it larger than any existing row. + if src_row is None: + if dst_row is None: + return 0 + return 1 + elif dst_row is None: + return -1 + + for k in self.pkey_list: + v1 = src_row[k] + v2 = dst_row[k] + if v1 < v2: + return -1 + elif v1 > v2: + return 1 + return 0 + diff --git a/python/londiste/setup.py b/python/londiste/setup.py new file mode 100644 index 00000000..ed44b093 --- /dev/null +++ b/python/londiste/setup.py @@ -0,0 +1,580 @@ +#! /usr/bin/env python + +"""Londiste setup and sanity checker. + +""" +import sys, os, skytools +from installer import * + +__all__ = ['ProviderSetup', 'SubscriberSetup'] + +def find_column_types(curs, table): + table_oid = skytools.get_table_oid(curs, table) + if table_oid == None: + return None + + key_sql = """ + SELECT k.attname FROM pg_index i, pg_attribute k + WHERE i.indrelid = %d AND k.attrelid = i.indexrelid + AND i.indisprimary AND k.attnum > 0 AND NOT k.attisdropped + """ % table_oid + + # find columns + q = """ + SELECT a.attname as name, + CASE WHEN k.attname IS NOT NULL + THEN 'k' ELSE 'v' END AS type + FROM pg_attribute a LEFT JOIN (%s) k ON (k.attname = a.attname) + WHERE a.attrelid = %d AND a.attnum > 0 AND NOT a.attisdropped + ORDER BY a.attnum + """ % (key_sql, table_oid) + curs.execute(q) + rows = curs.dictfetchall() + return rows + +def make_type_string(col_rows): + res = map(lambda x: x['type'], col_rows) + return "".join(res) + +class CommonSetup(skytools.DBScript): + def __init__(self, args): + skytools.DBScript.__init__(self, 'londiste', args) + self.set_single_loop(1) + self.pidfile = self.pidfile + ".setup" + + self.pgq_queue_name = self.cf.get("pgq_queue_name") + self.consumer_id = self.cf.get("pgq_consumer_id", self.job_name) + self.fake = self.cf.getint('fake', 0) + + if len(self.args) < 3: + self.log.error("need subcommand") + sys.exit(1) + + def run(self): + self.admin() + + def fetch_provider_table_list(self, curs): + q = """select table_name, trigger_name + from londiste.provider_get_table_list(%s)""" + curs.execute(q, [self.pgq_queue_name]) + return curs.dictfetchall() + + def get_provider_table_list(self): + src_db = self.get_database('provider_db') + src_curs = src_db.cursor() + list = self.fetch_provider_table_list(src_curs) + src_db.commit() + res = [] + for row in list: + res.append(row['table_name']) + return res + + def get_provider_seqs(self, curs): + q = """SELECT * from londiste.provider_get_seq_list(%s)""" + curs.execute(q, [self.pgq_queue_name]) + res = [] + for row in curs.fetchall(): + res.append(row[0]) + return res + + def get_all_seqs(self, curs): + q = """SELECT n.nspname || '.'|| c.relname + from pg_class c, pg_namespace n + where n.oid = c.relnamespace + and c.relkind = 'S' + order by 1""" + curs.execute(q) + res = [] + for row in curs.fetchall(): + res.append(row[0]) + return res + + def check_provider_queue(self): + src_db = self.get_database('provider_db') + src_curs = src_db.cursor() + q = "select count(1) from pgq.get_queue_info(%s)" + src_curs.execute(q, [self.pgq_queue_name]) + ok = src_curs.fetchone()[0] + src_db.commit() + + if not ok: + self.log.error('Event queue does not exist yet') + sys.exit(1) + + def fetch_subscriber_tables(self, curs): + q = "select * from londiste.subscriber_get_table_list(%s)" + curs.execute(q, [self.pgq_queue_name]) + return curs.dictfetchall() + + def get_subscriber_table_list(self): + dst_db = self.get_database('subscriber_db') + dst_curs = dst_db.cursor() + list = self.fetch_subscriber_tables(dst_curs) + dst_db.commit() + res = [] + for row in list: + res.append(row['table_name']) + return res + + def init_optparse(self, parser=None): + p = skytools.DBScript.init_optparse(self, parser) + p.add_option("--expect-sync", action="store_true", dest="expect_sync", + help = "no copy needed", default=False) + p.add_option("--force", action="store_true", + help="force", default=False) + return p + + +# +# Provider commands +# + +class ProviderSetup(CommonSetup): + + def admin(self): + cmd = self.args[2] + if cmd == "tables": + self.provider_show_tables() + elif cmd == "add": + self.provider_add_tables(self.args[3:]) + elif cmd == "remove": + self.provider_remove_tables(self.args[3:]) + elif cmd == "add-seq": + for seq in self.args[3:]: + self.provider_add_seq(seq) + self.provider_notify_change() + elif cmd == "remove-seq": + for seq in self.args[3:]: + self.provider_remove_seq(seq) + self.provider_notify_change() + elif cmd == "install": + self.provider_install() + elif cmd == "seqs": + self.provider_list_seqs() + else: + self.log.error('bad subcommand') + sys.exit(1) + + def provider_list_seqs(self): + src_db = self.get_database('provider_db') + src_curs = src_db.cursor() + list = self.get_provider_seqs(src_curs) + src_db.commit() + + for seq in list: + print seq + + def provider_install(self): + src_db = self.get_database('provider_db') + src_curs = src_db.cursor() + install_provider(src_curs, self.log) + + # create event queue + q = "select pgq.create_queue(%s)" + self.exec_provider(q, [self.pgq_queue_name]) + + def provider_add_tables(self, table_list): + self.check_provider_queue() + + cur_list = self.get_provider_table_list() + for tbl in table_list: + if tbl.find('.') < 0: + tbl = "public." + tbl + if tbl not in cur_list: + self.log.info('Adding %s' % tbl) + self.provider_add_table(tbl) + else: + self.log.info("Table %s already added" % tbl) + self.provider_notify_change() + + def provider_remove_tables(self, table_list): + self.check_provider_queue() + + cur_list = self.get_provider_table_list() + for tbl in table_list: + if tbl.find('.') < 0: + tbl = "public." + tbl + if tbl not in cur_list: + self.log.info('%s already removed' % tbl) + else: + self.log.info("Removing %s" % tbl) + self.provider_remove_table(tbl) + self.provider_notify_change() + + def provider_add_table(self, tbl): + q = "select londiste.provider_add_table(%s, %s)" + self.exec_provider(q, [self.pgq_queue_name, tbl]) + + def provider_remove_table(self, tbl): + q = "select londiste.provider_remove_table(%s, %s)" + self.exec_provider(q, [self.pgq_queue_name, tbl]) + + def provider_show_tables(self): + self.check_provider_queue() + list = self.get_provider_table_list() + for tbl in list: + print tbl + + def provider_notify_change(self): + q = "select londiste.provider_notify_change(%s)" + self.exec_provider(q, [self.pgq_queue_name]) + + def provider_add_seq(self, seq): + seq = skytools.fq_name(seq) + q = "select londiste.provider_add_seq(%s, %s)" + self.exec_provider(q, [self.pgq_queue_name, seq]) + + def provider_remove_seq(self, seq): + seq = skytools.fq_name(seq) + q = "select londiste.provider_remove_seq(%s, %s)" + self.exec_provider(q, [self.pgq_queue_name, seq]) + + def exec_provider(self, sql, args): + src_db = self.get_database('provider_db') + src_curs = src_db.cursor() + + src_curs.execute(sql, args) + + if self.fake: + src_db.rollback() + else: + src_db.commit() + +# +# Subscriber commands +# + +class SubscriberSetup(CommonSetup): + + def admin(self): + cmd = self.args[2] + if cmd == "tables": + self.subscriber_show_tables() + elif cmd == "missing": + self.subscriber_missing_tables() + elif cmd == "add": + self.subscriber_add_tables(self.args[3:]) + elif cmd == "remove": + self.subscriber_remove_tables(self.args[3:]) + elif cmd == "resync": + self.subscriber_resync_tables(self.args[3:]) + elif cmd == "register": + self.subscriber_register() + elif cmd == "unregister": + self.subscriber_unregister() + elif cmd == "install": + self.subscriber_install() + elif cmd == "check": + self.check_tables(self.get_provider_table_list()) + elif cmd == "fkeys": + self.collect_fkeys(self.get_provider_table_list()) + elif cmd == "seqs": + self.subscriber_list_seqs() + elif cmd == "add-seq": + self.subscriber_add_seq(self.args[3:]) + elif cmd == "remove-seq": + self.subscriber_remove_seq(self.args[3:]) + else: + self.log.error('bad subcommand: ' + cmd) + sys.exit(1) + + def collect_fkeys(self, table_list): + dst_db = self.get_database('subscriber_db') + dst_curs = dst_db.cursor() + + oid_list = [] + for tbl in table_list: + try: + oid = skytools.get_table_oid(dst_curs, tbl) + if oid: + oid_list.append(str(oid)) + except: + pass + if len(oid_list) == 0: + print "No tables" + return + oid_str = ",".join(oid_list) + + q = "SELECT n.nspname || '.' || t.relname as tbl, c.conname as con,"\ + " pg_get_constraintdef(c.oid) as def"\ + " FROM pg_constraint c, pg_class t, pg_namespace n"\ + " WHERE c.contype = 'f' and c.conrelid in (%s)"\ + " AND t.oid = c.conrelid AND n.oid = t.relnamespace" % oid_str + dst_curs.execute(q) + res = dst_curs.dictfetchall() + dst_db.commit() + + print "-- dropping" + for row in res: + q = "ALTER TABLE ONLY %(tbl)s DROP CONSTRAINT %(con)s;" + print q % row + print "-- creating" + for row in res: + q = "ALTER TABLE ONLY %(tbl)s ADD CONSTRAINT %(con)s %(def)s;" + print q % row + + def check_tables(self, table_list): + src_db = self.get_database('provider_db') + src_curs = src_db.cursor() + dst_db = self.get_database('subscriber_db') + dst_curs = dst_db.cursor() + + failed = 0 + for tbl in table_list: + self.log.info('Checking %s' % tbl) + if not skytools.exists_table(src_curs, tbl): + self.log.error('Table %s missing from provider side' % tbl) + failed += 1 + elif not skytools.exists_table(dst_curs, tbl): + self.log.error('Table %s missing from subscriber side' % tbl) + failed += 1 + else: + failed += self.check_table_columns(src_curs, dst_curs, tbl) + failed += self.check_table_triggers(dst_curs, tbl) + + src_db.commit() + dst_db.commit() + + return failed + + def check_table_triggers(self, dst_curs, tbl): + oid = skytools.get_table_oid(dst_curs, tbl) + if not oid: + self.log.error('Table %s not found' % tbl) + return 1 + q = "select count(1) from pg_trigger where tgrelid = %s" + dst_curs.execute(q, [oid]) + got = dst_curs.fetchone()[0] + if got: + self.log.error('found trigger on table %s (%s)' % (tbl, str(oid))) + return 1 + else: + return 0 + + def check_table_columns(self, src_curs, dst_curs, tbl): + src_colrows = find_column_types(src_curs, tbl) + dst_colrows = find_column_types(dst_curs, tbl) + + src_cols = make_type_string(src_colrows) + dst_cols = make_type_string(dst_colrows) + if src_cols.find('k') < 0: + self.log.error('provider table %s has no primary key (%s)' % ( + tbl, src_cols)) + return 1 + if dst_cols.find('k') < 0: + self.log.error('subscriber table %s has no primary key (%s)' % ( + tbl, dst_cols)) + return 1 + + if src_cols != dst_cols: + self.log.warning('table %s structure is not same (%s/%s)'\ + ', trying to continue' % (tbl, src_cols, dst_cols)) + + err = 0 + for row in src_colrows: + found = 0 + for row2 in dst_colrows: + if row2['name'] == row['name']: + found = 1 + break + if not found: + err = 1 + self.log.error('%s: column %s on provider not on subscriber' + % (tbl, row['name'])) + elif row['type'] != row2['type']: + err = 1 + self.log.error('%s: pk different on column %s' + % (tbl, row['name'])) + + return err + + def subscriber_install(self): + dst_db = self.get_database('subscriber_db') + dst_curs = dst_db.cursor() + + install_subscriber(dst_curs, self.log) + + if self.fake: + self.log.debug('rollback') + dst_db.rollback() + else: + self.log.debug('commit') + dst_db.commit() + + def subscriber_register(self): + src_db = self.get_database('provider_db') + src_curs = src_db.cursor() + src_curs.execute("select pgq.register_consumer(%s, %s)", + [self.pgq_queue_name, self.consumer_id]) + src_db.commit() + + def subscriber_unregister(self): + q = "select londiste.subscriber_set_table_state(%s, %s, NULL, NULL)" + + dst_db = self.get_database('subscriber_db') + dst_curs = dst_db.cursor() + tbl_rows = self.fetch_subscriber_tables(dst_curs) + for row in tbl_rows: + dst_curs.execute(q, [self.pgq_queue_name, row['table_name']]) + dst_db.commit() + + src_db = self.get_database('provider_db') + src_curs = src_db.cursor() + src_curs.execute("select pgq.unregister_consumer(%s, %s)", + [self.pgq_queue_name, self.consumer_id]) + src_db.commit() + + def subscriber_show_tables(self): + list = self.get_subscriber_table_list() + for tbl in list: + print tbl + + def subscriber_missing_tables(self): + provider_tables = self.get_provider_table_list() + subscriber_tables = self.get_subscriber_table_list() + for tbl in provider_tables: + if tbl not in subscriber_tables: + print tbl + + def subscriber_add_tables(self, table_list): + provider_tables = self.get_provider_table_list() + subscriber_tables = self.get_subscriber_table_list() + + err = 0 + for tbl in table_list: + tbl = skytools.fq_name(tbl) + if tbl not in provider_tables: + err = 1 + self.log.error("Table %s not attached to queue" % tbl) + if err: + if self.options.force: + self.log.warning('--force used, ignoring errors') + else: + sys.exit(1) + + err = self.check_tables(table_list) + if err: + if self.options.force: + self.log.warning('--force used, ignoring errors') + else: + sys.exit(1) + + for tbl in table_list: + tbl = skytools.fq_name(tbl) + if tbl in subscriber_tables: + self.log.info("Table %s already added" % tbl) + else: + self.log.info("Adding %s" % tbl) + self.subscriber_add_one_table(tbl) + + def subscriber_remove_tables(self, table_list): + subscriber_tables = self.get_subscriber_table_list() + for tbl in table_list: + tbl = skytools.fq_name(tbl) + if tbl in subscriber_tables: + self.subscriber_remove_one_table(tbl) + else: + self.log.info("Table %s already removed" % tbl) + + def subscriber_resync_tables(self, table_list): + dst_db = self.get_database('subscriber_db') + dst_curs = dst_db.cursor() + list = self.fetch_subscriber_tables(dst_curs) + for tbl in table_list: + tbl = skytools.fq_name(tbl) + tbl_row = None + for row in list: + if row['table_name'] == tbl: + tbl_row = row + break + if not tbl_row: + self.log.warning("Table %s not found" % tbl) + elif tbl_row['merge_state'] != 'ok': + self.log.warning("Table %s is not in stable state" % tbl) + else: + self.log.info("Resyncing %s" % tbl) + q = "select londiste.subscriber_set_table_state(%s, %s, NULL, NULL)" + dst_curs.execute(q, [self.pgq_queue_name, tbl]) + dst_db.commit() + + def subscriber_add_one_table(self, tbl): + q = "select londiste.subscriber_add_table(%s, %s)" + + dst_db = self.get_database('subscriber_db') + dst_curs = dst_db.cursor() + dst_curs.execute(q, [self.pgq_queue_name, tbl]) + if self.options.expect_sync: + q = "select londiste.subscriber_set_table_state(%s, %s, null, 'ok')" + dst_curs.execute(q, [self.pgq_queue_name, tbl]) + dst_db.commit() + + def subscriber_remove_one_table(self, tbl): + q = "select londiste.subscriber_remove_table(%s, %s)" + + dst_db = self.get_database('subscriber_db') + dst_curs = dst_db.cursor() + dst_curs.execute(q, [self.pgq_queue_name, tbl]) + dst_db.commit() + + def get_subscriber_seq_list(self): + dst_db = self.get_database('subscriber_db') + dst_curs = dst_db.cursor() + q = "SELECT * from londiste.subscriber_get_seq_list(%s)" + dst_curs.execute(q, [self.pgq_queue_name]) + list = dst_curs.fetchall() + dst_db.commit() + res = [] + for row in list: + res.append(row[0]) + return res + + def subscriber_list_seqs(self): + list = self.get_subscriber_seq_list() + for seq in list: + print seq + + def subscriber_add_seq(self, seq_list): + src_db = self.get_database('provider_db') + src_curs = src_db.cursor() + dst_db = self.get_database('subscriber_db') + dst_curs = dst_db.cursor() + + prov_list = self.get_provider_seqs(src_curs) + src_db.commit() + + full_list = self.get_all_seqs(dst_curs) + cur_list = self.get_subscriber_seq_list() + + for seq in seq_list: + seq = skytools.fq_name(seq) + if seq not in prov_list: + self.log.error('Seq %s does not exist on provider side' % seq) + continue + if seq not in full_list: + self.log.error('Seq %s does not exist on subscriber side' % seq) + continue + if seq in cur_list: + self.log.info('Seq %s already subscribed' % seq) + continue + + self.log.info('Adding sequence: %s' % seq) + q = "select londiste.subscriber_add_seq(%s, %s)" + dst_curs.execute(q, [self.pgq_queue_name, seq]) + + dst_db.commit() + + def subscriber_remove_seq(self, seq_list): + dst_db = self.get_database('subscriber_db') + dst_curs = dst_db.cursor() + cur_list = self.get_subscriber_seq_list() + + for seq in seq_list: + seq = skytools.fq_name(seq) + if seq not in cur_list: + self.log.warning('Seq %s not subscribed') + else: + self.log.info('Removing sequence: %s' % seq) + q = "select londiste.subscriber_remove_seq(%s, %s)" + dst_curs.execute(q, [self.pgq_queue_name, seq]) + dst_db.commit() + diff --git a/python/londiste/syncer.py b/python/londiste/syncer.py new file mode 100644 index 00000000..eaee3468 --- /dev/null +++ b/python/londiste/syncer.py @@ -0,0 +1,177 @@ + +"""Catch moment when tables are in sync on master and slave. +""" + +import sys, time, skytools + +class Syncer(skytools.DBScript): + """Walks tables in primary key order and checks if data matches.""" + + def __init__(self, args): + skytools.DBScript.__init__(self, 'londiste', args) + self.set_single_loop(1) + + self.pgq_queue_name = self.cf.get("pgq_queue_name") + self.pgq_consumer_id = self.cf.get('pgq_consumer_id', self.job_name) + + if self.pidfile: + self.pidfile += ".repair" + + def init_optparse(self, p=None): + p = skytools.DBScript.init_optparse(self, p) + p.add_option("--force", action="store_true", help="ignore lag") + return p + + def check_consumer(self, src_db): + src_curs = src_db.cursor() + + # before locking anything check if consumer is working ok + q = "select extract(epoch from ticker_lag) from pgq.get_queue_list()"\ + " where queue_name = %s" + src_curs.execute(q, [self.pgq_queue_name]) + ticker_lag = src_curs.fetchone()[0] + q = "select extract(epoch from lag)"\ + " from pgq.get_consumer_list()"\ + " where queue_name = %s"\ + " and consumer_name = %s" + src_curs.execute(q, [self.pgq_queue_name, self.pgq_consumer_id]) + res = src_curs.fetchall() + src_db.commit() + + if len(res) == 0: + self.log.error('No such consumer') + sys.exit(1) + consumer_lag = res[0][0] + + if consumer_lag > ticker_lag + 10 and not self.options.force: + self.log.error('Consumer lagging too much, cannot proceed') + sys.exit(1) + + def get_subscriber_table_state(self): + dst_db = self.get_database('subscriber_db') + dst_curs = dst_db.cursor() + q = "select * from londiste.subscriber_get_table_list(%s)" + dst_curs.execute(q, [self.pgq_queue_name]) + res = dst_curs.dictfetchall() + dst_db.commit() + return res + + def work(self): + src_loc = self.cf.get('provider_db') + lock_db = self.get_database('provider_db', cache='lock_db') + src_db = self.get_database('provider_db') + dst_db = self.get_database('subscriber_db') + + self.check_consumer(src_db) + + state_list = self.get_subscriber_table_state() + state_map = {} + full_list = [] + for ts in state_list: + name = ts['table_name'] + full_list.append(name) + state_map[name] = ts + + if len(self.args) > 2: + tlist = self.args[2:] + else: + tlist = full_list + + for tbl in tlist: + if not tbl in state_map: + self.log.warning('Table not subscribed: %s' % tbl) + continue + st = state_map[tbl] + if st['merge_state'] != 'ok': + self.log.info('Table %s not synced yet, no point' % tbl) + continue + self.check_table(tbl, lock_db, src_db, dst_db) + lock_db.commit() + src_db.commit() + dst_db.commit() + + def check_table(self, tbl, lock_db, src_db, dst_db): + """Get transaction to same state, then process.""" + + + lock_curs = lock_db.cursor() + src_curs = src_db.cursor() + dst_curs = dst_db.cursor() + + if not skytools.exists_table(src_curs, tbl): + self.log.warning("Table %s does not exist on provider side" % tbl) + return + if not skytools.exists_table(dst_curs, tbl): + self.log.warning("Table %s does not exist on subscriber side" % tbl) + return + + # lock table in separate connection + self.log.info('Locking %s' % tbl) + lock_db.commit() + lock_curs.execute("LOCK TABLE %s IN SHARE MODE" % tbl) + lock_time = time.time() + + # now wait until consumer has updated target table until locking + self.log.info('Syncing %s' % tbl) + + # consumer must get futher than this tick + src_curs.execute("select pgq.ticker(%s)", [self.pgq_queue_name]) + tick_id = src_curs.fetchone()[0] + src_db.commit() + # avoid depending on ticker by inserting second tick also + time.sleep(0.1) + src_curs.execute("select pgq.ticker(%s)", [self.pgq_queue_name]) + src_db.commit() + src_curs.execute("select to_char(now(), 'YYYY-MM-DD HH24:MI:SS.MS')") + tpos = src_curs.fetchone()[0] + src_db.commit() + # now wait + while 1: + time.sleep(0.2) + + q = """select now() - lag > %s, now(), lag + from pgq.get_consumer_list() + where consumer_name = %s + and queue_name = %s""" + src_curs.execute(q, [tpos, self.pgq_consumer_id, self.pgq_queue_name]) + res = src_curs.fetchall() + src_db.commit() + + if len(res) == 0: + raise Exception('No such consumer') + + row = res[0] + self.log.debug("tpos=%s now=%s lag=%s ok=%s" % (tpos, row[1], row[2], row[0])) + if row[0]: + break + + # loop max 10 secs + if time.time() > lock_time + 10 and not self.options.force: + self.log.error('Consumer lagging too much, exiting') + lock_db.rollback() + sys.exit(1) + + # take snapshot on provider side + src_curs.execute("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE") + src_curs.execute("SELECT 1") + + # take snapshot on subscriber side + dst_db.commit() + dst_curs.execute("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE") + dst_curs.execute("SELECT 1") + + # release lock + lock_db.commit() + + # do work + self.process_sync(tbl, src_db, dst_db) + + # done + src_db.commit() + dst_db.commit() + + def process_sync(self, tbl, src_db, dst_db): + """It gets 2 connections in state where tbl should be in same state. + """ + raise Exception('process_sync not implemented') + diff --git a/python/londiste/table_copy.py b/python/londiste/table_copy.py new file mode 100644 index 00000000..1754baaf --- /dev/null +++ b/python/londiste/table_copy.py @@ -0,0 +1,107 @@ +#! /usr/bin/env python + +"""Do a full table copy. + +For internal usage. +""" + +import sys, os, skytools + +from skytools.dbstruct import * +from playback import * + +__all__ = ['CopyTable'] + +class CopyTable(Replicator): + def __init__(self, args, copy_thread = 1): + Replicator.__init__(self, args) + + if copy_thread: + self.pidfile += ".copy" + self.consumer_id += "_copy" + self.copy_thread = 1 + + def init_optparse(self, parser=None): + p = Replicator.init_optparse(self, parser) + p.add_option("--skip-truncate", action="store_true", dest="skip_truncate", + help = "avoid truncate", default=False) + return p + + def do_copy(self, tbl_stat): + src_db = self.get_database('provider_db') + dst_db = self.get_database('subscriber_db') + + # it should not matter to pgq + src_db.commit() + dst_db.commit() + + # change to SERIALIZABLE isolation level + src_db.set_isolation_level(2) + src_db.commit() + + # initial sync copy + src_curs = src_db.cursor() + dst_curs = dst_db.cursor() + + self.log.info("Starting full copy of %s" % tbl_stat.name) + + # find dst struct + src_struct = TableStruct(src_curs, tbl_stat.name) + dst_struct = TableStruct(dst_curs, tbl_stat.name) + + # check if columns match + dlist = dst_struct.get_column_list() + for c in src_struct.get_column_list(): + if c not in dlist: + raise Exception('Column %s does not exist on dest side' % c) + + # drop unnecessary stuff + objs = T_CONSTRAINT | T_INDEX | T_TRIGGER | T_RULE + dst_struct.drop(dst_curs, objs, log = self.log) + + # do truncate & copy + self.real_copy(src_curs, dst_curs, tbl_stat.name) + + # get snapshot + src_curs.execute("select get_current_snapshot()") + snapshot = src_curs.fetchone()[0] + src_db.commit() + + # restore READ COMMITTED behaviour + src_db.set_isolation_level(1) + src_db.commit() + + # create previously dropped objects + dst_struct.create(dst_curs, objs, log = self.log) + + # set state + tbl_stat.change_snapshot(snapshot) + if self.copy_thread: + tbl_stat.change_state(TABLE_CATCHING_UP) + else: + tbl_stat.change_state(TABLE_OK) + self.save_table_state(dst_curs) + dst_db.commit() + + def real_copy(self, srccurs, dstcurs, tablename): + "Main copy logic." + + # drop data + if self.options.skip_truncate: + self.log.info("%s: skipping truncate" % tablename) + else: + self.log.info("%s: truncating" % tablename) + dstcurs.execute("truncate " + tablename) + + # do copy + self.log.info("%s: start copy" % tablename) + col_list = skytools.get_table_columns(srccurs, tablename) + stats = skytools.full_copy(tablename, srccurs, dstcurs, col_list) + if stats: + self.log.info("%s: copy finished: %d bytes, %d rows" % ( + tablename, stats[0], stats[1])) + +if __name__ == '__main__': + script = CopyTable(sys.argv[1:]) + script.start() + diff --git a/python/pgq/__init__.py b/python/pgq/__init__.py new file mode 100644 index 00000000..f0e9c1a6 --- /dev/null +++ b/python/pgq/__init__.py @@ -0,0 +1,6 @@ +"""PgQ framework for Python.""" + +from pgq.event import * +from pgq.consumer import * +from pgq.producer import * + diff --git a/python/pgq/consumer.py b/python/pgq/consumer.py new file mode 100644 index 00000000..bd49dccf --- /dev/null +++ b/python/pgq/consumer.py @@ -0,0 +1,410 @@ + +"""PgQ consumer framework for Python. + +API problems(?): + - process_event() and process_batch() should have db as argument. + - should ev.tag*() update db immidiately? + +""" + +import sys, time, skytools + +from pgq.event import * + +__all__ = ['Consumer', 'RemoteConsumer', 'SerialConsumer'] + +class Consumer(skytools.DBScript): + """Consumer base class. + """ + + def __init__(self, service_name, db_name, args): + """Initialize new consumer. + + @param service_name: service_name for DBScript + @param db_name: name of database for get_database() + @param args: cmdline args for DBScript + """ + + skytools.DBScript.__init__(self, service_name, args) + + self.db_name = db_name + self.reg_list = [] + self.consumer_id = self.cf.get("pgq_consumer_id", self.job_name) + self.pgq_queue_name = self.cf.get("pgq_queue_name") + + def attach(self): + """Attach consumer to interesting queues.""" + res = self.register_consumer(self.pgq_queue_name) + return res + + def detach(self): + """Detach consumer from all queues.""" + tmp = self.reg_list[:] + for q in tmp: + self.unregister_consumer(q) + + def process_event(self, db, event): + """Process one event. + + Should be overrided by user code. + + Event should be tagged as done, retry or failed. + If not, it will be tagged as for retry. + """ + raise Exception("needs to be implemented") + + def process_batch(self, db, batch_id, event_list): + """Process all events in batch. + + By default calls process_event for each. + Can be overrided by user code. + + Events should be tagged as done, retry or failed. + If not, they will be tagged as for retry. + """ + for ev in event_list: + self.process_event(db, ev) + + def work(self): + """Do the work loop, once (internal).""" + + if len(self.reg_list) == 0: + self.log.debug("Attaching") + self.attach() + + db = self.get_database(self.db_name) + curs = db.cursor() + + data_avail = 0 + for queue in self.reg_list: + self.stat_start() + + # acquire batch + batch_id = self._load_next_batch(curs, queue) + db.commit() + if batch_id == None: + continue + data_avail = 1 + + # load events + list = self._load_batch_events(curs, batch_id, queue) + db.commit() + + # process events + self._launch_process_batch(db, batch_id, list) + + # done + self._finish_batch(curs, batch_id, list) + db.commit() + self.stat_end(len(list)) + + # if false, script sleeps + return data_avail + + def register_consumer(self, queue_name): + db = self.get_database(self.db_name) + cx = db.cursor() + cx.execute("select pgq.register_consumer(%s, %s)", + [queue_name, self.consumer_id]) + res = cx.fetchone()[0] + db.commit() + + self.reg_list.append(queue_name) + + return res + + def unregister_consumer(self, queue_name): + db = self.get_database(self.db_name) + cx = db.cursor() + cx.execute("select pgq.unregister_consumer(%s, %s)", + [queue_name, self.consumer_id]) + db.commit() + + self.reg_list.remove(queue_name) + + def _launch_process_batch(self, db, batch_id, list): + self.process_batch(db, batch_id, list) + + def _load_batch_events(self, curs, batch_id, queue_name): + """Fetch all events for this batch.""" + + # load events + sql = "select * from pgq.get_batch_events(%d)" % batch_id + curs.execute(sql) + rows = curs.dictfetchall() + + # map them to python objects + list = [] + for r in rows: + ev = Event(queue_name, r) + list.append(ev) + + return list + + def _load_next_batch(self, curs, queue_name): + """Allocate next batch. (internal)""" + + q = "select pgq.next_batch(%s, %s)" + curs.execute(q, [queue_name, self.consumer_id]) + return curs.fetchone()[0] + + def _finish_batch(self, curs, batch_id, list): + """Tag events and notify that the batch is done.""" + + retry = failed = 0 + for ev in list: + if ev.status == EV_FAILED: + self._tag_failed(curs, batch_id, ev) + failed += 1 + elif ev.status == EV_RETRY: + self._tag_retry(curs, batch_id, ev) + retry += 1 + curs.execute("select pgq.finish_batch(%s)", [batch_id]) + + def _tag_failed(self, curs, batch_id, ev): + """Tag event as failed. (internal)""" + curs.execute("select pgq.event_failed(%s, %s, %s)", + [batch_id, ev.id, ev.fail_reason]) + + def _tag_retry(self, cx, batch_id, ev): + """Tag event for retry. (internal)""" + cx.execute("select pgq.event_retry(%s, %s, %s)", + [batch_id, ev.id, ev.retry_time]) + + def get_batch_info(self, batch_id): + """Get info about batch. + + @return: Return value is a dict of: + + - queue_name: queue name + - consumer_name: consumers name + - batch_start: batch start time + - batch_end: batch end time + - tick_id: end tick id + - prev_tick_id: start tick id + - lag: how far is batch_end from current moment. + """ + db = self.get_database(self.db_name) + cx = db.cursor() + q = "select queue_name, consumer_name, batch_start, batch_end,"\ + " prev_tick_id, tick_id, lag"\ + " from pgq.get_batch_info(%s)" + cx.execute(q, [batch_id]) + row = cx.dictfetchone() + db.commit() + return row + + def stat_start(self): + self.stat_batch_start = time.time() + + def stat_end(self, count): + t = time.time() + self.stat_add('count', count) + self.stat_add('duration', t - self.stat_batch_start) + + +class RemoteConsumer(Consumer): + """Helper for doing event processing in another database. + + Requires that whole batch is processed in one TX. + """ + + def __init__(self, service_name, db_name, remote_db, args): + Consumer.__init__(self, service_name, db_name, args) + self.remote_db = remote_db + + def process_batch(self, db, batch_id, event_list): + """Process all events in batch. + + By default calls process_event for each. + """ + dst_db = self.get_database(self.remote_db) + curs = dst_db.cursor() + + if self.is_last_batch(curs, batch_id): + for ev in event_list: + ev.tag_done() + return + + self.process_remote_batch(db, batch_id, event_list, dst_db) + + self.set_last_batch(curs, batch_id) + dst_db.commit() + + def is_last_batch(self, dst_curs, batch_id): + """Helper function to keep track of last successful batch + in external database. + """ + q = "select pgq_ext.is_batch_done(%s, %s)" + dst_curs.execute(q, [ self.consumer_id, batch_id ]) + return dst_curs.fetchone()[0] + + def set_last_batch(self, dst_curs, batch_id): + """Helper function to set last successful batch + in external database. + """ + q = "select pgq_ext.set_batch_done(%s, %s)" + dst_curs.execute(q, [ self.consumer_id, batch_id ]) + + def process_remote_batch(self, db, batch_id, event_list, dst_db): + raise Exception('process_remote_batch not implemented') + +class SerialConsumer(Consumer): + """Consumer that applies batches sequentially in second database. + + Requirements: + - Whole batch in one TX. + - Must not use retry queue. + + Features: + - Can detect if several batches are already applied to dest db. + - If some ticks are lost. allows to seek back on queue. + Whether it succeeds, depends on pgq configuration. + """ + + def __init__(self, service_name, db_name, remote_db, args): + Consumer.__init__(self, service_name, db_name, args) + self.remote_db = remote_db + self.dst_completed_table = "pgq_ext.completed_tick" + self.cur_batch_info = None + + def startup(self): + if self.options.rewind: + self.rewind() + sys.exit(0) + if self.options.reset: + self.dst_reset() + sys.exit(0) + return Consumer.startup(self) + + def init_optparse(self, parser = None): + p = Consumer.init_optparse(self, parser) + p.add_option("--rewind", action = "store_true", + help = "change queue position according to destination") + p.add_option("--reset", action = "store_true", + help = "reset queue pos on destination side") + return p + + def process_batch(self, db, batch_id, event_list): + """Process all events in batch. + """ + + dst_db = self.get_database(self.remote_db) + curs = dst_db.cursor() + + self.cur_batch_info = self.get_batch_info(batch_id) + + # check if done + if self.is_batch_done(curs): + for ev in event_list: + ev.tag_done() + return + + # actual work + self.process_remote_batch(db, batch_id, event_list, dst_db) + + # make sure no retry events + for ev in event_list: + if ev.status == EV_RETRY: + raise Exception("SerialConsumer must not use retry queue") + + # finish work + self.set_batch_done(curs) + dst_db.commit() + + def is_batch_done(self, dst_curs): + """Helper function to keep track of last successful batch + in external database. + """ + + prev_tick = self.cur_batch_info['prev_tick_id'] + + q = "select last_tick_id from %s where consumer_id = %%s" % ( + self.dst_completed_table ,) + dst_curs.execute(q, [self.consumer_id]) + res = dst_curs.fetchone() + + if not res or not res[0]: + # seems this consumer has not run yet against dst_db + return False + dst_tick = res[0] + + if prev_tick == dst_tick: + # on track + return False + + if prev_tick < dst_tick: + self.log.warning('Got tick %d, dst has %d - skipping' % (prev_tick, dst_tick)) + return True + else: + self.log.error('Got tick %d, dst has %d - ticks lost' % (prev_tick, dst_tick)) + raise Exception('Lost ticks') + + def set_batch_done(self, dst_curs): + """Helper function to set last successful batch + in external database. + """ + tick_id = self.cur_batch_info['tick_id'] + q = "delete from %s where consumer_id = %%s; "\ + "insert into %s (consumer_id, last_tick_id) values (%%s, %%s)" % ( + self.dst_completed_table, + self.dst_completed_table) + dst_curs.execute(q, [ self.consumer_id, + self.consumer_id, tick_id ]) + + def attach(self): + new = Consumer.attach(self) + if new: + self.clean_completed_tick() + + def detach(self): + """If detaching, also clean completed tick table on dest.""" + + Consumer.detach(self) + self.clean_completed_tick() + + def clean_completed_tick(self): + self.log.info("removing completed tick from dst") + dst_db = self.get_database(self.remote_db) + dst_curs = dst_db.cursor() + + q = "delete from %s where consumer_id = %%s" % ( + self.dst_completed_table,) + dst_curs.execute(q, [self.consumer_id]) + dst_db.commit() + + def process_remote_batch(self, db, batch_id, event_list, dst_db): + raise Exception('process_remote_batch not implemented') + + def rewind(self): + self.log.info("Rewinding queue") + src_db = self.get_database(self.db_name) + dst_db = self.get_database(self.remote_db) + src_curs = src_db.cursor() + dst_curs = dst_db.cursor() + + q = "select last_tick_id from %s where consumer_id = %%s" % ( + self.dst_completed_table,) + dst_curs.execute(q, [self.consumer_id]) + row = dst_curs.fetchone() + if row: + dst_tick = row[0] + q = "select pgq.register_consumer(%s, %s, %s)" + src_curs.execute(q, [self.pgq_queue_name, self.consumer_id, dst_tick]) + else: + self.log.warning('No tick found on dst side') + + dst_db.commit() + src_db.commit() + + def dst_reset(self): + self.log.info("Resetting queue tracking on dst side") + dst_db = self.get_database(self.remote_db) + dst_curs = dst_db.cursor() + + q = "delete from %s where consumer_id = %%s" % ( + self.dst_completed_table,) + dst_curs.execute(q, [self.consumer_id]) + dst_db.commit() + + diff --git a/python/pgq/event.py b/python/pgq/event.py new file mode 100644 index 00000000..d7b2d7ee --- /dev/null +++ b/python/pgq/event.py @@ -0,0 +1,60 @@ + +"""PgQ event container. +""" + +__all__ = ('EV_RETRY', 'EV_DONE', 'EV_FAILED', 'Event') + +# Event status codes +EV_RETRY = 0 +EV_DONE = 1 +EV_FAILED = 2 + +_fldmap = { + 'ev_id': 'ev_id', + 'ev_txid': 'ev_txid', + 'ev_time': 'ev_time', + 'ev_type': 'ev_type', + 'ev_data': 'ev_data', + 'ev_extra1': 'ev_extra1', + 'ev_extra2': 'ev_extra2', + 'ev_extra3': 'ev_extra3', + 'ev_extra4': 'ev_extra4', + + 'id': 'ev_id', + 'txid': 'ev_txid', + 'time': 'ev_time', + 'type': 'ev_type', + 'data': 'ev_data', + 'extra1': 'ev_extra1', + 'extra2': 'ev_extra2', + 'extra3': 'ev_extra3', + 'extra4': 'ev_extra4', +} + +class Event(object): + """Event data for consumers. + + Consumer is supposed to tag them after processing. + If not, events will stay in retry queue. + """ + def __init__(self, queue_name, row): + self._event_row = row + self.status = EV_RETRY + self.retry_time = 60 + self.fail_reason = "Buggy consumer" + self.queue_name = queue_name + + def __getattr__(self, key): + return self._event_row[_fldmap[key]] + + def tag_done(self): + self.status = EV_DONE + + def tag_retry(self, retry_time = 60): + self.status = EV_RETRY + self.retry_time = retry_time + + def tag_failed(self, reason): + self.status = EV_FAILED + self.fail_reason = reason + diff --git a/python/pgq/maint.py b/python/pgq/maint.py new file mode 100644 index 00000000..4636f74f --- /dev/null +++ b/python/pgq/maint.py @@ -0,0 +1,99 @@ +"""PgQ maintenance functions.""" + +import skytools, time + +def get_pgq_api_version(curs): + q = "select count(1) from pg_proc p, pg_namespace n"\ + " where n.oid = p.pronamespace and n.nspname='pgq'"\ + " and p.proname='version';" + curs.execute(q) + if not curs.fetchone()[0]: + return '1.0.0' + + curs.execute("select pgq.version()") + return curs.fetchone()[0] + +def version_ge(curs, want_ver): + """Check is db version of pgq is greater than want_ver.""" + db_ver = get_pgq_api_version(curs) + want_tuple = map(int, want_ver.split('.')) + db_tuple = map(int, db_ver.split('.')) + if db_tuple[0] != want_tuple[0]: + raise Exception('Wrong major version') + if db_tuple[1] >= want_tuple[1]: + return 1 + return 0 + +class MaintenanceJob(skytools.DBScript): + """Periodic maintenance.""" + def __init__(self, ticker, args): + skytools.DBScript.__init__(self, 'pgqadm', args) + self.ticker = ticker + self.last_time = 0 # start immidiately + self.last_ticks = 0 + self.clean_ticks = 1 + self.maint_delay = 5*60 + + def startup(self): + # disable regular DBScript startup() + pass + + def reload(self): + skytools.DBScript.reload(self) + + # force loop_delay + self.loop_delay = 5 + + self.maint_delay = 60 * self.cf.getfloat('maint_delay_min', 5) + self.maint_delay = self.cf.getfloat('maint_delay', self.maint_delay) + + def work(self): + t = time.time() + if self.last_time + self.maint_delay > t: + return + + self.do_maintenance() + + self.last_time = t + duration = time.time() - t + self.stat_add('maint_duration', duration) + + def do_maintenance(self): + """Helper function for running maintenance.""" + + db = self.get_database('db', autocommit=1) + cx = db.cursor() + + if skytools.exists_function(cx, "pgq.maint_rotate_tables_step1", 1): + # rotate each queue in own TX + q = "select queue_name from pgq.get_queue_info()" + cx.execute(q) + for row in cx.fetchall(): + cx.execute("select pgq.maint_rotate_tables_step1(%s)", [row[0]]) + res = cx.fetchone()[0] + if res: + self.log.info('Rotating %s' % row[0]) + else: + cx.execute("select pgq.maint_rotate_tables_step1();") + + # finish rotation + cx.execute("select pgq.maint_rotate_tables_step2();") + + # move retry events to main queue in small blocks + rcount = 0 + while 1: + cx.execute('select pgq.maint_retry_events();') + res = cx.fetchone()[0] + rcount += res + if res == 0: + break + if rcount: + self.log.info('Got %d events for retry' % rcount) + + # vacuum tables that are needed + cx.execute('set maintenance_work_mem = 32768') + cx.execute('select * from pgq.maint_tables_to_vacuum()') + for row in cx.fetchall(): + cx.execute('vacuum %s;' % row[0]) + + diff --git a/python/pgq/producer.py b/python/pgq/producer.py new file mode 100644 index 00000000..81e1ca4f --- /dev/null +++ b/python/pgq/producer.py @@ -0,0 +1,41 @@ + +"""PgQ producer helpers for Python. +""" + +import skytools + +_fldmap = { + 'id': 'ev_id', + 'time': 'ev_time', + 'type': 'ev_type', + 'data': 'ev_data', + 'extra1': 'ev_extra1', + 'extra2': 'ev_extra2', + 'extra3': 'ev_extra3', + 'extra4': 'ev_extra4', + + 'ev_id': 'ev_id', + 'ev_time': 'ev_time', + 'ev_type': 'ev_type', + 'ev_data': 'ev_data', + 'ev_extra1': 'ev_extra1', + 'ev_extra2': 'ev_extra2', + 'ev_extra3': 'ev_extra3', + 'ev_extra4': 'ev_extra4', +} + +def bulk_insert_events(curs, rows, fields, queue_name): + q = "select pgq.current_event_table(%s)" + curs.execute(q, [queue_name]) + tbl = curs.fetchone()[0] + db_fields = map(_fldmap.get, fields) + skytools.magic_insert(curs, tbl, rows, db_fields) + +def insert_event(curs, queue, ev_type, ev_data, + extra1=None, extra2=None, + extra3=None, extra4=None): + q = "select pgq.insert_event(%s, %s, %s, %s, %s, %s, %s)" + curs.execute(q, [queue, ev_type, ev_data, + extra1, extra2, extra3, extra4]) + return curs.fetchone()[0] + diff --git a/python/pgq/status.py b/python/pgq/status.py new file mode 100644 index 00000000..2214045f --- /dev/null +++ b/python/pgq/status.py @@ -0,0 +1,93 @@ + +"""Status display. +""" + +import sys, os, skytools + +def ival(data, as = None): + "Format interval for output" + if not as: + as = data.split('.')[-1] + numfmt = 'FM9999999' + expr = "coalesce(to_char(extract(epoch from %s), '%s') || 's', 'NULL') as %s" + return expr % (data, numfmt, as) + +class PGQStatus(skytools.DBScript): + def __init__(self, args, check = 0): + skytools.DBScript.__init__(self, 'pgqadm', args) + + self.show_status() + + sys.exit(0) + + def show_status(self): + db = self.get_database("db", autocommit=1) + cx = db.cursor() + + cx.execute("show server_version") + pgver = cx.fetchone()[0] + cx.execute("select pgq.version()") + qver = cx.fetchone()[0] + print "Postgres version: %s PgQ version: %s" % (pgver, qver) + + q = """select f.queue_name, f.num_tables, %s, %s, %s, + q.queue_ticker_max_lag, q.queue_ticker_max_amount, + q.queue_ticker_idle_interval + from pgq.get_queue_info() f, pgq.queue q + where q.queue_name = f.queue_name""" % ( + ival('f.rotation_delay'), + ival('f.ticker_lag'), + ) + cx.execute(q) + event_rows = cx.dictfetchall() + + q = """select queue_name, consumer_name, %s, %s, %s + from pgq.get_consumer_info()""" % ( + ival('lag'), + ival('last_seen'), + ) + cx.execute(q) + consumer_rows = cx.dictfetchall() + + print "\n%-32s %s %9s %13s %6s" % ('Event queue', + 'Rotation', 'Ticker', 'TLag') + print '-' * 78 + for ev_row in event_rows: + tck = "%s/%ss/%ss" % (ev_row['queue_ticker_max_amount'], + ev_row['queue_ticker_max_lag'], + ev_row['queue_ticker_idle_interval']) + rot = "%s/%s" % (ev_row['queue_ntables'], ev_row['queue_rotation_period']) + print "%-39s%7s %9s %13s %6s" % ( + ev_row['queue_name'], + rot, + tck, + ev_row['ticker_lag'], + ) + print '-' * 78 + print "\n%-42s %9s %9s" % ( + 'Consumer', 'Lag', 'LastSeen') + print '-' * 78 + for ev_row in event_rows: + cons = self.pick_consumers(ev_row, consumer_rows) + self.show_queue(ev_row, cons) + print '-' * 78 + db.commit() + + def show_consumer(self, cons): + print " %-48s %9s %9s" % ( + cons['consumer_name'], + cons['lag'], cons['last_seen']) + def show_queue(self, ev_row, consumer_rows): + print "%(queue_name)s:" % ev_row + for cons in consumer_rows: + self.show_consumer(cons) + + + def pick_consumers(self, ev_row, consumer_rows): + res = [] + for con in consumer_rows: + if con['queue_name'] != ev_row['queue_name']: + continue + res.append(con) + return res + diff --git a/python/pgq/ticker.py b/python/pgq/ticker.py new file mode 100644 index 00000000..c218eaf1 --- /dev/null +++ b/python/pgq/ticker.py @@ -0,0 +1,172 @@ +"""PgQ ticker. + +It will also launch maintenance job. +""" + +import sys, os, time, threading +import skytools + +from maint import MaintenanceJob + +__all__ = ['SmartTicker'] + +def is_txid_sane(curs): + curs.execute("select get_current_txid()") + txid = curs.fetchone()[0] + + # on 8.2 theres no such table + if not skytools.exists_table(curs, 'txid.epoch'): + return 1 + + curs.execute("select epoch, last_value from txid.epoch") + epoch, last_val = curs.fetchone() + stored_val = (epoch << 32) | last_val + + if stored_val <= txid: + return 1 + else: + return 0 + +class QueueStatus(object): + def __init__(self, name): + self.queue_name = name + self.seq_name = None + self.idle_period = 60 + self.max_lag = 3 + self.max_count = 200 + self.last_tick_time = 0 + self.last_count = 0 + self.quiet_count = 0 + + def set_data(self, row): + self.seq_name = row['queue_event_seq'] + self.idle_period = row['queue_ticker_idle_period'] + self.max_lag = row['queue_ticker_max_lag'] + self.max_count = row['queue_ticker_max_count'] + + def need_tick(self, cur_count, cur_time): + # check if tick is needed + need_tick = 0 + lag = cur_time - self.last_tick_time + + if cur_count == self.last_count: + # totally idle database + + # don't go immidiately to big delays, as seq grows before commit + if self.quiet_count < 5: + if lag >= self.max_lag: + need_tick = 1 + self.quiet_count += 1 + else: + if lag >= self.idle_period: + need_tick = 1 + else: + self.quiet_count = 0 + # somewhat loaded machine + if cur_count - self.last_count >= self.max_count: + need_tick = 1 + elif lag >= self.max_lag: + need_tick = 1 + if need_tick: + self.last_tick_time = cur_time + self.last_count = cur_count + return need_tick + +class SmartTicker(skytools.DBScript): + last_tick_event = 0 + last_tick_time = 0 + quiet_count = 0 + tick_count = 0 + maint_thread = None + + def __init__(self, args): + skytools.DBScript.__init__(self, 'pgqadm', args) + + self.ticker_log_time = 0 + self.ticker_log_delay = 5*60 + self.queue_map = {} + self.refresh_time = 0 + + def reload(self): + skytools.DBScript.reload(self) + self.ticker_log_delay = self.cf.getfloat("ticker_log_delay", 5*60) + + def startup(self): + if self.maint_thread: + return + + db = self.get_database("db", autocommit = 1) + cx = db.cursor() + ok = is_txid_sane(cx) + if not ok: + self.log.error('txid in bad state') + sys.exit(1) + + self.maint_thread = MaintenanceJob(self, [self.cf.filename]) + t = threading.Thread(name = 'maint_thread', + target = self.maint_thread.run) + t.setDaemon(1) + t.start() + + def refresh_queues(self, cx): + q = "select queue_name, queue_event_seq, queue_ticker_idle_period,"\ + " queue_ticker_max_lag, queue_ticker_max_count"\ + " from pgq.queue"\ + " where not queue_external_ticker" + cx.execute(q) + new_map = {} + data_list = [] + from_list = [] + for row in cx.dictfetchall(): + queue_name = row['queue_name'] + try: + que = self.queue_map[queue_name] + except KeyError, x: + que = QueueStatus(queue_name) + que.set_data(row) + new_map[queue_name] = que + + p1 = "'%s', %s.last_value" % (queue_name, que.seq_name) + data_list.append(p1) + from_list.append(que.seq_name) + + self.queue_map = new_map + self.seq_query = "select %s from %s" % ( + ", ".join(data_list), + ", ".join(from_list)) + + if len(from_list) == 0: + self.seq_query = None + + self.refresh_time = time.time() + + def work(self): + db = self.get_database("db", autocommit = 1) + cx = db.cursor() + + cur_time = time.time() + + if cur_time >= self.refresh_time + 30: + self.refresh_queues(cx) + + if not self.seq_query: + return + + # now check seqs + cx.execute(self.seq_query) + res = cx.fetchone() + pos = 0 + while pos < len(res): + id = res[pos] + val = res[pos + 1] + pos += 2 + que = self.queue_map[id] + if que.need_tick(val, cur_time): + cx.execute("select pgq.ticker(%s)", [que.queue_name]) + self.tick_count += 1 + + if cur_time > self.ticker_log_time + self.ticker_log_delay: + self.ticker_log_time = cur_time + self.stat_add('ticks', self.tick_count) + self.tick_count = 0 + diff --git a/python/pgqadm.py b/python/pgqadm.py new file mode 100755 index 00000000..78f513dc --- /dev/null +++ b/python/pgqadm.py @@ -0,0 +1,162 @@ +#! /usr/bin/env python + +"""PgQ ticker and maintenance. +""" + +import sys +import skytools + +from pgq.ticker import SmartTicker +from pgq.status import PGQStatus +#from pgq.admin import PGQAdmin + +"""TODO: +pgqadm ini check +""" + +command_usage = """ +%prog [options] INI CMD [subcmd args] + +commands: + ticker start ticking & maintenance process + + status show overview of queue health + check show problematic consumers + + install install code into db + create QNAME create queue + drop QNAME drop queue + register QNAME CONS install code into db + unregister QNAME CONS install code into db + config QNAME [VAR=VAL] show or change queue config +""" + +config_allowed_list = [ + 'queue_ticker_max_lag', 'queue_ticker_max_amount', + 'queue_ticker_idle_interval', 'queue_rotation_period'] + +class PGQAdmin(skytools.DBScript): + def __init__(self, args): + skytools.DBScript.__init__(self, 'pgqadm', args) + self.set_single_loop(1) + + if len(self.args) < 2: + print "need command" + sys.exit(1) + + int_cmds = { + 'create': self.create_queue, + 'drop': self.drop_queue, + 'register': self.register, + 'unregister': self.unregister, + 'install': self.installer, + 'config': self.change_config, + } + + cmd = self.args[1] + if cmd == "ticker": + script = SmartTicker(args) + elif cmd == "status": + script = PGQStatus(args) + elif cmd == "check": + script = PGQStatus(args, check = 1) + elif cmd in int_cmds: + script = None + self.work = int_cmds[cmd] + else: + print "unknown command" + sys.exit(1) + + if self.pidfile: + self.pidfile += ".admin" + self.run_script = script + + def start(self): + if self.run_script: + self.run_script.start() + else: + skytools.DBScript.start(self) + + def init_optparse(self, parser=None): + p = skytools.DBScript.init_optparse(self, parser) + p.set_usage(command_usage.strip()) + return p + + def installer(self): + objs = [ + skytools.DBLanguage("plpgsql"), + skytools.DBLanguage("plpythonu"), + skytools.DBFunction("get_current_txid", 0, sql_file="txid.sql"), + skytools.DBSchema("pgq", sql_file="pgq.sql"), + ] + + db = self.get_database('db') + curs = db.cursor() + skytools.db_install(curs, objs, self.log) + db.commit() + + def create_queue(self): + qname = self.args[2] + self.log.info('Creating queue: %s' % qname) + self.exec_sql("select pgq.create_queue(%s)", [qname]) + + def drop_queue(self): + qname = self.args[2] + self.log.info('Dropping queue: %s' % qname) + self.exec_sql("select pgq.drop_queue(%s)", [qname]) + + def register(self): + qname = self.args[2] + cons = self.args[3] + self.log.info('Registering consumer %s on queue %s' % (cons, qname)) + self.exec_sql("select pgq.register_consumer(%s, %s)", [qname, cons]) + + def unregister(self): + qname = self.args[2] + cons = self.args[3] + self.log.info('Unregistering consumer %s from queue %s' % (cons, qname)) + self.exec_sql("select pgq.unregister_consumer(%s, %s)", [qname, cons]) + + def change_config(self): + qname = self.args[2] + if len(self.args) == 3: + self.show_config(qname) + return + alist = [] + for el in self.args[3:]: + k, v = el.split('=') + if k not in config_allowed_list: + raise Exception('unknown config var: '+k) + expr = "%s=%s" % (k, skytools.quote_literal(v)) + alist.append(expr) + self.log.info('Change queue %s config to: %s' % (qname, ", ".join(alist))) + sql = "update pgq.queue set %s where queue_name = %s" % ( + ", ".join(alist), skytools.quote_literal(qname)) + self.exec_sql(sql, []) + + def exec_sql(self, q, args): + self.log.debug(q) + db = self.get_database('db') + curs = db.cursor() + curs.execute(q, args) + db.commit() + + def show_config(self, qname): + klist = ",".join(config_allowed_list) + q = "select * from pgq.queue where queue_name = %s" + db = self.get_database('db') + curs = db.cursor() + curs.execute(q, [qname]) + res = curs.dictfetchone() + db.commit() + + print qname + for k in config_allowed_list: + print " %s=%s" % (k, res[k]) + +if __name__ == '__main__': + script = PGQAdmin(sys.argv[1:]) + script.start() + + + diff --git a/python/skytools/__init__.py b/python/skytools/__init__.py new file mode 100644 index 00000000..ed2b39bc --- /dev/null +++ b/python/skytools/__init__.py @@ -0,0 +1,10 @@ + +"""Tools for Python database scripts.""" + +from config import * +from dbstruct import * +from gzlog import * +from quoting import * +from scripting import * +from sqltools import * + diff --git a/python/skytools/config.py b/python/skytools/config.py new file mode 100644 index 00000000..de420322 --- /dev/null +++ b/python/skytools/config.py @@ -0,0 +1,139 @@ + +"""Nicer config class.""" + +import sys, os, ConfigParser, socket + +__all__ = ['Config'] + +class Config(object): + """Bit improved ConfigParser. + + Additional features: + - Remembers section. + - Acceps defaults in get() functions. + - List value support. + """ + def __init__(self, main_section, filename, sane_config = 1): + """Initialize Config and read from file. + + @param sane_config: chooses between ConfigParser/SafeConfigParser. + """ + defs = { + 'job_name': main_section, + 'service_name': main_section, + 'host_name': socket.gethostname(), + } + if not os.path.isfile(filename): + raise Exception('Config file not found: '+filename) + + self.filename = filename + self.sane_config = sane_config + if sane_config: + self.cf = ConfigParser.SafeConfigParser(defs) + else: + self.cf = ConfigParser.ConfigParser(defs) + self.cf.read(filename) + self.main_section = main_section + if not self.cf.has_section(main_section): + raise Exception("Wrong config file, no section '%s'"%main_section) + + def reload(self): + """Re-reads config file.""" + self.cf.read(self.filename) + + def get(self, key, default=None): + """Reads string value, if not set then default.""" + try: + return self.cf.get(self.main_section, key) + except ConfigParser.NoOptionError, det: + if default == None: + raise Exception("Config value not set: " + key) + return default + + def getint(self, key, default=None): + """Reads int value, if not set then default.""" + try: + return self.cf.getint(self.main_section, key) + except ConfigParser.NoOptionError, det: + if default == None: + raise Exception("Config value not set: " + key) + return default + + def getboolean(self, key, default=None): + """Reads boolean value, if not set then default.""" + try: + return self.cf.getboolean(self.main_section, key) + except ConfigParser.NoOptionError, det: + if default == None: + raise Exception("Config value not set: " + key) + return default + + def getfloat(self, key, default=None): + """Reads float value, if not set then default.""" + try: + return self.cf.getfloat(self.main_section, key) + except ConfigParser.NoOptionError, det: + if default == None: + raise Exception("Config value not set: " + key) + return default + + def getlist(self, key, default=None): + """Reads comma-separated list from key.""" + try: + s = self.cf.get(self.main_section, key).strip() + res = [] + if not s: + return res + for v in s.split(","): + res.append(v.strip()) + return res + except ConfigParser.NoOptionError, det: + if default == None: + raise Exception("Config value not set: " + key) + return default + + def getfile(self, key, default=None): + """Reads filename from config. + + In addition to reading string value, expands ~ to user directory. + """ + fn = self.get(key, default) + if fn == "" or fn == "-": + return fn + # simulate that the cwd is script location + #path = os.path.dirname(sys.argv[0]) + # seems bad idea, cwd should be cwd + + fn = os.path.expanduser(fn) + + return fn + + def get_wildcard(self, key, values=[], default=None): + """Reads a wildcard property from conf and returns its string value, if not set then default.""" + + orig_key = key + keys = [key] + + for wild in values: + key = key.replace('*', wild, 1) + keys.append(key) + keys.reverse() + + for key in keys: + try: + return self.cf.get(self.main_section, key) + except ConfigParser.NoOptionError, det: + pass + + if default == None: + raise Exception("Config value not set: " + orig_key) + return default + + def sections(self): + """Returns list of sections in config file, excluding DEFAULT.""" + return self.cf.sections() + + def clone(self, main_section): + """Return new Config() instance with new main section on same config file.""" + return Config(main_section, self.filename, self.sane_config) + diff --git a/python/skytools/dbstruct.py b/python/skytools/dbstruct.py new file mode 100644 index 00000000..22333429 --- /dev/null +++ b/python/skytools/dbstruct.py @@ -0,0 +1,380 @@ +"""Find table structure and allow CREATE/DROP elements from it. +""" + +import sys, re + +from sqltools import fq_name_parts, get_table_oid + +__all__ = ['TableStruct', + 'T_TABLE', 'T_CONSTRAINT', 'T_INDEX', 'T_TRIGGER', + 'T_RULE', 'T_GRANT', 'T_OWNER', 'T_PKEY', 'T_ALL'] + +T_TABLE = 1 << 0 +T_CONSTRAINT = 1 << 1 +T_INDEX = 1 << 2 +T_TRIGGER = 1 << 3 +T_RULE = 1 << 4 +T_GRANT = 1 << 5 +T_OWNER = 1 << 6 +T_PKEY = 1 << 20 # special, one of constraints +T_ALL = ( T_TABLE | T_CONSTRAINT | T_INDEX + | T_TRIGGER | T_RULE | T_GRANT | T_OWNER ) + +# +# Utility functions +# + +def find_new_name(curs, name): + """Create new object name for case the old exists. + + Needed when creating a new table besides old one. + """ + # cut off previous numbers + m = re.search('_[0-9]+$', name) + if m: + name = name[:m.start()] + + # now loop + for i in range(1, 1000): + tname = "%s_%d" % (name, i) + q = "select count(1) from pg_class where relname = %s" + curs.execute(q, [tname]) + if curs.fetchone()[0] == 0: + return tname + + # failed + raise Exception('find_new_name failed') + +def rx_replace(rx, sql, new_part): + """Find a regex match and replace that part with new_part.""" + m = re.search(rx, sql, re.I) + if not m: + raise Exception('rx_replace failed') + p1 = sql[:m.start()] + p2 = sql[m.end():] + return p1 + new_part + p2 + +# +# Schema objects +# + +class TElem(object): + """Keeps info about one metadata object.""" + SQL = "" + type = 0 + def get_create_sql(self, curs): + """Return SQL statement for creating or None if not supported.""" + return None + def get_drop_sql(self, curs): + """Return SQL statement for dropping or None of not supported.""" + return None + +class TConstraint(TElem): + """Info about constraint.""" + type = T_CONSTRAINT + SQL = """ + SELECT conname as name, pg_get_constraintdef(oid) as def, contype + FROM pg_constraint WHERE conrelid = %(oid)s + """ + def __init__(self, table_name, row): + self.table_name = table_name + self.name = row['name'] + self.defn = row['def'] + self.contype = row['contype'] + + # tag pkeys + if self.contype == 'p': + self.type += T_PKEY + + def get_create_sql(self, curs, new_table_name=None): + fmt = "ALTER TABLE ONLY %s ADD CONSTRAINT %s %s;" + if new_table_name: + name = self.name + if self.contype in ('p', 'u'): + name = find_new_name(curs, self.name) + sql = fmt % (new_table_name, name, self.defn) + else: + sql = fmt % (self.table_name, self.name, self.defn) + return sql + + def get_drop_sql(self, curs): + fmt = "ALTER TABLE ONLY %s DROP CONSTRAINT %s;" + sql = fmt % (self.table_name, self.name) + return sql + +class TIndex(TElem): + """Info about index.""" + type = T_INDEX + SQL = """ + SELECT n.nspname || '.' || c.relname as name, + pg_get_indexdef(i.indexrelid) as defn + FROM pg_index i, pg_class c, pg_namespace n + WHERE c.oid = i.indexrelid AND i.indrelid = %(oid)s + AND n.oid = c.relnamespace + AND NOT EXISTS + (select objid from pg_depend + where classid = %(pg_class_oid)s + and objid = c.oid + and deptype = 'i') + """ + def __init__(self, table_name, row): + self.name = row['name'] + self.defn = row['defn'] + ';' + + def get_create_sql(self, curs, new_table_name = None): + if not new_table_name: + return self.defn + name = find_new_name(curs, self.name) + pnew = "INDEX %s ON %s " % (name, new_table_name) + rx = r"\bINDEX[ ][a-z0-9._]+[ ]ON[ ][a-z0-9._]+[ ]" + sql = rx_replace(rx, self.defn, pnew) + return sql + def get_drop_sql(self, curs): + return 'DROP INDEX %s;' % self.name + +class TRule(TElem): + """Info about rule.""" + type = T_RULE + SQL = """ + SELECT rulename as name, pg_get_ruledef(oid) as def + FROM pg_rewrite + WHERE ev_class = %(oid)s AND rulename <> '_RETURN'::name + """ + def __init__(self, table_name, row, new_name = None): + self.table_name = table_name + self.name = row['name'] + self.defn = row['def'] + + def get_create_sql(self, curs, new_table_name = None): + if not new_table_name: + return self.defn + rx = r"\bTO[ ][a-z0-9._]+[ ]DO[ ]" + pnew = "TO %s DO " % new_table_name + return rx_replace(rx, self.defn, pnew) + + def get_drop_sql(self, curs): + return 'DROP RULE %s ON %s' % (self.name, self.table_name) + +class TTrigger(TElem): + """Info about trigger.""" + type = T_TRIGGER + SQL = """ + SELECT tgname as name, pg_get_triggerdef(oid) as def + FROM pg_trigger + WHERE tgrelid = %(oid)s AND NOT tgisconstraint + """ + def __init__(self, table_name, row): + self.table_name = table_name + self.name = row['name'] + self.defn = row['def'] + ';' + + def get_create_sql(self, curs, new_table_name = None): + if not new_table_name: + return self.defn + + rx = r"\bON[ ][a-z0-9._]+[ ]" + pnew = "ON %s " % new_table_name + return rx_replace(rx, self.defn, pnew) + + def get_drop_sql(self, curs): + return 'DROP TRIGGER %s ON %s' % (self.name, self.table_name) + +class TOwner(TElem): + """Info about table owner.""" + type = T_OWNER + SQL = """ + SELECT pg_get_userbyid(relowner) as owner FROM pg_class + WHERE oid = %(oid)s + """ + def __init__(self, table_name, row, new_name = None): + self.table_name = table_name + self.name = 'Owner' + self.owner = row['owner'] + + def get_create_sql(self, curs, new_name = None): + if not new_name: + new_name = self.table_name + return 'ALTER TABLE %s OWNER TO %s;' % (new_name, self.owner) + +class TGrant(TElem): + """Info about permissions.""" + type = T_GRANT + SQL = "SELECT relacl FROM pg_class where oid = %(oid)s" + acl_map = { + 'r': 'SELECT', 'w': 'UPDATE', 'a': 'INSERT', 'd': 'DELETE', + 'R': 'RULE', 'x': 'REFERENCES', 't': 'TRIGGER', 'X': 'EXECUTE', + 'U': 'USAGE', 'C': 'CREATE', 'T': 'TEMPORARY' + } + def acl_to_grants(self, acl): + if acl == "arwdRxt": # ALL for tables + return "ALL" + return ", ".join([ self.acl_map[c] for c in acl ]) + + def parse_relacl(self, relacl): + if relacl is None: + return [] + if len(relacl) > 0 and relacl[0] == '{' and relacl[-1] == '}': + relacl = relacl[1:-1] + list = [] + for f in relacl.split(','): + user, tmp = f.strip('"').split('=') + acl, who = tmp.split('/') + list.append((user, acl, who)) + return list + + def __init__(self, table_name, row, new_name = None): + self.name = table_name + self.acl_list = self.parse_relacl(row['relacl']) + + def get_create_sql(self, curs, new_name = None): + if not new_name: + new_name = self.name + + list = [] + for user, acl, who in self.acl_list: + astr = self.acl_to_grants(acl) + sql = "GRANT %s ON %s TO %s;" % (astr, new_name, user) + list.append(sql) + return "\n".join(list) + + def get_drop_sql(self, curs): + list = [] + for user, acl, who in self.acl_list: + sql = "REVOKE ALL FROM %s ON %s;" % (user, self.name) + list.append(sql) + return "\n".join(list) + +class TColumn(TElem): + """Info about table column.""" + SQL = """ + select a.attname as name, + a.attname || ' ' + || format_type(a.atttypid, a.atttypmod) + || case when a.attnotnull then ' not null' else '' end + || case when a.atthasdef then ' ' || d.adsrc else '' end + as def + from pg_attribute a left join pg_attrdef d + on (d.adrelid = a.attrelid and d.adnum = a.attnum) + where a.attrelid = %(oid)s + and not a.attisdropped + and a.attnum > 0 + order by a.attnum; + """ + def __init__(self, table_name, row): + self.name = row['name'] + self.column_def = row['def'] + +class TTable(TElem): + """Info about table only (columns).""" + type = T_TABLE + def __init__(self, table_name, col_list): + self.name = table_name + self.col_list = col_list + + def get_create_sql(self, curs, new_name = None): + if not new_name: + new_name = self.name + sql = "create table %s (" % new_name + sep = "\n\t" + for c in self.col_list: + sql += sep + c.column_def + sep = ",\n\t" + sql += "\n);" + return sql + + def get_drop_sql(self, curs): + return "DROP TABLE %s;" % self.name + +# +# Main table object, loads all the others +# + +class TableStruct(object): + """Collects and manages all info about table. + + Allow to issue CREATE/DROP statements about any + group of elements. + """ + def __init__(self, curs, table_name): + """Initializes class by loading info about table_name from database.""" + + self.table_name = table_name + + # fill args + schema, name = fq_name_parts(table_name) + args = { + 'schema': schema, + 'table': name, + 'oid': get_table_oid(curs, table_name), + 'pg_class_oid': get_table_oid(curs, 'pg_catalog.pg_class'), + } + + # load table struct + self.col_list = self._load_elem(curs, args, TColumn) + self.object_list = [ TTable(table_name, self.col_list) ] + + # load additional objects + to_load = [TConstraint, TIndex, TTrigger, TRule, TGrant, TOwner] + for eclass in to_load: + self.object_list += self._load_elem(curs, args, eclass) + + def _load_elem(self, curs, args, eclass): + list = [] + curs.execute(eclass.SQL % args) + for row in curs.dictfetchall(): + list.append(eclass(self.table_name, row)) + return list + + def create(self, curs, objs, new_table_name = None, log = None): + """Issues CREATE statements for requested set of objects. + + If new_table_name is giver, creates table under that name + and also tries to rename all indexes/constraints that conflict + with existing table. + """ + + for o in self.object_list: + if o.type & objs: + sql = o.get_create_sql(curs, new_table_name) + if not sql: + continue + if log: + log.info('Creating %s' % o.name) + log.debug(sql) + curs.execute(sql) + + def drop(self, curs, objs, log = None): + """Issues DROP statements for requested set of objects.""" + for o in self.object_list: + if o.type & objs: + sql = o.get_drop_sql(curs) + if not sql: + continue + if log: + log.info('Dropping %s' % o.name) + log.debug(sql) + curs.execute(sql) + + def get_column_list(self): + """Returns list of column names the table has.""" + + res = [] + for c in self.col_list: + res.append(c.name) + return res + +def test(): + import psycopg + db = psycopg.connect("dbname=fooz") + curs = db.cursor() + + s = TableStruct(curs, "public.data1") + + s.drop(curs, T_ALL) + s.create(curs, T_ALL) + s.create(curs, T_ALL, "data1_new") + s.create(curs, T_PKEY) + +if __name__ == '__main__': + test() + diff --git a/python/skytools/gzlog.py b/python/skytools/gzlog.py new file mode 100644 index 00000000..558e2813 --- /dev/null +++ b/python/skytools/gzlog.py @@ -0,0 +1,39 @@ + +"""Atomic append of gzipped data. + +The point is - if several gzip streams are concated, they +are read back as one whose stream. +""" + +import gzip +from cStringIO import StringIO + +__all__ = ['gzip_append'] + +# +# gzip storage +# +def gzip_append(filename, data, level = 6): + """Append a block of data to file with safety checks.""" + + # compress data + buf = StringIO() + g = gzip.GzipFile(fileobj = buf, compresslevel = level, mode = "w") + g.write(data) + g.close() + zdata = buf.getvalue() + + # append, safely + f = open(filename, "a+", 0) + f.seek(0, 2) + pos = f.tell() + try: + f.write(zdata) + f.close() + except Exception, ex: + # rollback on error + f.seek(pos, 0) + f.truncate() + f.close() + raise ex + diff --git a/python/skytools/quoting.py b/python/skytools/quoting.py new file mode 100644 index 00000000..96b0b022 --- /dev/null +++ b/python/skytools/quoting.py @@ -0,0 +1,156 @@ +# quoting.py + +"""Various helpers for string quoting/unquoting.""" + +import psycopg, urllib, re + +# +# SQL quoting +# + +def quote_literal(s): + """Quote a literal value for SQL. + + Surronds it with single-quotes. + """ + + if s == None: + return "null" + s = psycopg.QuotedString(str(s)) + return str(s) + +def quote_copy(s): + """Quoting for copy command.""" + + if s == None: + return "\\N" + s = str(s) + s = s.replace("\\", "\\\\") + s = s.replace("\t", "\\t") + s = s.replace("\n", "\\n") + s = s.replace("\r", "\\r") + return s + +def quote_bytea_raw(s): + """Quoting for bytea parser.""" + + if s == None: + return None + return s.replace("\\", "\\\\").replace("\0", "\\000") + +def quote_bytea_literal(s): + """Quote bytea for regular SQL.""" + + return quote_literal(quote_bytea_raw(s)) + +def quote_bytea_copy(s): + """Quote bytea for COPY.""" + + return quote_copy(quote_bytea_raw(s)) + +def quote_statement(sql, dict): + """Quote whose statement. + + Data values are taken from dict. + """ + xdict = {} + for k, v in dict.items(): + xdict[k] = quote_literal(v) + return sql % xdict + +# +# quoting for JSON strings +# + +_jsre = re.compile(r'[\x00-\x1F\\/"]') +_jsmap = { "\b": "\\b", "\f": "\\f", "\n": "\\n", "\r": "\\r", + "\t": "\\t", "\\": "\\\\", '"': '\\"', + "/": "\\/", # to avoid html attacks +} + +def _json_quote_char(m): + c = m.group(0) + try: + return _jsmap[c] + except KeyError: + return r"\u%04x" % ord(c) + +def quote_json(s): + """JSON style quoting.""" + if s is None: + return "null" + return '"%s"' % _jsre.sub(_json_quote_char, s) + +# +# Database specific urlencode and urldecode. +# + +def db_urlencode(dict): + """Database specific urlencode. + + Encode None as key without '='. That means that in "foo&bar=", + foo is NULL and bar is empty string. + """ + + elem_list = [] + for k, v in dict.items(): + if v is None: + elem = urllib.quote_plus(str(k)) + else: + elem = urllib.quote_plus(str(k)) + '=' + urllib.quote_plus(str(v)) + elem_list.append(elem) + return '&'.join(elem_list) + +def db_urldecode(qs): + """Database specific urldecode. + + Decode key without '=' as None. + This also does not support one key several times. + """ + + res = {} + for elem in qs.split('&'): + if not elem: + continue + pair = elem.split('=', 1) + name = urllib.unquote_plus(pair[0]) + if len(pair) == 1: + res[name] = None + else: + res[name] = urllib.unquote_plus(pair[1]) + return res + +# +# Remove C-like backslash escapes +# + +_esc_re = r"\\([0-7][0-7][0-7]|.)" +_esc_rc = re.compile(_esc_re) +_esc_map = { + 't': '\t', + 'n': '\n', + 'r': '\r', + 'a': '\a', + 'b': '\b', + "'": "'", + '"': '"', + '\\': '\\', +} + +def _sub_unescape(m): + v = m.group(1) + if len(v) == 1: + return _esc_map[v] + else: + return chr(int(v, 8)) + +def unescape(val): + """Removes C-style escapes from string.""" + return _esc_rc.sub(_sub_unescape, val) + +def unescape_copy(val): + """Removes C-style escapes, also converts "\N" to None.""" + if val == r"\N": + return None + return unescape(val) + diff --git a/python/skytools/scripting.py b/python/skytools/scripting.py new file mode 100644 index 00000000..cf976801 --- /dev/null +++ b/python/skytools/scripting.py @@ -0,0 +1,523 @@ + +"""Useful functions and classes for database scripts.""" + +import sys, os, signal, psycopg, optparse, traceback, time +import logging, logging.handlers, logging.config + +from skytools.config import * +import skytools.skylog + +__all__ = ['daemonize', 'run_single_process', 'DBScript', + 'I_AUTOCOMMIT', 'I_READ_COMMITTED', 'I_SERIALIZABLE'] + +# +# daemon mode +# + +def daemonize(): + """Turn the process into daemon. + + Goes background and disables all i/o. + """ + + # launch new process, kill parent + pid = os.fork() + if pid != 0: + os._exit(0) + + # start new session + os.setsid() + + # stop i/o + fd = os.open("/dev/null", os.O_RDWR) + os.dup2(fd, 0) + os.dup2(fd, 1) + os.dup2(fd, 2) + if fd > 2: + os.close(fd) + +# +# Pidfile locking+cleanup & daemonization combined +# + +def _write_pidfile(pidfile): + pid = os.getpid() + f = open(pidfile, 'w') + f.write(str(pid)) + f.close() + +def run_single_process(runnable, daemon, pidfile): + """Run runnable class, possibly daemonized, locked on pidfile.""" + + # check if another process is running + if pidfile and os.path.isfile(pidfile): + print "Pidfile exists, another process running?" + sys.exit(1) + + # daemonize if needed and write pidfile + if daemon: + daemonize() + if pidfile: + _write_pidfile(pidfile) + + # Catch SIGTERM to cleanup pidfile + def sigterm_hook(signum, frame): + try: + os.remove(pidfile) + except: pass + sys.exit(0) + # attach it to signal + if pidfile: + signal.signal(signal.SIGTERM, sigterm_hook) + + # run + try: + runnable.run() + finally: + # another try of cleaning up + if pidfile: + try: + os.remove(pidfile) + except: pass + +# +# logging setup +# + +_log_config_done = 0 +_log_init_done = {} + +def _init_log(job_name, cf, log_level): + """Logging setup happens here.""" + global _log_init_done, _log_config_done + + got_skylog = 0 + use_skylog = cf.getint("use_skylog", 0) + + # load logging config if needed + if use_skylog and not _log_config_done: + # python logging.config braindamage: + # cannot specify external classess without such hack + logging.skylog = skytools.skylog + + # load general config + list = ['skylog.ini', '~/.skylog.ini', '/etc/skylog.ini'] + for fn in list: + fn = os.path.expanduser(fn) + if os.path.isfile(fn): + defs = {'job_name': job_name} + logging.config.fileConfig(fn, defs) + got_skylog = 1 + break + _log_config_done = 1 + if not got_skylog: + sys.stderr.write("skylog.ini not found!\n") + sys.exit(1) + + # avoid duplicate logging init for job_name + log = logging.getLogger(job_name) + if job_name in _log_init_done: + return log + _log_init_done[job_name] = 1 + + # compatibility: specify ini file in script config + logfile = cf.getfile("logfile", "") + if logfile: + fmt = logging.Formatter('%(asctime)s %(process)s %(levelname)s %(message)s') + size = cf.getint('log_size', 10*1024*1024) + num = cf.getint('log_count', 3) + hdlr = logging.handlers.RotatingFileHandler( + logfile, 'a', size, num) + hdlr.setFormatter(fmt) + log.addHandler(hdlr) + + # if skylog.ini is disabled or not available, log at least to stderr + if not got_skylog: + hdlr = logging.StreamHandler() + fmt = logging.Formatter('%(asctime)s %(process)s %(levelname)s %(message)s') + hdlr.setFormatter(fmt) + log.addHandler(hdlr) + + log.setLevel(log_level) + + return log + +#: how old connections need to be closed +DEF_CONN_AGE = 20*60 # 20 min + +#: isolation level not set +I_DEFAULT = -1 + +#: isolation level constant for AUTOCOMMIT +I_AUTOCOMMIT = 0 +#: isolation level constant for READ COMMITTED +I_READ_COMMITTED = 1 +#: isolation level constant for SERIALIZABLE +I_SERIALIZABLE = 2 + +class DBCachedConn(object): + """Cache a db connection.""" + def __init__(self, name, loc, max_age = DEF_CONN_AGE): + self.name = name + self.loc = loc + self.conn = None + self.conn_time = 0 + self.max_age = max_age + self.autocommit = -1 + self.isolation_level = -1 + + def get_connection(self, autocommit = 0, isolation_level = -1): + # autocommit overrider isolation_level + if autocommit: + isolation_level = I_AUTOCOMMIT + + # default isolation_level is READ COMMITTED + if isolation_level < 0: + isolation_level = I_READ_COMMITTED + + # new conn? + if not self.conn: + self.isolation_level = isolation_level + self.conn = psycopg.connect(self.loc) + + self.conn.set_isolation_level(isolation_level) + self.conn_time = time.time() + else: + if self.isolation_level != isolation_level: + raise Exception("Conflict in isolation_level") + + # done + return self.conn + + def refresh(self): + if not self.conn: + return + #for row in self.conn.notifies(): + # if row[0].lower() == "reload": + # self.reset() + # return + if not self.max_age: + return + if time.time() - self.conn_time >= self.max_age: + self.reset() + + def reset(self): + if not self.conn: + return + + # drop reference + conn = self.conn + self.conn = None + + if self.isolation_level == I_AUTOCOMMIT: + return + + # rollback & close + try: + conn.rollback() + except: pass + try: + conn.close() + except: pass + +class DBScript(object): + """Base class for database scripts. + + Handles logging, daemonizing, config, errors. + """ + service_name = None + job_name = None + cf = None + log = None + + def __init__(self, service_name, args): + """Script setup. + + User class should override work() and optionally __init__(), startup(), + reload(), reset() and init_optparse(). + + NB: in case of daemon, the __init__() and startup()/work() will be + run in different processes. So nothing fancy should be done in __init__(). + + @param service_name: unique name for script. + It will be also default job_name, if not specified in config. + @param args: cmdline args (sys.argv[1:]), but can be overrided + """ + self.service_name = service_name + self.db_cache = {} + self.go_daemon = 0 + self.do_single_loop = 0 + self.looping = 1 + self.need_reload = 1 + self.stat_dict = {} + self.log_level = logging.INFO + self.work_state = 1 + + # parse command line + parser = self.init_optparse() + self.options, self.args = parser.parse_args(args) + + # check args + if self.options.daemon: + self.go_daemon = 1 + if self.options.quiet: + self.log_level = logging.WARNING + if self.options.verbose: + self.log_level = logging.DEBUG + if len(self.args) < 1: + print "need config file" + sys.exit(1) + conf_file = self.args[0] + + # load config + self.cf = Config(self.service_name, conf_file) + self.job_name = self.cf.get("job_name", self.service_name) + self.pidfile = self.cf.getfile("pidfile", '') + + self.reload() + + # init logging + self.log = _init_log(self.job_name, self.cf, self.log_level) + + # send signal, if needed + if self.options.cmd == "kill": + self.send_signal(signal.SIGTERM) + elif self.options.cmd == "stop": + self.send_signal(signal.SIGINT) + elif self.options.cmd == "reload": + self.send_signal(signal.SIGHUP) + + def init_optparse(self, parser = None): + """Initialize a OptionParser() instance that will be used to + parse command line arguments. + + Note that it can be overrided both directions - either DBScript + will initialize a instance and passes to user code or user can + initialize and then pass to DBScript.init_optparse(). + + @param parser: optional OptionParser() instance, + where DBScript should attachs its own arguments. + @return: initialized OptionParser() instance. + """ + if parser: + p = parser + else: + p = optparse.OptionParser() + p.set_usage("%prog [options] INI") + # generic options + p.add_option("-q", "--quiet", action="store_true", + help = "make program silent") + p.add_option("-v", "--verbose", action="store_true", + help = "make program verbose") + p.add_option("-d", "--daemon", action="store_true", + help = "go background") + + # control options + g = optparse.OptionGroup(p, 'control running process') + g.add_option("-r", "--reload", + action="store_const", const="reload", dest="cmd", + help = "reload config (send SIGHUP)") + g.add_option("-s", "--stop", + action="store_const", const="stop", dest="cmd", + help = "stop program safely (send SIGINT)") + g.add_option("-k", "--kill", + action="store_const", const="kill", dest="cmd", + help = "kill program immidiately (send SIGTERM)") + p.add_option_group(g) + + return p + + def send_signal(self, sig): + if not self.pidfile: + self.log.warning("No pidfile in config, nothing todo") + sys.exit(0) + if not os.path.isfile(self.pidfile): + self.log.warning("No pidfile, process not running") + sys.exit(0) + pid = int(open(self.pidfile, "r").read()) + os.kill(pid, sig) + sys.exit(0) + + def set_single_loop(self, do_single_loop): + """Changes whether the script will loop or not.""" + self.do_single_loop = do_single_loop + + def start(self): + """This will launch main processing thread.""" + if self.go_daemon: + if not self.pidfile: + self.log.error("Daemon needs pidfile") + sys.exit(1) + run_single_process(self, self.go_daemon, self.pidfile) + + def stop(self): + """Safely stops processing loop.""" + self.looping = 0 + + def reload(self): + "Reload config." + self.cf.reload() + self.loop_delay = self.cf.getfloat("loop_delay", 1.0) + + def hook_sighup(self, sig, frame): + "Internal SIGHUP handler. Minimal code here." + self.need_reload = 1 + + def hook_sigint(self, sig, frame): + "Internal SIGINT handler. Minimal code here." + self.stop() + + def stat_add(self, key, value): + self.stat_put(key, value) + + def stat_put(self, key, value): + """Sets a stat value.""" + self.stat_dict[key] = value + + def stat_increase(self, key, increase = 1): + """Increases a stat value.""" + if key in self.stat_dict: + self.stat_dict[key] += increase + else: + self.stat_dict[key] = increase + + def send_stats(self): + "Send statistics to log." + + res = [] + for k, v in self.stat_dict.items(): + res.append("%s: %s" % (k, str(v))) + + if len(res) == 0: + return + + logmsg = "{%s}" % ", ".join(res) + self.log.info(logmsg) + self.stat_dict = {} + + def get_database(self, dbname, autocommit = 0, isolation_level = -1, + cache = None, max_age = DEF_CONN_AGE): + """Load cached database connection. + + User must not store it permanently somewhere, + as all connections will be invalidated on reset. + """ + + if not cache: + cache = dbname + if cache in self.db_cache: + dbc = self.db_cache[cache] + else: + loc = self.cf.get(dbname) + dbc = DBCachedConn(cache, loc, max_age) + self.db_cache[cache] = dbc + + return dbc.get_connection(autocommit, isolation_level) + + def close_database(self, dbname): + """Explicitly close a cached connection. + + Next call to get_database() will reconnect. + """ + if dbname in self.db_cache: + dbc = self.db_cache[dbname] + dbc.reset() + + def reset(self): + "Something bad happened, reset all connections." + for dbc in self.db_cache.values(): + dbc.reset() + self.db_cache = {} + + def run(self): + "Thread main loop." + + # run startup, safely + try: + self.startup() + except KeyboardInterrupt, det: + raise + except SystemExit, det: + raise + except Exception, det: + exc, msg, tb = sys.exc_info() + self.log.fatal("Job %s crashed: %s: '%s' (%s: %s)" % ( + self.job_name, str(exc), str(msg).rstrip(), + str(tb), repr(traceback.format_tb(tb)))) + del tb + self.reset() + sys.exit(1) + + while self.looping: + # reload config, if needed + if self.need_reload: + self.reload() + self.need_reload = 0 + + # do some work + work = self.run_once() + + # send stats that was added + self.send_stats() + + # reconnect if needed + for dbc in self.db_cache.values(): + dbc.refresh() + + # exit if needed + if self.do_single_loop: + self.log.debug("Only single loop requested, exiting") + break + + # remember work state + self.work_state = work + # should sleep? + if not work: + try: + time.sleep(self.loop_delay) + except Exception, d: + self.log.debug("sleep failed: "+str(d)) + sys.exit(0) + + def run_once(self): + "Run users work function, safely." + try: + return self.work() + except SystemExit, d: + self.send_stats() + self.log.info("got SystemExit(%s), exiting" % str(d)) + self.reset() + raise d + except KeyboardInterrupt, d: + self.send_stats() + self.log.info("got KeyboardInterrupt, exiting") + self.reset() + sys.exit(1) + except Exception, d: + self.send_stats() + exc, msg, tb = sys.exc_info() + self.log.fatal("Job %s crashed: %s: '%s' (%s: %s)" % ( + self.job_name, str(exc), str(msg).rstrip(), + str(tb), repr(traceback.format_tb(tb)))) + del tb + self.reset() + if self.looping: + time.sleep(20) + return 1 + + def work(self): + "Here should user's processing happen." + raise Exception("Nothing implemented?") + + def startup(self): + """Will be called just before entering main loop. + + In case of daemon, if will be called in same process as work(), + unlike __init__(). + """ + + # set signals + signal.signal(signal.SIGHUP, self.hook_sighup) + signal.signal(signal.SIGINT, self.hook_sigint) + + diff --git a/python/skytools/skylog.py b/python/skytools/skylog.py new file mode 100644 index 00000000..2f6344ae --- /dev/null +++ b/python/skytools/skylog.py @@ -0,0 +1,173 @@ +"""Our log handlers for Python's logging package. +""" + +import sys, os, time, socket, psycopg +import logging, logging.handlers + +from quoting import quote_json + +# configurable file logger +class EasyRotatingFileHandler(logging.handlers.RotatingFileHandler): + """Easier setup for RotatingFileHandler.""" + def __init__(self, filename, maxBytes = 10*1024*1024, backupCount = 3): + """Args same as for RotatingFileHandler, but in filename '~' is expanded.""" + fn = os.path.expanduser(filename) + logging.handlers.RotatingFileHandler.__init__(self, fn, maxBytes=maxBytes, backupCount=backupCount) + +# send JSON message over UDP +class UdpLogServerHandler(logging.handlers.DatagramHandler): + """Sends log records over UDP to logserver in JSON format.""" + + # map logging levels to logserver levels + _level_map = { + logging.DEBUG : 'DEBUG', + logging.INFO : 'INFO', + logging.WARNING : 'WARN', + logging.ERROR : 'ERROR', + logging.CRITICAL: 'FATAL', + } + + # JSON message template + _log_template = '{\n\t'\ + '"logger": "skytools.UdpLogServer",\n\t'\ + '"timestamp": %.0f,\n\t'\ + '"level": "%s",\n\t'\ + '"thread": null,\n\t'\ + '"message": %s,\n\t'\ + '"properties": {"application":"%s", "hostname":"%s"}\n'\ + '}' + + # cut longer msgs + MAXMSG = 1024 + + def makePickle(self, record): + """Create message in JSON format.""" + # get & cut msg + msg = self.format(record) + if len(msg) > self.MAXMSG: + msg = msg[:self.MAXMSG] + txt_level = self._level_map.get(record.levelno, "ERROR") + pkt = self._log_template % (time.time()*1000, txt_level, + quote_json(msg), record.name, socket.gethostname()) + return pkt + +class LogDBHandler(logging.handlers.SocketHandler): + """Sends log records into PostgreSQL server. + + Additionally, does some statistics aggregating, + to avoid overloading log server. + + It subclasses SocketHandler to get throtthling for + failed connections. + """ + + # map codes to string + _level_map = { + logging.DEBUG : 'DEBUG', + logging.INFO : 'INFO', + logging.WARNING : 'WARNING', + logging.ERROR : 'ERROR', + logging.CRITICAL: 'FATAL', + } + + def __init__(self, connect_string): + """ + Initializes the handler with a specific connection string. + """ + + logging.handlers.SocketHandler.__init__(self, None, None) + self.closeOnError = 1 + + self.connect_string = connect_string + + self.stat_cache = {} + self.stat_flush_period = 60 + # send first stat line immidiately + self.last_stat_flush = 0 + + def createSocket(self): + try: + logging.handlers.SocketHandler.createSocket(self) + except: + self.sock = self.makeSocket() + + def makeSocket(self): + """Create server connection. + In this case its not socket but psycopg conection.""" + + db = psycopg.connect(self.connect_string) + db.autocommit(1) + return db + + def emit(self, record): + """Process log record.""" + + # we do not want log debug messages + if record.levelno < logging.INFO: + return + + try: + self.process_rec(record) + except (SystemExit, KeyboardInterrupt): + raise + except: + self.handleError(record) + + def process_rec(self, record): + """Aggregate stats if needed, and send to logdb.""" + # render msg + msg = self.format(record) + + # dont want to send stats too ofter + if record.levelno == logging.INFO and msg and msg[0] == "{": + self.aggregate_stats(msg) + if time.time() - self.last_stat_flush >= self.stat_flush_period: + self.flush_stats(record.name) + return + + if record.levelno < logging.INFO: + self.flush_stats(record.name) + + # dont send more than one line + ln = msg.find('\n') + if ln > 0: + msg = msg[:ln] + + txt_level = self._level_map.get(record.levelno, "ERROR") + self.send_to_logdb(record.name, txt_level, msg) + + def aggregate_stats(self, msg): + """Sum stats together, to lessen load on logdb.""" + + msg = msg[1:-1] + for rec in msg.split(", "): + k, v = rec.split(": ") + agg = self.stat_cache.get(k, 0) + if v.find('.') >= 0: + agg += float(v) + else: + agg += int(v) + self.stat_cache[k] = agg + + def flush_stats(self, service): + """Send awuired stats to logdb.""" + res = [] + for k, v in self.stat_cache.items(): + res.append("%s: %s" % (k, str(v))) + if len(res) > 0: + logmsg = "{%s}" % ", ".join(res) + self.send_to_logdb(service, "INFO", logmsg) + self.stat_cache = {} + self.last_stat_flush = time.time() + + def send_to_logdb(self, service, type, msg): + """Actual sending is done here.""" + + if self.sock is None: + self.createSocket() + + if self.sock: + logcur = self.sock.cursor() + query = "select * from log.add(%s, %s, %s)" + logcur.execute(query, [type, service, msg]) + diff --git a/python/skytools/sqltools.py b/python/skytools/sqltools.py new file mode 100644 index 00000000..75e209f1 --- /dev/null +++ b/python/skytools/sqltools.py @@ -0,0 +1,398 @@ + +"""Database tools.""" + +import os +from cStringIO import StringIO +from quoting import quote_copy, quote_literal + +# +# Fully qualified table name +# + +def fq_name_parts(tbl): + "Return fully qualified name parts." + + tmp = tbl.split('.') + if len(tmp) == 1: + return ('public', tbl) + elif len(tmp) == 2: + return tmp + else: + raise Exception('Syntax error in table name:'+tbl) + +def fq_name(tbl): + "Return fully qualified name." + return '.'.join(fq_name_parts(tbl)) + +# +# info about table +# +def get_table_oid(curs, table_name): + schema, name = fq_name_parts(table_name) + q = """select c.oid from pg_namespace n, pg_class c + where c.relnamespace = n.oid + and n.nspname = %s and c.relname = %s""" + curs.execute(q, [schema, name]) + res = curs.fetchall() + if len(res) == 0: + raise Exception('Table not found: '+table_name) + return res[0][0] + +def get_table_pkeys(curs, tbl): + oid = get_table_oid(curs, tbl) + q = "SELECT k.attname FROM pg_index i, pg_attribute k"\ + " WHERE i.indrelid = %s AND k.attrelid = i.indexrelid"\ + " AND i.indisprimary AND k.attnum > 0 AND NOT k.attisdropped"\ + " ORDER BY k.attnum" + curs.execute(q, [oid]) + return map(lambda x: x[0], curs.fetchall()) + +def get_table_columns(curs, tbl): + oid = get_table_oid(curs, tbl) + q = "SELECT k.attname FROM pg_attribute k"\ + " WHERE k.attrelid = %s"\ + " AND k.attnum > 0 AND NOT k.attisdropped"\ + " ORDER BY k.attnum" + curs.execute(q, [oid]) + return map(lambda x: x[0], curs.fetchall()) + +# +# exist checks +# +def exists_schema(curs, schema): + q = "select count(1) from pg_namespace where nspname = %s" + curs.execute(q, [schema]) + res = curs.fetchone() + return res[0] + +def exists_table(curs, table_name): + schema, name = fq_name_parts(table_name) + q = """select count(1) from pg_namespace n, pg_class c + where c.relnamespace = n.oid and c.relkind = 'r' + and n.nspname = %s and c.relname = %s""" + curs.execute(q, [schema, name]) + res = curs.fetchone() + return res[0] + +def exists_type(curs, type_name): + schema, name = fq_name_parts(type_name) + q = """select count(1) from pg_namespace n, pg_type t + where t.typnamespace = n.oid + and n.nspname = %s and t.typname = %s""" + curs.execute(q, [schema, name]) + res = curs.fetchone() + return res[0] + +def exists_function(curs, function_name, nargs): + # this does not check arg types, so may match several functions + schema, name = fq_name_parts(function_name) + q = """select count(1) from pg_namespace n, pg_proc p + where p.pronamespace = n.oid and p.pronargs = %s + and n.nspname = %s and p.proname = %s""" + curs.execute(q, [nargs, schema, name]) + res = curs.fetchone() + return res[0] + +def exists_language(curs, lang_name): + q = """select count(1) from pg_language + where lanname = %s""" + curs.execute(q, [lang_name]) + res = curs.fetchone() + return res[0] + +# +# Support for PostgreSQL snapshot +# + +class Snapshot(object): + "Represents a PostgreSQL snapshot." + + def __init__(self, str): + "Create snapshot from string." + + self.sn_str = str + tmp = str.split(':') + if len(tmp) != 3: + raise Exception('Unknown format for snapshot') + self.xmin = int(tmp[0]) + self.xmax = int(tmp[1]) + self.txid_list = [] + if tmp[2] != "": + for s in tmp[2].split(','): + self.txid_list.append(int(s)) + + def contains(self, txid): + "Is txid visible in snapshot." + + txid = int(txid) + + if txid < self.xmin: + return True + if txid >= self.xmax: + return False + if txid in self.txid_list: + return False + return True + +# +# Copy helpers +# + +def _gen_dict_copy(tbl, row, fields): + tmp = [] + for f in fields: + v = row[f] + tmp.append(quote_copy(v)) + return "\t".join(tmp) + +def _gen_dict_insert(tbl, row, fields): + tmp = [] + for f in fields: + v = row[f] + tmp.append(quote_literal(v)) + fmt = "insert into %s (%s) values (%s);" + return fmt % (tbl, ",".join(fields), ",".join(tmp)) + +def _gen_list_copy(tbl, row, fields): + tmp = [] + for i in range(len(fields)): + v = row[i] + tmp.append(quote_copy(v)) + return "\t".join(tmp) + +def _gen_list_insert(tbl, row, fields): + tmp = [] + for i in range(len(fields)): + v = row[i] + tmp.append(quote_literal(v)) + fmt = "insert into %s (%s) values (%s);" + return fmt % (tbl, ",".join(fields), ",".join(tmp)) + +def magic_insert(curs, tablename, data, fields = None, use_insert = 0): + """Copy/insert a list of dict/list data to database. + + If curs == None, then the copy or insert statements are returned + as string. For list of dict the field list is optional, as its + possible to guess them from dict keys. + """ + if len(data) == 0: + return + + # decide how to process + if type(data[0]) == type({}): + if fields == None: + fields = data[0].keys() + if use_insert: + row_func = _gen_dict_insert + else: + row_func = _gen_dict_copy + else: + if fields == None: + raise Exception("Non-dict data needs field list") + if use_insert: + row_func = _gen_list_insert + else: + row_func = _gen_list_copy + + # init processing + buf = StringIO() + if curs == None and use_insert == 0: + fmt = "COPY %s (%s) FROM STDIN;\n" + buf.write(fmt % (tablename, ",".join(fields))) + + # process data + for row in data: + buf.write(row_func(tablename, row, fields)) + buf.write("\n") + + # if user needs only string, return it + if curs == None: + if use_insert == 0: + buf.write("\\.\n") + return buf.getvalue() + + # do the actual copy/inserts + if use_insert: + curs.execute(buf.getvalue()) + else: + buf.seek(0) + hdr = "%s (%s)" % (tablename, ",".join(fields)) + curs.copy_from(buf, hdr) + +def db_copy_from_dict(curs, tablename, dict_list, fields = None): + """Do a COPY FROM STDIN using list of dicts as source.""" + + if len(dict_list) == 0: + return + + if fields == None: + fields = dict_list[0].keys() + + buf = StringIO() + for dat in dict_list: + row = [] + for k in fields: + row.append(quote_copy(dat[k])) + buf.write("\t".join(row)) + buf.write("\n") + + buf.seek(0) + hdr = "%s (%s)" % (tablename, ",".join(fields)) + + curs.copy_from(buf, hdr) + +def db_copy_from_list(curs, tablename, row_list, fields): + """Do a COPY FROM STDIN using list of lists as source.""" + + if len(row_list) == 0: + return + + if fields == None or len(fields) == 0: + raise Exception('Need field list') + + buf = StringIO() + for dat in row_list: + row = [] + for i in range(len(fields)): + row.append(quote_copy(dat[i])) + buf.write("\t".join(row)) + buf.write("\n") + + buf.seek(0) + hdr = "%s (%s)" % (tablename, ",".join(fields)) + + curs.copy_from(buf, hdr) + +# +# Full COPY of table from one db to another +# + +class CopyPipe(object): + "Splits one big COPY to chunks." + + def __init__(self, dstcurs, tablename, limit = 512*1024, cancel_func=None): + self.tablename = tablename + self.dstcurs = dstcurs + self.buf = StringIO() + self.limit = limit + self.cancel_func = None + self.total_rows = 0 + self.total_bytes = 0 + + def write(self, data): + "New data from psycopg" + + self.total_bytes += len(data) + self.total_rows += data.count("\n") + + if self.buf.tell() >= self.limit: + pos = data.find('\n') + if pos >= 0: + # split at newline + p1 = data[:pos + 1] + p2 = data[pos + 1:] + self.buf.write(p1) + self.flush() + + data = p2 + + self.buf.write(data) + + def flush(self): + "Send data out." + + if self.cancel_func: + self.cancel_func() + + if self.buf.tell() > 0: + self.buf.seek(0) + self.dstcurs.copy_from(self.buf, self.tablename) + self.buf.seek(0) + self.buf.truncate() + +def full_copy(tablename, src_curs, dst_curs, column_list = []): + """COPY table from one db to another.""" + + if column_list: + hdr = "%s (%s)" % (tablename, ",".join(column_list)) + else: + hdr = tablename + buf = CopyPipe(dst_curs, hdr) + src_curs.copy_to(buf, hdr) + buf.flush() + + return (buf.total_bytes, buf.total_rows) + + +# +# SQL installer +# + +class DBObject(object): + """Base class for installable DB objects.""" + name = None + sql = None + sql_file = None + def __init__(self, name, sql = None, sql_file = None): + self.name = name + self.sql = sql + self.sql_file = sql_file + def get_sql(self): + if self.sql: + return self.sql + if self.sql_file: + if self.sql_file[0] == "/": + fn = self.sql_file + else: + contrib_list = [ + "/opt/pgsql/share/contrib", + "/usr/share/postgresql/8.0/contrib", + "/usr/share/postgresql/8.0/contrib", + "/usr/share/postgresql/8.1/contrib", + "/usr/share/postgresql/8.2/contrib", + ] + for dir in contrib_list: + fn = os.path.join(dir, self.sql_file) + if os.path.isfile(fn): + return open(fn, "r").read() + raise Exception('File not found: '+self.sql_file) + raise Exception('object not defined') + def create(self, curs): + curs.execute(self.get_sql()) + +class DBSchema(DBObject): + """Handles db schema.""" + def exists(self, curs): + return exists_schema(curs, self.name) + +class DBTable(DBObject): + """Handles db table.""" + def exists(self, curs): + return exists_table(curs, self.name) + +class DBFunction(DBObject): + """Handles db function.""" + def __init__(self, name, nargs, sql = None, sql_file = None): + DBObject.__init__(self, name, sql, sql_file) + self.nargs = nargs + def exists(self, curs): + return exists_function(curs, self.name, self.nargs) + +class DBLanguage(DBObject): + """Handles db language.""" + def __init__(self, name): + DBObject.__init__(self, name, sql = "create language %s" % name) + def exists(self, curs): + return exists_language(curs, self.name) + +def db_install(curs, list, log = None): + """Installs list of objects into db.""" + for obj in list: + if not obj.exists(curs): + if log: + log.info('Installing %s' % obj.name) + obj.create(curs) + else: + if log: + log.info('%s is installed' % obj.name) + diff --git a/python/walmgr.py b/python/walmgr.py new file mode 100755 index 00000000..8f43fd6d --- /dev/null +++ b/python/walmgr.py @@ -0,0 +1,648 @@ +#! /usr/bin/env python + +"""WALShipping manager. + +walmgr [-n] COMMAND + +Master commands: + setup Configure PostgreSQL for WAL archiving + backup Copies all master data to slave + sync Copies in-progress WALs to slave + syncdaemon Daemon mode for regular syncing + stop Stop archiving - de-configure PostgreSQL + +Slave commands: + restore Stop postmaster, move new data dir to right + location and start postmaster in playback mode. + boot Stop playback, accept queries. + pause Just wait, don't play WAL-s + continue Start playing WAL-s again + +Internal commands: + xarchive archive one WAL file (master) + xrestore restore one WAL file (slave) + +Switches: + -n no action, just print commands +""" + +import os, sys, skytools, getopt, re, signal, time, traceback + +MASTER = 1 +SLAVE = 0 + +def usage(err): + if err > 0: + print >>sys.stderr, __doc__ + else: + print __doc__ + sys.exit(err) + +class WalMgr(skytools.DBScript): + def __init__(self, wtype, cf_file, not_really, internal = 0, go_daemon = 0): + self.not_really = not_really + self.pg_backup = 0 + + if wtype == MASTER: + service_name = "wal-master" + else: + service_name = "wal-slave" + + if not os.path.isfile(cf_file): + print "Config not found:", cf_file + sys.exit(1) + + if go_daemon: + s_args = ["-d", cf_file] + else: + s_args = [cf_file] + + skytools.DBScript.__init__(self, service_name, s_args, + force_logfile = internal) + + def pg_start_backup(self, code): + q = "select pg_start_backup('FullBackup')" + self.log.info("Execute SQL: %s; [%s]" % (q, self.cf.get("master_db"))) + if self.not_really: + self.pg_backup = 1 + return + db = self.get_database("master_db") + db.cursor().execute(q) + db.commit() + self.close_database("master_db") + self.pg_backup = 1 + + def pg_stop_backup(self): + if not self.pg_backup: + return + + q = "select pg_stop_backup()" + self.log.debug("Execute SQL: %s; [%s]" % (q, self.cf.get("master_db"))) + if self.not_really: + return + db = self.get_database("master_db") + db.cursor().execute(q) + db.commit() + self.close_database("master_db") + + def signal_postmaster(self, data_dir, sgn): + pidfile = os.path.join(data_dir, "postmaster.pid") + if not os.path.isfile(pidfile): + self.log.info("postmaster is not running") + return + buf = open(pidfile, "r").readline() + pid = int(buf.strip()) + self.log.debug("Signal %d to process %d" % (sgn, pid)) + if not self.not_really: + os.kill(pid, sgn) + + def exec_big_rsync(self, cmdline): + cmd = "' '".join(cmdline) + self.log.debug("Execute big rsync cmd: '%s'" % (cmd)) + if self.not_really: + return + res = os.spawnvp(os.P_WAIT, cmdline[0], cmdline) + if res == 24: + self.log.info("Some files vanished, but thats OK") + elif res != 0: + self.log.fatal("exec failed, res=%d" % res) + self.pg_stop_backup() + sys.exit(1) + + def exec_cmd(self, cmdline): + cmd = "' '".join(cmdline) + self.log.debug("Execute cmd: '%s'" % (cmd)) + if self.not_really: + return + res = os.spawnvp(os.P_WAIT, cmdline[0], cmdline) + if res != 0: + self.log.fatal("exec failed, res=%d" % res) + sys.exit(1) + + def chdir(self, loc): + self.log.debug("chdir: '%s'" % (loc)) + if self.not_really: + return + try: + os.chdir(loc) + except os.error: + self.log.fatal("CHDir failed") + self.pg_stop_backup() + sys.exit(1) + + def get_last_complete(self): + """Get the name of last xarchived segment.""" + + data_dir = self.cf.get("master_data") + fn = os.path.join(data_dir, ".walshipping.last") + try: + last = open(fn, "r").read().strip() + return last + except: + self.log.info("Failed to read %s" % fn) + return None + + def set_last_complete(self, last): + """Set the name of last xarchived segment.""" + + data_dir = self.cf.get("master_data") + fn = os.path.join(data_dir, ".walshipping.last") + fn_tmp = fn + ".new" + try: + f = open(fn_tmp, "w") + f.write(last) + f.close() + os.rename(fn_tmp, fn) + except: + self.log.fatal("Cannot write to %s" % fn) + + def master_setup(self): + self.log.info("Configuring WAL archiving") + + script = os.path.abspath(sys.argv[0]) + cf_file = os.path.abspath(self.cf.filename) + cf_val = "%s %s %s" % (script, cf_file, "xarchive %p %f") + + self.master_configure_archiving(cf_val) + + def master_stop(self): + self.log.info("Disabling WAL archiving") + + self.master_configure_archiving('') + + def master_configure_archiving(self, cf_val): + cf_file = self.cf.get("master_config") + data_dir = self.cf.get("master_data") + r_active = re.compile("^[ ]*archive_command[ ]*=[ ]*'(.*)'.*$", re.M) + r_disabled = re.compile("^.*archive_command.*$", re.M) + + cf_full = "archive_command = '%s'" % cf_val + + if not os.path.isfile(cf_file): + self.log.fatal("Config file not found: %s" % cf_file) + self.log.info("Using config file: %s", cf_file) + + buf = open(cf_file, "r").read() + m = r_active.search(buf) + if m: + old_val = m.group(1) + if old_val == cf_val: + self.log.debug("postmaster already configured") + else: + self.log.debug("found active but different conf") + newbuf = "%s%s%s" % (buf[:m.start()], cf_full, buf[m.end():]) + self.change_config(cf_file, newbuf) + else: + m = r_disabled.search(buf) + if m: + self.log.debug("found disabled value") + newbuf = "%s\n%s%s" % (buf[:m.end()], cf_full, buf[m.end():]) + self.change_config(cf_file, newbuf) + else: + self.log.debug("found no value") + newbuf = "%s\n%s\n\n" % (buf, cf_full) + self.change_config(cf_file, newbuf) + + self.log.info("Sending SIGHUP to postmaster") + self.signal_postmaster(data_dir, signal.SIGHUP) + self.log.info("Done") + + def change_config(self, cf_file, buf): + cf_old = cf_file + ".old" + cf_new = cf_file + ".new" + + if self.not_really: + cf_new = "/tmp/postgresql.conf.new" + open(cf_new, "w").write(buf) + self.log.info("Showing diff") + os.system("diff -u %s %s" % (cf_file, cf_new)) + self.log.info("Done diff") + os.remove(cf_new) + return + + # polite method does not work, as usually not enough perms for it + if 0: + open(cf_new, "w").write(buf) + bak = open(cf_file, "r").read() + open(cf_old, "w").write(bak) + os.rename(cf_new, cf_file) + else: + open(cf_file, "w").write(buf) + + def remote_mkdir(self, remdir): + tmp = remdir.split(":", 1) + if len(tmp) != 2: + raise Exception("cannot find hostname") + host, path = tmp + cmdline = ["ssh", host, "mkdir", "-p", path] + self.exec_cmd(cmdline) + + def master_backup(self): + """Copy master data directory to slave.""" + + data_dir = self.cf.get("master_data") + dst_loc = self.cf.get("full_backup") + if dst_loc[-1] != "/": + dst_loc += "/" + + self.pg_start_backup("FullBackup") + + master_spc_dir = os.path.join(data_dir, "pg_tblspc") + slave_spc_dir = dst_loc + "tmpspc" + + # copy data + self.chdir(data_dir) + cmdline = ["rsync", "-a", "--delete", + "--exclude", ".*", + "--exclude", "*.pid", + "--exclude", "*.opts", + "--exclude", "*.conf", + "--exclude", "*.conf.*", + "--exclude", "pg_xlog", + ".", dst_loc] + self.exec_big_rsync(cmdline) + + # copy tblspc first, to test + if os.path.isdir(master_spc_dir): + self.log.info("Checking tablespaces") + list = os.listdir(master_spc_dir) + if len(list) > 0: + self.remote_mkdir(slave_spc_dir) + for tblspc in list: + if tblspc[0] == ".": + continue + tfn = os.path.join(master_spc_dir, tblspc) + if not os.path.islink(tfn): + self.log.info("Suspicious pg_tblspc entry: "+tblspc) + continue + spc_path = os.path.realpath(tfn) + self.log.info("Got tablespace %s: %s" % (tblspc, spc_path)) + dstfn = slave_spc_dir + "/" + tblspc + + try: + os.chdir(spc_path) + except Exception, det: + self.log.warning("Broken link:" + str(det)) + continue + cmdline = ["rsync", "-a", "--delete", + "--exclude", ".*", + ".", dstfn] + self.exec_big_rsync(cmdline) + + # copy pg_xlog + self.chdir(data_dir) + cmdline = ["rsync", "-a", + "--exclude", "*.done", + "--exclude", "*.backup", + "--delete", "pg_xlog", dst_loc] + self.exec_big_rsync(cmdline) + + self.pg_stop_backup() + + self.log.info("Full backup successful") + + def master_xarchive(self, srcpath, srcname): + """Copy a complete WAL segment to slave.""" + + start_time = time.time() + self.log.debug("%s: start copy", srcname) + + self.set_last_complete(srcname) + + dst_loc = self.cf.get("completed_wals") + if dst_loc[-1] != "/": + dst_loc += "/" + + # copy data + cmdline = ["rsync", "-t", srcpath, dst_loc] + self.exec_cmd(cmdline) + + self.log.debug("%s: done", srcname) + end_time = time.time() + self.stat_add('count', 1) + self.stat_add('duration', end_time - start_time) + + def master_sync(self): + """Copy partial WAL segments.""" + + data_dir = self.cf.get("master_data") + xlog_dir = os.path.join(data_dir, "pg_xlog") + dst_loc = self.cf.get("partial_wals") + if dst_loc[-1] != "/": + dst_loc += "/" + + files = os.listdir(xlog_dir) + files.sort() + + last = self.get_last_complete() + if last: + self.log.info("%s: last complete" % last) + else: + self.log.info("last complete not found, copying all") + + for fn in files: + # check if interesting file + if len(fn) < 10: + continue + if fn[0] < "0" or fn[0] > '9': + continue + if fn.find(".") > 0: + continue + # check if to old + if last: + dot = last.find(".") + if dot > 0: + xlast = last[:dot] + if fn < xlast: + continue + else: + if fn <= last: + continue + + # got interesting WAL + xlog = os.path.join(xlog_dir, fn) + # copy data + cmdline = ["rsync", "-t", xlog, dst_loc] + self.exec_cmd(cmdline) + + self.log.info("Partial copy done") + + def slave_xrestore(self, srcname, dstpath): + loop = 1 + while loop: + try: + self.slave_xrestore_unsafe(srcname, dstpath) + loop = 0 + except SystemExit, d: + sys.exit(1) + except Exception, d: + exc, msg, tb = sys.exc_info() + self.log.fatal("xrestore %s crashed: %s: '%s' (%s: %s)" % ( + srcname, str(exc), str(msg).rstrip(), + str(tb), repr(traceback.format_tb(tb)))) + time.sleep(10) + self.log.info("Re-exec: %s", repr(sys.argv)) + os.execv(sys.argv[0], sys.argv) + + def slave_xrestore_unsafe(self, srcname, dstpath): + srcdir = self.cf.get("completed_wals") + partdir = self.cf.get("partial_wals") + keep_old_logs = self.cf.getint("keep_old_logs", 0) + pausefile = os.path.join(srcdir, "PAUSE") + stopfile = os.path.join(srcdir, "STOP") + srcfile = os.path.join(srcdir, srcname) + partfile = os.path.join(partdir, srcname) + + # loop until srcfile or stopfile appears + while 1: + if os.path.isfile(pausefile): + self.log.info("pause requested, sleeping") + time.sleep(20) + continue + + if os.path.isfile(srcfile): + self.log.info("%s: Found" % srcname) + break + + # ignore .history files + unused, ext = os.path.splitext(srcname) + if ext == ".history": + self.log.info("%s: not found, ignoring" % srcname) + sys.exit(1) + + # if stopping, include also partial wals + if os.path.isfile(stopfile): + if os.path.isfile(partfile): + self.log.info("%s: found partial" % srcname) + srcfile = partfile + break + else: + self.log.info("%s: not found, stopping" % srcname) + sys.exit(1) + + # nothing to do, sleep + self.log.debug("%s: not found, sleeping" % srcname) + time.sleep(20) + + # got one, copy it + cmdline = ["cp", srcfile, dstpath] + self.exec_cmd(cmdline) + + self.log.debug("%s: copy done, cleanup" % srcname) + self.slave_cleanup(srcname) + + # it would be nice to have apply time too + self.stat_add('count', 1) + + def slave_startup(self): + data_dir = self.cf.get("slave_data") + full_dir = self.cf.get("full_backup") + stop_cmd = self.cf.get("slave_stop_cmd", "") + start_cmd = self.cf.get("slave_start_cmd") + pidfile = os.path.join(data_dir, "postmaster.pid") + + # stop postmaster if ordered + if stop_cmd and os.path.isfile(pidfile): + self.log.info("Stopping postmaster: " + stop_cmd) + if not self.not_really: + os.system(stop_cmd) + time.sleep(3) + + # is it dead? + if os.path.isfile(pidfile): + self.log.fatal("Postmaster still running. Cannot continue.") + sys.exit(1) + + # find name for data backup + i = 0 + while 1: + bak = "%s.%d" % (data_dir, i) + if not os.path.isdir(bak): + break + i += 1 + + # move old data away + if os.path.isdir(data_dir): + self.log.info("Move %s to %s" % (data_dir, bak)) + if not self.not_really: + os.rename(data_dir, bak) + + # move new data + self.log.info("Move %s to %s" % (full_dir, data_dir)) + if not self.not_really: + os.rename(full_dir, data_dir) + else: + data_dir = full_dir + + # re-link tablespaces + spc_dir = os.path.join(data_dir, "pg_tblspc") + tmp_dir = os.path.join(data_dir, "tmpspc") + if os.path.isdir(spc_dir) and os.path.isdir(tmp_dir): + self.log.info("Linking tablespaces to temporary location") + + # don't look into spc_dir, thus allowing + # user to move them before. re-link only those + # that are still in tmp_dir + list = os.listdir(tmp_dir) + list.sort() + + for d in list: + if d[0] == ".": + continue + link_loc = os.path.join(spc_dir, d) + link_dst = os.path.join(tmp_dir, d) + self.log.info("Linking tablespace %s to %s" % (d, link_dst)) + if not self.not_really: + if os.path.islink(link_loc): + os.remove(link_loc) + os.symlink(link_dst, link_loc) + + # write recovery.conf + rconf = os.path.join(data_dir, "recovery.conf") + script = os.path.abspath(sys.argv[0]) + cf_file = os.path.abspath(self.cf.filename) + conf = "\nrestore_command = '%s %s %s'\n" % ( + script, cf_file, 'xrestore %f "%p"') + self.log.info("Write %s" % rconf) + if self.not_really: + print conf + else: + f = open(rconf, "w") + f.write(conf) + f.close() + + # remove stopfile + srcdir = self.cf.get("completed_wals") + stopfile = os.path.join(srcdir, "STOP") + if os.path.isfile(stopfile): + self.log.info("Removing stopfile: "+stopfile) + if not self.not_really: + os.remove(stopfile) + + # run database in recovery mode + self.log.info("Starting postmaster: " + start_cmd) + if not self.not_really: + os.system(start_cmd) + + def slave_boot(self): + srcdir = self.cf.get("completed_wals") + stopfile = os.path.join(srcdir, "STOP") + open(stopfile, "w").write("1") + self.log.info("Stopping recovery mode") + + def slave_pause(self): + srcdir = self.cf.get("completed_wals") + pausefile = os.path.join(srcdir, "PAUSE") + open(pausefile, "w").write("1") + self.log.info("Pausing recovery mode") + + def slave_continue(self): + srcdir = self.cf.get("completed_wals") + pausefile = os.path.join(srcdir, "PAUSE") + if os.path.isfile(pausefile): + os.remove(pausefile) + self.log.info("Continuing with recovery") + else: + self.log.info("Recovery not paused?") + + def slave_cleanup(self, last_applied): + completed_wals = self.cf.get("completed_wals") + partial_wals = self.cf.get("partial_wals") + + self.log.debug("cleaning completed wals since %s" % last_applied) + last = self.del_wals(completed_wals, last_applied) + if last: + if os.path.isdir(partial_wals): + self.log.debug("cleaning partial wals since %s" % last) + self.del_wals(partial_wals, last) + else: + self.log.warning("partial_wals dir does not exist: %s" + % partial_wals) + self.log.debug("cleaning done") + + def del_wals(self, path, last): + dot = last.find(".") + if dot > 0: + last = last[:dot] + list = os.listdir(path) + list.sort() + cur_last = None + n = len(list) + for i in range(n): + fname = list[i] + full = os.path.join(path, fname) + if fname[0] < "0" or fname[0] > "9": + continue + + ok_del = 0 + if fname < last: + self.log.debug("deleting %s" % full) + os.remove(full) + cur_last = fname + return cur_last + + def work(self): + self.master_sync() + +def main(): + try: + opts, args = getopt.getopt(sys.argv[1:], "nh") + except getopt.error, det: + print det + usage(1) + not_really = 0 + for o, v in opts: + if o == "-n": + not_really = 1 + elif o == "-h": + usage(0) + if len(args) < 2: + usage(1) + ini = args[0] + cmd = args[1] + + if cmd == "setup": + script = WalMgr(MASTER, ini, not_really) + script.master_setup() + elif cmd == "stop": + script = WalMgr(MASTER, ini, not_really) + script.master_stop() + elif cmd == "backup": + script = WalMgr(MASTER, ini, not_really) + script.master_backup() + elif cmd == "xarchive": + if len(args) != 4: + print >> sys.stderr, "usage: walmgr INI xarchive %p %f" + sys.exit(1) + script = WalMgr(MASTER, ini, not_really, 1) + script.master_xarchive(args[2], args[3]) + elif cmd == "sync": + script = WalMgr(MASTER, ini, not_really) + script.master_sync() + elif cmd == "syncdaemon": + script = WalMgr(MASTER, ini, not_really, go_daemon=1) + script.start() + elif cmd == "xrestore": + if len(args) != 4: + print >> sys.stderr, "usage: walmgr INI xrestore %p %f" + sys.exit(1) + script = WalMgr(SLAVE, ini, not_really, 1) + script.slave_xrestore(args[2], args[3]) + elif cmd == "restore": + script = WalMgr(SLAVE, ini, not_really) + script.slave_startup() + elif cmd == "boot": + script = WalMgr(SLAVE, ini, not_really) + script.slave_boot() + elif cmd == "pause": + script = WalMgr(SLAVE, ini, not_really) + script.slave_pause() + elif cmd == "continue": + script = WalMgr(SLAVE, ini, not_really) + script.slave_continue() + else: + usage(1) + +if __name__ == '__main__': + main() + diff --git a/scripts/bulk_loader.ini.templ b/scripts/bulk_loader.ini.templ new file mode 100644 index 00000000..187c1be2 --- /dev/null +++ b/scripts/bulk_loader.ini.templ @@ -0,0 +1,13 @@ +[bulk_loader] +job_name = bizgres_loader + +src_db = dbname=bulksrc +dst_db = dbname=bulkdst + +pgq_queue_name = xx + +use_skylog = 1 + +logfile = ~/log/%(job_name)s.log +pidfile = ~/pid/%(job_name)s.pid + diff --git a/scripts/bulk_loader.py b/scripts/bulk_loader.py new file mode 100755 index 00000000..a098787e --- /dev/null +++ b/scripts/bulk_loader.py @@ -0,0 +1,181 @@ +#! /usr/bin/env python + +"""Bulkloader for slow databases (Bizgres). + +Idea is following: + - Script reads from queue a batch of urlencoded row changes. + Inserts/updates/deletes, maybe many per one row. + - It changes them to minimal amount of DELETE commands + followed by big COPY of new data. + - One side-effect is that total order of how rows appear + changes, but per-row changes will be kept in order. + +The speedup from the COPY will happen only if the batches are +large enough. So the ticks should happen only after couple +of minutes. + +""" + +import sys, os, pgq, skytools + +def mk_delete_sql(tbl, key_list, data): + """ generate delete command """ + whe_list = [] + for k in key_list: + whe_list.append("%s = %s" % (k, skytools.quote_literal(data[k]))) + whe_str = " and ".join(whe_list) + return "delete from %s where %s;" % (tbl, whe_str) + +class TableCache(object): + """Per-table data hander.""" + + def __init__(self, tbl): + """Init per-batch table data cache.""" + self.name = tbl + self.ev_list = [] + self.pkey_map = {} + self.pkey_list = [] + self.pkey_str = None + self.col_list = None + + def add_event(self, ev): + """Store new event.""" + + # op & data + ev.op = ev.type[0] + ev.row = skytools.db_urldecode(ev.data) + + # get pkey column names + if self.pkey_str is None: + self.pkey_str = ev.type.split(':')[1] + if self.pkey_str: + self.pkey_list = self.pkey_str.split(',') + + # get pkey value + if self.pkey_str: + pk_data = [] + for k in self.pkey_list: + pk_data.append(ev.row[k]) + ev.pk_data = tuple(pk_data) + elif ev.op == 'I': + # fake pkey, just to get them spread out + ev.pk_data = ev.id + else: + raise Exception('non-pk tables not supported: %s' % ev.extra1) + + # get full column list, detect added columns + if not self.col_list: + self.col_list = ev.row.keys() + elif self.col_list != ev.row.keys(): + # ^ supposedly python guarantees same order in keys() + + # find new columns + for c in ev.row.keys(): + if c not in self.col_list: + for oldev in self.ev_list: + oldev.row[c] = None + self.col_list = ev.row.keys() + + # add to list + self.ev_list.append(ev) + + # keep all versions of row data + if ev.pk_data in self.pkey_map: + self.pkey_map[ev.pk_data].append(ev) + else: + self.pkey_map[ev.pk_data] = [ev] + + def finish(self): + """Got all data, prepare for insertion.""" + + del_list = [] + copy_list = [] + for ev_list in self.pkey_map.values(): + # rewrite list of I/U/D events to + # optional DELETE and optional INSERT/COPY command + exists_before = -1 + exists_after = 1 + for ev in ev_list: + if ev.op == "I": + if exists_before < 0: + exists_before = 0 + exists_after = 1 + elif ev.op == "U": + if exists_before < 0: + exists_before = 1 + #exists_after = 1 # this shouldnt be needed + elif ev.op == "D": + if exists_before < 0: + exists_before = 1 + exists_after = 0 + else: + raise Exception('unknown event type: %s' % ev.op) + + # skip short-lived rows + if exists_before == 0 and exists_after == 0: + continue + + # take last event + ev = ev_list[-1] + + # generate needed commands + if exists_before: + del_list.append(mk_delete_sql(self.name, self.pkey_list, ev.row)) + if exists_after: + copy_list.append(ev.row) + + # reorder cols + new_list = self.pkey_list[:] + for k in self.col_list: + if k not in self.pkey_list: + new_list.append(k) + + return del_list, new_list, copy_list + +class BulkLoader(pgq.SerialConsumer): + def __init__(self, args): + pgq.SerialConsumer.__init__(self, "bulk_loader", "src_db", "dst_db", args) + + def process_remote_batch(self, batch_id, ev_list, dst_db): + """Content dispatcher.""" + + # add events to per-table caches + tables = {} + for ev in ev_list: + tbl = ev.extra1 + + if not tbl in tables: + tables[tbl] = TableCache(tbl) + cache = tables[tbl] + cache.add_event(ev) + ev.tag_done() + + # then process them + for tbl, cache in tables.items(): + self.process_one_table(dst_db, tbl, cache) + + def process_one_table(self, dst_db, tbl, cache): + self.log.debug("process_one_table: %s" % tbl) + del_list, col_list, copy_list = cache.finish() + curs = dst_db.cursor() + + if not skytools.exists_table(curs, tbl): + self.log.warning("Ignoring events for table: %s" % tbl) + return + + if len(del_list) > 0: + self.log.info("Deleting %d rows from %s" % (len(del_list), tbl)) + + q = " ".join(del_list) + self.log.debug(q) + curs.execute(q) + + if len(copy_list) > 0: + self.log.info("Copying %d rows into %s" % (len(copy_list), tbl)) + self.log.debug("COPY %s (%s)" % (tbl, ','.join(col_list))) + skytools.magic_insert(curs, tbl, copy_list, col_list) + +if __name__ == '__main__': + script = BulkLoader(sys.argv[1:]) + script.start() + diff --git a/scripts/catsql.py b/scripts/catsql.py new file mode 100755 index 00000000..94fbacd8 --- /dev/null +++ b/scripts/catsql.py @@ -0,0 +1,141 @@ +#! /usr/bin/env python + +"""Prints out SQL files with psql command execution. + +Supported psql commands: \i, \cd, \q +Others are skipped. + +Aditionally does some pre-processing for NDoc. +NDoc is looks nice but needs some hand-holding. + +Bug: + +- function def end detection searches for 'as'/'is' but does not check + word boundaries - finds them even in function name. That means in + main conf, as/is must be disabled and $ ' added. This script can + remove the unnecessary AS from output. + +Niceties: + +- Ndoc includes function def in output only if def is after comment. + But for SQL functions its better to have it after def. + This script can swap comment and def. + +- Optionally remove CREATE FUNCTION (OR REPLACE) from def to + keep it shorter in doc. + +Note: + +- NDoc compares real function name and name in comment. if differ, + it decides detection failed. + +""" + +import sys, os, re, getopt + +def usage(x): + print "usage: catsql [--ndoc] FILE [FILE ...]" + sys.exit(x) + +# NDoc specific changes +cf_ndoc = 0 + +# compile regexes +func_re = r"create\s+(or\s+replace\s+)?function\s+" +func_rc = re.compile(func_re, re.I) +comm_rc = re.compile(r"^\s*([#]\s*)?(?P<com>--.*)", re.I) +end_rc = re.compile(r"\b([;]|begin|declare|end)\b", re.I) +as_rc = re.compile(r"\s+as\s+", re.I) +cmd_rc = re.compile(r"^\\([a-z]*)(\s+.*)?", re.I) + +# conversion func +def fix_func(ln): + # if ndoc, replace AS with ' ' + if cf_ndoc: + return as_rc.sub(' ', ln) + else: + return ln + +# got function def +def proc_func(f, ln): + # remove CREATE OR REPLACE + if cf_ndoc: + ln = func_rc.sub('', ln) + + ln = fix_func(ln) + pre_list = [ln] + comm_list = [] + n_comm = 0 + while 1: + ln = f.readline() + if not ln: + break + + com = None + if cf_ndoc: + com = comm_rc.search(ln) + if cf_ndoc and com: + pos = com.start('com') + comm_list.append(ln[pos:]) + elif end_rc.search(ln): + break + elif len(comm_list) > 0: + break + else: + pre_list.append(fix_func(ln)) + + if len(comm_list) > 2: + map(sys.stdout.write, comm_list) + map(sys.stdout.write, pre_list) + else: + map(sys.stdout.write, pre_list) + map(sys.stdout.write, comm_list) + if ln: + sys.stdout.write(fix_func(ln)) + +def cat_file(fn): + sys.stdout.write("\n") + f = open(fn) + while 1: + ln = f.readline() + if not ln: + break + m = cmd_rc.search(ln) + if m: + cmd = m.group(1) + if cmd == "i": # include a file + fn2 = m.group(2).strip() + cat_file(fn2) + elif cmd == "q": # quit + sys.exit(0) + elif cmd == "cd": # chdir + dir = m.group(2).strip() + os.chdir(dir) + else: # skip all others + pass + else: + if func_rc.search(ln): # function header + proc_func(f, ln) + else: # normal sql + sys.stdout.write(ln) + sys.stdout.write("\n") + +def main(): + global cf_ndoc + + try: + opts, args = getopt.gnu_getopt(sys.argv[1:], 'h', ['ndoc']) + except getopt.error, d: + print d + usage(1) + for o, v in opts: + if o == "-h": + usage(0) + elif o == "--ndoc": + cf_ndoc = 1 + for fn in args: + cat_file(fn) + +if __name__ == '__main__': + main() + diff --git a/scripts/cube_dispatcher.ini.templ b/scripts/cube_dispatcher.ini.templ new file mode 100644 index 00000000..dea70697 --- /dev/null +++ b/scripts/cube_dispatcher.ini.templ @@ -0,0 +1,23 @@ +[cube_dispatcher] +job_name = some_queue_to_cube + +src_db = dbname=sourcedb_test +dst_db = dbname=dataminedb_test + +pgq_queue_name = udata.some_queue + +logfile = ~/log/%(job_name)s.log +pidfile = ~/pid/%(job_name)s.pid + +# how many rows are kept: keep_latest, keep_all +mode = keep_latest + +# to_char() fmt for table suffix +#dateformat = YYYY_MM_DD +# following disables table suffixes: +#dateformat = + +part_template = + create table _DEST_TABLE (like _PARENT); + alter table only _DEST_TABLE add primary key (_PKEY); + diff --git a/scripts/cube_dispatcher.py b/scripts/cube_dispatcher.py new file mode 100755 index 00000000..d59ac300 --- /dev/null +++ b/scripts/cube_dispatcher.py @@ -0,0 +1,175 @@ +#! /usr/bin/env python + +# it accepts urlencoded rows for multiple tables from queue +# and insert them into actual tables, with partitioning on tick time + +import sys, os, pgq, skytools + +DEF_CREATE = """ +create table _DEST_TABLE (like _PARENT); +alter table only _DEST_TABLE add primary key (_PKEY); +""" + +class CubeDispatcher(pgq.SerialConsumer): + def __init__(self, args): + pgq.SerialConsumer.__init__(self, "cube_dispatcher", "src_db", "dst_db", args) + + self.dateformat = self.cf.get('dateformat', 'YYYY_MM_DD') + + self.part_template = self.cf.get('part_template', DEF_CREATE) + + mode = self.cf.get('mode', 'keep_latest') + if mode == 'keep_latest': + self.keep_latest = 1 + elif mode == 'keep_all': + self.keep_latest = 0 + else: + self.log.fatal('wrong mode setting') + sys.exit(1) + + def get_part_date(self, batch_id): + if not self.dateformat: + return None + + # fetch and format batch date + src_db = self.get_database('src_db') + curs = src_db.cursor() + q = 'select to_char(batch_end, %s) from pgq.get_batch_info(%s)' + curs.execute(q, [self.dateformat, batch_id]) + src_db.commit() + return curs.fetchone()[0] + + def process_remote_batch(self, batch_id, ev_list, dst_db): + + # actual processing + date_str = self.get_part_date(batch_id) + self.dispatch(dst_db, ev_list, self.get_part_date(batch_id)) + + # tag as done + for ev in ev_list: + ev.tag_done() + + def dispatch(self, dst_db, ev_list, date_str): + """Actual event processing.""" + + # get tables and sql + tables = {} + sql_list = [] + for ev in ev_list: + if date_str: + tbl = "%s_%s" % (ev.extra1, date_str) + else: + tbl = ev.extra1 + + if not tbl in tables: + tables[tbl] = self.get_table_info(ev, tbl) + + sql = self.make_sql(tbl, ev) + sql_list.append(sql) + + # create tables if needed + self.check_tables(dst_db, tables) + + # insert into data tables + curs = dst_db.cursor() + block = [] + for sql in sql_list: + self.log.debug(sql) + block.append(sql) + if len(block) > 100: + curs.execute("\n".join(block)) + block = [] + if len(block) > 0: + curs.execute("\n".join(block)) + + def get_table_info(self, ev, tbl): + inf = { + 'parent': ev.extra1, + 'table': tbl, + 'key_list': ev.type.split(':')[1] + } + return inf + + def make_sql(self, tbl, ev): + """Return SQL statement(s) for that event.""" + + # parse data + data = skytools.db_urldecode(ev.data) + + # parse tbl info + op, keys = ev.type.split(':') + key_list = keys.split(',') + if self.keep_latest and len(key_list) == 0: + raise Exception('No pkey on table %s' % tbl) + + # generate sql + if op in ('I', 'U'): + if self.keep_latest: + sql = "%s %s" % (self.mk_delete_sql(tbl, key_list, data), + self.mk_insert_sql(tbl, key_list, data)) + else: + sql = self.mk_insert_sql(tbl, key_list, data) + elif op == "D": + if not self.keep_latest: + raise Exception('Delete op not supported if mode=keep_all') + + sql = self.mk_delete_sql(tbl, key_list, data) + else: + raise Exception('Unknown row op: %s' % op) + return sql + + def mk_delete_sql(self, tbl, key_list, data): + # generate delete command + whe_list = [] + for k in key_list: + whe_list.append("%s = %s" % (k, skytools.quote_literal(data[k]))) + whe_str = " and ".join(whe_list) + return "delete from %s where %s;" % (tbl, whe_str) + + def mk_insert_sql(self, tbl, key_list, data): + # generate insert command + col_list = [] + val_list = [] + for c, v in data.items(): + col_list.append(c) + val_list.append(skytools.quote_literal(v)) + col_str = ",".join(col_list) + val_str = ",".join(val_list) + return "insert into %s (%s) values (%s);" % ( + tbl, col_str, val_str) + + def check_tables(self, dcon, tables): + """Checks that tables needed for copy are there. If not + then creates them. + + Used by other procedures to ensure that table is there + before they start inserting. + + The commits should not be dangerous, as we haven't done anything + with cdr's yet, so they should still be in one TX. + + Although it would be nicer to have a lock for table creation. + """ + + dcur = dcon.cursor() + exist_map = {} + for tbl, inf in tables.items(): + if skytools.exists_table(dcur, tbl): + continue + + sql = self.part_template + sql = sql.replace('_DEST_TABLE', inf['table']) + sql = sql.replace('_PARENT', inf['parent']) + sql = sql.replace('_PKEY', inf['key_list']) + # be similar to table_dispatcher + schema_table = inf['table'].replace(".", "__") + sql = sql.replace('_SCHEMA_TABLE', schema_table) + + dcur.execute(sql) + dcon.commit() + self.log.info('%s: Created table %s' % (self.job_name, tbl)) + +if __name__ == '__main__': + script = CubeDispatcher(sys.argv[1:]) + script.start() + diff --git a/scripts/queue_mover.ini.templ b/scripts/queue_mover.ini.templ new file mode 100644 index 00000000..8d1ff5f1 --- /dev/null +++ b/scripts/queue_mover.ini.templ @@ -0,0 +1,14 @@ +[queue_mover] +job_name = queue_mover_test + +src_db = dbname=sourcedb_test +dst_db = dbname=dataminedb_test + +pgq_queue_name = source_queue +dst_queue_name = dest_queue + +logfile = ~/log/%(job_name)s.log +pidfile = ~/pid/%(job_name)s.pid + +use_skylog = 0 + diff --git a/scripts/queue_mover.py b/scripts/queue_mover.py new file mode 100755 index 00000000..129728a3 --- /dev/null +++ b/scripts/queue_mover.py @@ -0,0 +1,30 @@ +#! /usr/bin/env python + +# this script simply mover events from one queue to another + +import sys, os, pgq, skytools + +class QueueMover(pgq.SerialConsumer): + def __init__(self, args): + pgq.SerialConsumer.__init__(self, "queue_mover", "src_db", "dst_db", args) + + self.dst_queue_name = self.cf.get("dst_queue_name") + + def process_remote_batch(self, db, batch_id, ev_list, dst_db): + + # load data + rows = [] + for ev in ev_list: + data = [ev.type, ev.data, ev.extra1, ev.extra2, ev.extra3, ev.extra4, ev.time] + rows.append(data) + ev.tag_done() + fields = ['type', 'data', 'extra1', 'extra2', 'extra3', 'extra4', 'time'] + + # insert data + curs = dst_db.cursor() + pgq.bulk_insert_events(curs, rows, fields, self.dst_queue_name) + +if __name__ == '__main__': + script = QueueMover(sys.argv[1:]) + script.start() + diff --git a/scripts/queue_splitter.ini.templ b/scripts/queue_splitter.ini.templ new file mode 100644 index 00000000..68c5ccbb --- /dev/null +++ b/scripts/queue_splitter.ini.templ @@ -0,0 +1,13 @@ +[queue_splitter] +job_name = queue_splitter_test + +src_db = dbname=sourcedb_test +dst_db = dbname=destdb_test + +pgq_queue_name = source_queue + +logfile = ~/log/%(job_name)s.log +pidfile = ~/pid/%(job_name)s.pid + +use_skylog = 0 + diff --git a/scripts/queue_splitter.py b/scripts/queue_splitter.py new file mode 100755 index 00000000..c6714ca0 --- /dev/null +++ b/scripts/queue_splitter.py @@ -0,0 +1,33 @@ +#! /usr/bin/env python + +# puts events into queue specified by field from 'queue_field' config parameter + +import sys, os, pgq, skytools + +class QueueSplitter(pgq.SerialConsumer): + def __init__(self, args): + pgq.SerialConsumer.__init__(self, "queue_splitter", "src_db", "dst_db", args) + + def process_remote_batch(self, db, batch_id, ev_list, dst_db): + cache = {} + queue_field = self.cf.get('queue_field', 'extra1') + for ev in ev_list: + row = [ev.type, ev.data, ev.extra1, ev.extra2, ev.extra3, ev.extra4, ev.time] + queue = ev.__getattr__(queue_field) + if queue not in cache: + cache[queue] = [] + cache[queue].append(row) + ev.tag_done() + + # should match the composed row + fields = ['type', 'data', 'extra1', 'extra2', 'extra3', 'extra4', 'time'] + + # now send them to right queues + curs = dst_db.cursor() + for queue, rows in cache.items(): + pgq.bulk_insert_events(curs, rows, fields, queue) + +if __name__ == '__main__': + script = QueueSplitter(sys.argv[1:]) + script.start() + diff --git a/scripts/scriptmgr.ini.templ b/scripts/scriptmgr.ini.templ new file mode 100644 index 00000000..7fa1419d --- /dev/null +++ b/scripts/scriptmgr.ini.templ @@ -0,0 +1,43 @@ + +[scriptmgr] +job_name = scriptmgr_cphdb5 +config_list = ~/dbscripts/conf/*.ini, ~/random/conf/*.ini +logfile = ~/log/%(job_name)s.log +pidfile = ~/pid/%(job_name)s.pid +#use_skylog = 1 + +# +# defaults for services +# +[DEFAULT] +cwd = ~/dbscripts +args = -v + +# +# service descriptions +# + +[cube_dispatcher] +script = cube_dispatcher.py + +[table_dispatcher] +script = table_dispatcher.py + +[bulk_loader] +script = bulk_loader.py + +[londiste] +script = londiste.py +args = replay + +[pgqadm] +script = pgqadm.py +args = ticker + +# +# services to be ignored +# + +[log_checker] +disabled = 1 + diff --git a/scripts/scriptmgr.py b/scripts/scriptmgr.py new file mode 100755 index 00000000..2ee742b2 --- /dev/null +++ b/scripts/scriptmgr.py @@ -0,0 +1,220 @@ +#! /usr/bin/env python + +"""Bulk start/stop of scripts. + +Reads a bunch of config files and maps them to scripts, then handles those. +""" + +import sys, os, skytools, signal, glob, ConfigParser, time + +command_usage = """ +%prog [options] INI CMD [subcmd args] + +commands: + start [-a | jobname ..] start a job + stop [-a | jobname ..] stop a job + restart [-a | jobname ..] restart job(s) + reload [-a | jobname ..] send reload signal + status +""" + +def job_sort_cmp(j1, j2): + d1 = j1['service'] + j1['job_name'] + d2 = j2['service'] + j2['job_name'] + if d1 < d2: return -1 + elif d1 > d2: return 1 + else: return 0 + +class ScriptMgr(skytools.DBScript): + def init_optparse(self, p = None): + p = skytools.DBScript.init_optparse(self, p) + p.add_option("-a", "--all", action="store_true", help="apply command to all jobs") + p.set_usage(command_usage.strip()) + return p + + def load_jobs(self): + self.svc_list = [] + self.svc_map = {} + self.config_list = [] + + # load services + svc_list = self.cf.sections() + svc_list.remove(self.service_name) + for svc_name in svc_list: + cf = self.cf.clone(svc_name) + disabled = cf.getboolean('disabled', 0) + defscript = None + if disabled: + defscript = '/disabled' + svc = { + 'service': svc_name, + 'script': cf.getfile('script', defscript), + 'cwd': cf.getfile('cwd'), + 'disabled': cf.getboolean('disabled', 0), + 'args': cf.get('args', ''), + } + self.svc_list.append(svc) + self.svc_map[svc_name] = svc + + # generate config list + for tmp in self.cf.getlist('config_list'): + tmp = os.path.expanduser(tmp) + tmp = os.path.expandvars(tmp) + for fn in glob.glob(tmp): + self.config_list.append(fn) + + # read jobs + for fn in self.config_list: + raw = ConfigParser.SafeConfigParser({'job_name':'?', 'service_name':'?'}) + raw.read(fn) + + # skip its own config + if raw.has_section(self.service_name): + continue + + got = 0 + for sect in raw.sections(): + if sect in self.svc_map: + got = 1 + self.add_job(fn, sect) + if not got: + self.log.warning('Cannot find service for %s' % fn) + + def add_job(self, cf_file, service_name): + svc = self.svc_map[service_name] + cf = skytools.Config(service_name, cf_file) + disabled = svc['disabled'] + if not disabled: + disabled = cf.getboolean('disabled', 0) + job = { + 'disabled': disabled, + 'config': cf_file, + 'cwd': svc['cwd'], + 'script': svc['script'], + 'args': svc['args'], + 'service': svc['service'], + 'job_name': cf.get('job_name'), + 'pidfile': cf.getfile('pidfile'), + } + self.job_list.append(job) + self.job_map[job['job_name']] = job + + def cmd_status(self): + for job in self.job_list: + os.chdir(job['cwd']) + cf = skytools.Config(job['service'], job['config']) + pidfile = cf.getfile('pidfile') + name = job['job_name'] + svc = job['service'] + if job['disabled']: + name += " (disabled)" + + if os.path.isfile(pidfile): + print " OK [%s] %s" % (svc, name) + else: + print " STOPPED [%s] %s" % (svc, name) + + def cmd_info(self): + for job in self.job_list: + print job + + def cmd_start(self, job_name): + job = self.job_map[job_name] + if job['disabled']: + self.log.info("Skipping %s" % job_name) + return 0 + self.log.info('Starting %s' % job_name) + os.chdir(job['cwd']) + pidfile = job['pidfile'] + if os.path.isfile(pidfile): + self.log.warning("Script %s seems running") + return 0 + cmd = "%(script)s %(config)s %(args)s -d" % job + res = os.system(cmd) + self.log.debug(res) + if res != 0: + self.log.error('startup failed: %s' % job_name) + return 1 + else: + return 0 + + def cmd_stop(self, job_name): + job = self.job_map[job_name] + if job['disabled']: + self.log.info("Skipping %s" % job_name) + return + self.log.info('Stopping %s' % job_name) + self.signal_job(job, signal.SIGINT) + + def cmd_reload(self, job_name): + job = self.job_map[job_name] + if job['disabled']: + self.log.info("Skipping %s" % job_name) + return + self.log.info('Reloading %s' % job_name) + self.signal_job(job, signal.SIGHUP) + + def signal_job(self, job, sig): + os.chdir(job['cwd']) + pidfile = job['pidfile'] + if os.path.isfile(pidfile): + pid = int(open(pidfile).read()) + os.kill(pid, sig) + else: + self.log.warning("Job %s not running" % job['job_name']) + + def work(self): + self.set_single_loop(1) + self.job_list = [] + self.job_map = {} + self.load_jobs() + + if len(self.args) < 2: + print "need command" + sys.exit(1) + + jobs = self.args[2:] + if len(jobs) == 0 and self.options.all: + for job in self.job_list: + jobs.append(job['job_name']) + + self.job_list.sort(job_sort_cmp) + + cmd = self.args[1] + if cmd == "status": + self.cmd_status() + return + elif cmd == "info": + self.cmd_info() + return + + if len(jobs) == 0: + print "no jobs given?" + sys.exit(1) + + if cmd == "start": + err = 0 + for n in jobs: + err += self.cmd_start(n) + if err > 0: + self.log.error('some scripts failed') + sys.exit(1) + elif cmd == "stop": + for n in jobs: + self.cmd_stop(n) + elif cmd == "restart": + for n in jobs: + self.cmd_stop(n) + time.sleep(2) + self.cmd_start(n) + elif cmd == "reload": + for n in self.jobs: + self.cmd_reload(n) + else: + print "unknown command:", cmd + sys.exit(1) + +if __name__ == '__main__': + script = ScriptMgr('scriptmgr', sys.argv[1:]) + script.start() + diff --git a/scripts/table_dispatcher.ini.templ b/scripts/table_dispatcher.ini.templ new file mode 100644 index 00000000..131dd7fe --- /dev/null +++ b/scripts/table_dispatcher.ini.templ @@ -0,0 +1,31 @@ +[udata_dispatcher] +job_name = test_move + +src_db = dbname=sourcedb_test +dst_db = dbname=dataminedb_test + +pgq_queue_name = OrderLog + +logfile = ~/log/%(job_name)s.log +pidfile = ~/pid/%(job_name)s.pid + +# where to put data. when partitioning, will be used as base name +dest_table = orders + +# date field with will be used for partitioning +# special value: _EVTIME - event creation time +part_column = start_date + +#fields = * +#fields = id, name +#fields = id:newid, name, bar:baz + + +# template used for creating partition tables +# _DEST_TABLE +part_template = + create table _DEST_TABLE () inherits (orders); + alter table only _DEST_TABLE add constraint _DEST_TABLE_pkey primary key (id); + grant select on _DEST_TABLE to group reporting; + + diff --git a/scripts/table_dispatcher.py b/scripts/table_dispatcher.py new file mode 100755 index 00000000..054ced9a --- /dev/null +++ b/scripts/table_dispatcher.py @@ -0,0 +1,124 @@ +#! /usr/bin/env python + +# it loads urlencoded rows for one trable from queue and inserts +# them into actual tables, with optional partitioning + +import sys, os, pgq, skytools + +DEST_TABLE = "_DEST_TABLE" +SCHEMA_TABLE = "_SCHEMA_TABLE" + +class TableDispatcher(pgq.SerialConsumer): + def __init__(self, args): + pgq.SerialConsumer.__init__(self, "table_dispatcher", "src_db", "dst_db", args) + + self.part_template = self.cf.get("part_template", '') + self.dest_table = self.cf.get("dest_table") + self.part_field = self.cf.get("part_field", '') + self.part_method = self.cf.get("part_method", 'daily') + if self.part_method not in ('daily', 'monthly'): + raise Exception('bad part_method') + + if self.cf.get("fields", "*") == "*": + self.field_map = None + else: + self.field_map = {} + for fval in self.cf.getlist('fields'): + tmp = fval.split(':') + if len(tmp) == 1: + self.field_map[tmp[0]] = tmp[0] + else: + self.field_map[tmp[0]] = tmp[1] + + def process_remote_batch(self, batch_id, ev_list, dst_db): + if len(ev_list) == 0: + return + + # actual processing + self.dispatch(dst_db, ev_list) + + # tag as done + for ev in ev_list: + ev.tag_done() + + def dispatch(self, dst_db, ev_list): + """Generic dispatcher.""" + + # load data + tables = {} + for ev in ev_list: + row = skytools.db_urldecode(ev.data) + + # guess dest table + if self.part_field: + if self.part_field == "_EVTIME": + partval = str(ev.creation_date) + else: + partval = str(row[self.part_field]) + partval = partval.split(' ')[0] + date = partval.split('-') + if self.part_method == 'monthly': + date = date[:2] + suffix = '_'.join(date) + tbl = "%s_%s" % (self.dest_table, suffix) + else: + tbl = self.dest_table + + # map fields + if self.field_map is None: + dstrow = row + else: + dstrow = {} + for k, v in self.field_map.items(): + dstrow[v] = row[k] + + # add row into table + if not tbl in tables: + tables[tbl] = [dstrow] + else: + tables[tbl].append(dstrow) + + # create tables if needed + self.check_tables(dst_db, tables) + + # insert into data tables + curs = dst_db.cursor() + for tbl, tbl_rows in tables.items(): + skytools.magic_insert(curs, tbl, tbl_rows) + + def check_tables(self, dcon, tables): + """Checks that tables needed for copy are there. If not + then creates them. + + Used by other procedures to ensure that table is there + before they start inserting. + + The commits should not be dangerous, as we haven't done anything + with cdr's yet, so they should still be in one TX. + + Although it would be nicer to have a lock for table creation. + """ + + dcur = dcon.cursor() + exist_map = {} + for tbl in tables.keys(): + if not skytools.exists_table(dcur, tbl): + if not self.part_template: + raise Exception('Dest table does not exists and no way to create it.') + + sql = self.part_template + sql = sql.replace(DEST_TABLE, tbl) + + # we do this to make sure that constraints for + # tables who contain a schema will still work + schema_table = tbl.replace(".", "__") + sql = sql.replace(SCHEMA_TABLE, schema_table) + + dcur.execute(sql) + dcon.commit() + self.log.info('%s: Created table %s' % (self.job_name, tbl)) + +if __name__ == '__main__': + script = TableDispatcher(sys.argv[1:]) + script.start() + diff --git a/setup.py b/setup.py new file mode 100755 index 00000000..26e59ff4 --- /dev/null +++ b/setup.py @@ -0,0 +1,36 @@ +#! /usr/bin/env python + +from distutils.core import setup + +import re +buf = open("configure.ac","r").read(256) +m = re.search("AC_INIT[(][^,]*,\s+([^)]*)[)]", buf) +ac_ver = m.group(1) + +setup( + name = "skytools", + license = "BSD", + version = ac_ver, + maintainer = "Marko Kreen", + maintainer_email = "[email protected]", + url = "http://pgfoundry.org/projects/skytools/", + package_dir = {'': 'python'}, + packages = ['skytools', 'londiste', 'pgq'], + scripts = ['python/londiste.py', 'python/pgqadm.py', 'python/walmgr.py', + 'scripts/cube_dispatcher.py', 'scripts/queue_mover.py', + 'scripts/table_dispatcher.py', 'scripts/bulk_loader.py', + 'scripts/scriptmgr.py', 'scripts/queue_splitter.py'], + data_files = [ ('share/doc/skytools/conf', [ + 'python/conf/londiste.ini', + 'python/conf/pgqadm.ini', + 'python/conf/wal-master.ini', + 'python/conf/wal-slave.ini', + 'scripts/queue_mover.ini.templ', + 'scripts/queue_splitter.ini.templ', + 'scripts/cube_dispatcher.ini.templ', + 'scripts/table_dispatcher.ini.templ', + 'scripts/bulk_loader.ini.templ', + 'scripts/scriptmgr.ini.templ', + ])] +) + diff --git a/source.cfg b/source.cfg new file mode 100644 index 00000000..b3dc1683 --- /dev/null +++ b/source.cfg @@ -0,0 +1,13 @@ +# what to include in source distribution + +# MANIFEST.in for Python Distutils + +include Makefile COPYRIGHT README NEWS config.mak.in configure configure.ac source.cfg + +recursive-include sql *.sql Makefile *.out *.in *.[ch] README* *.in +recursive-include python/conf *.ini +recursive-include scripts *.py *.templ +recursive-include debian changelog packages.in +recursive-include doc * +recursive-include tests * + diff --git a/sql/Makefile b/sql/Makefile new file mode 100644 index 00000000..3ea6c12d --- /dev/null +++ b/sql/Makefile @@ -0,0 +1,10 @@ + +include ../config.mak + +SUBDIRS = logtriga londiste pgq pgq_ext txid + +all install clean distclean installcheck: + for dir in $(SUBDIRS); do \ + make -C $$dir $@ DESTDIR=$(DESTDIR) || exit $?; \ + done + diff --git a/sql/logtriga/Makefile b/sql/logtriga/Makefile new file mode 100644 index 00000000..7f055489 --- /dev/null +++ b/sql/logtriga/Makefile @@ -0,0 +1,12 @@ + +include ../../config.mak + +MODULE_big = logtriga +SRCS = logtriga.c textbuf.c +OBJS = $(SRCS:.c=.o) +DATA_built = logtriga.sql + +REGRESS = logtriga + +include $(PGXS) + diff --git a/sql/logtriga/README.logtriga b/sql/logtriga/README.logtriga new file mode 100644 index 00000000..747aaede --- /dev/null +++ b/sql/logtriga/README.logtriga @@ -0,0 +1,47 @@ + +logtriga - generic table changes logger +======================================= + +logtriga provides generic table changes logging trigger. +It prepares partial SQL statement about a change and +gives it to user query. + +Usage +----- + + CREATE TRIGGER foo_log AFTER INSERT OR UPDATE OR DELETE ON foo_tbl + FOR EACH ROW EXECUTE PROCEDURE logtriga(column_types, query); + +Where column_types is a string where each charater defines type of +that column. Known types: + + * k - one of primary key columns for table. + * v - data column + * i - uninteresting column, to be ignored. + +Trigger function prepares 2 string arguments for query and executes it. + + * $1 - Operation type: I/U/D. + * $2 - Partial SQL for event playback. + + * INSERT INTO FOO_TBL (field, list) values (val1, val2) + * UPDATE FOO_TBL SET field1 = val1, field2 = val2 where key1 = kval1 + * DELETE FROM FOO_TBL WHERE key1 = keyval1 + +The upper-case part is left out. + +Example +------- + +Following query emulates Slony-I behaviour: + + insert into SL_SCHEMA.sl_log_1 + (log_origin, log_xid, log_tableid, + log_actionseq, log_cmdtype, log_cmddata) + values (CLUSTER_IDENT, SL_SCHEMA.getCurrentXid(), TABLE_OID, + nextval('SL_SCHEMA.sl_action_seq'), $1, $2) + +The upper-case strings should be replaced with actual values +on trigger creation. + + diff --git a/sql/logtriga/expected/logtriga.out b/sql/logtriga/expected/logtriga.out new file mode 100644 index 00000000..64daf912 --- /dev/null +++ b/sql/logtriga/expected/logtriga.out @@ -0,0 +1,95 @@ +-- init +\set ECHO none +create table rtest ( + id integer primary key, + dat text +); +NOTICE: CREATE TABLE / PRIMARY KEY will create implicit index "rtest_pkey" for table "rtest" +create table clog ( + id serial, + op text, + data text +); +NOTICE: CREATE TABLE will create implicit sequence "clog_id_seq" for serial column "clog.id" +create trigger rtest_triga after insert or update or delete on rtest +for each row execute procedure logtriga('kv', +'insert into clog (op, data) values ($1, $2)'); +-- simple test +insert into rtest values (1, 'value1'); +update rtest set dat = 'value2'; +delete from rtest; +select * from clog; delete from clog; + id | op | data +----+----+-------------------------------- + 1 | I | (id,dat) values ('1','value1') + 2 | U | dat='value2' where id='1' + 3 | D | id='1' +(3 rows) + +-- test new fields +alter table rtest add column dat2 text; +insert into rtest values (1, 'value1'); +update rtest set dat = 'value2'; +delete from rtest; +select * from clog; delete from clog; + id | op | data +----+----+-------------------------------- + 4 | I | (id,dat) values ('1','value1') + 5 | U | dat='value2' where id='1' + 6 | D | id='1' +(3 rows) + +-- test field rename +alter table rtest alter column dat type integer using 0; +insert into rtest values (1, '666', 'newdat'); +update rtest set dat = 5; +delete from rtest; +select * from clog; delete from clog; + id | op | data +----+----+----------------------------- + 7 | I | (id,dat) values ('1','666') + 8 | U | dat='5' where id='1' + 9 | D | id='1' +(3 rows) + +-- test field ignore +drop trigger rtest_triga on rtest; +create trigger rtest_triga after insert or update or delete on rtest +for each row execute procedure logtriga('kiv', +'insert into clog (op, data) values ($1, $2)'); +insert into rtest values (1, '666', 'newdat'); +update rtest set dat = 5, dat2 = 'newdat2'; +update rtest set dat = 6; +delete from rtest; +select * from clog; delete from clog; + id | op | data +----+----+--------------------------------- + 10 | I | (id,dat2) values ('1','newdat') + 11 | U | dat2='newdat2' where id='1' + 12 | D | id='1' +(3 rows) + +-- test wrong key +drop trigger rtest_triga on rtest; +create trigger rtest_triga after insert or update or delete on rtest +for each row execute procedure logtriga('vik', +'insert into clog (op, data) values ($1, $2)'); +insert into rtest values (1, 0, 'non-null'); +insert into rtest values (2, 0, NULL); +update rtest set dat2 = 'non-null2' where id=1; +update rtest set dat2 = NULL where id=1; +update rtest set dat2 = 'new-nonnull' where id=2; +ERROR: logtriga: Unexpected NULL key value +delete from rtest where id=1; +ERROR: logtriga: Unexpected NULL key value +delete from rtest where id=2; +ERROR: logtriga: Unexpected NULL key value +select * from clog; delete from clog; + id | op | data +----+----+---------------------------------------- + 13 | I | (id,dat2) values ('1','non-null') + 14 | I | (id,dat2) values ('2',null) + 15 | U | dat2='non-null2' where dat2='non-null' + 16 | U | dat2=NULL where dat2='non-null2' +(4 rows) + diff --git a/sql/logtriga/logtriga.c b/sql/logtriga/logtriga.c new file mode 100644 index 00000000..af284b8d --- /dev/null +++ b/sql/logtriga/logtriga.c @@ -0,0 +1,500 @@ +/* ---------------------------------------------------------------------- + * logtriga.c + * + * Generic trigger for logging table changes. + * Based on Slony-I log trigger. + * Does not depend on event storage. + * + * Copyright (c) 2003-2006, PostgreSQL Global Development Group + * Author: Jan Wieck, Afilias USA INC. + * + * Generalized by Marko Kreen. + * ---------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "executor/spi.h" +#include "commands/trigger.h" +#include "catalog/pg_operator.h" +#include "utils/typcache.h" + +#include "textbuf.h" + +PG_FUNCTION_INFO_V1(logtriga); +Datum logtriga(PG_FUNCTION_ARGS); + +#ifdef PG_MODULE_MAGIC +PG_MODULE_MAGIC; +#endif + +/* + * There may be several plans to be cached. + * + * FIXME: plans are kept in singe-linked list + * so not very fast access. Probably they should be + * handled more intelligently. + */ +typedef struct PlanCache PlanCache; + +struct PlanCache { + PlanCache *next; + char *query; + void *plan; +}; + +/* + * Cache result allocations. + */ +typedef struct ArgCache +{ + TBuf *op_type; + TBuf *op_data; +} ArgCache; + + +static PlanCache *plan_cache = NULL; +static ArgCache *arg_cache = NULL; + +/* + * Cache helpers + */ + +static void *get_plan(const char *query) +{ + PlanCache *c; + void *plan; + Oid plan_types[2]; + + for (c = plan_cache; c; c = c->next) + if (strcmp(query, c->query) == 0) + return c->plan; + + /* + * Plan not cached, prepare new plan then. + */ + plan_types[0] = TEXTOID; + plan_types[1] = TEXTOID; + plan = SPI_saveplan(SPI_prepare(query, 2, plan_types)); + if (plan == NULL) + elog(ERROR, "logtriga: SPI_prepare() failed"); + + /* create cache object */ + c = malloc(sizeof(*c)); + if (!c) + elog(ERROR, "logtriga: no memory for plan cache"); + + c->plan = plan; + c->query = strdup(query); + + /* insert at start */ + c->next = plan_cache; + plan_cache = c; + + return plan; +} + +static ArgCache * +get_arg_cache(void) +{ + if (arg_cache == NULL) { + ArgCache *a = malloc(sizeof(*a)); + if (!a) + elog(ERROR, "logtriga: no memory"); + memset(a, 0, sizeof(*a)); + a->op_type = tbuf_alloc(8); + a->op_data = tbuf_alloc(8192); + arg_cache = a; + } + return arg_cache; +} + +static void +append_key_eq(TBuf *tbuf, const char *col_ident, const char *col_value) +{ + if (col_value == NULL) + elog(ERROR, "logtriga: Unexpected NULL key value"); + + tbuf_encode_cstring(tbuf, col_ident, "quote_ident"); + tbuf_append_char(tbuf, '='); + tbuf_encode_cstring(tbuf, col_value, "quote_literal"); +} + +static void +append_normal_eq(TBuf *tbuf, const char *col_ident, const char *col_value) +{ + tbuf_encode_cstring(tbuf, col_ident, "quote_ident"); + tbuf_append_char(tbuf, '='); + if (col_value != NULL) + tbuf_encode_cstring(tbuf, col_value, "quote_literal"); + else + tbuf_append_cstring(tbuf, "NULL"); +} + +static void process_insert(ArgCache *cs, TriggerData *tg, char *attkind) +{ + HeapTuple new_row = tg->tg_trigtuple; + TupleDesc tupdesc = tg->tg_relation->rd_att; + int i; + int need_comma = false; + int attkind_idx; + + /* + * INSERT + * + * op_type = 'I' op_data = ("non-NULL-col" [, ...]) values ('value' [, + * ...]) + */ + tbuf_append_cstring(cs->op_type, "I"); + + /* + * Specify all the columns + */ + tbuf_append_char(cs->op_data, '('); + attkind_idx = -1; + for (i = 0; i < tg->tg_relation->rd_att->natts; i++) + { + char *col_ident; + + /* Skip dropped columns */ + if (tupdesc->attrs[i]->attisdropped) + continue; + + /* Check if allowed by colstring */ + attkind_idx++; + if (attkind[attkind_idx] == '\0') + break; + if (attkind[attkind_idx] == 'i') + continue; + + if (need_comma) + tbuf_append_char(cs->op_data, ','); + else + need_comma = true; + + /* quote column name */ + col_ident = SPI_fname(tupdesc, i + 1); + tbuf_encode_cstring(cs->op_data, col_ident, "quote_ident"); + } + + /* + * Append the string ") values (" + */ + tbuf_append_cstring(cs->op_data, ") values ("); + + /* + * Append the values + */ + need_comma = false; + attkind_idx = -1; + for (i = 0; i < tg->tg_relation->rd_att->natts; i++) + { + char *col_value; + + /* Skip dropped columns */ + if (tupdesc->attrs[i]->attisdropped) + continue; + + /* Check if allowed by colstring */ + attkind_idx++; + if (attkind[attkind_idx] == '\0') + break; + if (attkind[attkind_idx] == 'i') + continue; + + if (need_comma) + tbuf_append_char(cs->op_data, ','); + else + need_comma = true; + + /* quote column value */ + col_value = SPI_getvalue(new_row, tupdesc, i + 1); + if (col_value == NULL) + tbuf_append_cstring(cs->op_data, "null"); + else + tbuf_encode_cstring(cs->op_data, col_value, "quote_literal"); + } + + /* + * Terminate and done + */ + tbuf_append_char(cs->op_data, ')'); +} + +static int process_update(ArgCache *cs, TriggerData *tg, char *attkind) +{ + HeapTuple old_row = tg->tg_trigtuple; + HeapTuple new_row = tg->tg_newtuple; + TupleDesc tupdesc = tg->tg_relation->rd_att; + Datum old_value; + Datum new_value; + bool old_isnull; + bool new_isnull; + + char *col_ident; + char *col_value; + int i; + int need_comma = false; + int need_and = false; + int attkind_idx; + int ignore_count = 0; + + /* + * UPDATE + * + * op_type = 'U' op_data = "col_ident"='value' [, ...] where "pk_ident" = + * 'value' [ and ...] + */ + tbuf_append_cstring(cs->op_type, "U"); + + attkind_idx = -1; + for (i = 0; i < tg->tg_relation->rd_att->natts; i++) + { + /* + * Ignore dropped columns + */ + if (tupdesc->attrs[i]->attisdropped) + continue; + + attkind_idx++; + if (attkind[attkind_idx] == '\0') + break; + + old_value = SPI_getbinval(old_row, tupdesc, i + 1, &old_isnull); + new_value = SPI_getbinval(new_row, tupdesc, i + 1, &new_isnull); + + /* + * If old and new value are NULL, the column is unchanged + */ + if (old_isnull && new_isnull) + continue; + + /* + * If both are NOT NULL, we need to compare the values and skip + * setting the column if equal + */ + if (!old_isnull && !new_isnull) + { + Oid opr_oid; + FmgrInfo *opr_finfo_p; + + /* + * Lookup the equal operators function call info using the + * typecache if available + */ + TypeCacheEntry *type_cache; + + type_cache = lookup_type_cache(SPI_gettypeid(tupdesc, i + 1), + TYPECACHE_EQ_OPR | TYPECACHE_EQ_OPR_FINFO); + opr_oid = type_cache->eq_opr; + if (opr_oid == ARRAY_EQ_OP) + opr_oid = InvalidOid; + else + opr_finfo_p = &(type_cache->eq_opr_finfo); + + /* + * If we have an equal operator, use that to do binary + * comparision. Else get the string representation of both + * attributes and do string comparision. + */ + if (OidIsValid(opr_oid)) + { + if (DatumGetBool(FunctionCall2(opr_finfo_p, + old_value, new_value))) + continue; + } + else + { + char *old_strval = SPI_getvalue(old_row, tupdesc, i + 1); + char *new_strval = SPI_getvalue(new_row, tupdesc, i + 1); + + if (strcmp(old_strval, new_strval) == 0) + continue; + } + } + + if (attkind[attkind_idx] == 'i') + { + /* this change should be ignored */ + ignore_count++; + continue; + } + + if (need_comma) + tbuf_append_char(cs->op_data, ','); + else + need_comma = true; + + col_ident = SPI_fname(tupdesc, i + 1); + col_value = SPI_getvalue(new_row, tupdesc, i + 1); + + append_normal_eq(cs->op_data, col_ident, col_value); + } + + /* + * It can happen that the only UPDATE an application does is to set a + * column to the same value again. In that case, we'd end up here with + * no columns in the SET clause yet. We add the first key column here + * with it's old value to simulate the same for the replication + * engine. + */ + if (!need_comma) + { + /* there was change in ignored columns, skip whole event */ + if (ignore_count > 0) + return 0; + + for (i = 0, attkind_idx = -1; i < tg->tg_relation->rd_att->natts; i++) + { + if (tupdesc->attrs[i]->attisdropped) + continue; + + attkind_idx++; + if (attkind[attkind_idx] == 'k') + break; + } + col_ident = SPI_fname(tupdesc, i + 1); + col_value = SPI_getvalue(old_row, tupdesc, i + 1); + + append_key_eq(cs->op_data, col_ident, col_value); + } + + tbuf_append_cstring(cs->op_data, " where "); + + for (i = 0, attkind_idx = -1; i < tg->tg_relation->rd_att->natts; i++) + { + /* + * Ignore dropped columns + */ + if (tupdesc->attrs[i]->attisdropped) + continue; + + attkind_idx++; + if (attkind[attkind_idx] != 'k') + continue; + if (attkind[attkind_idx] == '\0') + break; + col_ident = SPI_fname(tupdesc, i + 1); + col_value = SPI_getvalue(old_row, tupdesc, i + 1); + + if (need_and) + tbuf_append_cstring(cs->op_data, " and "); + else + need_and = true; + + append_key_eq(cs->op_data, col_ident, col_value); + } + return 1; +} + +static void process_delete(ArgCache *cs, TriggerData *tg, char *attkind) +{ + HeapTuple old_row = tg->tg_trigtuple; + TupleDesc tupdesc = tg->tg_relation->rd_att; + char *col_ident; + char *col_value; + int i; + int need_and = false; + int attkind_idx; + + /* + * DELETE + * + * op_type = 'D' op_data = "pk_ident"='value' [and ...] + */ + tbuf_append_cstring(cs->op_type, "D"); + + for (i = 0, attkind_idx = -1; i < tg->tg_relation->rd_att->natts; i++) + { + if (tupdesc->attrs[i]->attisdropped) + continue; + + attkind_idx++; + if (attkind[attkind_idx] != 'k') + continue; + if (attkind[attkind_idx] == '\0') + break; + col_ident = SPI_fname(tupdesc, i + 1); + col_value = SPI_getvalue(old_row, tupdesc, i + 1); + + if (need_and) + tbuf_append_cstring(cs->op_data, " and "); + else + need_and = true; + + append_key_eq(cs->op_data, col_ident, col_value); + } +} + +Datum logtriga(PG_FUNCTION_ARGS) +{ + TriggerData *tg; + Datum argv[2]; + int rc; + ArgCache *cs; + char *attkind; + char *query; + int need_event = 1; + + /* + * Get the trigger call context + */ + if (!CALLED_AS_TRIGGER(fcinfo)) + elog(ERROR, "logtriga not called as trigger"); + tg = (TriggerData *) (fcinfo->context); + + /* + * Check all logTrigger() calling conventions + */ + if (!TRIGGER_FIRED_AFTER(tg->tg_event)) + elog(ERROR, "logtriga must be fired AFTER"); + if (!TRIGGER_FIRED_FOR_ROW(tg->tg_event)) + elog(ERROR, "logtriga must be fired FOR EACH ROW"); + if (tg->tg_trigger->tgnargs != 2) + elog(ERROR, "logtriga must be defined with 2 args"); + + /* + * Connect to the SPI manager + */ + if ((rc = SPI_connect()) < 0) + elog(ERROR, "logtriga: SPI_connect() failed"); + + cs = get_arg_cache(); + + tbuf_reset(cs->op_type); + tbuf_reset(cs->op_data); + + /* + * Get all the trigger arguments + */ + attkind = tg->tg_trigger->tgargs[0]; + query = tg->tg_trigger->tgargs[1]; + + if (strchr(attkind, 'k') == NULL) + elog(ERROR, "logtriga: need at least one key column"); + + /* + * Determine cmdtype and op_data depending on the command type + */ + if (TRIGGER_FIRED_BY_INSERT(tg->tg_event)) + process_insert(cs, tg, attkind); + else if (TRIGGER_FIRED_BY_UPDATE(tg->tg_event)) + need_event = process_update(cs, tg, attkind); + else if (TRIGGER_FIRED_BY_DELETE(tg->tg_event)) + process_delete(cs, tg, attkind); + else + elog(ERROR, "logtriga fired for unhandled event"); + + /* + * Construct the parameter array and insert the log row. + */ + if (need_event) + { + argv[0] = PointerGetDatum(tbuf_look_text(cs->op_type)); + argv[1] = PointerGetDatum(tbuf_look_text(cs->op_data)); + SPI_execp(get_plan(query), argv, NULL, 0); + } + SPI_finish(); + return PointerGetDatum(NULL); +} + diff --git a/sql/logtriga/logtriga.sql.in b/sql/logtriga/logtriga.sql.in new file mode 100644 index 00000000..7bd36e7f --- /dev/null +++ b/sql/logtriga/logtriga.sql.in @@ -0,0 +1,10 @@ + +-- usage: logtriga(flds, query) +-- +-- query should include 2 args: +-- $1 - for op type I/U/D, +-- $2 - for op data + +CREATE OR REPLACE FUNCTION logtriga() RETURNS trigger +AS 'MODULE_PATHNAME', 'logtriga' LANGUAGE C; + diff --git a/sql/logtriga/sql/logtriga.sql b/sql/logtriga/sql/logtriga.sql new file mode 100644 index 00000000..f0acbf53 --- /dev/null +++ b/sql/logtriga/sql/logtriga.sql @@ -0,0 +1,74 @@ +-- init +\set ECHO none +\i logtriga.sql +\set ECHO all + +create table rtest ( + id integer primary key, + dat text +); + +create table clog ( + id serial, + op text, + data text +); + +create trigger rtest_triga after insert or update or delete on rtest +for each row execute procedure logtriga('kv', +'insert into clog (op, data) values ($1, $2)'); + +-- simple test +insert into rtest values (1, 'value1'); +update rtest set dat = 'value2'; +delete from rtest; +select * from clog; delete from clog; + +-- test new fields +alter table rtest add column dat2 text; +insert into rtest values (1, 'value1'); +update rtest set dat = 'value2'; +delete from rtest; +select * from clog; delete from clog; + +-- test field rename +alter table rtest alter column dat type integer using 0; +insert into rtest values (1, '666', 'newdat'); +update rtest set dat = 5; +delete from rtest; +select * from clog; delete from clog; + + +-- test field ignore +drop trigger rtest_triga on rtest; +create trigger rtest_triga after insert or update or delete on rtest +for each row execute procedure logtriga('kiv', +'insert into clog (op, data) values ($1, $2)'); + +insert into rtest values (1, '666', 'newdat'); +update rtest set dat = 5, dat2 = 'newdat2'; +update rtest set dat = 6; +delete from rtest; +select * from clog; delete from clog; + + +-- test wrong key +drop trigger rtest_triga on rtest; +create trigger rtest_triga after insert or update or delete on rtest +for each row execute procedure logtriga('vik', +'insert into clog (op, data) values ($1, $2)'); + +insert into rtest values (1, 0, 'non-null'); +insert into rtest values (2, 0, NULL); +update rtest set dat2 = 'non-null2' where id=1; +update rtest set dat2 = NULL where id=1; +update rtest set dat2 = 'new-nonnull' where id=2; + +delete from rtest where id=1; +delete from rtest where id=2; + +select * from clog; delete from clog; + + + + diff --git a/sql/logtriga/textbuf.c b/sql/logtriga/textbuf.c new file mode 100644 index 00000000..1c7b5d33 --- /dev/null +++ b/sql/logtriga/textbuf.c @@ -0,0 +1,334 @@ + +#include <postgres.h> +#include "funcapi.h" +#include "mb/pg_wchar.h" +#include "parser/keywords.h" + +#if 1 +#define talloc(len) malloc(len) +#define trealloc(p, len) realloc(p, len) +#define tfree(p) free(p) +#else +#define talloc(len) palloc(len) +#define trealloc(p, len) repalloc(p, len) +#define tfree(p) pfree(p) +#endif + +#include "textbuf.h" + +struct TBuf { + text *data; + int size; +}; + +static void request_avail(TBuf *tbuf, int len) +{ + int newlen = tbuf->size; + int need = VARSIZE(tbuf->data) + len; + if (need < newlen) + return; + while (need > newlen) + newlen *= 2; + tbuf->data = trealloc(tbuf->data, newlen); + tbuf->size = newlen; +} + +static inline char *get_endp(TBuf *tbuf) +{ + char *p = VARDATA(tbuf->data); + int len = VARSIZE(tbuf->data) - VARHDRSZ; + return p + len; +} + +static inline void inc_used(TBuf *tbuf, int len) +{ + VARATT_SIZEP(tbuf->data) += len; +} + +static void tbuf_init(TBuf *tbuf, int start_size) +{ + if (start_size < VARHDRSZ) + start_size = VARHDRSZ; + tbuf->data = talloc(start_size); + tbuf->size = start_size; + VARATT_SIZEP(tbuf->data) = VARHDRSZ; +} + +TBuf *tbuf_alloc(int start_size) +{ + TBuf *res; + res = talloc(sizeof(TBuf)); + tbuf_init(res, start_size); + return res; +} + +void tbuf_free(TBuf *tbuf) +{ + if (tbuf->data) + tfree(tbuf->data); + tfree(tbuf); +} + +int tbuf_get_size(TBuf *tbuf) +{ + return VARSIZE(tbuf->data) - VARHDRSZ; +} + +void tbuf_reset(TBuf *tbuf) +{ + VARATT_SIZEP(tbuf->data) = VARHDRSZ; +} + +const text *tbuf_look_text(TBuf *tbuf) +{ + return tbuf->data; +} + +const char *tbuf_look_cstring(TBuf *tbuf) +{ + char *p; + request_avail(tbuf, 1); + p = get_endp(tbuf); + *p = 0; + return VARDATA(tbuf->data); +} + +void tbuf_append_cstring(TBuf *tbuf, const char *str) +{ + int len = strlen(str); + request_avail(tbuf, len); + memcpy(get_endp(tbuf), str, len); + inc_used(tbuf, len); +} + +void tbuf_append_text(TBuf *tbuf, const text *str) +{ + int len = VARSIZE(str) - VARHDRSZ; + request_avail(tbuf, len); + memcpy(get_endp(tbuf), VARDATA(str), len); + inc_used(tbuf, len); +} + +void tbuf_append_char(TBuf *tbuf, char chr) +{ + char *p; + request_avail(tbuf, 1); + p = get_endp(tbuf); + *p = chr; + inc_used(tbuf, 1); +} + +text *tbuf_steal_text(TBuf *tbuf) +{ + text *data = tbuf->data; + tbuf->data = NULL; + return data; +} + +static const char b64tbl[] = +"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +static int b64encode(char *dst, const uint8 *src, int srclen) +{ + char *p = dst; + const uint8 *s = src, *end = src + srclen; + int pos = 2; + uint32 buf = 0; + + while (s < end) { + buf |= (unsigned char) *s << (pos << 3); + pos--; + s++; + /* write it out */ + if (pos < 0) { + *p++ = b64tbl[ (buf >> 18) & 0x3f ]; + *p++ = b64tbl[ (buf >> 12) & 0x3f ]; + *p++ = b64tbl[ (buf >> 6) & 0x3f ]; + *p++ = b64tbl[ buf & 0x3f ]; + pos = 2; + buf = 0; + } + } + if (pos != 2) { + *p++ = b64tbl[ (buf >> 18) & 0x3f ]; + *p++ = b64tbl[ (buf >> 12) & 0x3f ]; + *p++ = (pos == 0) ? b64tbl[ (buf >> 6) & 0x3f ] : '='; + *p++ = '='; + } + return p - dst; +} + +static const char hextbl[] = "0123456789abcdef"; +static int urlencode(char *dst, const uint8 *src, int srclen) +{ + const uint8 *end = src + srclen; + char *p = dst; + while (src < end) { + if (*src == '=') + *p++ = '+'; + else if ((*src >= '0' && *src <= '9') + || (*src >= 'A' && *src <= 'Z') + || (*src >= 'a' && *src <= 'z')) + *p++ = *src; + else { + *p++ = '%'; + *p++ = hextbl[*src >> 4]; + *p++ = hextbl[*src & 15]; + } + } + return p - dst; +} + +static int quote_literal(char *dst, const uint8 *src, int srclen) +{ + const uint8 *cp1; + char *cp2; + int wl; + + cp1 = src; + cp2 = dst; + + *cp2++ = '\''; + while (srclen > 0) + { + if ((wl = pg_mblen(cp1)) != 1) + { + srclen -= wl; + + while (wl-- > 0) + *cp2++ = *cp1++; + continue; + } + + if (*cp1 == '\'') + *cp2++ = '\''; + if (*cp1 == '\\') + *cp2++ = '\\'; + *cp2++ = *cp1++; + srclen--; + } + + *cp2++ = '\''; + + return cp2 - dst; +} + + +/* + * slon_quote_identifier - Quote an identifier only if needed + * + * When quotes are needed, we palloc the required space; slightly + * space-wasteful but well worth it for notational simplicity. + * + * Version: pgsql/src/backend/utils/adt/ruleutils.c,v 1.188 2005/01/13 17:19:10 + */ +static int +quote_ident(char *dst, const uint8 *src, int srclen) +{ + /* + * Can avoid quoting if ident starts with a lowercase letter or + * underscore and contains only lowercase letters, digits, and + * underscores, *and* is not any SQL keyword. Otherwise, supply + * quotes. + */ + int nquotes = 0; + bool safe; + const char *ptr; + char *optr; + char ident[NAMEDATALEN + 1]; + + /* expect idents be not bigger than NAMEDATALEN */ + if (srclen > NAMEDATALEN) + srclen = NAMEDATALEN; + memcpy(ident, src, srclen); + ident[srclen] = 0; + + /* + * would like to use <ctype.h> macros here, but they might yield + * unwanted locale-specific results... + */ + safe = ((ident[0] >= 'a' && ident[0] <= 'z') || ident[0] == '_'); + + for (ptr = ident; *ptr; ptr++) + { + char ch = *ptr; + + if ((ch >= 'a' && ch <= 'z') || + (ch >= '0' && ch <= '9') || + (ch == '_')) + continue; /* okay */ + + safe = false; + if (ch == '"') + nquotes++; + } + + if (safe) + { + /* + * Check for keyword. This test is overly strong, since many of + * the "keywords" known to the parser are usable as column names, + * but the parser doesn't provide any easy way to test for whether + * an identifier is safe or not... so be safe not sorry. + * + * Note: ScanKeywordLookup() does case-insensitive comparison, but + * that's fine, since we already know we have all-lower-case. + */ + if (ScanKeywordLookup(ident) != NULL) + safe = false; + } + + optr = dst; + if (!safe) + *optr++ = '"'; + + for (ptr = ident; *ptr; ptr++) + { + char ch = *ptr; + + if (ch == '"') + *optr++ = '"'; + *optr++ = ch; + } + if (!safe) + *optr++ = '"'; + + return optr - dst; +} + + +void tbuf_encode_cstring(TBuf *tbuf, + const char *str, + const char *encoding) +{ + if (str == NULL) + elog(ERROR, "tbuf_encode_cstring: NULL"); + tbuf_encode_data(tbuf, (const uint8 *)str, strlen(str), encoding); +} + +void tbuf_encode_data(TBuf *tbuf, + const uint8 *data, int len, + const char *encoding) +{ + int dlen = 0; + char *dst; + if (strcmp(encoding, "url") == 0) { + request_avail(tbuf, len*3); + dst = get_endp(tbuf); + dlen = urlencode(dst, data, len); + } else if (strcmp(encoding, "base64") == 0) { + request_avail(tbuf, (len + 2) * 4 / 3); + dst = get_endp(tbuf); + dlen = b64encode(dst, data, len); + } else if (strcmp(encoding, "quote_literal") == 0) { + request_avail(tbuf, len * 2 + 2); + dst = get_endp(tbuf); + dlen = quote_literal(dst, data, len); + } else if (strcmp(encoding, "quote_ident") == 0) { + request_avail(tbuf, len * 2 + 2); + dst = get_endp(tbuf); + dlen = quote_ident(dst, data, len); + } else + elog(ERROR, "bad encoding"); + inc_used(tbuf, dlen); +} + diff --git a/sql/logtriga/textbuf.h b/sql/logtriga/textbuf.h new file mode 100644 index 00000000..acdff685 --- /dev/null +++ b/sql/logtriga/textbuf.h @@ -0,0 +1,26 @@ +struct TBuf; + +typedef struct TBuf TBuf; + +TBuf *tbuf_alloc(int start_size); +void tbuf_free(TBuf *tbuf); +int tbuf_get_size(TBuf *tbuf); +void tbuf_reset(TBuf *tbuf); + +const text *tbuf_look_text(TBuf *tbuf); +const char *tbuf_look_cstring(TBuf *tbuf); + +void tbuf_append_cstring(TBuf *tbuf, const char *str); +void tbuf_append_text(TBuf *tbuf, const text *str); +void tbuf_append_char(TBuf *tbuf, char chr); + +text *tbuf_steal_text(TBuf *tbuf); + +void tbuf_encode_cstring(TBuf *tbuf, + const char *str, + const char *encoding); + +void tbuf_encode_data(TBuf *tbuf, + const uint8 *data, int len, + const char *encoding); + diff --git a/sql/londiste/Makefile b/sql/londiste/Makefile new file mode 100644 index 00000000..154da071 --- /dev/null +++ b/sql/londiste/Makefile @@ -0,0 +1,20 @@ + +DATA_built = londiste.sql londiste.upgrade.sql +DOCS = README.londiste + +FUNCS = $(wildcard functions/*.sql) +SRCS = structure/tables.sql structure/types.sql $(FUNCS) + +REGRESS = londiste_install londiste_denytrigger londiste_provider londiste_subscriber +REGRESS_OPTS = --load-language=plpythonu --load-language=plpgsql + +include ../../config.mak + +include $(PGXS) + +londiste.sql: $(SRCS) + cat $(SRCS) > $@ + +londiste.upgrade.sql: $(FUNCS) + cat $(FUNCS) > $@ + diff --git a/sql/londiste/README.londiste b/sql/londiste/README.londiste new file mode 100644 index 00000000..5104f4ff --- /dev/null +++ b/sql/londiste/README.londiste @@ -0,0 +1,29 @@ + +londiste database backend +-------------------------- + +Provider side: +-------------- + +londiste.provider_table +londiste.provider_seq + + +Subscriber side +--------------- + +table londiste.completed +table londiste.subscriber_table +table londiste.subscriber_seq + + +Open issues +------------ + +- notify behaviour +- should notify-s given to db for processing? +- link init functions +- switchover +- are set_last_tick()/get_last_tick() functions needed anymore? +- typecheck for add_table()? + diff --git a/sql/londiste/expected/londiste_denytrigger.out b/sql/londiste/expected/londiste_denytrigger.out new file mode 100644 index 00000000..4fe2f408 --- /dev/null +++ b/sql/londiste/expected/londiste_denytrigger.out @@ -0,0 +1,40 @@ +create table denytest ( val integer); +insert into denytest values (1); +create trigger xdeny after insert or update or delete +on denytest for each row execute procedure londiste.deny_trigger(); +insert into denytest values (2); +ERROR: ('Changes no allowed on this table',) +update denytest set val = 2; +ERROR: ('Changes no allowed on this table',) +delete from denytest; +ERROR: ('Changes no allowed on this table',) +select londiste.disable_deny_trigger(true); + disable_deny_trigger +---------------------- + t +(1 row) + +update denytest set val = 2; +select londiste.disable_deny_trigger(true); + disable_deny_trigger +---------------------- + t +(1 row) + +update denytest set val = 2; +select londiste.disable_deny_trigger(false); + disable_deny_trigger +---------------------- + f +(1 row) + +update denytest set val = 2; +ERROR: ('Changes no allowed on this table',) +select londiste.disable_deny_trigger(false); + disable_deny_trigger +---------------------- + f +(1 row) + +update denytest set val = 2; +ERROR: ('Changes no allowed on this table',) diff --git a/sql/londiste/expected/londiste_install.out b/sql/londiste/expected/londiste_install.out new file mode 100644 index 00000000..e4527e08 --- /dev/null +++ b/sql/londiste/expected/londiste_install.out @@ -0,0 +1 @@ +\set ECHO off diff --git a/sql/londiste/expected/londiste_provider.out b/sql/londiste/expected/londiste_provider.out new file mode 100644 index 00000000..1c081936 --- /dev/null +++ b/sql/londiste/expected/londiste_provider.out @@ -0,0 +1,135 @@ +set client_min_messages = 'warning'; +-- +-- tables +-- +create table testdata ( + id serial primary key, + data text +); +create table testdata_nopk ( + id serial, + data text +); +select londiste.provider_add_table('pqueue', 'public.testdata_nopk'); +ERROR: need key column +CONTEXT: PL/pgSQL function "provider_add_table" line 2 at return +select londiste.provider_add_table('pqueue', 'public.testdata'); +ERROR: no such event queue +CONTEXT: PL/pgSQL function "provider_add_table" line 2 at return +select pgq.create_queue('pqueue'); + create_queue +-------------- + 1 +(1 row) + +select londiste.provider_add_table('pqueue', 'public.testdata'); + provider_add_table +-------------------- + 1 +(1 row) + +select londiste.provider_add_table('pqueue', 'public.testdata'); +ERROR: duplicate key violates unique constraint "provider_table_pkey" +CONTEXT: SQL statement "INSERT INTO londiste.provider_table (queue_name, table_name, trigger_name) values ( $1 , $2 , $3 )" +PL/pgSQL function "provider_add_table" line 23 at SQL statement +PL/pgSQL function "provider_add_table" line 2 at return +select londiste.provider_refresh_trigger('pqueue', 'public.testdata'); + provider_refresh_trigger +-------------------------- + 1 +(1 row) + +select * from londiste.provider_get_table_list('pqueue'); + table_name | trigger_name +-----------------+--------------- + public.testdata | pqueue_logger +(1 row) + +select londiste.provider_remove_table('pqueue', 'public.nonexist'); +ERROR: no such table registered +select londiste.provider_remove_table('pqueue', 'public.testdata'); + provider_remove_table +----------------------- + 1 +(1 row) + +select * from londiste.provider_get_table_list('pqueue'); + table_name | trigger_name +------------+-------------- +(0 rows) + +-- +-- seqs +-- +select * from londiste.provider_get_seq_list('pqueue'); + provider_get_seq_list +----------------------- +(0 rows) + +select londiste.provider_add_seq('pqueue', 'public.no_seq'); +ERROR: seq not found +CONTEXT: PL/pgSQL function "find_seq_oid" line 2 at return +SQL statement "SELECT 1 from pg_class where oid = londiste.find_seq_oid( $1 )" +PL/pgSQL function "provider_add_seq" line 10 at perform +select londiste.provider_add_seq('pqueue', 'public.testdata_id_seq'); + provider_add_seq +------------------ + 0 +(1 row) + +select londiste.provider_add_seq('pqueue', 'public.testdata_id_seq'); +ERROR: duplicate key violates unique constraint "provider_seq_pkey" +CONTEXT: SQL statement "INSERT INTO londiste.provider_seq (queue_name, seq_name) values ( $1 , $2 )" +PL/pgSQL function "provider_add_seq" line 16 at SQL statement +select * from londiste.provider_get_seq_list('pqueue'); + provider_get_seq_list +------------------------ + public.testdata_id_seq +(1 row) + +select londiste.provider_remove_seq('pqueue', 'public.testdata_id_seq'); + provider_remove_seq +--------------------- + 0 +(1 row) + +select londiste.provider_remove_seq('pqueue', 'public.testdata_id_seq'); +ERROR: seq not attached +select * from londiste.provider_get_seq_list('pqueue'); + provider_get_seq_list +----------------------- +(0 rows) + +-- +-- linked queue +-- +select londiste.provider_add_table('pqueue', 'public.testdata'); + provider_add_table +-------------------- + 1 +(1 row) + +insert into londiste.link (source, dest) values ('mqueue', 'pqueue'); +select londiste.provider_add_table('pqueue', 'public.testdata'); +ERROR: Linked queue, manipulation not allowed +CONTEXT: PL/pgSQL function "provider_add_table" line 2 at return +select londiste.provider_remove_table('pqueue', 'public.testdata'); +ERROR: Linked queue, manipulation not allowed +select londiste.provider_add_seq('pqueue', 'public.testdata_id_seq'); +ERROR: Linked queue, cannot modify +select londiste.provider_remove_seq('pqueue', 'public.testdata_seq'); +ERROR: Linked queue, cannot modify +-- +-- cleanup +-- +delete from londiste.link; +drop table testdata; +drop table testdata_nopk; +delete from londiste.provider_seq; +delete from londiste.provider_table; +select pgq.drop_queue('pqueue'); + drop_queue +------------ + 1 +(1 row) + diff --git a/sql/londiste/expected/londiste_subscriber.out b/sql/londiste/expected/londiste_subscriber.out new file mode 100644 index 00000000..7ec6944e --- /dev/null +++ b/sql/londiste/expected/londiste_subscriber.out @@ -0,0 +1,128 @@ +set client_min_messages = 'warning'; +create table testdata ( + id serial primary key, + data text +); +-- +-- tables +-- +select londiste.subscriber_add_table('pqueue', 'public.testdata_nopk'); + subscriber_add_table +---------------------- + 0 +(1 row) + +select londiste.subscriber_add_table('pqueue', 'public.testdata'); + subscriber_add_table +---------------------- + 0 +(1 row) + +select pgq.create_queue('pqueue'); + create_queue +-------------- + 1 +(1 row) + +select londiste.subscriber_add_table('pqueue', 'public.testdata'); +ERROR: duplicate key violates unique constraint "subscriber_table_pkey" +CONTEXT: SQL statement "INSERT INTO londiste.subscriber_table (queue_name, table_name) values ( $1 , $2 )" +PL/pgSQL function "subscriber_add_table" line 2 at SQL statement +select londiste.subscriber_add_table('pqueue', 'public.testdata'); +ERROR: duplicate key violates unique constraint "subscriber_table_pkey" +CONTEXT: SQL statement "INSERT INTO londiste.subscriber_table (queue_name, table_name) values ( $1 , $2 )" +PL/pgSQL function "subscriber_add_table" line 2 at SQL statement +select * from londiste.subscriber_get_table_list('pqueue'); + table_name | merge_state | snapshot | trigger_name +----------------------+-------------+----------+-------------- + public.testdata_nopk | | | + public.testdata | | | +(2 rows) + +select londiste.subscriber_remove_table('pqueue', 'public.nonexist'); +ERROR: no such table +select londiste.subscriber_remove_table('pqueue', 'public.testdata'); + subscriber_remove_table +------------------------- + 0 +(1 row) + +select * from londiste.subscriber_get_table_list('pqueue'); + table_name | merge_state | snapshot | trigger_name +----------------------+-------------+----------+-------------- + public.testdata_nopk | | | +(1 row) + +-- +-- seqs +-- +select * from londiste.subscriber_get_seq_list('pqueue'); + subscriber_get_seq_list +------------------------- +(0 rows) + +select londiste.subscriber_add_seq('pqueue', 'public.no_seq'); + subscriber_add_seq +-------------------- + 0 +(1 row) + +select londiste.subscriber_add_seq('pqueue', 'public.testdata_id_seq'); + subscriber_add_seq +-------------------- + 0 +(1 row) + +select londiste.subscriber_add_seq('pqueue', 'public.testdata_id_seq'); +ERROR: duplicate key violates unique constraint "subscriber_seq_pkey" +CONTEXT: SQL statement "INSERT INTO londiste.subscriber_seq (queue_name, seq_name) values ( $1 , $2 )" +PL/pgSQL function "subscriber_add_seq" line 4 at SQL statement +select * from londiste.subscriber_get_seq_list('pqueue'); + subscriber_get_seq_list +------------------------- + public.no_seq + public.testdata_id_seq +(2 rows) + +select londiste.subscriber_remove_seq('pqueue', 'public.testdata_id_seq'); + subscriber_remove_seq +----------------------- + 0 +(1 row) + +select londiste.subscriber_remove_seq('pqueue', 'public.testdata_id_seq'); +ERROR: no such seq? +select * from londiste.subscriber_get_seq_list('pqueue'); + subscriber_get_seq_list +------------------------- + public.no_seq +(1 row) + +-- +-- linked queue +-- +select londiste.subscriber_add_table('pqueue', 'public.testdata'); + subscriber_add_table +---------------------- + 0 +(1 row) + +insert into londiste.link (source, dest) values ('mqueue', 'pqueue'); +select londiste.subscriber_add_table('pqueue', 'public.testdata'); +ERROR: duplicate key violates unique constraint "subscriber_table_pkey" +CONTEXT: SQL statement "INSERT INTO londiste.subscriber_table (queue_name, table_name) values ( $1 , $2 )" +PL/pgSQL function "subscriber_add_table" line 2 at SQL statement +select londiste.subscriber_remove_table('pqueue', 'public.testdata'); + subscriber_remove_table +------------------------- + 0 +(1 row) + +select londiste.subscriber_add_seq('pqueue', 'public.testdata_id_seq'); + subscriber_add_seq +-------------------- + 0 +(1 row) + +select londiste.subscriber_remove_seq('pqueue', 'public.testdata_seq'); +ERROR: no such seq? diff --git a/sql/londiste/functions/londiste.denytrigger.sql b/sql/londiste/functions/londiste.denytrigger.sql new file mode 100644 index 00000000..5f69ec05 --- /dev/null +++ b/sql/londiste/functions/londiste.denytrigger.sql @@ -0,0 +1,19 @@ + +create or replace function londiste.deny_trigger() +returns trigger as $$ + if 'undeny' in GD: + return 'OK' + plpy.error('Changes no allowed on this table') +$$ language plpythonu; + +create or replace function londiste.disable_deny_trigger(i_allow boolean) +returns boolean as $$ + if args[0]: + GD['undeny'] = 1 + return True + else: + if 'undeny' in GD: + del GD['undeny'] + return False +$$ language plpythonu; + diff --git a/sql/londiste/functions/londiste.find_column_types.sql b/sql/londiste/functions/londiste.find_column_types.sql new file mode 100644 index 00000000..52f8864a --- /dev/null +++ b/sql/londiste/functions/londiste.find_column_types.sql @@ -0,0 +1,26 @@ +create or replace function londiste.find_column_types(tbl text) +returns text as $$ +declare + res text; + col record; + tbl_oid oid; +begin + tbl_oid := londiste.find_table_oid(tbl); + res := ''; + for col in + SELECT CASE WHEN k.attname IS NOT NULL THEN 'k' ELSE 'v' END AS type + FROM pg_attribute a LEFT JOIN ( + SELECT k.attname FROM pg_index i, pg_attribute k + WHERE i.indrelid = tbl_oid AND k.attrelid = i.indexrelid + AND i.indisprimary AND k.attnum > 0 AND NOT k.attisdropped + ) k ON (k.attname = a.attname) + WHERE a.attrelid = tbl_oid AND a.attnum > 0 AND NOT a.attisdropped + ORDER BY a.attnum + loop + res := res || col.type; + end loop; + + return res; +end; +$$ language plpgsql; + diff --git a/sql/londiste/functions/londiste.find_table_oid.sql b/sql/londiste/functions/londiste.find_table_oid.sql new file mode 100644 index 00000000..907b71ba --- /dev/null +++ b/sql/londiste/functions/londiste.find_table_oid.sql @@ -0,0 +1,49 @@ +create or replace function londiste.find_rel_oid(tbl text, kind text) +returns oid as $$ +declare + res oid; + pos integer; + schema text; + name text; +begin + pos := position('.' in tbl); + if pos > 0 then + schema := substring(tbl for pos - 1); + name := substring(tbl from pos + 1); + else + schema := 'public'; + name := tbl; + end if; + select c.oid into res + from pg_namespace n, pg_class c + where c.relnamespace = n.oid + and c.relkind = kind + and n.nspname = schema and c.relname = name; + if not found then + if kind = 'r' then + raise exception 'table not found'; + elsif kind = 'S' then + raise exception 'seq not found'; + else + raise exception 'weird relkind'; + end if; + end if; + + return res; +end; +$$ language plpgsql; + +create or replace function londiste.find_table_oid(tbl text) +returns oid as $$ +begin + return londiste.find_rel_oid(tbl, 'r'); +end; +$$ language plpgsql; + +create or replace function londiste.find_seq_oid(tbl text) +returns oid as $$ +begin + return londiste.find_rel_oid(tbl, 'S'); +end; +$$ language plpgsql; + diff --git a/sql/londiste/functions/londiste.get_last_tick.sql b/sql/londiste/functions/londiste.get_last_tick.sql new file mode 100644 index 00000000..e50d6e99 --- /dev/null +++ b/sql/londiste/functions/londiste.get_last_tick.sql @@ -0,0 +1,13 @@ + +create or replace function londiste.get_last_tick(i_consumer text) +returns bigint as $$ +declare + res bigint; +begin + select last_tick_id into res + from londiste.completed + where consumer_id = i_consumer; + return res; +end; +$$ language plpgsql security definer; + diff --git a/sql/londiste/functions/londiste.link.sql b/sql/londiste/functions/londiste.link.sql new file mode 100644 index 00000000..befedf04 --- /dev/null +++ b/sql/londiste/functions/londiste.link.sql @@ -0,0 +1,112 @@ + +create or replace function londiste.link_source(i_dst_name text) +returns text as $$ +declare + res text; +begin + select source into res from londiste.link + where dest = i_dst_name; + return res; +end; +$$ language plpgsql security definer; + +create or replace function londiste.link_dest(i_source_name text) +returns text as $$ +declare + res text; +begin + select dest into res from londiste.link + where source = i_source_name; + return res; +end; +$$ language plpgsql security definer; + +create or replace function londiste.cmp_list(list1 text, a_queue text, a_table text, a_field text) +returns boolean as $$ +declare + sql text; + tmp record; + list2 text; +begin + sql := 'select ' || a_field || ' as name from ' || a_table + || ' where queue_name = ' || quote_literal(a_queue) + || ' order by 1'; + list2 := ''; + for tmp in execute sql loop + if list2 = '' then + list2 := tmp.name; + else + list2 := list2 || ',' || tmp.name; + end if; + end loop; + return list1 = list2; +end; +$$ language plpgsql; + +create or replace function londiste.link(i_source_name text, i_dest_name text, prov_tick_id bigint, prov_tbl_list text, prov_seq_list text) +returns text as $$ +declare + tmp text; + list text; + tick_seq text; + external boolean; + last_tick bigint; +begin + -- check if all matches + if not londiste.cmp_list(prov_tbl_list, i_source_name, + 'londiste.subscriber_table', 'table_name') + then + raise exception 'not all tables copied into subscriber'; + end if; + if not londiste.cmp_list(prov_seq_list, i_source_name, + 'londiste.subscriber_seq', 'seq_name') + then + raise exception 'not all seqs copied into subscriber'; + end if; + if not londiste.cmp_list(prov_seq_list, i_dest_name, + 'londiste.provider_table', 'table_name') + then + raise exception 'linked provider queue does not have all tables'; + end if; + if not londiste.cmp_list(prov_seq_list, i_dest_name, + 'londiste.provider_seq', 'seq_name') + then + raise exception 'linked provider queue does not have all seqs'; + end if; + + -- check pgq + select queue_external_ticker, queue_tick_seq into external, tick_seq + from pgq.queue where queue_name = i_dest_name; + if not found then + raise exception 'dest queue does not exist'; + end if; + if external then + raise exception 'dest queue has already external_ticker turned on?'; + end if; + + if nextval(tick_seq) >= prov_tick_id then + raise exception 'dest queue ticks larger'; + end if; + + update pgq.queue set queue_external_ticker = true + where queue_name = i_dest_name; + + insert into londiste.link (source, dest) values (i_source_name, i_dest_name); + + return null; +end; +$$ language plpgsql security definer; + +create or replace function londiste.link_del(i_source_name text, i_dest_name text) +returns text as $$ +begin + delete from londiste.link + where source = i_source_name + and dest = i_dest_name; + if not found then + raise exception 'no suck link'; + end if; + return null; +end; +$$ language plpgsql security definer; + diff --git a/sql/londiste/functions/londiste.provider_add_seq.sql b/sql/londiste/functions/londiste.provider_add_seq.sql new file mode 100644 index 00000000..6658ef63 --- /dev/null +++ b/sql/londiste/functions/londiste.provider_add_seq.sql @@ -0,0 +1,27 @@ + +create or replace function londiste.provider_add_seq( + i_queue_name text, i_seq_name text) +returns integer as $$ +declare + link text; +begin + -- check if linked queue + link := londiste.link_source(i_queue_name); + if link is not null then + raise exception 'Linked queue, cannot modify'; + end if; + + perform 1 from pg_class + where oid = londiste.find_seq_oid(i_seq_name); + if not found then + raise exception 'seq not found'; + end if; + + insert into londiste.provider_seq (queue_name, seq_name) + values (i_queue_name, i_seq_name); + perform londiste.provider_notify_change(i_queue_name); + + return 0; +end; +$$ language plpgsql security definer; + diff --git a/sql/londiste/functions/londiste.provider_add_table.sql b/sql/londiste/functions/londiste.provider_add_table.sql new file mode 100644 index 00000000..354b5572 --- /dev/null +++ b/sql/londiste/functions/londiste.provider_add_table.sql @@ -0,0 +1,48 @@ +create or replace function londiste.provider_add_table( + i_queue_name text, + i_table_name text, + i_col_types text +) returns integer strict as $$ +declare + tgname text; + sql text; +begin + if londiste.link_source(i_queue_name) is not null then + raise exception 'Linked queue, manipulation not allowed'; + end if; + + if position('k' in i_col_types) < 1 then + raise exception 'need key column'; + end if; + if position('.' in i_table_name) < 1 then + raise exception 'need fully-qualified table name'; + end if; + select queue_name into tgname + from pgq.queue where queue_name = i_queue_name; + if not found then + raise exception 'no such event queue'; + end if; + + tgname := i_queue_name || '_logger'; + tgname := replace(lower(tgname), '.', '_'); + insert into londiste.provider_table + (queue_name, table_name, trigger_name) + values (i_queue_name, i_table_name, tgname); + + perform londiste.provider_create_trigger( + i_queue_name, i_table_name, i_col_types); + + return 1; +end; +$$ language plpgsql security definer; + +create or replace function londiste.provider_add_table( + i_queue_name text, + i_table_name text +) returns integer as $$ +begin + return londiste.provider_add_table(i_queue_name, i_table_name, + londiste.find_column_types(i_table_name)); +end; +$$ language plpgsql; + diff --git a/sql/londiste/functions/londiste.provider_create_trigger.sql b/sql/londiste/functions/londiste.provider_create_trigger.sql new file mode 100644 index 00000000..8b16f3be --- /dev/null +++ b/sql/londiste/functions/londiste.provider_create_trigger.sql @@ -0,0 +1,33 @@ + +create or replace function londiste.provider_create_trigger( + i_queue_name text, + i_table_name text, + i_col_types text +) returns integer strict as $$ +declare + tgname text; + sql text; +begin + select trigger_name into tgname + from londiste.provider_table + where queue_name = i_queue_name + and table_name = i_table_name; + if not found then + raise exception 'table not found'; + end if; + + sql := 'select pgq.insert_event(' + || quote_literal(i_queue_name) + || ', $1, $2, ' + || quote_literal(i_table_name) + || ', NULL, NULL, NULL)'; + execute 'create trigger ' || tgname + || ' after insert or update or delete on ' + || i_table_name + || ' for each row execute procedure logtriga($arg1$' + || i_col_types || '$arg1$, $arg2$' || sql || '$arg2$)'; + + return 1; +end; +$$ language plpgsql security definer; + diff --git a/sql/londiste/functions/londiste.provider_get_seq_list.sql b/sql/londiste/functions/londiste.provider_get_seq_list.sql new file mode 100644 index 00000000..3c053fca --- /dev/null +++ b/sql/londiste/functions/londiste.provider_get_seq_list.sql @@ -0,0 +1,17 @@ + +create or replace function londiste.provider_get_seq_list(i_queue_name text) +returns setof text as $$ +declare + rec record; +begin + for rec in + select seq_name from londiste.provider_seq + where queue_name = i_queue_name + order by nr + loop + return next rec.seq_name; + end loop; + return; +end; +$$ language plpgsql security definer; + diff --git a/sql/londiste/functions/londiste.provider_get_table_list.sql b/sql/londiste/functions/londiste.provider_get_table_list.sql new file mode 100644 index 00000000..9627802c --- /dev/null +++ b/sql/londiste/functions/londiste.provider_get_table_list.sql @@ -0,0 +1,18 @@ + +create or replace function londiste.provider_get_table_list(i_queue text) +returns setof londiste.ret_provider_table_list as $$ +declare + rec londiste.ret_provider_table_list%rowtype; +begin + for rec in + select table_name, trigger_name + from londiste.provider_table + where queue_name = i_queue + order by nr + loop + return next rec; + end loop; + return; +end; +$$ language plpgsql security definer; + diff --git a/sql/londiste/functions/londiste.provider_notify_change.sql b/sql/londiste/functions/londiste.provider_notify_change.sql new file mode 100644 index 00000000..65505fb0 --- /dev/null +++ b/sql/londiste/functions/londiste.provider_notify_change.sql @@ -0,0 +1,26 @@ + +create or replace function londiste.provider_notify_change(i_queue_name text) +returns integer as $$ +declare + res text; + tbl record; +begin + res := ''; + for tbl in + select table_name from londiste.provider_table + where queue_name = i_queue_name + order by nr + loop + if res = '' then + res := tbl.table_name; + else + res := res || ',' || tbl.table_name; + end if; + end loop; + + perform pgq.insert_event(i_queue_name, 'T', res); + + return 1; +end; +$$ language plpgsql; + diff --git a/sql/londiste/functions/londiste.provider_refresh_trigger.sql b/sql/londiste/functions/londiste.provider_refresh_trigger.sql new file mode 100644 index 00000000..fe361c19 --- /dev/null +++ b/sql/londiste/functions/londiste.provider_refresh_trigger.sql @@ -0,0 +1,44 @@ + +create or replace function londiste.provider_refresh_trigger( + i_queue_name text, + i_table_name text, + i_col_types text +) returns integer strict as $$ +declare + t_name text; + tbl_oid oid; +begin + select trigger_name into t_name + from londiste.provider_table + where queue_name = i_queue_name + and table_name = i_table_name; + if not found then + raise exception 'table not found'; + end if; + + tbl_oid := londiste.find_table_oid(i_table_name); + perform 1 from pg_trigger + where tgrelid = tbl_oid + and tgname = t_name; + if found then + execute 'drop trigger ' || t_name || ' on ' || i_table_name; + end if; + + perform londiste.provider_create_trigger(i_queue_name, i_table_name, i_col_types); + + return 1; +end; +$$ language plpgsql security definer; + +create or replace function londiste.provider_refresh_trigger( + i_queue_name text, + i_table_name text +) returns integer strict as $$ +begin + return londiste.provider_refresh_trigger(i_queue_name, i_table_name, + londiste.find_column_types(i_table_name)); +end; +$$ language plpgsql security definer; + + + diff --git a/sql/londiste/functions/londiste.provider_remove_seq.sql b/sql/londiste/functions/londiste.provider_remove_seq.sql new file mode 100644 index 00000000..47754b84 --- /dev/null +++ b/sql/londiste/functions/londiste.provider_remove_seq.sql @@ -0,0 +1,26 @@ + +create or replace function londiste.provider_remove_seq( + i_queue_name text, i_seq_name text) +returns integer as $$ +declare + link text; +begin + -- check if linked queue + link := londiste.link_source(i_queue_name); + if link is not null then + raise exception 'Linked queue, cannot modify'; + end if; + + delete from londiste.provider_seq + where queue_name = i_queue_name + and seq_name = i_seq_name; + if not found then + raise exception 'seq not attached'; + end if; + + perform londiste.provider_notify_change(i_queue_name); + + return 0; +end; +$$ language plpgsql security definer; + diff --git a/sql/londiste/functions/londiste.provider_remove_table.sql b/sql/londiste/functions/londiste.provider_remove_table.sql new file mode 100644 index 00000000..6143bb2c --- /dev/null +++ b/sql/londiste/functions/londiste.provider_remove_table.sql @@ -0,0 +1,30 @@ + +create or replace function londiste.provider_remove_table( + i_queue_name text, + i_table_name text +) returns integer as $$ +declare + tgname text; +begin + if londiste.link_source(i_queue_name) is not null then + raise exception 'Linked queue, manipulation not allowed'; + end if; + + select trigger_name into tgname from londiste.provider_table + where queue_name = i_queue_name + and table_name = i_table_name; + if not found then + raise exception 'no such table registered'; + end if; + + execute 'drop trigger ' || tgname || ' on ' || i_table_name; + + delete from londiste.provider_table + where queue_name = i_queue_name + and table_name = i_table_name; + + return 1; +end; +$$ language plpgsql security definer; + + diff --git a/sql/londiste/functions/londiste.set_last_tick.sql b/sql/londiste/functions/londiste.set_last_tick.sql new file mode 100644 index 00000000..61378a84 --- /dev/null +++ b/sql/londiste/functions/londiste.set_last_tick.sql @@ -0,0 +1,18 @@ + +create or replace function londiste.set_last_tick( + i_consumer text, + i_tick_id bigint) +returns integer as $$ +begin + update londiste.completed + set last_tick_id = i_tick_id + where consumer_id = i_consumer; + if not found then + insert into londiste.completed (consumer_id, last_tick_id) + values (i_consumer, i_tick_id); + end if; + + return 1; +end; +$$ language plpgsql security definer; + diff --git a/sql/londiste/functions/londiste.subscriber_add_seq.sql b/sql/londiste/functions/londiste.subscriber_add_seq.sql new file mode 100644 index 00000000..c144e47c --- /dev/null +++ b/sql/londiste/functions/londiste.subscriber_add_seq.sql @@ -0,0 +1,23 @@ + +create or replace function londiste.subscriber_add_seq( + i_queue_name text, i_seq_name text) +returns integer as $$ +declare + link text; +begin + insert into londiste.subscriber_seq (queue_name, seq_name) + values (i_queue_name, i_seq_name); + + -- update linked queue if needed + link := londiste.link_dest(i_queue_name); + if link is not null then + insert into londiste.provider_seq + (queue_name, seq_name) + values (link, i_seq_name); + perform londiste.provider_notify_change(link); + end if; + + return 0; +end; +$$ language plpgsql security definer; + diff --git a/sql/londiste/functions/londiste.subscriber_add_table.sql b/sql/londiste/functions/londiste.subscriber_add_table.sql new file mode 100644 index 00000000..d5a73314 --- /dev/null +++ b/sql/londiste/functions/londiste.subscriber_add_table.sql @@ -0,0 +1,14 @@ + +create or replace function londiste.subscriber_add_table( + i_queue_name text, i_table text) +returns integer as $$ +begin + insert into londiste.subscriber_table (queue_name, table_name) + values (i_queue_name, i_table); + + -- linked queue is updated, when the table is copied + + return 0; +end; +$$ language plpgsql security definer; + diff --git a/sql/londiste/functions/londiste.subscriber_get_seq_list.sql b/sql/londiste/functions/londiste.subscriber_get_seq_list.sql new file mode 100644 index 00000000..1d218f48 --- /dev/null +++ b/sql/londiste/functions/londiste.subscriber_get_seq_list.sql @@ -0,0 +1,17 @@ + +create or replace function londiste.subscriber_get_seq_list(i_queue_name text) +returns setof text as $$ +declare + rec record; +begin + for rec in + select seq_name from londiste.subscriber_seq + where queue_name = i_queue_name + order by nr + loop + return next rec.seq_name; + end loop; + return; +end; +$$ language plpgsql security definer; + diff --git a/sql/londiste/functions/londiste.subscriber_get_table_list.sql b/sql/londiste/functions/londiste.subscriber_get_table_list.sql new file mode 100644 index 00000000..4a17d5da --- /dev/null +++ b/sql/londiste/functions/londiste.subscriber_get_table_list.sql @@ -0,0 +1,35 @@ + +create or replace function londiste.subscriber_get_table_list(i_queue_name text) +returns setof londiste.ret_subscriber_table as $$ +declare + rec londiste.ret_subscriber_table%rowtype; +begin + for rec in + select table_name, merge_state, snapshot, trigger_name + from londiste.subscriber_table + where queue_name = i_queue_name + order by nr + loop + return next rec; + end loop; + return; +end; +$$ language plpgsql security definer; + +-- compat +create or replace function londiste.get_table_state(i_queue text) +returns setof londiste.subscriber_table as $$ +declare + rec londiste.subscriber_table%rowtype; +begin + for rec in + select * from londiste.subscriber_table + where queue_name = i_queue + order by nr + loop + return next rec; + end loop; + return; +end; +$$ language plpgsql security definer; + diff --git a/sql/londiste/functions/londiste.subscriber_remove_seq.sql b/sql/londiste/functions/londiste.subscriber_remove_seq.sql new file mode 100644 index 00000000..f8715a49 --- /dev/null +++ b/sql/londiste/functions/londiste.subscriber_remove_seq.sql @@ -0,0 +1,27 @@ + +create or replace function londiste.subscriber_remove_seq( + i_queue_name text, i_seq_name text) +returns integer as $$ +declare + link text; +begin + delete from londiste.subscriber_seq + where queue_name = i_queue_name + and seq_name = i_seq_name; + if not found then + raise exception 'no such seq?'; + end if; + + -- update linked queue if needed + link := londiste.link_dest(i_queue_name); + if link is not null then + delete from londiste.provider_seq + where queue_name = link + and seq_name = i_seq_name; + perform londiste.provider_notify_change(link); + end if; + + return 0; +end; +$$ language plpgsql security definer; + diff --git a/sql/londiste/functions/londiste.subscriber_remove_table.sql b/sql/londiste/functions/londiste.subscriber_remove_table.sql new file mode 100644 index 00000000..49af4053 --- /dev/null +++ b/sql/londiste/functions/londiste.subscriber_remove_table.sql @@ -0,0 +1,27 @@ + +create or replace function londiste.subscriber_remove_table( + i_queue_name text, i_table text) +returns integer as $$ +declare + link text; +begin + delete from londiste.subscriber_table + where queue_name = i_queue_name + and table_name = i_table; + if not found then + raise exception 'no such table'; + end if; + + -- sync link + link := londiste.link_dest(i_queue_name); + if link is not null then + delete from londiste.provider_table + where queue_name = link + and table_name = i_table; + perform londiste.provider_notify_change(link); + end if; + + return 0; +end; +$$ language plpgsql; + diff --git a/sql/londiste/functions/londiste.subscriber_set_table_state.sql b/sql/londiste/functions/londiste.subscriber_set_table_state.sql new file mode 100644 index 00000000..cab12444 --- /dev/null +++ b/sql/londiste/functions/londiste.subscriber_set_table_state.sql @@ -0,0 +1,58 @@ + +create or replace function londiste.subscriber_set_table_state( + i_queue_name text, + i_table_name text, + i_snapshot text, + i_merge_state text) +returns integer as $$ +declare + link text; + ok integer; +begin + update londiste.subscriber_table + set snapshot = i_snapshot, + merge_state = i_merge_state + where queue_name = i_queue_name + and table_name = i_table_name; + if not found then + raise exception 'no such table'; + end if; + + -- sync link state also + link := londiste.link_dest(i_queue_name); + if link then + select * from londiste.provider_table + where queue_name = linkdst + and table_name = i_table_name; + if found then + if i_merge_state is null or i_merge_state <> 'ok' then + delete from londiste.provider_table + where queue_name = link + and table_name = i_table_name; + perform londiste.notify_change(link); + end if; + else + if i_merge_state = 'ok' then + insert into londiste.provider_table (queue_name, table_name) + values (link, i_table_name); + perform londiste.notify_change(link); + end if; + end if; + end if; + + return 1; +end; +$$ language plpgsql security definer; + +create or replace function londiste.set_table_state( + i_queue_name text, + i_table_name text, + i_snapshot text, + i_merge_state text) +returns integer as $$ +begin + return londiste.subscriber_set_table_state(i_queue_name, i_table_name, i_snapshot, i_merge_state); +end; +$$ language plpgsql security definer; + + diff --git a/sql/londiste/sql/londiste_denytrigger.sql b/sql/londiste/sql/londiste_denytrigger.sql new file mode 100644 index 00000000..dad81ffc --- /dev/null +++ b/sql/londiste/sql/londiste_denytrigger.sql @@ -0,0 +1,19 @@ + +create table denytest ( val integer); +insert into denytest values (1); +create trigger xdeny after insert or update or delete +on denytest for each row execute procedure londiste.deny_trigger(); + +insert into denytest values (2); +update denytest set val = 2; +delete from denytest; + +select londiste.disable_deny_trigger(true); +update denytest set val = 2; +select londiste.disable_deny_trigger(true); +update denytest set val = 2; +select londiste.disable_deny_trigger(false); +update denytest set val = 2; +select londiste.disable_deny_trigger(false); +update denytest set val = 2; + diff --git a/sql/londiste/sql/londiste_install.sql b/sql/londiste/sql/londiste_install.sql new file mode 100644 index 00000000..4637659f --- /dev/null +++ b/sql/londiste/sql/londiste_install.sql @@ -0,0 +1,8 @@ +\set ECHO off +set log_error_verbosity = 'terse'; +\i ../txid/txid.sql +\i ../pgq/pgq.sql +\i ../logtriga/logtriga.sql +\i londiste.sql +\set ECHO all + diff --git a/sql/londiste/sql/londiste_provider.sql b/sql/londiste/sql/londiste_provider.sql new file mode 100644 index 00000000..74075383 --- /dev/null +++ b/sql/londiste/sql/londiste_provider.sql @@ -0,0 +1,68 @@ + +set client_min_messages = 'warning'; + +-- +-- tables +-- +create table testdata ( + id serial primary key, + data text +); +create table testdata_nopk ( + id serial, + data text +); + +select londiste.provider_add_table('pqueue', 'public.testdata_nopk'); +select londiste.provider_add_table('pqueue', 'public.testdata'); + +select pgq.create_queue('pqueue'); +select londiste.provider_add_table('pqueue', 'public.testdata'); +select londiste.provider_add_table('pqueue', 'public.testdata'); + +select londiste.provider_refresh_trigger('pqueue', 'public.testdata'); + +select * from londiste.provider_get_table_list('pqueue'); + +select londiste.provider_remove_table('pqueue', 'public.nonexist'); +select londiste.provider_remove_table('pqueue', 'public.testdata'); + +select * from londiste.provider_get_table_list('pqueue'); + +-- +-- seqs +-- + +select * from londiste.provider_get_seq_list('pqueue'); +select londiste.provider_add_seq('pqueue', 'public.no_seq'); +select londiste.provider_add_seq('pqueue', 'public.testdata_id_seq'); +select londiste.provider_add_seq('pqueue', 'public.testdata_id_seq'); +select * from londiste.provider_get_seq_list('pqueue'); +select londiste.provider_remove_seq('pqueue', 'public.testdata_id_seq'); +select londiste.provider_remove_seq('pqueue', 'public.testdata_id_seq'); +select * from londiste.provider_get_seq_list('pqueue'); + +-- +-- linked queue +-- +select londiste.provider_add_table('pqueue', 'public.testdata'); +insert into londiste.link (source, dest) values ('mqueue', 'pqueue'); + + +select londiste.provider_add_table('pqueue', 'public.testdata'); +select londiste.provider_remove_table('pqueue', 'public.testdata'); + +select londiste.provider_add_seq('pqueue', 'public.testdata_id_seq'); +select londiste.provider_remove_seq('pqueue', 'public.testdata_seq'); + +-- +-- cleanup +-- + +delete from londiste.link; +drop table testdata; +drop table testdata_nopk; +delete from londiste.provider_seq; +delete from londiste.provider_table; +select pgq.drop_queue('pqueue'); + diff --git a/sql/londiste/sql/londiste_subscriber.sql b/sql/londiste/sql/londiste_subscriber.sql new file mode 100644 index 00000000..0583a395 --- /dev/null +++ b/sql/londiste/sql/londiste_subscriber.sql @@ -0,0 +1,53 @@ + +set client_min_messages = 'warning'; + +create table testdata ( + id serial primary key, + data text +); + +-- +-- tables +-- + +select londiste.subscriber_add_table('pqueue', 'public.testdata_nopk'); +select londiste.subscriber_add_table('pqueue', 'public.testdata'); + +select pgq.create_queue('pqueue'); +select londiste.subscriber_add_table('pqueue', 'public.testdata'); +select londiste.subscriber_add_table('pqueue', 'public.testdata'); + +select * from londiste.subscriber_get_table_list('pqueue'); + +select londiste.subscriber_remove_table('pqueue', 'public.nonexist'); +select londiste.subscriber_remove_table('pqueue', 'public.testdata'); + +select * from londiste.subscriber_get_table_list('pqueue'); + +-- +-- seqs +-- + +select * from londiste.subscriber_get_seq_list('pqueue'); +select londiste.subscriber_add_seq('pqueue', 'public.no_seq'); +select londiste.subscriber_add_seq('pqueue', 'public.testdata_id_seq'); +select londiste.subscriber_add_seq('pqueue', 'public.testdata_id_seq'); +select * from londiste.subscriber_get_seq_list('pqueue'); +select londiste.subscriber_remove_seq('pqueue', 'public.testdata_id_seq'); +select londiste.subscriber_remove_seq('pqueue', 'public.testdata_id_seq'); +select * from londiste.subscriber_get_seq_list('pqueue'); + +-- +-- linked queue +-- +select londiste.subscriber_add_table('pqueue', 'public.testdata'); +insert into londiste.link (source, dest) values ('mqueue', 'pqueue'); + + +select londiste.subscriber_add_table('pqueue', 'public.testdata'); +select londiste.subscriber_remove_table('pqueue', 'public.testdata'); + +select londiste.subscriber_add_seq('pqueue', 'public.testdata_id_seq'); +select londiste.subscriber_remove_seq('pqueue', 'public.testdata_seq'); + + diff --git a/sql/londiste/structure/tables.sql b/sql/londiste/structure/tables.sql new file mode 100644 index 00000000..83ac24f4 --- /dev/null +++ b/sql/londiste/structure/tables.sql @@ -0,0 +1,53 @@ +set default_with_oids = 'off'; + +create schema londiste; +grant usage on schema londiste to public; + +create table londiste.provider_table ( + nr serial not null, + queue_name text not null, + table_name text not null, + trigger_name text, + primary key (queue_name, table_name) +); + +create table londiste.provider_seq ( + nr serial not null, + queue_name text not null, + seq_name text not null, + primary key (queue_name, seq_name) +); + +create table londiste.completed ( + consumer_id text not null, + last_tick_id bigint not null, + + primary key (consumer_id) +); + +create table londiste.link ( + source text not null, + dest text not null, + primary key (source), + unique (dest) +); + +create table londiste.subscriber_table ( + nr serial not null, + queue_name text not null, + table_name text not null, + snapshot text, + merge_state text, + trigger_name text, + + primary key (queue_name, table_name) +); + +create table londiste.subscriber_seq ( + nr serial not null, + queue_name text not null, + seq_name text not null, + + primary key (queue_name, seq_name) +); + diff --git a/sql/londiste/structure/types.sql b/sql/londiste/structure/types.sql new file mode 100644 index 00000000..e5d64655 --- /dev/null +++ b/sql/londiste/structure/types.sql @@ -0,0 +1,13 @@ + +create type londiste.ret_provider_table_list as ( + table_name text, + trigger_name text +); + +create type londiste.ret_subscriber_table as ( + table_name text, + merge_state text, + snapshot text, + trigger_name text +); + diff --git a/sql/pgq/Makefile b/sql/pgq/Makefile new file mode 100644 index 00000000..6ea78852 --- /dev/null +++ b/sql/pgq/Makefile @@ -0,0 +1,48 @@ + +DOCS = README.pgq +DATA_built = pgq.sql +DATA = structure/uninstall_pgq.sql + +SRCS = $(wildcard structure/*.sql) \ + $(wildcard functions/*.sql) \ + $(wildcard triggers/*.sql) + +REGRESS = pgq_init logutriga sqltriga +REGRESS_OPTS = --load-language=plpythonu --load-language=plpgsql + +PGXS = $(shell pg_config --pgxs) +include $(PGXS) + +NDOC = NaturalDocs +NDOCARGS = -r -o html docs/html -p docs -i docs/sql +CATSQL = ../../scripts/catsql.py + +pgq.sql: $(SRCS) + $(CATSQL) structure/install.sql > $@ + +dox: cleandox + mkdir -p docs/html + mkdir -p docs/sql + $(CATSQL) --ndoc structure/tables.sql structure/types.sql > docs/sql/schema.sql + $(CATSQL) --ndoc structure/func_public.sql > docs/sql/external.sql + $(CATSQL) --ndoc structure/func_internal.sql > docs/sql/internal.sql + $(CATSQL) --ndoc structure/triggers.sql > docs/sql/triggers.sql + $(NDOC) $(NDOCARGS) + +cleandox: + rm -rf docs/html docs/Data docs/sql + +clean: cleandox + +test: + #-dropdb pgq + #createdb pgq + #psql -f structure/pgq.sql pgq + make installcheck || { less regression.diffs; exit 1; } + +upload: dox + rsync -az docs/html structure functions data1:public_html/pgq/ + make cleandox + rsync -az catsql.py Makefile docs data1:public_html/pgq/ + make dox + diff --git a/sql/pgq/README.pgq b/sql/pgq/README.pgq new file mode 100644 index 00000000..b8757161 --- /dev/null +++ b/sql/pgq/README.pgq @@ -0,0 +1,19 @@ + +Schema overview +=============== + +pgq.consumer consumer name <> id mapping +pgq.queue queue information +pgq.subscription consumer registrations +pgq.tick snapshots that group events into batches +pgq.retry_queue events to be retried +pgq.failed_queue events that have failed +pgq.event_* data tables + +Random ideas +============ + +- all ticker logic in DB (plpython) +- more funcs in plpython +- insert_event in C (way to get rid of plpython) + diff --git a/sql/pgq/docs/Languages.txt b/sql/pgq/docs/Languages.txt new file mode 100644 index 00000000..aa9ce802 --- /dev/null +++ b/sql/pgq/docs/Languages.txt @@ -0,0 +1,113 @@ +Format: 1.35 + +# This is the Natural Docs languages file for this project. If you change +# anything here, it will apply to THIS PROJECT ONLY. If you'd like to change +# something for all your projects, edit the Languages.txt in Natural Docs' +# Config directory instead. + + +# You can prevent certain file extensions from being scanned like this: +# Ignore Extensions: [extension] [extension] ... + + +#------------------------------------------------------------------------------- +# SYNTAX: +# +# Unlike other Natural Docs configuration files, in this file all comments +# MUST be alone on a line. Some languages deal with the # character, so you +# cannot put comments on the same line as content. +# +# Also, all lists are separated with spaces, not commas, again because some +# languages may need to use them. +# +# Language: [name] +# Alter Language: [name] +# Defines a new language or alters an existing one. Its name can use any +# characters. If any of the properties below have an add/replace form, you +# must use that when using Alter Language. +# +# The language Shebang Script is special. It's entry is only used for +# extensions, and files with those extensions have their shebang (#!) lines +# read to determine the real language of the file. Extensionless files are +# always treated this way. +# +# The language Text File is also special. It's treated as one big comment +# so you can put Natural Docs content in them without special symbols. Also, +# if you don't specify a package separator, ignored prefixes, or enum value +# behavior, it will copy those settings from the language that is used most +# in the source tree. +# +# Extensions: [extension] [extension] ... +# [Add/Replace] Extensions: [extension] [extension] ... +# Defines the file extensions of the language's source files. You can +# redefine extensions found in the main languages file. You can use * to +# mean any undefined extension. +# +# Shebang Strings: [string] [string] ... +# [Add/Replace] Shebang Strings: [string] [string] ... +# Defines a list of strings that can appear in the shebang (#!) line to +# designate that it's part of the language. You can redefine strings found +# in the main languages file. +# +# Ignore Prefixes in Index: [prefix] [prefix] ... +# [Add/Replace] Ignored Prefixes in Index: [prefix] [prefix] ... +# +# Ignore [Topic Type] Prefixes in Index: [prefix] [prefix] ... +# [Add/Replace] Ignored [Topic Type] Prefixes in Index: [prefix] [prefix] ... +# Specifies prefixes that should be ignored when sorting symbols in an +# index. Can be specified in general or for a specific topic type. +# +#------------------------------------------------------------------------------ +# For basic language support only: +# +# Line Comments: [symbol] [symbol] ... +# Defines a space-separated list of symbols that are used for line comments, +# if any. +# +# Block Comments: [opening sym] [closing sym] [opening sym] [closing sym] ... +# Defines a space-separated list of symbol pairs that are used for block +# comments, if any. +# +# Package Separator: [symbol] +# Defines the default package separator symbol. The default is a dot. +# +# [Topic Type] Prototype Enders: [symbol] [symbol] ... +# When defined, Natural Docs will attempt to get a prototype from the code +# immediately following the topic type. It stops when it reaches one of +# these symbols. Use \n for line breaks. +# +# Line Extender: [symbol] +# Defines the symbol that allows a prototype to span multiple lines if +# normally a line break would end it. +# +# Enum Values: [global|under type|under parent] +# Defines how enum values are referenced. The default is global. +# global - Values are always global, referenced as 'value'. +# under type - Values are under the enum type, referenced as +# 'package.enum.value'. +# under parent - Values are under the enum's parent, referenced as +# 'package.value'. +# +# Perl Package: [perl package] +# Specifies the Perl package used to fine-tune the language behavior in ways +# too complex to do in this file. +# +#------------------------------------------------------------------------------ +# For full language support only: +# +# Full Language Support: [perl package] +# Specifies the Perl package that has the parsing routines necessary for full +# language support. +# +#------------------------------------------------------------------------------- + +# The following languages are defined in the main file, if you'd like to alter +# them: +# +# Text File, Shebang Script, C/C++, C#, Java, JavaScript, Perl, Python, +# PHP, SQL, Visual Basic, Pascal, Assembly, Ada, Tcl, Ruby, Makefile, +# ActionScript, ColdFusion, R, Fortran + +# If you add a language that you think would be useful to other developers +# and should be included in Natural Docs by default, please e-mail it to +# languages [at] naturaldocs [dot] org. diff --git a/sql/pgq/docs/Menu.txt b/sql/pgq/docs/Menu.txt new file mode 100644 index 00000000..c4b66684 --- /dev/null +++ b/sql/pgq/docs/Menu.txt @@ -0,0 +1,43 @@ +Format: 1.35 + + +Title: PgQ +SubTitle: Database API + +# You can add a footer to your documentation like this: +# Footer: [text] +# If you want to add a copyright notice, this would be the place to do it. + + +# -------------------------------------------------------------------------- +# +# Cut and paste the lines below to change the order in which your files +# appear on the menu. Don't worry about adding or removing files, Natural +# Docs will take care of that. +# +# You can further organize the menu by grouping the entries. Add a +# "Group: [name] {" line to start a group, and add a "}" to end it. +# +# You can add text and web links to the menu by adding "Text: [text]" and +# "Link: [name] ([URL])" lines, respectively. +# +# The formatting and comments are auto-generated, so don't worry about +# neatness when editing the file. Natural Docs will clean it up the next +# time it is run. When working with groups, just deal with the braces and +# forget about the indentation and comments. +# +# -------------------------------------------------------------------------- + + +File: Public Functions (external.sql) +File: Public Triggers (triggers.sql) +File: Internal Functions (internal.sql) +File: Internal Tables (schema.sql) + +Group: Index { + + Index: Everything + Database Table Index: Database Tables + Function Index: Functions + } # Group: Index + diff --git a/sql/pgq/docs/Topics.txt b/sql/pgq/docs/Topics.txt new file mode 100644 index 00000000..da6181df --- /dev/null +++ b/sql/pgq/docs/Topics.txt @@ -0,0 +1,107 @@ +Format: 1.35 + +# This is the Natural Docs topics file for this project. If you change anything +# here, it will apply to THIS PROJECT ONLY. If you'd like to change something +# for all your projects, edit the Topics.txt in Natural Docs' Config directory +# instead. + + +# If you'd like to prevent keywords from being recognized by Natural Docs, you +# can do it like this: +# Ignore Keywords: [keyword], [keyword], ... +# +# Or you can use the list syntax like how they are defined: +# Ignore Keywords: +# [keyword] +# [keyword], [plural keyword] +# ... + + +#------------------------------------------------------------------------------- +# SYNTAX: +# +# Topic Type: [name] +# Alter Topic Type: [name] +# Creates a new topic type or alters one from the main file. Each type gets +# its own index and behavior settings. Its name can have letters, numbers, +# spaces, and these charaters: - / . ' +# +# Plural: [name] +# Sets the plural name of the topic type, if different. +# +# Keywords: +# [keyword] +# [keyword], [plural keyword] +# ... +# Defines or adds to the list of keywords for the topic type. They may only +# contain letters, numbers, and spaces and are not case sensitive. Plural +# keywords are used for list topics. You can redefine keywords found in the +# main topics file. +# +# Index: [yes|no] +# Whether the topics get their own index. Defaults to yes. Everything is +# included in the general index regardless of this setting. +# +# Scope: [normal|start|end|always global] +# How the topics affects scope. Defaults to normal. +# normal - Topics stay within the current scope. +# start - Topics start a new scope for all the topics beneath it, +# like class topics. +# end - Topics reset the scope back to global for all the topics +# beneath it. +# always global - Topics are defined as global, but do not change the scope +# for any other topics. +# +# Class Hierarchy: [yes|no] +# Whether the topics are part of the class hierarchy. Defaults to no. +# +# Variable Type: [yes|no] +# Whether the topics can be a variable type. Defaults to no. +# +# Page Title If First: [yes|no] +# Whether the topic's title becomes the page title if it's the first one in +# a file. Defaults to no. +# +# Break Lists: [yes|no] +# Whether list topics should be broken into individual topics in the output. +# Defaults to no. +# +# Can Group With: [type], [type], ... +# Defines a list of topic types that this one can possibly be grouped with. +# Defaults to none. +#------------------------------------------------------------------------------- + +# The following topics are defined in the main file, if you'd like to alter +# their behavior or add keywords: +# +# Generic, Class, Interface, Section, File, Group, Function, Variable, +# Property, Type, Constant, Enumeration, Event, Delegate, Macro, +# Database, Database Table, Database View, Database Index, Database +# Cursor, Database Trigger, Cookie, Build Target + +# If you add something that you think would be useful to other developers +# and should be included in Natural Docs by default, please e-mail it to +# topics [at] naturaldocs [dot] org. + + +Topic Type: Schema + + Plural: Schemas + Index: No + Scope: Start + Class Hierarchy: Yes + + Keywords: + schema, schemas + + +Alter Topic Type: Function + + Add Keywords: + public function + internal function + + +Alter Topic Type: File + + Index: No diff --git a/sql/pgq/expected/logutriga.out b/sql/pgq/expected/logutriga.out new file mode 100644 index 00000000..6c7f9b14 --- /dev/null +++ b/sql/pgq/expected/logutriga.out @@ -0,0 +1,22 @@ +create or replace function pgq.insert_event(que text, ev_type text, ev_data text, x1 text, x2 text, x3 text, x4 text) +returns bigint as $$ +begin + raise notice 'insert_event(%, %, %, %)', que, ev_type, ev_data, x1; + return 1; +end; +$$ language plpgsql; +create table udata ( + id serial primary key, + txt text, + bin bytea +); +NOTICE: CREATE TABLE will create implicit sequence "udata_id_seq" for serial column "udata.id" +NOTICE: CREATE TABLE / PRIMARY KEY will create implicit index "udata_pkey" for table "udata" +create trigger utest AFTER insert or update or delete ON udata +for each row execute procedure pgq.logutriga('udata_que'); +insert into udata (txt) values ('text1'); +ERROR: plpython: function "logutriga" failed +DETAIL: exceptions.NameError: global name 'rowdata' is not defined +insert into udata (bin) values (E'bi\tn\\000bin'); +ERROR: plpython: function "logutriga" failed +DETAIL: exceptions.NameError: global name 'rowdata' is not defined diff --git a/sql/pgq/expected/pgq_init.out b/sql/pgq/expected/pgq_init.out new file mode 100644 index 00000000..67b555ca --- /dev/null +++ b/sql/pgq/expected/pgq_init.out @@ -0,0 +1,253 @@ +\set ECHO none +select * from pgq.maint_tables_to_vacuum(); + maint_tables_to_vacuum +------------------------ + pgq.subscription + pgq.queue + pgq.tick + pgq.retry_queue +(4 rows) + +select * from pgq.maint_retry_events(); + maint_retry_events +-------------------- + 0 +(1 row) + +select pgq.create_queue('tmpqueue'); + create_queue +-------------- + 1 +(1 row) + +select pgq.register_consumer('tmpqueue', 'consumer'); + register_consumer +------------------- + 1 +(1 row) + +select pgq.unregister_consumer('tmpqueue', 'consumer'); + unregister_consumer +--------------------- + 1 +(1 row) + +select pgq.drop_queue('tmpqueue'); + drop_queue +------------ + 1 +(1 row) + +select pgq.create_queue('myqueue'); + create_queue +-------------- + 1 +(1 row) + +select pgq.register_consumer('myqueue', 'consumer'); + register_consumer +------------------- + 1 +(1 row) + +select pgq.next_batch('myqueue', 'consumer'); + next_batch +------------ + +(1 row) + +select pgq.next_batch('myqueue', 'consumer'); + next_batch +------------ + +(1 row) + +select pgq.ticker(); + ticker +-------- + 1 +(1 row) + +select pgq.next_batch('myqueue', 'consumer'); + next_batch +------------ + 1 +(1 row) + +select pgq.next_batch('myqueue', 'consumer'); + next_batch +------------ + 1 +(1 row) + +select queue_name, consumer_name, prev_tick_id, tick_id, lag from pgq.get_batch_info(1); + queue_name | consumer_name | prev_tick_id | tick_id | lag +------------+---------------+--------------+---------+------------- + myqueue | consumer | 1 | 2 | @ 0.00 secs +(1 row) + +select queue_name from pgq.get_queue_info() limit 0; + queue_name +------------ +(0 rows) + +select queue_name, consumer_name from pgq.get_consumer_info() limit 0; + queue_name | consumer_name +------------+--------------- +(0 rows) + +select pgq.finish_batch(1); + finish_batch +-------------- + 1 +(1 row) + +select pgq.finish_batch(1); +WARNING: finish_batch: batch 1 not found + finish_batch +-------------- + 0 +(1 row) + +select pgq.ticker(); + ticker +-------- + 1 +(1 row) + +select pgq.next_batch('myqueue', 'consumer'); + next_batch +------------ + 2 +(1 row) + +select * from pgq.batch_event_tables(2); + batch_event_tables +-------------------- + pgq.event_2_0 +(1 row) + +select * from pgq.get_batch_events(2); + ev_id | ev_time | ev_txid | ev_retry | ev_type | ev_data | ev_extra1 | ev_extra2 | ev_extra3 | ev_extra4 +-------+---------+---------+----------+---------+---------+-----------+-----------+-----------+----------- +(0 rows) + +select pgq.finish_batch(2); + finish_batch +-------------- + 1 +(1 row) + +select pgq.insert_event('myqueue', 'r1', 'data'); + insert_event +-------------- + 1 +(1 row) + +select pgq.insert_event('myqueue', 'r2', 'data'); + insert_event +-------------- + 2 +(1 row) + +select pgq.insert_event('myqueue', 'r3', 'data'); + insert_event +-------------- + 3 +(1 row) + +select pgq.current_event_table('myqueue'); + current_event_table +--------------------- + pgq.event_2_0 +(1 row) + +select pgq.ticker(); + ticker +-------- + 1 +(1 row) + +select pgq.next_batch('myqueue', 'consumer'); + next_batch +------------ + 3 +(1 row) + +select ev_id,ev_retry,ev_type,ev_data,ev_extra1,ev_extra2,ev_extra3,ev_extra4 from pgq.get_batch_events(3); + ev_id | ev_retry | ev_type | ev_data | ev_extra1 | ev_extra2 | ev_extra3 | ev_extra4 +-------+----------+---------+---------+-----------+-----------+-----------+----------- + 1 | | r1 | data | | | | + 2 | | r2 | data | | | | + 3 | | r3 | data | | | | +(3 rows) + +select * from pgq.failed_event_list('myqueue', 'consumer'); + ev_failed_reason | ev_failed_time | ev_id | ev_time | ev_txid | ev_owner | ev_retry | ev_type | ev_data | ev_extra1 | ev_extra2 | ev_extra3 | ev_extra4 +------------------+----------------+-------+---------+---------+----------+----------+---------+---------+-----------+-----------+-----------+----------- +(0 rows) + +select pgq.event_failed(3, 1, 'failure test'); + event_failed +-------------- + 1 +(1 row) + +select pgq.event_failed(3, 1, 'failure test'); + event_failed +-------------- + 0 +(1 row) + +select pgq.event_retry(3, 2, 0); + event_retry +------------- + 1 +(1 row) + +select pgq.event_retry(3, 2, 0); + event_retry +------------- + 0 +(1 row) + +select pgq.finish_batch(3); + finish_batch +-------------- + 1 +(1 row) + +select ev_failed_reason, ev_id, ev_txid, ev_retry, ev_type, ev_data + from pgq.failed_event_list('myqueue', 'consumer'); + ev_failed_reason | ev_id | ev_txid | ev_retry | ev_type | ev_data +------------------+-------+---------+----------+---------+--------- + failure test | 1 | | 0 | r1 | data +(1 row) + +select ev_failed_reason, ev_id, ev_txid, ev_retry, ev_type, ev_data + from pgq.failed_event_list('myqueue', 'consumer', 0, 1); + ev_failed_reason | ev_id | ev_txid | ev_retry | ev_type | ev_data +------------------+-------+---------+----------+---------+--------- +(0 rows) + +select * from pgq.failed_event_count('myqueue', 'consumer'); + failed_event_count +-------------------- + 1 +(1 row) + +select * from pgq.failed_event_delete('myqueue', 'consumer', 0); +ERROR: event not found +select pgq.event_retry_raw('myqueue', 'consumer', now(), 666, now(), 0, + 'rawtest', 'data', null, null, null, null); + event_retry_raw +----------------- + 666 +(1 row) + +select pgq.ticker(); + ticker +-------- + 1 +(1 row) + diff --git a/sql/pgq/expected/sqltriga.out b/sql/pgq/expected/sqltriga.out new file mode 100644 index 00000000..8e396212 --- /dev/null +++ b/sql/pgq/expected/sqltriga.out @@ -0,0 +1,86 @@ +-- start testing +create table rtest ( + id integer primary key, + dat text +); +NOTICE: CREATE TABLE / PRIMARY KEY will create implicit index "rtest_pkey" for table "rtest" +create trigger rtest_triga after insert or update or delete on rtest +for each row execute procedure pgq.sqltriga('que'); +-- simple test +insert into rtest values (1, 'value1'); +NOTICE: insert_event(que, I, (dat,id) values ('value1','1'), public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +update rtest set dat = 'value2'; +NOTICE: insert_event(que, U, dat='value2' where id='1', public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +delete from rtest; +NOTICE: insert_event(que, D, id='1', public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +-- test new fields +alter table rtest add column dat2 text; +insert into rtest values (1, 'value1'); +NOTICE: insert_event(que, I, (dat,id) values ('value1','1'), public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +update rtest set dat = 'value2'; +NOTICE: insert_event(que, U, dat='value2' where id='1', public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +delete from rtest; +NOTICE: insert_event(que, D, id='1', public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +-- test field ignore +drop trigger rtest_triga on rtest; +create trigger rtest_triga after insert or update or delete on rtest +for each row execute procedure pgq.sqltriga('que2', 'ignore=dat2'); +insert into rtest values (1, '666', 'newdat'); +NOTICE: insert_event(que2, I, (dat,id) values ('666','1'), public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +update rtest set dat = 5, dat2 = 'newdat2'; +NOTICE: insert_event(que2, U, dat='5' where id='1', public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +update rtest set dat = 6; +NOTICE: insert_event(que2, U, dat='6' where id='1', public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +delete from rtest; +NOTICE: insert_event(que2, D, id='1', public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +-- test hashed pkey +drop trigger rtest_triga on rtest; +create trigger rtest_triga after insert or update or delete on rtest +for each row execute procedure pgq.sqltriga('que2', 'ignore=dat2&pkey=dat,hashtext(dat)'); +insert into rtest values (1, '666', 'newdat'); +NOTICE: insert_event(que2, I, (dat,id) values ('666','1'), public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +update rtest set dat = 5, dat2 = 'newdat2'; +NOTICE: insert_event(que2, U, dat='5' where dat='5' and hashtext(dat) = hashtext('5'), public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +update rtest set dat = 6; +NOTICE: insert_event(que2, U, dat='6' where dat='6' and hashtext(dat) = hashtext('6'), public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +delete from rtest; +NOTICE: insert_event(que2, D, dat='6' and hashtext(dat) = hashtext('6'), public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +-- test wrong key +drop trigger rtest_triga on rtest; +create trigger rtest_triga after insert or update or delete on rtest +for each row execute procedure pgq.sqltriga('que3'); +insert into rtest values (1, 0, 'non-null'); +NOTICE: insert_event(que3, I, (dat,id) values ('0','1'), public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +insert into rtest values (2, 0, NULL); +NOTICE: insert_event(que3, I, (dat,id) values ('0','2'), public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +update rtest set dat2 = 'non-null2' where id=1; +NOTICE: insert_event(que3, U, id='1' where id='1', public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +update rtest set dat2 = NULL where id=1; +NOTICE: insert_event(que3, U, id='1' where id='1', public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +update rtest set dat2 = 'new-nonnull' where id=2; +NOTICE: insert_event(que3, U, id='2' where id='2', public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +delete from rtest where id=1; +NOTICE: insert_event(que3, D, id='1', public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +delete from rtest where id=2; +NOTICE: insert_event(que3, D, id='2', public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" diff --git a/sql/pgq/functions/pgq.batch_event_sql.sql b/sql/pgq/functions/pgq.batch_event_sql.sql new file mode 100644 index 00000000..825de5b6 --- /dev/null +++ b/sql/pgq/functions/pgq.batch_event_sql.sql @@ -0,0 +1,106 @@ +create or replace function pgq.batch_event_sql(x_batch_id bigint) +returns text as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.batch_event_sql(1) +-- Creates SELECT statement that fetches events for this batch. +-- +-- Parameters: +-- x_batch_id - ID of a active batch. +-- +-- Returns: +-- SQL statement +-- ---------------------------------------------------------------------- +declare + rec record; + sql text; + tbl text; + arr text; + part text; + select_fields text; + retry_expr text; + batch record; +begin + select s.sub_last_tick, s.sub_next_tick, s.sub_id, s.sub_queue, + get_snapshot_xmax(last.tick_snapshot) as tx_start, + get_snapshot_xmax(cur.tick_snapshot) as tx_end, + last.tick_snapshot as last_snapshot, + cur.tick_snapshot as cur_snapshot + into batch + from pgq.subscription s, pgq.tick last, pgq.tick cur + where s.sub_batch = x_batch_id + and last.tick_queue = s.sub_queue + and last.tick_id = s.sub_last_tick + and cur.tick_queue = s.sub_queue + and cur.tick_id = s.sub_next_tick; + if not found then + raise exception 'batch not found'; + end if; + + -- load older transactions + arr := ''; + for rec in + -- active tx-es in prev_snapshot that were committed in cur_snapshot + select id1 from + get_snapshot_active(batch.last_snapshot) id1 left join + get_snapshot_active(batch.cur_snapshot) id2 on (id1 = id2) + where id2 is null + order by 1 desc + loop + -- try to avoid big IN expression, so try to include nearby + -- tx'es into range + if batch.tx_start - 100 <= rec.id1 then + batch.tx_start := rec.id1; + else + if arr = '' then + arr := rec.id1; + else + arr := arr || ',' || rec.id1; + end if; + end if; + end loop; + + -- must match pgq.event_template + select_fields := 'select ev_id, ev_time, ev_txid, ev_retry, ev_type,' + || ' ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4'; + retry_expr := ' and (ev_owner is null or ev_owner = ' + || batch.sub_id || ')'; + + -- now generate query that goes over all potential tables + sql := ''; + for rec in + select xtbl from pgq.batch_event_tables(x_batch_id) xtbl + loop + tbl := rec.xtbl; + -- this gets newer queries that definitely are not in prev_snapshot + part := select_fields + || ' from pgq.tick cur, pgq.tick last, ' || tbl || ' ev ' + || ' where cur.tick_id = ' || batch.sub_next_tick + || ' and cur.tick_queue = ' || batch.sub_queue + || ' and last.tick_id = ' || batch.sub_last_tick + || ' and last.tick_queue = ' || batch.sub_queue + || ' and ev.ev_txid >= ' || batch.tx_start + || ' and ev.ev_txid <= ' || batch.tx_end + || ' and txid_in_snapshot(ev.ev_txid, cur.tick_snapshot)' + || ' and not txid_in_snapshot(ev.ev_txid, last.tick_snapshot)' + || retry_expr; + -- now include older tx-es, that were ongoing + -- at the time of prev_snapshot + if arr <> '' then + part := part || ' union all ' + || select_fields || ' from ' || tbl || ' ev ' + || ' where ev.ev_txid in (' || arr || ')' + || retry_expr; + end if; + if sql = '' then + sql := part; + else + sql := sql || ' union all ' || part; + end if; + end loop; + if sql = '' then + raise exception 'could not construct sql for batch %', x_batch_id; + end if; + return sql || ' order by 1'; +end; +$$ language plpgsql; -- no perms needed + diff --git a/sql/pgq/functions/pgq.batch_event_tables.sql b/sql/pgq/functions/pgq.batch_event_tables.sql new file mode 100644 index 00000000..f6bdc309 --- /dev/null +++ b/sql/pgq/functions/pgq.batch_event_tables.sql @@ -0,0 +1,67 @@ +create or replace function pgq.batch_event_tables(x_batch_id bigint) +returns setof text as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.batch_event_tables(1) +-- +-- Returns set of table names where this batch events may reside. +-- +-- Parameters: +-- x_batch_id - ID of a active batch. +-- ---------------------------------------------------------------------- +declare + nr integer; + tbl text; + use_prev integer; + use_next integer; + batch record; +begin + select + get_snapshot_xmin(last.tick_snapshot) as tx_min, -- absolute minimum + get_snapshot_xmax(cur.tick_snapshot) as tx_max, -- absolute maximum + q.queue_data_pfx, q.queue_ntables, + q.queue_cur_table, q.queue_switch_step1, q.queue_switch_step2 + into batch + from pgq.tick last, pgq.tick cur, pgq.subscription s, pgq.queue q + where cur.tick_id = s.sub_next_tick + and cur.tick_queue = s.sub_queue + and last.tick_id = s.sub_last_tick + and last.tick_queue = s.sub_queue + and s.sub_batch = x_batch_id + and q.queue_id = s.sub_queue; + if not found then + raise exception 'Cannot find data for batch %', x_batch_id; + end if; + + -- if its definitely not in one or other, look into both + if batch.tx_max < batch.queue_switch_step1 then + use_prev := 1; + use_next := 0; + elsif batch.queue_switch_step2 is not null + and (batch.tx_min > batch.queue_switch_step2) + then + use_prev := 0; + use_next := 1; + else + use_prev := 1; + use_next := 1; + end if; + + if use_prev then + nr := batch.queue_cur_table - 1; + if nr < 0 then + nr := batch.queue_ntables - 1; + end if; + tbl := batch.queue_data_pfx || '_' || nr; + return next tbl; + end if; + + if use_next then + tbl := batch.queue_data_pfx || '_' || batch.queue_cur_table; + return next tbl; + end if; + + return; +end; +$$ language plpgsql; -- no perms needed + + diff --git a/sql/pgq/functions/pgq.create_queue.sql b/sql/pgq/functions/pgq.create_queue.sql new file mode 100644 index 00000000..927a48cb --- /dev/null +++ b/sql/pgq/functions/pgq.create_queue.sql @@ -0,0 +1,71 @@ +create or replace function pgq.create_queue(i_queue_name text) +returns integer as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.create_queue(1) +-- +-- Creates new queue with given name. +-- +-- Returns: +-- 0 - queue already exists +-- 1 - queue created +-- ---------------------------------------------------------------------- +declare + tblpfx text; + tblname text; + idxpfx text; + idxname text; + sql text; + id integer; + tick_seq text; + ev_seq text; + n_tables integer; +begin + if i_queue_name is null then + raise exception 'Invalid NULL value'; + end if; + + -- check if exists + perform 1 from pgq.queue where queue_name = i_queue_name; + if found then + return 0; + end if; + + -- insert event + id := nextval('pgq.queue_queue_id_seq'); + tblpfx := 'pgq.event_' || id; + idxpfx := 'event_' || id; + tick_seq := 'pgq.event_' || id || '_tick_seq'; + ev_seq := 'pgq.event_' || id || '_id_seq'; + insert into pgq.queue (queue_id, queue_name, + queue_data_pfx, queue_event_seq, queue_tick_seq) + values (id, i_queue_name, tblpfx, ev_seq, tick_seq); + + select queue_ntables into n_tables from pgq.queue + where queue_id = id; + + -- create seqs + execute 'CREATE SEQUENCE ' || tick_seq; + execute 'CREATE SEQUENCE ' || ev_seq; + + -- create data tables + execute 'CREATE TABLE ' || tblpfx || ' () ' + || ' INHERITS (pgq.event_template)'; + for i in 0 .. (n_tables - 1) loop + tblname := tblpfx || '_' || i; + idxname := idxpfx || '_' || i; + execute 'CREATE TABLE ' || tblname || ' () ' + || ' INHERITS (' || tblpfx || ')'; + execute 'ALTER TABLE ' || tblname || ' ALTER COLUMN ev_id ' + || ' SET DEFAULT nextval(' || quote_literal(ev_seq) || ')'; + execute 'create index ' || idxname || '_txid_idx on ' + || tblname || ' (ev_txid)'; + end loop; + + perform pgq.grant_perms(i_queue_name); + + perform pgq.ticker(i_queue_name); + + return 1; +end; +$$ language plpgsql security definer; + diff --git a/sql/pgq/functions/pgq.current_event_table.sql b/sql/pgq/functions/pgq.current_event_table.sql new file mode 100644 index 00000000..b20dac03 --- /dev/null +++ b/sql/pgq/functions/pgq.current_event_table.sql @@ -0,0 +1,25 @@ +create or replace function pgq.current_event_table(x_queue_name text) +returns text as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.current_event_table(1) +-- +-- Return active event table for particular queue. +-- +-- Note: +-- The result is valid only during current transaction. +-- +-- Parameters: +-- x_queue_name - Queue name. +-- ---------------------------------------------------------------------- +declare + res text; +begin + select queue_data_pfx || '_' || queue_cur_table into res + from pgq.queue where queue_name = x_queue_name; + if not found then + raise exception 'Event queue not found'; + end if; + return res; +end; +$$ language plpgsql; -- no perms needed + diff --git a/sql/pgq/functions/pgq.drop_queue.sql b/sql/pgq/functions/pgq.drop_queue.sql new file mode 100644 index 00000000..3819a914 --- /dev/null +++ b/sql/pgq/functions/pgq.drop_queue.sql @@ -0,0 +1,56 @@ +create or replace function pgq.drop_queue(x_queue_name text) +returns integer as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.drop_queue(1) +-- +-- Drop queue and all associated tables. +-- No consumers must be listening on the queue. +-- +-- ---------------------------------------------------------------------- +declare + tblname text; + q record; + num integer; +begin + -- check ares + if x_queue_name is null then + raise exception 'Invalid NULL value'; + end if; + + -- check if exists + select * into q from pgq.queue + where queue_name = x_queue_name; + if not found then + raise exception 'No such event queue'; + end if; + + -- check if no consumers + select count(*) into num from pgq.subscription + where sub_queue = q.queue_id; + if num > 0 then + raise exception 'cannot drop queue, consumers still attached'; + end if; + + -- drop data tables + for i in 0 .. (q.queue_ntables - 1) loop + tblname := q.queue_data_pfx || '_' || i; + execute 'DROP TABLE ' || tblname; + end loop; + execute 'DROP TABLE ' || q.queue_data_pfx; + + -- delete ticks + delete from pgq.tick where tick_queue = q.queue_id; + + -- drop seqs + -- FIXME: any checks needed here? + execute 'DROP SEQUENCE ' || q.queue_tick_seq; + execute 'DROP SEQUENCE ' || q.queue_event_seq; + + -- delete event + delete from pgq.queue + where queue_name = x_queue_name; + + return 1; +end; +$$ language plpgsql security definer; + diff --git a/sql/pgq/functions/pgq.event_failed.sql b/sql/pgq/functions/pgq.event_failed.sql new file mode 100644 index 00000000..fbcf3c86 --- /dev/null +++ b/sql/pgq/functions/pgq.event_failed.sql @@ -0,0 +1,41 @@ +create or replace function pgq.event_failed( + x_batch_id bigint, + x_event_id bigint, + x_reason text) +returns integer as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.event_failed(3) +-- +-- Copies the event to failed queue. Can be looked later. +-- +-- Parameters: +-- x_batch_id - ID of active batch. +-- x_event_id - Event id +-- x_reason - Text to associate with event. +-- +-- Returns: +-- 0 if event was already in queue, 1 otherwise. +-- ---------------------------------------------------------------------- +begin + insert into pgq.failed_queue (ev_failed_reason, ev_failed_time, + ev_id, ev_time, ev_txid, ev_owner, ev_retry, ev_type, ev_data, + ev_extra1, ev_extra2, ev_extra3, ev_extra4) + select x_reason, now(), + ev_id, ev_time, NULL, sub_id, coalesce(ev_retry, 0), + ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4 + from pgq.get_batch_events(x_batch_id), + pgq.subscription + where sub_batch = x_batch_id + and ev_id = x_event_id; + if not found then + raise exception 'event not found'; + end if; + return 1; + +-- dont worry if the event is already in queue +exception + when unique_violation then + return 0; +end; +$$ language plpgsql security definer; + diff --git a/sql/pgq/functions/pgq.event_retry.sql b/sql/pgq/functions/pgq.event_retry.sql new file mode 100644 index 00000000..c0259745 --- /dev/null +++ b/sql/pgq/functions/pgq.event_retry.sql @@ -0,0 +1,68 @@ +create or replace function pgq.event_retry( + x_batch_id bigint, + x_event_id bigint, + x_retry_time timestamptz) +returns integer as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.event_retry(3) +-- +-- Put the event into retry queue, to be processed later again. +-- +-- Parameters: +-- x_batch_id - ID of active batch. +-- x_event_id - event id +-- x_retry_time - Time when the event should be put back into queue +-- +-- Returns: +-- nothing +-- ---------------------------------------------------------------------- +begin + insert into pgq.retry_queue (ev_retry_after, + ev_id, ev_time, ev_txid, ev_owner, ev_retry, ev_type, ev_data, + ev_extra1, ev_extra2, ev_extra3, ev_extra4) + select x_retry_time, + ev_id, ev_time, NULL, sub_id, coalesce(ev_retry, 0) + 1, + ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4 + from pgq.get_batch_events(x_batch_id), + pgq.subscription + where sub_batch = x_batch_id + and ev_id = x_event_id; + if not found then + raise exception 'event not found'; + end if; + return 1; + +-- dont worry if the event is already in queue +exception + when unique_violation then + return 0; +end; +$$ language plpgsql security definer; + + +create or replace function pgq.event_retry( + x_batch_id bigint, + x_event_id bigint, + x_retry_seconds integer) +returns integer as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.event_retry(3) +-- +-- Put the event into retry queue, to be processed later again. +-- +-- Parameters: +-- x_batch_id - ID of active batch. +-- x_event_id - event id +-- x_retry_seconds - Time when the event should be put back into queue +-- +-- Returns: +-- nothing +-- ---------------------------------------------------------------------- +declare + new_retry timestamptz; +begin + new_retry := current_timestamp + ((x_retry_seconds || ' seconds')::interval); + return pgq.event_retry(x_batch_id, x_event_id, new_retry); +end; +$$ language plpgsql security definer; + diff --git a/sql/pgq/functions/pgq.event_retry_raw.sql b/sql/pgq/functions/pgq.event_retry_raw.sql new file mode 100644 index 00000000..3a2efb29 --- /dev/null +++ b/sql/pgq/functions/pgq.event_retry_raw.sql @@ -0,0 +1,66 @@ +create or replace function pgq.event_retry_raw( + x_queue text, + x_consumer text, + x_retry_after timestamptz, + x_ev_id bigint, + x_ev_time timestamptz, + x_ev_retry integer, + x_ev_type text, + x_ev_data text, + x_ev_extra1 text, + x_ev_extra2 text, + x_ev_extra3 text, + x_ev_extra4 text) +returns bigint as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.event_retry_raw(12) +-- +-- Allows full control over what goes to retry queue. +-- +-- Parameters: +-- x_queue - name of the queue +-- x_consumer - name of the consumer +-- x_retry_after - when the event should be processed again +-- x_ev_id - event id +-- x_ev_time - creation time +-- x_ev_retry - retry count +-- x_ev_type - user data +-- x_ev_data - user data +-- x_ev_extra1 - user data +-- x_ev_extra2 - user data +-- x_ev_extra3 - user data +-- x_ev_extra4 - user data +-- +-- Returns: +-- Event ID. +-- ---------------------------------------------------------------------- +declare + q record; + id bigint; +begin + select sub_id, queue_event_seq into q + from pgq.consumer, pgq.queue, pgq.subscription + where queue_name = x_queue + and co_name = x_consumer + and sub_consumer = co_id + and sub_queue = queue_id; + if not found then + raise exception 'consumer not registered'; + end if; + + id := x_ev_id; + if id is null then + id := nextval(q.queue_event_seq); + end if; + + insert into pgq.retry_queue (ev_retry_after, + ev_id, ev_time, ev_owner, ev_retry, + ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4) + values (x_retry_after, x_ev_id, x_ev_time, q.sub_id, x_ev_retry, + x_ev_type, x_ev_data, x_ev_extra1, x_ev_extra2, + x_ev_extra3, x_ev_extra4); + + return id; +end; +$$ language plpgsql security definer; + diff --git a/sql/pgq/functions/pgq.failed_queue.sql b/sql/pgq/functions/pgq.failed_queue.sql new file mode 100644 index 00000000..0ae02043 --- /dev/null +++ b/sql/pgq/functions/pgq.failed_queue.sql @@ -0,0 +1,201 @@ + +create or replace function pgq.failed_event_list( + x_queue_name text, + x_consumer_name text) +returns setof pgq.failed_queue as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.failed_event_list(2) +-- +-- Get list of all failed events for one consumer. +-- +-- Parameters: +-- x_queue_name - Queue name +-- x_consumer_name - Consumer name +-- +-- Returns: +-- List of failed events. +-- ---------------------------------------------------------------------- +declare + rec pgq.failed_queue%rowtype; +begin + for rec in + select fq.* + from pgq.failed_queue fq, pgq.consumer, + pgq.queue, pgq.subscription + where queue_name = x_queue_name + and co_name = x_consumer_name + and sub_consumer = co_id + and sub_queue = queue_id + and ev_owner = sub_id + order by ev_id + loop + return next rec; + end loop; + return; +end; +$$ language plpgsql security definer; + +create or replace function pgq.failed_event_list( + x_queue_name text, + x_consumer_name text, + x_count integer, + x_offset integer) +returns setof pgq.failed_queue as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.failed_event_list(4) +-- +-- Get list of failed events, from offset and specific count. +-- +-- Parameters: +-- x_queue_name - Queue name +-- x_consumer_name - Consumer name +-- x_count - Max amount of events to fetch +-- x_offset - From this offset +-- +-- Returns: +-- List of failed events. +-- ---------------------------------------------------------------------- +declare + rec pgq.failed_queue%rowtype; +begin + for rec in + select fq.* + from pgq.failed_queue fq, pgq.consumer, + pgq.queue, pgq.subscription + where queue_name = x_queue_name + and co_name = x_consumer_name + and sub_consumer = co_id + and sub_queue = queue_id + and ev_owner = sub_id + order by ev_id + limit x_count + offset x_offset + loop + return next rec; + end loop; + return; +end; +$$ language plpgsql security definer; + +create or replace function pgq.failed_event_count( + x_queue_name text, + x_consumer_name text) +returns integer as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.failed_event_count(2) +-- +-- Get size of failed event queue. +-- +-- Parameters: +-- x_queue_name - Queue name +-- x_consumer_name - Consumer name +-- +-- Returns: +-- Number of failed events in failed event queue. +-- ---------------------------------------------------------------------- +declare + ret integer; +begin + select count(1) into ret + from pgq.failed_queue, pgq.consumer, pgq.queue, pgq.subscription + where queue_name = x_queue_name + and co_name = x_consumer_name + and sub_queue = queue_id + and sub_consumer = co_id + and ev_owner = sub_id; + return ret; +end; +$$ language plpgsql security definer; + +create or replace function pgq.failed_event_delete( + x_queue_name text, + x_consumer_name text, + x_event_id bigint) +returns integer as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.failed_event_delete(3) +-- +-- Delete specific event from failed event queue. +-- +-- Parameters: +-- x_queue_name - Queue name +-- x_consumer_name - Consumer name +-- x_event_id - Event ID +-- +-- Returns: +-- nothing +-- ---------------------------------------------------------------------- +declare + x_sub_id integer; +begin + select sub_id into x_sub_id + from pgq.subscription, pgq.consumer, pgq.queue + where queue_name = x_queue_name + and co_name = x_consumer_name + and sub_consumer = co_id + and sub_queue = queue_id; + if not found then + raise exception 'no such queue/consumer'; + end if; + + delete from pgq.failed_queue + where ev_owner = x_sub_id + and ev_id = x_event_id; + if not found then + raise exception 'event not found'; + end if; + + return 1; +end; +$$ language plpgsql security definer; + +create or replace function pgq.failed_event_retry( + x_queue_name text, + x_consumer_name text, + x_event_id bigint) +returns bigint as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.failed_event_retry(3) +-- +-- Insert specific event from failed queue to main queue. +-- +-- Parameters: +-- x_queue_name - Queue name +-- x_consumer_name - Consumer name +-- x_event_id - Event ID +-- +-- Returns: +-- nothing +-- ---------------------------------------------------------------------- +declare + ret bigint; + x_sub_id integer; +begin + select sub_id into x_sub_id + from pgq.subscription, pgq.consumer, pgq.queue + where queue_name = x_queue_name + and co_name = x_consumer_name + and sub_consumer = co_id + and sub_queue = queue_id; + if not found then + raise exception 'no such queue/consumer'; + end if; + + select pgq.insert_event_raw(x_queue_name, ev_id, ev_time, + ev_owner, ev_retry, ev_type, ev_data, + ev_extra1, ev_extra2, ev_extra3, ev_extra4) + into ret + from pgq.failed_queue, pgq.consumer, pgq.queue + where ev_owner = x_sub_id + and ev_id = x_event_id; + if not found then + raise exception 'event not found'; + end if; + + perform pgq.failed_event_delete(x_queue_name, x_consumer_name, x_event_id); + + return ret; +end; +$$ language plpgsql security definer; + + diff --git a/sql/pgq/functions/pgq.finish_batch.sql b/sql/pgq/functions/pgq.finish_batch.sql new file mode 100644 index 00000000..6ff4b28f --- /dev/null +++ b/sql/pgq/functions/pgq.finish_batch.sql @@ -0,0 +1,32 @@ + +create or replace function pgq.finish_batch( + x_batch_id bigint) +returns integer as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.finish_batch(1) +-- +-- Closes a batch. No more operations can be done with events +-- of this batch. +-- +-- Parameters: +-- x_batch_id - id of batch. +-- +-- Returns: +-- If batch 1 if batch was found, 0 otherwise. +-- ---------------------------------------------------------------------- +begin + update pgq.subscription + set sub_active = now(), + sub_last_tick = sub_next_tick, + sub_next_tick = null, + sub_batch = null + where sub_batch = x_batch_id; + if not found then + raise warning 'finish_batch: batch % not found', x_batch_id; + return 0; + end if; + + return 1; +end; +$$ language plpgsql security definer; + diff --git a/sql/pgq/functions/pgq.get_batch_events.sql b/sql/pgq/functions/pgq.get_batch_events.sql new file mode 100644 index 00000000..8166d519 --- /dev/null +++ b/sql/pgq/functions/pgq.get_batch_events.sql @@ -0,0 +1,26 @@ +create or replace function pgq.get_batch_events(x_batch_id bigint) +returns setof pgq.ret_batch_event as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.get_batch_events(1) +-- +-- Get all events in batch. +-- +-- Parameters: +-- x_batch_id - ID of active batch. +-- +-- Returns: +-- List of events. +-- ---------------------------------------------------------------------- +declare + rec pgq.ret_batch_event%rowtype; + sql text; +begin + sql := pgq.batch_event_sql(x_batch_id); + for rec in execute sql loop + return next rec; + end loop; + return; +end; +$$ language plpgsql; -- no perms needed + + diff --git a/sql/pgq/functions/pgq.get_batch_info.sql b/sql/pgq/functions/pgq.get_batch_info.sql new file mode 100644 index 00000000..617e588d --- /dev/null +++ b/sql/pgq/functions/pgq.get_batch_info.sql @@ -0,0 +1,36 @@ + +create or replace function pgq.get_batch_info(x_batch_id bigint) +returns pgq.ret_batch_info as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.get_batch_info(1) +-- +-- Returns detailed info about a batch. +-- +-- Parameters: +-- x_batch_id - id of a active batch. +-- +-- Returns: +-- Info +-- ---------------------------------------------------------------------- +declare + ret pgq.ret_batch_info%rowtype; +begin + select queue_name, co_name, + prev.tick_time as batch_start, + cur.tick_time as batch_end, + sub_last_tick, sub_next_tick, + current_timestamp - cur.tick_time as lag + into ret + from pgq.subscription, pgq.tick cur, pgq.tick prev, + pgq.queue, pgq.consumer + where sub_batch = x_batch_id + and prev.tick_id = sub_last_tick + and prev.tick_queue = sub_queue + and cur.tick_id = sub_next_tick + and cur.tick_queue = sub_queue + and queue_id = sub_queue + and co_id = sub_consumer; + return ret; +end; +$$ language plpgsql security definer; + diff --git a/sql/pgq/functions/pgq.get_consumer_info.sql b/sql/pgq/functions/pgq.get_consumer_info.sql new file mode 100644 index 00000000..3444421d --- /dev/null +++ b/sql/pgq/functions/pgq.get_consumer_info.sql @@ -0,0 +1,108 @@ + +------------------------------------------------------------------------- +create or replace function pgq.get_consumer_info() +returns setof pgq.ret_consumer_info as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.get_consumer_info(0) +-- +-- Desc +-- +-- Parameters: +-- arg - desc +-- +-- Returns: +-- desc +-- ---------------------------------------------------------------------- +declare + ret pgq.ret_consumer_info%rowtype; + i record; +begin + for i in select queue_name from pgq.queue order by 1 + loop + for ret in + select * from pgq.get_consumer_info(i.queue_name) + loop + return next ret; + end loop; + end loop; + return; +end; +$$ language plpgsql security definer; + + +------------------------------------------------------------------------- +create or replace function pgq.get_consumer_info(x_queue_name text) +returns setof pgq.ret_consumer_info as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.get_consumer_info(1) +-- +-- Desc +-- +-- Parameters: +-- arg - desc +-- +-- Returns: +-- desc +-- ---------------------------------------------------------------------- +declare + ret pgq.ret_consumer_info%rowtype; + tmp record; +begin + for tmp in + select queue_name, co_name + from pgq.queue, pgq.consumer, pgq.subscription + where queue_id = sub_queue + and co_id = sub_consumer + and queue_name = x_queue_name + order by 1, 2 + loop + for ret in + select * from pgq.get_consumer_info(tmp.queue_name, tmp.co_name) + loop + return next ret; + end loop; + end loop; + return; +end; +$$ language plpgsql security definer; + + +------------------------------------------------------------------------ +create or replace function pgq.get_consumer_info( + x_queue_name text, + x_consumer_name text) +returns setof pgq.ret_consumer_info as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.get_consumer_info(2) +-- +-- Get info about particular consumer on particular queue. +-- +-- Parameters: +-- x_queue_name - name of a queue. +-- x_consumer_name - name of a consumer +-- +-- Returns: +-- info +-- ---------------------------------------------------------------------- +declare + ret pgq.ret_consumer_info%rowtype; +begin + for ret in + select queue_name, co_name, + current_timestamp - tick_time as lag, + current_timestamp - sub_active as last_seen + from pgq.subscription, pgq.tick, pgq.queue, pgq.consumer + where tick_id = sub_last_tick + and queue_id = sub_queue + and tick_queue = sub_queue + and co_id = sub_consumer + and queue_name = x_queue_name + and co_name = x_consumer_name + order by 1,2 + loop + return next ret; + end loop; + return; +end; +$$ language plpgsql security definer; + diff --git a/sql/pgq/functions/pgq.get_queue_info.sql b/sql/pgq/functions/pgq.get_queue_info.sql new file mode 100644 index 00000000..097d1a20 --- /dev/null +++ b/sql/pgq/functions/pgq.get_queue_info.sql @@ -0,0 +1,51 @@ +create or replace function pgq.get_queue_info() +returns setof pgq.ret_queue_info as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.get_queue_info(0) +-- +-- Get info about all queues. +-- +-- Returns: +-- List of pgq.ret_queue_info records. +-- ---------------------------------------------------------------------- +declare + qname text; + ret pgq.ret_queue_info%rowtype; +begin + for qname in + select queue_name from pgq.queue order by 1 + loop + select * into ret from pgq.get_queue_info(qname); + return next ret; + end loop; + return; +end; +$$ language plpgsql security definer; + +create or replace function pgq.get_queue_info(qname text) +returns pgq.ret_queue_info as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.get_queue_info(1) +-- +-- Get info about particular queue. +-- +-- Returns: +-- One pgq.ret_queue_info record. +-- ---------------------------------------------------------------------- +declare + ret pgq.ret_queue_info%rowtype; +begin + select queue_name, queue_ntables, queue_cur_table, + queue_rotation_period, queue_switch_time, + queue_external_ticker, + queue_ticker_max_count, queue_ticker_max_lag, + queue_ticker_idle_period, + (select current_timestamp - tick_time + from pgq.tick where tick_queue = queue_id + order by tick_queue desc, tick_id desc limit 1 + ) as ticker_lag + into ret from pgq.queue where queue_name = qname; + return ret; +end; +$$ language plpgsql security definer; + diff --git a/sql/pgq/functions/pgq.grant_perms.sql b/sql/pgq/functions/pgq.grant_perms.sql new file mode 100644 index 00000000..d2c00837 --- /dev/null +++ b/sql/pgq/functions/pgq.grant_perms.sql @@ -0,0 +1,37 @@ +create or replace function pgq.grant_perms(x_queue_name text) +returns integer as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.grant_perms(1) +-- +-- Make event tables readable by public. +-- +-- Parameters: +-- x_queue_name - Name of the queue. +-- +-- Returns: +-- nothing +-- ---------------------------------------------------------------------- +declare + q record; + i integer; +begin + select * from pgq.queue into q + where queue_name = x_queue_name; + if not found then + raise exception 'Queue not found'; + end if; + execute 'grant select, update on ' + || q.queue_event_seq || ',' || q.queue_tick_seq + || ' to public'; + execute 'grant select on ' + || q.queue_data_pfx + || ' to public'; + for i in 0 .. q.queue_ntables - 1 loop + execute 'grant select, insert on ' + || q.queue_data_pfx || '_' || i + || ' to public'; + end loop; + return 1; +end; +$$ language plpgsql security definer; + diff --git a/sql/pgq/functions/pgq.insert_event.sql b/sql/pgq/functions/pgq.insert_event.sql new file mode 100644 index 00000000..2adfcbc0 --- /dev/null +++ b/sql/pgq/functions/pgq.insert_event.sql @@ -0,0 +1,49 @@ +create or replace function pgq.insert_event(queue_name text, ev_type text, ev_data text) +returns bigint as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.insert_event(3) +-- +-- Insert a event into queue. +-- +-- Parameters: +-- queue_name - Name of the queue +-- ev_type - User-specified type for the event +-- ev_data - User data for the event +-- +-- Returns: +-- Event ID +-- ---------------------------------------------------------------------- +begin + return pgq.insert_event(queue_name, ev_type, ev_data, null, null, null, null); +end; +$$ language plpgsql; -- event inserting needs no special perms + + + +create or replace function pgq.insert_event( + queue_name text, ev_type text, ev_data text, + ev_extra1 text, ev_extra2 text, ev_extra3 text, ev_extra4 text) +returns bigint as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.insert_event(7) +-- +-- Insert a event into queue with all the extra fields. +-- +-- Parameters: +-- queue_name - Name of the queue +-- ev_type - User-specified type for the event +-- ev_data - User data for the event +-- ev_extra1 - Extra data field for the event +-- ev_extra2 - Extra data field for the event +-- ev_extra3 - Extra data field for the event +-- ev_extra4 - Extra data field for the event +-- +-- Returns: +-- Event ID +-- ---------------------------------------------------------------------- +begin + return pgq.insert_event_raw(queue_name, null, now(), null, null, + ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4); +end; +$$ language plpgsql; -- event inserting needs no special perms + diff --git a/sql/pgq/functions/pgq.insert_event_raw.sql b/sql/pgq/functions/pgq.insert_event_raw.sql new file mode 100644 index 00000000..faca04a2 --- /dev/null +++ b/sql/pgq/functions/pgq.insert_event_raw.sql @@ -0,0 +1,87 @@ +create or replace function pgq.insert_event_raw( + queue_name text, ev_id bigint, ev_time timestamptz, + ev_owner integer, ev_retry integer, ev_type text, ev_data text, + ev_extra1 text, ev_extra2 text, ev_extra3 text, ev_extra4 text) +returns bigint as $$ +# -- ---------------------------------------------------------------------- +# -- Function: pgq.insert_event_raw(11) +# -- +# -- Actual event insertion. Used also by retry queue maintenance. +# -- +# -- Parameters: +# -- queue_name - Name of the queue +# -- ev_id - Event ID. If NULL, will be taken from seq. +# -- ev_time - Event creation time. +# -- ev_type - user data +# -- ev_data - user data +# -- ev_extra1 - user data +# -- ev_extra2 - user data +# -- ev_extra3 - user data +# -- ev_extra4 - user data +# -- +# -- Returns: +# -- Event ID. +# -- ---------------------------------------------------------------------- + + # load args + queue_name = args[0] + ev_id = args[1] + ev_time = args[2] + ev_owner = args[3] + ev_retry = args[4] + ev_type = args[5] + ev_data = args[6] + ev_extra1 = args[7] + ev_extra2 = args[8] + ev_extra3 = args[9] + ev_extra4 = args[10] + + if not "cf_plan" in SD: + # get current event table + q = "select queue_data_pfx, queue_cur_table, queue_event_seq "\ + " from pgq.queue where queue_name = $1" + SD["cf_plan"] = plpy.prepare(q, ["text"]) + + # get next id + q = "select nextval($1) as id" + SD["seq_plan"] = plpy.prepare(q, ["text"]) + + # get queue config + res = plpy.execute(SD["cf_plan"], [queue_name]) + if len(res) != 1: + plpy.error("Unknown event queue: %s" % (queue_name)) + tbl_prefix = res[0]["queue_data_pfx"] + cur_nr = res[0]["queue_cur_table"] + id_seq = res[0]["queue_event_seq"] + + # get id - bump seq even if id is given + res = plpy.execute(SD['seq_plan'], [id_seq]) + if ev_id is None: + ev_id = res[0]["id"] + + # create plan for insertion + ins_plan = None + ins_key = "ins.%s" % (queue_name) + if ins_key in SD: + nr, ins_plan = SD[ins_key] + if nr != cur_nr: + ins_plan = None + if ins_plan == None: + q = "insert into %s_%s (ev_id, ev_time, ev_owner, ev_retry,"\ + " ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4)"\ + " values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)" % ( + tbl_prefix, cur_nr) + types = ["int8", "timestamptz", "int4", "int4", "text", + "text", "text", "text", "text", "text"] + ins_plan = plpy.prepare(q, types) + SD[ins_key] = (cur_nr, ins_plan) + + # insert the event + plpy.execute(ins_plan, [ev_id, ev_time, ev_owner, ev_retry, ev_type, ev_data, + ev_extra1, ev_extra2, ev_extra3, ev_extra4]) + + # done + return ev_id + +$$ language plpythonu; -- event inserting needs no special perms + diff --git a/sql/pgq/functions/pgq.maint_retry_events.sql b/sql/pgq/functions/pgq.maint_retry_events.sql new file mode 100644 index 00000000..f3038b86 --- /dev/null +++ b/sql/pgq/functions/pgq.maint_retry_events.sql @@ -0,0 +1,42 @@ +create or replace function pgq.maint_retry_events() +returns integer as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.maint_retry_events(0) +-- +-- Moves retry events back to main queue. +-- +-- It moves small amount at a time. It should be called +-- until it returns 0 +-- +-- Parameters: +-- arg - desc +-- +-- Returns: +-- Number of events processed. +-- ---------------------------------------------------------------------- +declare + cnt integer; + rec record; +begin + cnt := 0; + for rec in + select pgq.insert_event_raw(queue_name, + ev_id, ev_time, ev_owner, ev_retry, ev_type, ev_data, + ev_extra1, ev_extra2, ev_extra3, ev_extra4), + ev_owner, ev_id + from pgq.retry_queue, pgq.queue, pgq.subscription + where ev_retry_after <= current_timestamp + and sub_id = ev_owner + and queue_id = sub_queue + order by ev_retry_after + limit 10 + loop + cnt := cnt + 1; + delete from pgq.retry_queue + where ev_owner = rec.ev_owner + and ev_id = rec.ev_id; + end loop; + return cnt; +end; +$$ language plpgsql; -- need admin access + diff --git a/sql/pgq/functions/pgq.maint_rotate_tables.sql b/sql/pgq/functions/pgq.maint_rotate_tables.sql new file mode 100644 index 00000000..0195b605 --- /dev/null +++ b/sql/pgq/functions/pgq.maint_rotate_tables.sql @@ -0,0 +1,98 @@ +create or replace function pgq.maint_rotate_tables_step1(i_queue_name text) +returns integer as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.maint_rotate_tables_step1(1) +-- +-- Rotate tables for one queue. +-- +-- Parameters: +-- i_queue_name - Name of the queue +-- +-- Returns: +-- nothing +-- ---------------------------------------------------------------------- +declare + badcnt integer; + cf record; + nr integer; + tbl text; +begin + -- check if needed and load record + select * from pgq.queue into cf + where queue_name = i_queue_name + and queue_rotation_period is not null + and queue_switch_step2 is not null + and queue_switch_time + queue_rotation_period < current_timestamp + for update; + if not found then + return 0; + end if; + + -- check if any consumer is on previous table + select coalesce(count(*), 0) into badcnt + from pgq.subscription, pgq.tick + where get_snapshot_xmin(tick_snapshot) < cf.queue_switch_step2 + and sub_queue = cf.queue_id + and tick_queue = cf.queue_id + and tick_id = (select tick_id from pgq.tick + where tick_id < sub_last_tick + and tick_queue = sub_queue + order by tick_queue desc, tick_id desc + limit 1); + if badcnt > 0 then + return 0; + end if; + + -- all is fine, calc next table number + nr := cf.queue_cur_table + 1; + if nr = cf.queue_ntables then + nr := 0; + end if; + tbl := cf.table_name || '_' || nr; + + -- there may be long lock on the table from pg_dump, + -- detect it and skip rotate then + begin + execute 'lock table ' || tbl || ' nowait'; + execute 'truncate ' || tbl; + exception + when lock_not_available then + raise warning 'truncate of % failed, skipping rotate', tbl; + return 0; + end; + + -- remember the moment + update pgq.queue + set queue_cur_table = nr, + queue_switch_time = current_timestamp, + queue_switch_step1 = get_current_txid(), + queue_switch_step2 = NULL + where queue_id = cf.queue_id; + + -- clean ticks - avoid partial batches + delete from pgq.tick + where tick_queue = cf.queue_id + and get_snapshot_xmin(snapshot) < cf.last_switch_step2; + + return 1; +end; +$$ language plpgsql; -- need admin access + +-- ---------------------------------------------------------------------- +-- Function: pgq.maint_rotate_tables_step2(0) +-- +-- It tag rotation as finished where needed. It should be +-- called in separate transaction than pgq.maint_rotate_tables_step1() +-- ---------------------------------------------------------------------- +create or replace function pgq.maint_rotate_tables_step2() +returns integer as $$ +-- visibility tracking. this should run in separate +-- tranaction than step1 +begin + update pgq.queue + set queue_switch_step2 = get_current_txid() + where queue_switch_step2 is null; + return 1; +end; +$$ language plpgsql; -- need admin access + diff --git a/sql/pgq/functions/pgq.maint_tables_to_vacuum.sql b/sql/pgq/functions/pgq.maint_tables_to_vacuum.sql new file mode 100644 index 00000000..920f68ec --- /dev/null +++ b/sql/pgq/functions/pgq.maint_tables_to_vacuum.sql @@ -0,0 +1,33 @@ +create or replace function pgq.maint_tables_to_vacuum() +returns setof text as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.maint_tables_to_vacuum(0) +-- +-- Returns list of tablenames that need frequent vacuuming. +-- +-- The goal is to avoid hardcoding them into maintenance process. +-- +-- Returns: +-- List of table names. +-- ---------------------------------------------------------------------- +begin + return next 'pgq.subscription'; + return next 'pgq.consumer'; + return next 'pgq.queue'; + return next 'pgq.tick'; + return next 'pgq.retry_queue'; + + -- vacuum also txid.epoch, if exists + perform 1 from pg_class t, pg_namespace n + where t.relname = 'epoch' + and n.nspname = 'txid' + and n.oid = t.relnamespace; + if found then + return next 'txid.epoch'; + end if; + + return; +end; +$$ language plpgsql; + + diff --git a/sql/pgq/functions/pgq.next_batch.sql b/sql/pgq/functions/pgq.next_batch.sql new file mode 100644 index 00000000..8d7d8f74 --- /dev/null +++ b/sql/pgq/functions/pgq.next_batch.sql @@ -0,0 +1,66 @@ +create or replace function pgq.next_batch(x_queue_name text, x_consumer_name text) +returns bigint as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.next_batch(2) +-- +-- Makes next block of events active. +-- +-- If it returns NULL, there is no events available in queue. +-- Consumer should sleep a bith then. +-- +-- Parameters: +-- x_queue_name - Name of the queue +-- x_consumer_name - Name of the consumer +-- +-- Returns: +-- Batch ID or NULL if there are no more events available. +-- ---------------------------------------------------------------------- +declare + next_tick bigint; + next_batch bigint; + errmsg text; + sub record; +begin + select sub_queue, sub_id, sub_last_tick, sub_batch into sub + from pgq.queue q, pgq.consumer c, pgq.subscription s + where q.queue_name = x_queue_name + and c.co_name = x_consumer_name + and s.sub_queue = q.queue_id + and s.sub_consumer = c.co_id; + if not found then + errmsg := 'Not subscriber to queue: ' + || coalesce(x_queue_name, 'NULL') + || '/' + || coalesce(x_consumer_name, 'NULL'); + raise exception '%', errmsg; + end if; + + -- has already active batch + if sub.sub_batch is not null then + return sub.sub_batch; + end if; + + -- find next tick + select tick_id into next_tick + from pgq.tick + where tick_id > sub.sub_last_tick + and tick_queue = sub.sub_queue + order by tick_queue asc, tick_id asc + limit 1; + if not found then + -- nothing to do + return null; + end if; + + -- get next batch + next_batch := nextval('pgq.batch_id_seq'); + update pgq.subscription + set sub_batch = next_batch, + sub_next_tick = next_tick, + sub_active = now() + where sub_id = sub.sub_id; + return next_batch; +end; +$$ language plpgsql security definer; + + diff --git a/sql/pgq/functions/pgq.register_consumer.sql b/sql/pgq/functions/pgq.register_consumer.sql new file mode 100644 index 00000000..7d387dab --- /dev/null +++ b/sql/pgq/functions/pgq.register_consumer.sql @@ -0,0 +1,120 @@ +create or replace function pgq.register_consumer( + x_queue_name text, + x_consumer_id text) +returns integer as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.register_consumer(2) +-- +-- Subscribe consumer on a queue. +-- +-- From this moment forward, consumer will see all events in the queue. +-- +-- Parameters: +-- x_queue_name - Name of queue +-- x_consumer_name - Name of consumer +-- +-- Returns: +-- 0 - if already registered +-- 1 - if new registration +-- ---------------------------------------------------------------------- +begin + return pgq.register_consumer(x_queue_name, x_consumer_id, NULL); +end; +$$ language plpgsql; -- no perms needed + + +create or replace function pgq.register_consumer( + x_queue_name text, + x_consumer_name text, + x_tick_pos bigint) +returns integer as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.register_consumer(3) +-- +-- Extended registration, allows to specify tick_id. +-- +-- Note: +-- For usage in special situations. +-- +-- Parameters: +-- x_queue_name - Name of a queue +-- x_consumer_name - Name of consumer +-- x_tick_pos - Tick ID +-- +-- Returns: +-- 0/1 whether consumer has already registered. +-- ---------------------------------------------------------------------- +declare + tmp text; + last_tick bigint; + x_queue_id integer; + x_consumer_id integer; + queue integer; + sub record; +begin + select queue_id into x_queue_id from pgq.queue + where queue_name = x_queue_name; + if not found then + raise exception 'Event queue not created yet'; + end if; + + -- get consumer and create if new + select co_id into x_consumer_id from pgq.consumer + where co_name = x_consumer_name; + if not found then + insert into pgq.consumer (co_name) values (x_consumer_name); + x_consumer_id := currval('pgq.consumer_co_id_seq'); + end if; + + -- if particular tick was requested, check if it exists + if x_tick_pos is not null then + perform 1 from pgq.tick + where tick_queue = x_queue_id + and tick_id = x_tick_pos; + if not found then + raise exception 'cannot reposition, tick not found: %', x_tick_pos; + end if; + end if; + + -- check if already registered + select sub_last_tick, sub_batch into sub + from pgq.subscription + where sub_consumer = x_consumer_id + and sub_queue = x_queue_id; + if found then + if sub.sub_batch is not null then + raise exception 'reposition while active not allowed'; + end if; + if x_tick_pos is not null then + -- update tick pos if requested + update pgq.subscription + set sub_last_tick = x_tick_pos + where sub_consumer = x_consumer_id + and sub_queue = x_queue_id; + end if; + -- already registered + return 0; + end if; + + -- new registration + if x_tick_pos is null then + -- start from current tick + select tick_id into last_tick from pgq.tick + where tick_queue = x_queue_id + order by tick_queue desc, tick_id desc + limit 1; + if not found then + raise exception 'No ticks for this queue. Please run ticker on database.'; + end if; + else + last_tick := x_tick_pos; + end if; + + -- register + insert into pgq.subscription (sub_queue, sub_consumer, sub_last_tick) + values (x_queue_id, x_consumer_id, last_tick); + return 1; +end; +$$ language plpgsql security definer; + + diff --git a/sql/pgq/functions/pgq.ticker.sql b/sql/pgq/functions/pgq.ticker.sql new file mode 100644 index 00000000..9489d195 --- /dev/null +++ b/sql/pgq/functions/pgq.ticker.sql @@ -0,0 +1,86 @@ +create or replace function pgq.ticker(i_queue_name text, i_tick_id bigint) +returns bigint as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.ticker(2) +-- +-- Insert a tick with a particular tick_id. +-- +-- For external tickers. +-- +-- Parameters: +-- i_queue_name - Name of the queue +-- i_tick_id - Id of new tick. +-- +-- Returns: +-- Tick id. +-- ---------------------------------------------------------------------- +begin + insert into pgq.tick (tick_queue, tick_id) + select queue_id, i_tick_id + from pgq.queue + where queue_name = i_queue_name + and queue_external_ticker; + if not found then + raise exception 'queue not found'; + end if; + return i_tick_id; +end; +$$ language plpgsql security definer; -- unsure about access + +create or replace function pgq.ticker(i_queue_name text) +returns bigint as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.ticker(1) +-- +-- Insert a tick with a tick_id from sequence. +-- +-- For pgqadm usage. +-- +-- Parameters: +-- i_queue_name - Name of the queue +-- +-- Returns: +-- Tick id. +-- ---------------------------------------------------------------------- +declare + res bigint; + ext boolean; + seq text; + q record; +begin + select queue_id, queue_tick_seq, queue_external_ticker into q + from pgq.queue where queue_name = i_queue_name; + if not found then + raise exception 'no such queue'; + end if; + + if q.queue_external_ticker then + raise exception 'This queue has external tick source.'; + end if; + + insert into pgq.tick (tick_queue, tick_id) + values (q.queue_id, nextval(q.queue_tick_seq)); + + res = currval(q.queue_tick_seq); + return res; +end; +$$ language plpgsql security definer; -- unsure about access + +create or replace function pgq.ticker() returns bigint as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.ticker(0) +-- +-- Creates ticks for all queues which dont have external ticker. +-- +-- Returns: +-- Number of queues that were processed. +-- ---------------------------------------------------------------------- +declare + res bigint; +begin + select count(pgq.ticker(queue_name)) into res + from pgq.queue where not queue_external_ticker; + return res; +end; +$$ language plpgsql security definer; + diff --git a/sql/pgq/functions/pgq.unregister_consumer.sql b/sql/pgq/functions/pgq.unregister_consumer.sql new file mode 100644 index 00000000..c97261d6 --- /dev/null +++ b/sql/pgq/functions/pgq.unregister_consumer.sql @@ -0,0 +1,44 @@ + +create or replace function pgq.unregister_consumer( + x_queue_name text, + x_consumer_name text) +returns integer as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.unregister_consumer(2) +-- +-- Unsubscriber consumer from the queue. Also consumer's failed +-- and retry events are deleted. +-- +-- Parameters: +-- x_queue_name - Name of the queue +-- x_consumer_name - Name of the consumer +-- +-- Returns: +-- nothing +-- ---------------------------------------------------------------------- +declare + x_sub_id integer; +begin + select sub_id into x_sub_id + from pgq.subscription, pgq.consumer, pgq.queue + where sub_queue = queue_id + and sub_consumer = co_id + and queue_name = x_queue_name + and co_name = x_consumer_name; + if not found then + raise exception 'consumer not registered on queue'; + end if; + + delete from pgq.retry_queue + where ev_owner = x_sub_id; + + delete from pgq.failed_queue + where ev_owner = x_sub_id; + + delete from pgq.subscription + where sub_id = x_sub_id; + + return 1; +end; +$$ language plpgsql security definer; + diff --git a/sql/pgq/functions/pgq.version.sql b/sql/pgq/functions/pgq.version.sql new file mode 100644 index 00000000..d48f9e18 --- /dev/null +++ b/sql/pgq/functions/pgq.version.sql @@ -0,0 +1,12 @@ +-- ---------------------------------------------------------------------- +-- Function: pgq.version(0) +-- +-- Returns verison string for pgq. +-- ---------------------------------------------------------------------- +create or replace function pgq.version() +returns text as $$ +begin + return '2.1'; +end; +$$ language plpgsql; + diff --git a/sql/pgq/sql/logutriga.sql b/sql/pgq/sql/logutriga.sql new file mode 100644 index 00000000..e2a90995 --- /dev/null +++ b/sql/pgq/sql/logutriga.sql @@ -0,0 +1,22 @@ + +create or replace function pgq.insert_event(que text, ev_type text, ev_data text, x1 text, x2 text, x3 text, x4 text) +returns bigint as $$ +begin + raise notice 'insert_event(%, %, %, %)', que, ev_type, ev_data, x1; + return 1; +end; +$$ language plpgsql; + +create table udata ( + id serial primary key, + txt text, + bin bytea +); + +create trigger utest AFTER insert or update or delete ON udata +for each row execute procedure pgq.logutriga('udata_que'); + +insert into udata (txt) values ('text1'); +insert into udata (bin) values (E'bi\tn\\000bin'); + + diff --git a/sql/pgq/sql/pgq_init.sql b/sql/pgq/sql/pgq_init.sql new file mode 100644 index 00000000..95f46459 --- /dev/null +++ b/sql/pgq/sql/pgq_init.sql @@ -0,0 +1,66 @@ + +\set ECHO none +\i ../txid/txid.sql +\i structure/install.sql + +\set ECHO all + +select * from pgq.maint_tables_to_vacuum(); +select * from pgq.maint_retry_events(); + +select pgq.create_queue('tmpqueue'); +select pgq.register_consumer('tmpqueue', 'consumer'); +select pgq.unregister_consumer('tmpqueue', 'consumer'); +select pgq.drop_queue('tmpqueue'); + +select pgq.create_queue('myqueue'); +select pgq.register_consumer('myqueue', 'consumer'); +select pgq.next_batch('myqueue', 'consumer'); +select pgq.next_batch('myqueue', 'consumer'); +select pgq.ticker(); +select pgq.next_batch('myqueue', 'consumer'); +select pgq.next_batch('myqueue', 'consumer'); + +select queue_name, consumer_name, prev_tick_id, tick_id, lag from pgq.get_batch_info(1); +select queue_name from pgq.get_queue_info() limit 0; +select queue_name, consumer_name from pgq.get_consumer_info() limit 0; + +select pgq.finish_batch(1); +select pgq.finish_batch(1); + +select pgq.ticker(); +select pgq.next_batch('myqueue', 'consumer'); +select * from pgq.batch_event_tables(2); +select * from pgq.get_batch_events(2); +select pgq.finish_batch(2); + +select pgq.insert_event('myqueue', 'r1', 'data'); +select pgq.insert_event('myqueue', 'r2', 'data'); +select pgq.insert_event('myqueue', 'r3', 'data'); +select pgq.current_event_table('myqueue'); +select pgq.ticker(); + +select pgq.next_batch('myqueue', 'consumer'); +select ev_id,ev_retry,ev_type,ev_data,ev_extra1,ev_extra2,ev_extra3,ev_extra4 from pgq.get_batch_events(3); + +select * from pgq.failed_event_list('myqueue', 'consumer'); + +select pgq.event_failed(3, 1, 'failure test'); +select pgq.event_failed(3, 1, 'failure test'); +select pgq.event_retry(3, 2, 0); +select pgq.event_retry(3, 2, 0); +select pgq.finish_batch(3); + +select ev_failed_reason, ev_id, ev_txid, ev_retry, ev_type, ev_data + from pgq.failed_event_list('myqueue', 'consumer'); + +select ev_failed_reason, ev_id, ev_txid, ev_retry, ev_type, ev_data + from pgq.failed_event_list('myqueue', 'consumer', 0, 1); + +select * from pgq.failed_event_count('myqueue', 'consumer'); +select * from pgq.failed_event_delete('myqueue', 'consumer', 0); + +select pgq.event_retry_raw('myqueue', 'consumer', now(), 666, now(), 0, + 'rawtest', 'data', null, null, null, null); + +select pgq.ticker(); diff --git a/sql/pgq/sql/sqltriga.sql b/sql/pgq/sql/sqltriga.sql new file mode 100644 index 00000000..49b86ee7 --- /dev/null +++ b/sql/pgq/sql/sqltriga.sql @@ -0,0 +1,58 @@ + +-- start testing +create table rtest ( + id integer primary key, + dat text +); + +create trigger rtest_triga after insert or update or delete on rtest +for each row execute procedure pgq.sqltriga('que'); + +-- simple test +insert into rtest values (1, 'value1'); +update rtest set dat = 'value2'; +delete from rtest; + +-- test new fields +alter table rtest add column dat2 text; +insert into rtest values (1, 'value1'); +update rtest set dat = 'value2'; +delete from rtest; + +-- test field ignore +drop trigger rtest_triga on rtest; +create trigger rtest_triga after insert or update or delete on rtest +for each row execute procedure pgq.sqltriga('que2', 'ignore=dat2'); + +insert into rtest values (1, '666', 'newdat'); +update rtest set dat = 5, dat2 = 'newdat2'; +update rtest set dat = 6; +delete from rtest; + +-- test hashed pkey +drop trigger rtest_triga on rtest; +create trigger rtest_triga after insert or update or delete on rtest +for each row execute procedure pgq.sqltriga('que2', 'ignore=dat2&pkey=dat,hashtext(dat)'); + +insert into rtest values (1, '666', 'newdat'); +update rtest set dat = 5, dat2 = 'newdat2'; +update rtest set dat = 6; +delete from rtest; + + +-- test wrong key +drop trigger rtest_triga on rtest; +create trigger rtest_triga after insert or update or delete on rtest +for each row execute procedure pgq.sqltriga('que3'); + +insert into rtest values (1, 0, 'non-null'); +insert into rtest values (2, 0, NULL); +update rtest set dat2 = 'non-null2' where id=1; +update rtest set dat2 = NULL where id=1; +update rtest set dat2 = 'new-nonnull' where id=2; + +delete from rtest where id=1; +delete from rtest where id=2; + + + diff --git a/sql/pgq/structure/func_internal.sql b/sql/pgq/structure/func_internal.sql new file mode 100644 index 00000000..f84bb195 --- /dev/null +++ b/sql/pgq/structure/func_internal.sql @@ -0,0 +1,23 @@ +-- Section: Internal Functions + +-- Group: Low-level event handling + +\i functions/pgq.batch_event_sql.sql +\i functions/pgq.batch_event_tables.sql +\i functions/pgq.event_retry_raw.sql +\i functions/pgq.insert_event_raw.sql + +-- Group: Ticker + +\i functions/pgq.ticker.sql + +-- Group: Periodic maintenence + +\i functions/pgq.maint_retry_events.sql +\i functions/pgq.maint_rotate_tables.sql +\i functions/pgq.maint_tables_to_vacuum.sql + +-- Group: Random utility functions + +\i functions/pgq.grant_perms.sql + diff --git a/sql/pgq/structure/func_public.sql b/sql/pgq/structure/func_public.sql new file mode 100644 index 00000000..4440a22b --- /dev/null +++ b/sql/pgq/structure/func_public.sql @@ -0,0 +1,36 @@ +-- Section: Public Functions + +-- Group: Queue creation + +\i functions/pgq.create_queue.sql +\i functions/pgq.drop_queue.sql + +-- Group: Event publishing + +\i functions/pgq.insert_event.sql +\i functions/pgq.current_event_table.sql + +-- Group: Subscribing to queue + +\i functions/pgq.register_consumer.sql +\i functions/pgq.unregister_consumer.sql + +-- Group: Batch processing + +\i functions/pgq.next_batch.sql +\i functions/pgq.get_batch_events.sql +\i functions/pgq.event_failed.sql +\i functions/pgq.event_retry.sql +\i functions/pgq.finish_batch.sql + +-- Group: General info functions + +\i functions/pgq.get_queue_info.sql +\i functions/pgq.get_consumer_info.sql +\i functions/pgq.version.sql +\i functions/pgq.get_batch_info.sql + +-- Group: Failed queue browsing + +\i functions/pgq.failed_queue.sql + diff --git a/sql/pgq/structure/install.sql b/sql/pgq/structure/install.sql new file mode 100644 index 00000000..b3c77cb6 --- /dev/null +++ b/sql/pgq/structure/install.sql @@ -0,0 +1,7 @@ + +\i structure/tables.sql +\i structure/types.sql +\i structure/func_internal.sql +\i structure/func_public.sql +\i structure/triggers.sql + diff --git a/sql/pgq/structure/tables.sql b/sql/pgq/structure/tables.sql new file mode 100644 index 00000000..fc56cc81 --- /dev/null +++ b/sql/pgq/structure/tables.sql @@ -0,0 +1,217 @@ +-- ---------------------------------------------------------------------- +-- Section: Internal Tables +-- +-- Map to Slony-I: +-- sl_node - pgq.consumer +-- sl_set - pgq.queue +-- sl_subscriber + sl_confirm - pgq.subscription +-- sl_event - pgq.tick +-- sl_setsync - pgq_ext.completed_* +-- sl_log_* - slony1 has per-cluster data tables, +-- here we do redirection in pgq.queue +-- to have per-queue data tables. +-- ---------------------------------------------------------------------- + +set client_min_messages = 'warning'; + +-- drop schema if exists pgq cascade; +create schema pgq; +grant usage on schema pgq to public; + +-- ---------------------------------------------------------------------- +-- Table: pgq.consumer +-- +-- Name to id lookup for consumers +-- +-- Columns: +-- co_id - consumer's id for internal usage +-- co_name - consumer's id for external usage +-- ---------------------------------------------------------------------- +create table pgq.consumer ( + co_id serial, + co_name text not null default 'fooz', + + constraint consumer_pkey primary key (co_id), + constraint consumer_name_uq UNIQUE (co_name) +); + + +-- ---------------------------------------------------------------------- +-- Table: pgq.queue +-- +-- Information about available queues +-- +-- Columns: +-- queue_id - queue id for internal usage +-- queue_name - queue name visible outside +-- queue_data - parent table for actual data tables +-- queue_switch_step1 - tx when rotation happened +-- queue_switch_step2 - tx after rotation was committed +-- queue_switch_time - time when switch happened +-- queue_ticker_max_count - batch should not contain more events +-- queue_ticker_max_lag - events should not age more +-- queue_ticker_idle_period - how often to tick when no events happen +-- ---------------------------------------------------------------------- +create table pgq.queue ( + queue_id serial, + queue_name text not null, + + queue_ntables integer not null default 3, + queue_cur_table integer not null default 0, + queue_rotation_period interval not null default '2 hours', + queue_switch_step1 bigint not null default get_current_txid(), + queue_switch_step2 bigint default get_current_txid(), + queue_switch_time timestamptz not null default now(), + + queue_external_ticker boolean not null default false, + queue_ticker_max_count integer not null default 500, + queue_ticker_max_lag interval not null default '3 seconds', + queue_ticker_idle_period interval not null default '1 minute', + + queue_data_pfx text not null, + queue_event_seq text not null, + queue_tick_seq text not null, + + constraint queue_pkey primary key (queue_id), + constraint queue_name_uq unique (queue_name) +); + +-- ---------------------------------------------------------------------- +-- Table: pgq.tick +-- +-- Snapshots for event batching +-- +-- Columns: +-- tick_queue - queue id whose tick it is +-- tick_id - ticks id (per-queue) +-- tick_time - time when tick happened +-- tick_snapshot +-- ---------------------------------------------------------------------- +create table pgq.tick ( + tick_queue int4 not null, + tick_id bigint not null, + tick_time timestamptz not null default now(), + tick_snapshot txid_snapshot not null default get_current_snapshot(), + + constraint tick_pkey primary key (tick_queue, tick_id), + constraint tick_queue_fkey foreign key (tick_queue) + references pgq.queue (queue_id) +); + +-- ---------------------------------------------------------------------- +-- Sequence: pgq.batch_id_seq +-- +-- Sequence for batch id's. +-- ---------------------------------------------------------------------- + +create sequence pgq.batch_id_seq; +-- ---------------------------------------------------------------------- +-- Table: pgq.subscription +-- +-- Consumer registration on a queue +-- +-- Columns: +-- +-- sub_id - subscription id for internal usage +-- sub_queue - queue id +-- sub_consumer - consumer's id +-- sub_tick - last tick the consumer processed +-- sub_batch - shortcut for queue_id/consumer_id/tick_id +-- sub_next_tick - +-- ---------------------------------------------------------------------- +create table pgq.subscription ( + sub_id serial not null, + sub_queue int4 not null, + sub_consumer int4 not null, + sub_last_tick bigint not null, + sub_active timestamptz not null default now(), + sub_batch bigint, + sub_next_tick bigint, + + constraint subscription_pkey primary key (sub_id), + constraint sub_queue_fkey foreign key (sub_queue) + references pgq.queue (queue_id), + constraint sub_consumer_fkey foreign key (sub_consumer) + references pgq.consumer (co_id) +); + + +-- ---------------------------------------------------------------------- +-- Table: pgq.event_template +-- +-- Parent table for all event tables +-- +-- Columns: +-- ev_id - event's id, supposed to be unique per queue +-- ev_time - when the event was inserted +-- ev_txid - transaction id which inserted the event +-- ev_owner - subscription id that wanted to retry this +-- ev_retry - how many times the event has been retried, NULL for new events +-- ev_type - consumer/producer can specify what the data fields contain +-- ev_data - data field +-- ev_extra1 - extra data field +-- ev_extra2 - extra data field +-- ev_extra3 - extra data field +-- ev_extra4 - extra data field +-- ---------------------------------------------------------------------- +create table pgq.event_template ( + ev_id bigint not null, + ev_time timestamptz not null, + + ev_txid bigint not null default get_current_txid(), + ev_owner int4, + ev_retry int4, + + ev_type text, + ev_data text, + ev_extra1 text, + ev_extra2 text, + ev_extra3 text, + ev_extra4 text +); + +-- ---------------------------------------------------------------------- +-- Table: pgq.retry_queue +-- +-- Events to be retried +-- +-- Columns: +-- ev_retry_after - time when it should be re-inserted to main queue +-- ---------------------------------------------------------------------- +create table pgq.retry_queue ( + ev_retry_after timestamptz not null, + + like pgq.event_template, + + constraint rq_pkey primary key (ev_owner, ev_id), + constraint rq_owner_fkey foreign key (ev_owner) + references pgq.subscription (sub_id) +); +alter table pgq.retry_queue alter column ev_owner set not null; +alter table pgq.retry_queue alter column ev_txid drop not null; +create index rq_retry_idx on pgq.retry_queue (ev_retry_after); + +-- ---------------------------------------------------------------------- +-- Table: pgq.failed_queue +-- +-- Events whose processing failed +-- +-- Columns: +-- ev_failed_reason - consumer's excuse for not processing +-- ev_failed_time - when it was tagged failed +-- ---------------------------------------------------------------------- +create table pgq.failed_queue ( + ev_failed_reason text, + ev_failed_time timestamptz not null, + + -- all event fields + like pgq.event_template, + + constraint fq_pkey primary key (ev_owner, ev_id), + constraint fq_owner_fkey foreign key (ev_owner) + references pgq.subscription (sub_id) +); +alter table pgq.failed_queue alter column ev_owner set not null; +alter table pgq.failed_queue alter column ev_txid drop not null; + + diff --git a/sql/pgq/structure/triggers.sql b/sql/pgq/structure/triggers.sql new file mode 100644 index 00000000..e732347f --- /dev/null +++ b/sql/pgq/structure/triggers.sql @@ -0,0 +1,8 @@ + +-- Section: Public Triggers + +-- Group: Trigger Functions + +\i triggers/pgq.logutriga.sql +\i triggers/pgq.sqltriga.sql + diff --git a/sql/pgq/structure/types.sql b/sql/pgq/structure/types.sql new file mode 100644 index 00000000..c89ce500 --- /dev/null +++ b/sql/pgq/structure/types.sql @@ -0,0 +1,47 @@ + +create type pgq.ret_queue_info as ( + queue_name text, + queue_ntables integer, + queue_cur_table integer, + queue_rotation_period interval, + queue_switch_time timestamptz, + queue_external_ticker boolean, + queue_ticker_max_count integer, + queue_ticker_max_lag interval, + queue_ticker_idle_period interval, + ticker_lag interval +); + +create type pgq.ret_consumer_info as ( + queue_name text, + consumer_name text, + lag interval, + last_seen interval +); + +create type pgq.ret_batch_info as ( + queue_name text, + consumer_name text, + batch_start timestamptz, + batch_end timestamptz, + prev_tick_id bigint, + tick_id bigint, + lag interval +); + + +create type pgq.ret_batch_event as ( + ev_id bigint, + ev_time timestamptz, + + ev_txid bigint, + ev_retry int4, + + ev_type text, + ev_data text, + ev_extra1 text, + ev_extra2 text, + ev_extra3 text, + ev_extra4 text +); + diff --git a/sql/pgq/triggers/pgq.logutriga.sql b/sql/pgq/triggers/pgq.logutriga.sql new file mode 100644 index 00000000..d4cb0145 --- /dev/null +++ b/sql/pgq/triggers/pgq.logutriga.sql @@ -0,0 +1,103 @@ + +create or replace function pgq.logutriga() +returns trigger as $$ +# -- ---------------------------------------------------------------------- +# -- Function: pgq.logutriga() +# -- +# -- Trigger function that puts row data urlencoded into queue. +# -- +# -- Trigger parameters: +# -- arg1 - queue name +# -- arg2 - optionally 'SKIP' +# -- +# -- Queue event fields: +# -- ev_type - I/U/D +# -- ev_data - column values urlencoded +# -- ev_extra1 - table name +# -- ev_extra2 - primary key columns +# -- +# -- Regular listen trigger example: +# -- > CREATE TRIGGER triga_nimi AFTER INSERT OR UPDATE ON customer +# -- > FOR EACH ROW EXECUTE PROCEDURE pgq.logutriga('qname'); +# -- +# -- Redirect trigger example: +# -- > CREATE TRIGGER triga_nimi AFTER INSERT OR UPDATE ON customer +# -- > FOR EACH ROW EXECUTE PROCEDURE pgq.logutriga('qname', 'SKIP'); +# -- ---------------------------------------------------------------------- + +# this triger takes 1 or 2 args: +# queue_name - destination queue +# option return code (OK, SKIP) SKIP means op won't happen +# copy-paste of db_urlencode from skytools.quoting +from urllib import quote_plus +def db_urlencode(dict): + elem_list = [] + for k, v in dict.items(): + if v is None: + elem = quote_plus(str(k)) + else: + elem = quote_plus(str(k)) + '=' + quote_plus(str(v)) + elem_list.append(elem) + return '&'.join(elem_list) + +# load args +queue_name = TD['args'][0] +if len(TD['args']) > 1: + ret_code = TD['args'][1] +else: + ret_code = 'OK' +table_oid = TD['relid'] + +# on first call init plans +if not 'init_done' in SD: + # find table name + q = "SELECT n.nspname || '.' || c.relname AS table_name"\ + " FROM pg_namespace n, pg_class c"\ + " WHERE n.oid = c.relnamespace AND c.oid = $1" + SD['name_plan'] = plpy.prepare(q, ['oid']) + + # find key columns + q = "SELECT k.attname FROM pg_index i, pg_attribute k"\ + " WHERE i.indrelid = $1 AND k.attrelid = i.indexrelid"\ + " AND i.indisprimary AND k.attnum > 0 AND NOT k.attisdropped"\ + " ORDER BY k.attnum" + SD['key_plan'] = plpy.prepare(q, ['oid']) + + # insert data + q = "SELECT pgq.insert_event($1, $2, $3, $4, $5, null, null)" + SD['ins_plan'] = plpy.prepare(q, ['text', 'text', 'text', 'text', 'text']) + + # shorter tags + SD['op_map'] = {'INSERT': 'I', 'UPDATE': 'U', 'DELETE': 'D'} + + # remember init + SD['init_done'] = 1 + +# load & cache table data +if table_oid in SD: + tbl_name, tbl_keys = SD[table_oid] +else: + res = plpy.execute(SD['name_plan'], [table_oid]) + tbl_name = res[0]['table_name'] + res = plpy.execute(SD['key_plan'], [table_oid]) + tbl_keys = ",".join(map(lambda x: x['attname'], res)) + + SD[table_oid] = (tbl_name, tbl_keys) + +# prepare args +if TD['event'] == 'DELETE': + data = db_urlencode(TD['old']) +else: + data = db_urlencode(TD['new']) + +# insert event +plpy.execute(SD['ins_plan'], [ + queue_name, + SD['op_map'][TD['event']], + data, tbl_name, tbl_keys]) + +# done +return ret_code + +$$ language plpythonu; + diff --git a/sql/pgq/triggers/pgq.sqltriga.sql b/sql/pgq/triggers/pgq.sqltriga.sql new file mode 100644 index 00000000..c978e19d --- /dev/null +++ b/sql/pgq/triggers/pgq.sqltriga.sql @@ -0,0 +1,195 @@ + +-- listen trigger: +-- create trigger triga_nimi after insert or update on customer +-- for each row execute procedure pgq.sqltriga('qname'); + +-- redirect trigger: +-- create trigger triga_nimi after insert or update on customer +-- for each row execute procedure pgq.sqltriga('qname', 'ret=SKIP'); + +create or replace function pgq.sqltriga() +returns trigger as $$ +# -- ---------------------------------------------------------------------- +# -- Function: pgq.sqltriga() +# -- +# -- Trigger function that puts row data in partial SQL form into queue. +# -- +# -- Parameters: +# -- arg1 - queue name +# -- arg2 - optional urlencoded options +# -- +# -- Extra options: +# -- +# -- ret - return value for function OK/SKIP +# -- pkey - override pkey fields, can be functions +# -- ignore - comma separated field names to ignore +# -- +# -- Queue event fields: +# -- ev_type - I/U/D +# -- ev_data - partial SQL statement +# -- ev_extra1 - table name +# -- +# -- ---------------------------------------------------------------------- +# this triger takes 1 or 2 args: +# queue_name - destination queue +# args - urlencoded dict of options: +# ret - return value: OK/SKIP +# pkey - comma-separated col names or funcs on cols +# simple: pkey=user,orderno +# hashed: pkey=user,hashtext(user) +# ignore - comma-separated col names to ignore + +# on first call init stuff +if not 'init_done' in SD: + # find table name plan + q = "SELECT n.nspname || '.' || c.relname AS table_name"\ + " FROM pg_namespace n, pg_class c"\ + " WHERE n.oid = c.relnamespace AND c.oid = $1" + SD['name_plan'] = plpy.prepare(q, ['oid']) + + # find key columns plan + q = "SELECT k.attname FROM pg_index i, pg_attribute k"\ + " WHERE i.indrelid = $1 AND k.attrelid = i.indexrelid"\ + " AND i.indisprimary AND k.attnum > 0 AND NOT k.attisdropped"\ + " ORDER BY k.attnum" + SD['key_plan'] = plpy.prepare(q, ['oid']) + + # data insertion + q = "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" + SD['ins_plan'] = plpy.prepare(q, ['text', 'text', 'text', 'text']) + + # shorter tags + SD['op_map'] = {'INSERT': 'I', 'UPDATE': 'U', 'DELETE': 'D'} + + # quoting + from psycopg import QuotedString + def quote(s): + if s is None: + return "null" + s = str(s) + return str(QuotedString(s)) + s = s.replace('\\', '\\\\') + s = s.replace("'", "''") + return "'%s'" % s + + # TableInfo class + import re, urllib + class TableInfo: + func_rc = re.compile("([^(]+) [(] ([^)]+) [)]", re.I | re.X) + def __init__(self, table_oid, options_txt): + res = plpy.execute(SD['name_plan'], [table_oid]) + self.name = res[0]['table_name'] + + self.parse_options(options_txt) + self.load_pkey() + + def recheck(self, options_txt): + if self.options_txt == options_txt: + return + self.parse_options(options_txt) + self.load_pkey() + + def parse_options(self, options_txt): + self.options = {'ret': 'OK'} + if options_txt: + for s in options_txt.split('&'): + k, v = s.split('=', 1) + self.options[k] = urllib.unquote_plus(v) + self.options_txt = options_txt + + def load_pkey(self): + self.pkey_list = [] + if not 'pkey' in self.options: + res = plpy.execute(SD['key_plan'], [table_oid]) + for krow in res: + col = krow['attname'] + expr = col + "=%s" + self.pkey_list.append( (col, expr) ) + else: + for a_pk in self.options['pkey'].split(','): + m = self.func_rc.match(a_pk) + if m: + col = m.group(2) + fn = m.group(1) + expr = "%s(%s) = %s(%%s)" % (fn, col, fn) + else: + # normal case + col = a_pk + expr = col + "=%s" + self.pkey_list.append( (col, expr) ) + if len(self.pkey_list) == 0: + plpy.error('sqltriga needs primary key on table') + + def get_insert_stmt(self, new): + col_list = [] + val_list = [] + for k, v in new.items(): + col_list.append(k) + val_list.append(quote(v)) + return "(%s) values (%s)" % (",".join(col_list), ",".join(val_list)) + + def get_update_stmt(self, old, new): + chg_list = [] + for k, v in new.items(): + ov = old[k] + if v == ov: + continue + chg_list.append("%s=%s" % (k, quote(v))) + if len(chg_list) == 0: + pk = self.pkey_list[0][0] + chg_list.append("%s=%s" % (pk, quote(new[pk]))) + return "%s where %s" % (",".join(chg_list), self.get_pkey_expr(new)) + + def get_pkey_expr(self, data): + exp_list = [] + for col, exp in self.pkey_list: + exp_list.append(exp % quote(data[col])) + return " and ".join(exp_list) + + SD['TableInfo'] = TableInfo + + # cache some functions + def proc_insert(tbl): + return tbl.get_insert_stmt(TD['new']) + def proc_update(tbl): + return tbl.get_update_stmt(TD['old'], TD['new']) + def proc_delete(tbl): + return tbl.get_pkey_expr(TD['old']) + SD['event_func'] = { + 'I': proc_insert, + 'U': proc_update, + 'D': proc_delete, + } + + # remember init + SD['init_done'] = 1 + + +# load args +table_oid = TD['relid'] +queue_name = TD['args'][0] +if len(TD['args']) > 1: + options_str = TD['args'][1] +else: + options_str = '' + +# load & cache table data +if table_oid in SD: + tbl = SD[table_oid] + tbl.recheck(options_str) +else: + tbl = SD['TableInfo'](table_oid, options_str) + SD[table_oid] = tbl + +# generate payload +op = SD['op_map'][TD['event']] +data = SD['event_func'][op](tbl) + +# insert event +plpy.execute(SD['ins_plan'], [queue_name, op, data, tbl.name]) + +# done +return tbl.options['ret'] + +$$ language plpythonu; + diff --git a/sql/pgq_ext/Makefile b/sql/pgq_ext/Makefile new file mode 100644 index 00000000..dc824924 --- /dev/null +++ b/sql/pgq_ext/Makefile @@ -0,0 +1,16 @@ + +DOCS = README.pgq_ext +DATA_built = pgq_ext.sql + +SRCS = structure/tables.sql functions/track_batch.sql functions/track_event.sql + +REGRESS = test_pgq_ext +REGRESS_OPTS = --load-language=plpgsql + +include ../../config.mak + +include $(PGXS) + +pgq_ext.sql: $(SRCS) + cat $(SRCS) > $@ + diff --git a/sql/pgq_ext/README.pgq_ext b/sql/pgq_ext/README.pgq_ext new file mode 100644 index 00000000..b2116b87 --- /dev/null +++ b/sql/pgq_ext/README.pgq_ext @@ -0,0 +1,52 @@ + +Track processed batches and events in target DB +================================================ + +Batch tracking is OK. + +Event tracking is OK if consumer does not use retry queue. + +Batch tracking +-------------- + +is_batch_done(consumer, batch) + +returns: + + true - batch is done already + false - batch is not done yet + +set_batch_done(consumer, batch) + +returns: + + true - tagging successful, batch was not done yet + false - batch was done already + +Event tracking +-------------- + +is_batch_done(consumer, batch, event) + +returns: + + true - event is done + false - event is not done yet + + +set_batch_done(consumer, batch, event) + +returns: + + true - tagging was successful, event was not done + false - event is done already + + +Fastvacuum +---------- + +pgq.ext.completed_batch +pgq.ext.completed_event +pgq.ext.completed_tick +pgq.ext.partial_batch + diff --git a/sql/pgq_ext/expected/test_pgq_ext.out b/sql/pgq_ext/expected/test_pgq_ext.out new file mode 100644 index 00000000..ccced856 --- /dev/null +++ b/sql/pgq_ext/expected/test_pgq_ext.out @@ -0,0 +1,85 @@ +\set ECHO off +-- +-- test batch tracking +-- +select pgq_ext.is_batch_done('c', 1); + is_batch_done +--------------- + f +(1 row) + +select pgq_ext.set_batch_done('c', 1); + set_batch_done +---------------- + t +(1 row) + +select pgq_ext.is_batch_done('c', 1); + is_batch_done +--------------- + t +(1 row) + +select pgq_ext.set_batch_done('c', 1); + set_batch_done +---------------- + f +(1 row) + +select pgq_ext.is_batch_done('c', 2); + is_batch_done +--------------- + f +(1 row) + +select pgq_ext.set_batch_done('c', 2); + set_batch_done +---------------- + t +(1 row) + +-- +-- test event tracking +-- +select pgq_ext.is_batch_done('c', 3); + is_batch_done +--------------- + f +(1 row) + +select pgq_ext.is_event_done('c', 3, 101); + is_event_done +--------------- + f +(1 row) + +select pgq_ext.set_event_done('c', 3, 101); + set_event_done +---------------- + t +(1 row) + +select pgq_ext.is_event_done('c', 3, 101); + is_event_done +--------------- + t +(1 row) + +select pgq_ext.set_event_done('c', 3, 101); + set_event_done +---------------- + f +(1 row) + +select pgq_ext.set_batch_done('c', 3); + set_batch_done +---------------- + t +(1 row) + +select * from pgq_ext.completed_event order by 1,2; + consumer_id | batch_id | event_id +-------------+----------+---------- + c | 3 | 101 +(1 row) + diff --git a/sql/pgq_ext/functions/track_batch.sql b/sql/pgq_ext/functions/track_batch.sql new file mode 100644 index 00000000..e77f590d --- /dev/null +++ b/sql/pgq_ext/functions/track_batch.sql @@ -0,0 +1,39 @@ + +create or replace function pgq_ext.is_batch_done( + a_consumer text, a_batch_id bigint) +returns boolean as $$ +declare + res boolean; +begin + select last_batch_id = a_batch_id + into res from pgq_ext.completed_batch + where consumer_id = a_consumer; + if not found then + return false; + end if; + return res; +end; +$$ language plpgsql security definer; + +create or replace function pgq_ext.set_batch_done( + a_consumer text, a_batch_id bigint) +returns boolean as $$ +begin + if pgq_ext.is_batch_done(a_consumer, a_batch_id) then + return false; + end if; + + if a_batch_id > 0 then + update pgq_ext.completed_batch + set last_batch_id = a_batch_id + where consumer_id = a_consumer; + if not found then + insert into pgq_ext.completed_batch (consumer_id, last_batch_id) + values (a_consumer, a_batch_id); + end if; + end if; + + return true; +end; +$$ language plpgsql security definer; + diff --git a/sql/pgq_ext/functions/track_event.sql b/sql/pgq_ext/functions/track_event.sql new file mode 100644 index 00000000..8e89f41e --- /dev/null +++ b/sql/pgq_ext/functions/track_event.sql @@ -0,0 +1,60 @@ + +create or replace function pgq_ext.is_event_done( + a_consumer text, + a_batch_id bigint, a_event_id bigint) +returns boolean as $$ +declare + res bigint; +begin + perform 1 from pgq_ext.completed_event + where consumer_id = a_consumer + and batch_id = a_batch_id + and event_id = a_event_id; + return found; +end; +$$ language plpgsql security definer; + +create or replace function pgq_ext.set_event_done( + a_consumer text, a_batch_id bigint, a_event_id bigint) +returns boolean as $$ +declare + old_batch bigint; +begin + -- check if done + perform 1 from pgq_ext.completed_event + where consumer_id = a_consumer + and batch_id = a_batch_id + and event_id = a_event_id; + if found then + return false; + end if; + + -- if batch changed, do cleanup + select cur_batch_id into old_batch + from pgq_ext.partial_batch + where consumer_id = a_consumer; + if not found then + -- first time here + insert into pgq_ext.partial_batch + (consumer_id, cur_batch_id) + values (a_consumer, a_batch_id); + elsif old_batch <> a_batch_id then + -- batch changed, that means old is finished on queue db + -- thus the tagged events are not needed anymore + delete from pgq_ext.completed_event + where consumer_id = a_consumer + and batch_id = old_batch; + -- remember current one + update pgq_ext.partial_batch + set cur_batch_id = a_batch_id + where consumer_id = a_consumer; + end if; + + -- tag as done + insert into pgq_ext.completed_event (consumer_id, batch_id, event_id) + values (a_consumer, a_batch_id, a_event_id); + + return true; +end; +$$ language plpgsql security definer; + diff --git a/sql/pgq_ext/sql/test_pgq_ext.sql b/sql/pgq_ext/sql/test_pgq_ext.sql new file mode 100644 index 00000000..d6f4eeaa --- /dev/null +++ b/sql/pgq_ext/sql/test_pgq_ext.sql @@ -0,0 +1,26 @@ +\set ECHO off +\i pgq_ext.sql +\set ECHO all + +-- +-- test batch tracking +-- +select pgq_ext.is_batch_done('c', 1); +select pgq_ext.set_batch_done('c', 1); +select pgq_ext.is_batch_done('c', 1); +select pgq_ext.set_batch_done('c', 1); +select pgq_ext.is_batch_done('c', 2); +select pgq_ext.set_batch_done('c', 2); + +-- +-- test event tracking +-- +select pgq_ext.is_batch_done('c', 3); +select pgq_ext.is_event_done('c', 3, 101); +select pgq_ext.set_event_done('c', 3, 101); +select pgq_ext.is_event_done('c', 3, 101); +select pgq_ext.set_event_done('c', 3, 101); +select pgq_ext.set_batch_done('c', 3); +select * from pgq_ext.completed_event order by 1,2; + + diff --git a/sql/pgq_ext/structure/tables.sql b/sql/pgq_ext/structure/tables.sql new file mode 100644 index 00000000..377353ba --- /dev/null +++ b/sql/pgq_ext/structure/tables.sql @@ -0,0 +1,48 @@ + +set client_min_messages = 'warning'; +set default_with_oids = 'off'; + +create schema pgq_ext; +grant usage on schema pgq_ext to public; + + +-- +-- batch tracking +-- +create table pgq_ext.completed_batch ( + consumer_id text not null, + last_batch_id bigint not null, + + primary key (consumer_id) +); + + +-- +-- event tracking +-- +create table pgq_ext.completed_event ( + consumer_id text not null, + batch_id bigint not null, + event_id bigint not null, + + primary key (consumer_id, batch_id, event_id) +); + +create table pgq_ext.partial_batch ( + consumer_id text not null, + cur_batch_id bigint not null, + + primary key (consumer_id) +); + +-- +-- tick tracking for SerialConsumer() +-- no access functions provided here +-- +create table pgq_ext.completed_tick ( + consumer_id text not null, + last_tick_id bigint not null, + + primary key (consumer_id) +); + diff --git a/sql/txid/Makefile b/sql/txid/Makefile new file mode 100644 index 00000000..b0c79925 --- /dev/null +++ b/sql/txid/Makefile @@ -0,0 +1,30 @@ + +MODULE_big = txid +SRCS = txid.c epoch.c +OBJS = $(SRCS:.c=.o) + +DATA_built = txid.sql +DATA = uninstall_txid.sql +DOCS = README.txid +EXTRA_CLEAN = txid.sql.in + +REGRESS = txid +REGRESS_OPTS = --load-language=plpgsql + +include ../../config.mak +include $(PGXS) + +# additional deps +txid.o: txid.h +epoch.o: txid.h + +# postgres >= manages epoch itself, so skip epoch tables +pgnew = $(shell test $(VERSION) "<" "8.2" && echo "false" || echo "true") +ifeq ($(pgnew),true) +TXID_SQL = txid.std.sql +else +TXID_SQL = txid.std.sql txid.schema.sql +endif +txid.sql.in: $(TXID_SQL) + cat $(TXID_SQL) > $@ + diff --git a/sql/txid/README.txid b/sql/txid/README.txid new file mode 100644 index 00000000..6cdac28a --- /dev/null +++ b/sql/txid/README.txid @@ -0,0 +1,65 @@ + +txid - 8 byte transaction ID's +============================== + +Based on xxid module from Slony-I. + +The goal is to make PostgreSQL internal transaction ID and snapshot +data usable externally. They cannot be used directly as the +internal 4-byte value wraps around and thus breaks indexing. + +This module extends the internal value with wraparound cound (epoch). +It uses relaxed method for wraparound check. There is a table +txid.epoch (epoch, last_value) which is used to check if the xid +is in current, next or previous epoch. It requires only occasional +read-write access - ca. after 100k - 500k transactions. + +Also it contains type 'txid_snapshot' and following functions: + + +get_current_txid() returns int8 + + Current transaction ID + +get_current_snapshot() returns txid_snapshot + + Current snapshot + +get_snapshot_xmin( snap ) returns int8 -- + + Smallest TXID in snapshot. TXID's smaller than this + are all visible in snapshot. + +get_snapshot_xmax( snap ) returns int8 + + Largest TXID in snapshot. TXID's starting from this one are + all invisible in snapshot. + +get_snapshot_values( snap ) setof int8 + + List of uncommitted TXID's in snapshot, that are invisible + in snapshot. Values are between xmin and xmax. + +txid_in_snapshot(id, snap) returns bool + + Is TXID visible in snapshot? + +txid_not_in_snapshot(id, snap) returns bool + + Is TXID invisible in snapshot? + + +Problems +-------- + +- it breaks when there are more than 2G tx'es between calls. + Fixed in 8.2 + +- functions that create new txid's should be 'security definers' + thus better protecting txid_epoch table. + +- After loading database from backup you should do: + + UPDATE txid.epoch SET epoch = epoch + 1, + last_value = (get_current_txid() & 4294967295); + diff --git a/sql/txid/epoch.c b/sql/txid/epoch.c new file mode 100644 index 00000000..a2cc28cd --- /dev/null +++ b/sql/txid/epoch.c @@ -0,0 +1,240 @@ +/*------------------------------------------------------------------------- + * epoch.c + * + * Detect current epoch. + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include <limits.h> + +#include "access/transam.h" +#include "executor/spi.h" +#include "miscadmin.h" +#include "catalog/pg_control.h" +#include "access/xlog.h" + +#include "txid.h" + +/* + * do a TransactionId -> txid conversion + */ +txid txid_convert_xid(TransactionId xid, TxidEpoch *state) +{ + uint64 epoch; + + /* avoid issues with the the special meaning of 0 */ + if (xid == InvalidTransactionId) + return MAX_INT64; + + /* return special xid's as-is */ + if (xid < FirstNormalTransactionId) + return xid; + + /* xid can on both sides on wrap-around */ + epoch = state->epoch; + if (TransactionIdPrecedes(xid, state->last_value)) { + if (xid > state->last_value) + epoch--; + } else if (TransactionIdFollows(xid, state->last_value)) { + if (xid < state->last_value) + epoch++; + } + return (epoch << 32) | xid; +} + +#if PG_CONTROL_VERSION >= 820 + +/* + * PostgreSQl 8.2 keeps track of epoch internally. + */ + +void txid_load_epoch(TxidEpoch *state, int try_write) +{ + TransactionId xid; + uint32 epoch; + + GetNextXidAndEpoch(&xid, &epoch); + + state->epoch = epoch; + state->last_value = xid; +} + +#else + +/* + * For older PostgreSQL keep epoch in table. + */ + +/* + * this caches the txid_epoch table. + * The struct should be updated only together with the table. + */ +static TxidEpoch epoch_state = { 0, 0 }; + +/* + * load values from txid_epoch table. + */ +static int load_epoch(void) +{ + HeapTuple row; + TupleDesc rdesc; + bool isnull = false; + Datum tmp; + int res; + uint64 db_epoch, db_value; + + res = SPI_connect(); + if (res < 0) + elog(ERROR, "cannot connect to SPI"); + + res = SPI_execute("select epoch, last_value from txid.epoch", true, 0); + if (res != SPI_OK_SELECT) + elog(ERROR, "load_epoch: select failed?"); + if (SPI_processed != 1) + elog(ERROR, "load_epoch: there must be exactly 1 row"); + + row = SPI_tuptable->vals[0]; + rdesc = SPI_tuptable->tupdesc; + + tmp = SPI_getbinval(row, rdesc, 1, &isnull); + if (isnull) + elog(ERROR, "load_epoch: epoch is NULL"); + db_epoch = DatumGetInt64(tmp); + + tmp = SPI_getbinval(row, rdesc, 2, &isnull); + if (isnull) + elog(ERROR, "load_epoch: last_value is NULL"); + db_value = DatumGetInt64(tmp); + + SPI_finish(); + + /* + * If the db has lesser values, then some updates were lost. + * + * Should that be special-cased? ATM just use db values. + * Thus immidiate update. + */ + epoch_state.epoch = db_epoch; + epoch_state.last_value = db_value; + return 1; +} + +/* + * updates last_value and epoch, if needed + */ +static void save_epoch(void) +{ + int res; + char qbuf[200]; + uint64 new_epoch, new_value; + TransactionId xid = GetTopTransactionId(); + TransactionId old_value; + + /* store old state */ + MemoryContext oldcontext = CurrentMemoryContext; + ResourceOwner oldowner = CurrentResourceOwner; + + /* + * avoid changing internal values. + */ + new_value = xid; + new_epoch = epoch_state.epoch; + old_value = (TransactionId)epoch_state.last_value; + if (xid < old_value) { + if (TransactionIdFollows(xid, old_value)) + new_epoch++; + else + return; + } + sprintf(qbuf, "update txid.epoch set epoch = %llu, last_value = %llu", + (unsigned long long)new_epoch, + (unsigned long long)new_value); + + /* + * The update may fail in case of SERIALIZABLE transaction. + * Try to catch the error and hide it. + */ + BeginInternalSubTransaction(NULL); + PG_TRY(); + { + /* do the update */ + res = SPI_connect(); + if (res < 0) + elog(ERROR, "cannot connect to SPI"); + res = SPI_execute(qbuf, false, 0); + SPI_finish(); + + ReleaseCurrentSubTransaction(); + } + PG_CATCH(); + { + /* we expect rollback to clean up inner SPI call */ + RollbackAndReleaseCurrentSubTransaction(); + FlushErrorState(); + res = -1; /* remember failure */ + } + PG_END_TRY(); + + /* restore old state */ + MemoryContextSwitchTo(oldcontext); + CurrentResourceOwner = oldowner; + + if (res < 0) + return; + + /* + * Seems the update was successful, update internal state too. + * + * There is a chance that the TX will be rollbacked, but then + * another backend will do the update, or this one at next + * checkpoint. + */ + epoch_state.epoch = new_epoch; + epoch_state.last_value = new_value; +} + +static void check_epoch(int update_prio) +{ + TransactionId xid = GetTopTransactionId(); + TransactionId recheck, tx_next; + int ok = 1; + + /* should not happen, but just in case */ + if (xid == InvalidTransactionId) + return; + + /* new backend */ + if (epoch_state.last_value == 0) + load_epoch(); + + /* try to avoid concurrent access */ + if (update_prio) + recheck = 50000 + 100 * (MyProcPid & 0x1FF); + else + recheck = 300000 + 1000 * (MyProcPid & 0x1FF); + + /* read table */ + tx_next = (TransactionId)epoch_state.last_value + recheck; + if (TransactionIdFollows(xid, tx_next)) + ok = load_epoch(); + + /* + * check if save is needed. last_value may be updated above. + */ + tx_next = (TransactionId)epoch_state.last_value + recheck; + if (!ok || TransactionIdFollows(xid, tx_next)) + save_epoch(); +} + +void txid_load_epoch(TxidEpoch *state, int try_write) +{ + check_epoch(try_write); + + state->epoch = epoch_state.epoch; + state->last_value = epoch_state.last_value; +} + + +#endif diff --git a/sql/txid/expected/txid.out b/sql/txid/expected/txid.out new file mode 100644 index 00000000..400f88c2 --- /dev/null +++ b/sql/txid/expected/txid.out @@ -0,0 +1,88 @@ +-- init +\set ECHO none +-- i/o +select '12:13:'::txid_snapshot; + txid_snapshot +--------------- + 12:13: +(1 row) + +select '12:13:1,2'::txid_snapshot; +ERROR: illegal txid_snapshot input format +-- errors +select '31:12:'::txid_snapshot; +ERROR: illegal txid_snapshot input format +select '0:1:'::txid_snapshot; +ERROR: illegal txid_snapshot input format +select '12:13:0'::txid_snapshot; +ERROR: illegal txid_snapshot input format +select '12:13:2,1'::txid_snapshot; +ERROR: illegal txid_snapshot input format +create table snapshot_test ( + nr integer, + snap txid_snapshot +); +insert into snapshot_test values (1, '12:13:'); +insert into snapshot_test values (2, '12:20:13,15,18'); +insert into snapshot_test values (3, '100001:100009:100005,100007,100008'); +select snap from snapshot_test order by nr; + snap +------------------------------------ + 12:13: + 12:20:13,15,18 + 100001:100009:100005,100007,100008 +(3 rows) + +select get_snapshot_xmin(snap), + get_snapshot_xmax(snap), + get_snapshot_active(snap) +from snapshot_test order by nr; + get_snapshot_xmin | get_snapshot_xmax | get_snapshot_active +-------------------+-------------------+--------------------- + 12 | 20 | 13 + 12 | 20 | 15 + 12 | 20 | 18 + 100001 | 100009 | 100005 + 100001 | 100009 | 100007 + 100001 | 100009 | 100008 +(6 rows) + +select id, txid_in_snapshot(id, snap), + txid_not_in_snapshot(id, snap) +from snapshot_test, generate_series(11, 21) id +where nr = 2; + id | txid_in_snapshot | txid_not_in_snapshot +----+------------------+---------------------- + 11 | t | f + 12 | t | f + 13 | f | t + 14 | t | f + 15 | f | t + 16 | t | f + 17 | t | f + 18 | f | t + 19 | t | f + 20 | f | t + 21 | f | t +(11 rows) + +-- test current values also +select get_current_txid() >= get_snapshot_xmin(get_current_snapshot()); + ?column? +---------- + t +(1 row) + +select get_current_txid() < get_snapshot_xmax(get_current_snapshot()); + ?column? +---------- + t +(1 row) + +select txid_in_snapshot(get_current_txid(), get_current_snapshot()), + txid_not_in_snapshot(get_current_txid(), get_current_snapshot()); + txid_in_snapshot | txid_not_in_snapshot +------------------+---------------------- + t | f +(1 row) + diff --git a/sql/txid/sql/txid.sql b/sql/txid/sql/txid.sql new file mode 100644 index 00000000..6009944b --- /dev/null +++ b/sql/txid/sql/txid.sql @@ -0,0 +1,43 @@ +-- init +\set ECHO none +\i txid.sql +\set ECHO all + +-- i/o +select '12:13:'::txid_snapshot; +select '12:13:1,2'::txid_snapshot; + +-- errors +select '31:12:'::txid_snapshot; +select '0:1:'::txid_snapshot; +select '12:13:0'::txid_snapshot; +select '12:13:2,1'::txid_snapshot; + +create table snapshot_test ( + nr integer, + snap txid_snapshot +); + +insert into snapshot_test values (1, '12:13:'); +insert into snapshot_test values (2, '12:20:13,15,18'); +insert into snapshot_test values (3, '100001:100009:100005,100007,100008'); + +select snap from snapshot_test order by nr; + +select get_snapshot_xmin(snap), + get_snapshot_xmax(snap), + get_snapshot_active(snap) +from snapshot_test order by nr; + +select id, txid_in_snapshot(id, snap), + txid_not_in_snapshot(id, snap) +from snapshot_test, generate_series(11, 21) id +where nr = 2; + +-- test current values also +select get_current_txid() >= get_snapshot_xmin(get_current_snapshot()); +select get_current_txid() < get_snapshot_xmax(get_current_snapshot()); + +select txid_in_snapshot(get_current_txid(), get_current_snapshot()), + txid_not_in_snapshot(get_current_txid(), get_current_snapshot()); + diff --git a/sql/txid/txid.c b/sql/txid/txid.c new file mode 100644 index 00000000..256d3608 --- /dev/null +++ b/sql/txid/txid.c @@ -0,0 +1,364 @@ +/*------------------------------------------------------------------------- + * txid.c + * + * Safe handling of transaction ID's. + * + * Copyright (c) 2003-2004, PostgreSQL Global Development Group + * Author: Jan Wieck, Afilias USA INC. + * + * 64-bit output: Marko Kreen, Skype Technologies + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include <limits.h> + +#include "access/xact.h" +#include "funcapi.h" + +#include "txid.h" + +#ifdef INT64_IS_BUSTED +#error txid needs working int64 +#endif + +#ifdef PG_MODULE_MAGIC +PG_MODULE_MAGIC; +#endif + +/* + * public functions + */ + +PG_FUNCTION_INFO_V1(txid_current); +PG_FUNCTION_INFO_V1(txid_snapshot_in); +PG_FUNCTION_INFO_V1(txid_snapshot_out); +PG_FUNCTION_INFO_V1(txid_in_snapshot); +PG_FUNCTION_INFO_V1(txid_not_in_snapshot); +PG_FUNCTION_INFO_V1(txid_current_snapshot); +PG_FUNCTION_INFO_V1(txid_snapshot_xmin); +PG_FUNCTION_INFO_V1(txid_snapshot_xmax); +PG_FUNCTION_INFO_V1(txid_snapshot_active); + +/* + * utility functions + */ + +static int _cmp_txid(const void *aa, const void *bb) +{ + const uint64 *a = aa; + const uint64 *b = bb; + if (*a < *b) + return -1; + if (*a > *b) + return 1; + return 0; +} + +static void sort_snapshot(TxidSnapshot *snap) +{ + qsort(snap->xip, snap->nxip, sizeof(txid), _cmp_txid); +} + +static TxidSnapshot * +parse_snapshot(const char *str) +{ + int a_size; + txid *xip; + + int a_used = 0; + txid xmin; + txid xmax; + txid last_val = 0, val; + TxidSnapshot *snap; + int size; + + char *endp; + + a_size = 1024; + xip = (txid *) palloc(sizeof(txid) * a_size); + + xmin = (txid) strtoull(str, &endp, 0); + if (*endp != ':') + elog(ERROR, "illegal txid_snapshot input format"); + str = endp + 1; + + xmax = (txid) strtoull(str, &endp, 0); + if (*endp != ':') + elog(ERROR, "illegal txid_snapshot input format"); + str = endp + 1; + + /* it should look sane */ + if (xmin >= xmax || xmin > MAX_INT64 || xmax > MAX_INT64 + || xmin == 0 || xmax == 0) + elog(ERROR, "illegal txid_snapshot input format"); + + while (*str != '\0') + { + if (a_used >= a_size) + { + a_size *= 2; + xip = (txid *) repalloc(xip, sizeof(txid) * a_size); + } + + /* read next value */ + if (*str == '\'') + { + str++; + val = (txid) strtoull(str, &endp, 0); + if (*endp != '\'') + elog(ERROR, "illegal txid_snapshot input format"); + str = endp + 1; + } + else + { + val = (txid) strtoull(str, &endp, 0); + str = endp; + } + + /* require the input to be in order */ + if (val < xmin || val <= last_val || val >= xmax) + elog(ERROR, "illegal txid_snapshot input format"); + + xip[a_used++] = val; + last_val = val; + + if (*str == ',') + str++; + else + { + if (*str != '\0') + elog(ERROR, "illegal txid_snapshot input format"); + } + } + + size = offsetof(TxidSnapshot, xip) + sizeof(txid) * a_used; + snap = (TxidSnapshot *) palloc(size); + snap->varsz = size; + snap->xmin = xmin; + snap->xmax = xmax; + snap->nxip = a_used; + if (a_used > 0) + memcpy(&(snap->xip[0]), xip, sizeof(txid) * a_used); + pfree(xip); + + return snap; +} + +/* + * Public functions + */ + +/* + * txid_current - Return the current transaction ID as txid + */ +Datum +txid_current(PG_FUNCTION_ARGS) +{ + txid val; + TxidEpoch state; + + txid_load_epoch(&state, 0); + + val = txid_convert_xid(GetTopTransactionId(), &state); + + PG_RETURN_INT64(val); +} + +/* + * txid_current_snapshot - return current snapshot + */ +Datum +txid_current_snapshot(PG_FUNCTION_ARGS) +{ + TxidSnapshot *snap; + unsigned num, i, size; + TxidEpoch state; + + if (SerializableSnapshot == NULL) + elog(ERROR, "get_current_snapshot: SerializableSnapshot == NULL"); + + txid_load_epoch(&state, 1); + + num = SerializableSnapshot->xcnt; + size = offsetof(TxidSnapshot, xip) + sizeof(txid) * num; + snap = palloc(size); + snap->varsz = size; + snap->xmin = txid_convert_xid(SerializableSnapshot->xmin, &state); + snap->xmax = txid_convert_xid(SerializableSnapshot->xmax, &state); + snap->nxip = num; + for (i = 0; i < num; i++) + snap->xip[i] = txid_convert_xid(SerializableSnapshot->xip[i], &state); + + /* we want then guaranteed ascending order */ + sort_snapshot(snap); + + PG_RETURN_POINTER(snap); +} + +/* + * txid_snapshot_in - input function for type txid_snapshot + */ +Datum +txid_snapshot_in(PG_FUNCTION_ARGS) +{ + TxidSnapshot *snap; + char *str = PG_GETARG_CSTRING(0); + + snap = parse_snapshot(str); + PG_RETURN_POINTER(snap); +} + +/* + * txid_snapshot_out - output function for type txid_snapshot + */ +Datum +txid_snapshot_out(PG_FUNCTION_ARGS) +{ + TxidSnapshot *snap = (TxidSnapshot *) PG_GETARG_VARLENA_P(0); + + char *str = palloc(60 + snap->nxip * 30); + char *cp = str; + int i; + + snprintf(str, 60, "%llu:%llu:", + (unsigned long long)snap->xmin, + (unsigned long long)snap->xmax); + cp = str + strlen(str); + + for (i = 0; i < snap->nxip; i++) + { + snprintf(cp, 30, "%llu%s", + (unsigned long long)snap->xip[i], + (i < snap->nxip - 1) ? "," : ""); + cp += strlen(cp); + } + + PG_RETURN_CSTRING(str); +} + + +/* + * txid_in_snapshot - is txid visible in snapshot ? + */ +Datum +txid_in_snapshot(PG_FUNCTION_ARGS) +{ + txid value = PG_GETARG_INT64(0); + TxidSnapshot *snap = (TxidSnapshot *) PG_GETARG_VARLENA_P(1); + int i; + int res = true; + + if (value < snap->xmin) + res = true; + else if (value >= snap->xmax) + res = false; + else + { + for (i = 0; i < snap->nxip; i++) + if (value == snap->xip[i]) + { + res = false; + break; + } + } + PG_FREE_IF_COPY(snap, 1); + PG_RETURN_BOOL(res); +} + + +/* + * txid_not_in_snapshot - is txid invisible in snapshot ? + */ +Datum +txid_not_in_snapshot(PG_FUNCTION_ARGS) +{ + txid value = PG_GETARG_INT64(0); + TxidSnapshot *snap = (TxidSnapshot *) PG_GETARG_VARLENA_P(1); + int i; + int res = false; + + if (value < snap->xmin) + res = false; + else if (value >= snap->xmax) + res = true; + else + { + for (i = 0; i < snap->nxip; i++) + if (value == snap->xip[i]) + { + res = true; + break; + } + } + PG_FREE_IF_COPY(snap, 1); + PG_RETURN_BOOL(res); +} + +/* + * txid_snapshot_xmin - return snapshot's xmin + */ +Datum +txid_snapshot_xmin(PG_FUNCTION_ARGS) +{ + TxidSnapshot *snap = (TxidSnapshot *) PG_GETARG_VARLENA_P(0); + txid res = snap->xmin; + PG_FREE_IF_COPY(snap, 0); + PG_RETURN_INT64(res); +} + +/* + * txid_snapshot_xmin - return snapshot's xmax + */ +Datum +txid_snapshot_xmax(PG_FUNCTION_ARGS) +{ + TxidSnapshot *snap = (TxidSnapshot *) PG_GETARG_VARLENA_P(0); + txid res = snap->xmax; + PG_FREE_IF_COPY(snap, 0); + PG_RETURN_INT64(res); +} + +/* remember state between function calls */ +struct snap_state { + int pos; + TxidSnapshot *snap; +}; + +/* + * txid_snapshot_active - returns uncommitted TXID's in snapshot. + */ +Datum +txid_snapshot_active(PG_FUNCTION_ARGS) +{ + FuncCallContext *fctx; + struct snap_state *state; + + if (SRF_IS_FIRSTCALL()) { + TxidSnapshot *snap; + int statelen; + + snap = (TxidSnapshot *) PG_GETARG_VARLENA_P(0); + + fctx = SRF_FIRSTCALL_INIT(); + statelen = sizeof(*state) + snap->varsz; + state = MemoryContextAlloc(fctx->multi_call_memory_ctx, statelen); + state->pos = 0; + state->snap = (TxidSnapshot *)((char *)state + sizeof(*state)); + memcpy(state->snap, snap, snap->varsz); + fctx->user_fctx = state; + + PG_FREE_IF_COPY(snap, 0); + } + fctx = SRF_PERCALL_SETUP(); + state = fctx->user_fctx; + if (state->pos < state->snap->nxip) { + Datum res = Int64GetDatum(state->snap->xip[state->pos]); + state->pos++; + SRF_RETURN_NEXT(fctx, res); + } else { + SRF_RETURN_DONE(fctx); + } +} + diff --git a/sql/txid/txid.h b/sql/txid/txid.h new file mode 100644 index 00000000..8c648754 --- /dev/null +++ b/sql/txid/txid.h @@ -0,0 +1,43 @@ +#ifndef _TXID_H_ +#define _TXID_H_ + +#define MAX_INT64 0x7FFFFFFFFFFFFFFFLL + +/* Use unsigned variant internally */ +typedef uint64 txid; + +typedef struct +{ + int32 varsz; + uint32 nxip; + txid xmin; + txid xmax; + txid xip[1]; +} TxidSnapshot; + + +typedef struct { + uint64 last_value; + uint64 epoch; +} TxidEpoch; + +/* internal functions */ +void txid_load_epoch(TxidEpoch *state, int try_write); +txid txid_convert_xid(TransactionId xid, TxidEpoch *state); + +/* public functions */ +Datum txid_current(PG_FUNCTION_ARGS); +Datum txid_current_snapshot(PG_FUNCTION_ARGS); + +Datum txid_snapshot_in(PG_FUNCTION_ARGS); +Datum txid_snapshot_out(PG_FUNCTION_ARGS); + +Datum txid_in_snapshot(PG_FUNCTION_ARGS); +Datum txid_not_in_snapshot(PG_FUNCTION_ARGS); +Datum txid_snapshot_xmin(PG_FUNCTION_ARGS); +Datum txid_snapshot_xmax(PG_FUNCTION_ARGS); +Datum txid_snapshot_active(PG_FUNCTION_ARGS); + + +#endif /* _TXID_H_ */ + diff --git a/sql/txid/txid.schema.sql b/sql/txid/txid.schema.sql new file mode 100644 index 00000000..b0a5b5a1 --- /dev/null +++ b/sql/txid/txid.schema.sql @@ -0,0 +1,50 @@ +-- ---------- +-- txid.sql +-- +-- SQL script for loading the transaction ID compatible datatype +-- +-- Copyright (c) 2003-2004, PostgreSQL Global Development Group +-- Author: Jan Wieck, Afilias USA INC. +-- +-- ---------- + +-- +-- now the epoch storage +-- + +CREATE SCHEMA txid; + +-- remember txid settings +-- use bigint so we can do arithmetic with it +create table txid.epoch ( + epoch bigint, + last_value bigint +); + +-- make sure there exist exactly one row +insert into txid.epoch values (0, 1); + + +-- then protect it +create function txid.epoch_guard() +returns trigger as $$ +begin + if TG_OP = 'UPDATE' then + -- epoch: allow only small increase + if NEW.epoch > OLD.epoch and NEW.epoch < (OLD.epoch + 3) then + return NEW; + end if; + -- last_value: allow only increase + if NEW.epoch = OLD.epoch and NEW.last_value > OLD.last_value then + return NEW; + end if; + end if; + raise exception 'bad operation on txid.epoch'; +end; +$$ language plpgsql; + +-- the trigger +create trigger epoch_guard_trigger +before insert or update or delete on txid.epoch +for each row execute procedure txid.epoch_guard(); + diff --git a/sql/txid/txid.std.sql b/sql/txid/txid.std.sql new file mode 100644 index 00000000..8ba34cbc --- /dev/null +++ b/sql/txid/txid.std.sql @@ -0,0 +1,77 @@ +-- ---------- +-- txid.sql +-- +-- SQL script for loading the transaction ID compatible datatype +-- +-- Copyright (c) 2003-2004, PostgreSQL Global Development Group +-- Author: Jan Wieck, Afilias USA INC. +-- +-- ---------- + +set client_min_messages = 'warning'; + +CREATE DOMAIN txid AS bigint CHECK (value > 0); + +-- +-- A special transaction snapshot data type for faster visibility checks +-- +CREATE OR REPLACE FUNCTION txid_snapshot_in(cstring) + RETURNS txid_snapshot + AS 'MODULE_PATHNAME' LANGUAGE C + IMMUTABLE STRICT; +CREATE OR REPLACE FUNCTION txid_snapshot_out(txid_snapshot) + RETURNS cstring + AS 'MODULE_PATHNAME' LANGUAGE C + IMMUTABLE STRICT; + +-- +-- The data type itself +-- +CREATE TYPE txid_snapshot ( + INPUT = txid_snapshot_in, + OUTPUT = txid_snapshot_out, + INTERNALLENGTH = variable, + STORAGE = extended, + ALIGNMENT = double +); + +CREATE OR REPLACE FUNCTION get_current_txid() + RETURNS bigint + AS 'MODULE_PATHNAME', 'txid_current' LANGUAGE C + SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION get_current_snapshot() + RETURNS txid_snapshot + AS 'MODULE_PATHNAME', 'txid_current_snapshot' LANGUAGE C + SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION get_snapshot_xmin(txid_snapshot) + RETURNS bigint + AS 'MODULE_PATHNAME', 'txid_snapshot_xmin' LANGUAGE C + IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION get_snapshot_xmax(txid_snapshot) + RETURNS bigint + AS 'MODULE_PATHNAME', 'txid_snapshot_xmax' LANGUAGE C + IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION get_snapshot_active(txid_snapshot) + RETURNS setof bigint + AS 'MODULE_PATHNAME', 'txid_snapshot_active' LANGUAGE C + IMMUTABLE STRICT; + + +-- +-- Special comparision functions used by the remote worker +-- for sync chunk selection +-- +CREATE OR REPLACE FUNCTION txid_in_snapshot(bigint, txid_snapshot) + RETURNS boolean + AS 'MODULE_PATHNAME', 'txid_in_snapshot' LANGUAGE C + IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION txid_not_in_snapshot(bigint, txid_snapshot) + RETURNS boolean + AS 'MODULE_PATHNAME', 'txid_not_in_snapshot' LANGUAGE C + IMMUTABLE STRICT; + diff --git a/sql/txid/uninstall_txid.sql b/sql/txid/uninstall_txid.sql new file mode 100644 index 00000000..17a88045 --- /dev/null +++ b/sql/txid/uninstall_txid.sql @@ -0,0 +1,10 @@ + +DROP DOMAIN txid; +DROP TYPE txid_snapshot cascade; +DROP SCHEMA txid CASCADE; +DROP FUNCTION get_current_txid(); +DROP FUNCTION get_snapshot_xmin(); +DROP FUNCTION get_snapshot_xmax(); +DROP FUNCTION get_snapshot_active(); + + diff --git a/tests/env.sh b/tests/env.sh new file mode 100644 index 00000000..05f61e8d --- /dev/null +++ b/tests/env.sh @@ -0,0 +1,6 @@ + +PYTHONPATH=../../python:$PYTHONPATH +PATH=../../python:../../scripts:$PATH +export PYTHONPATH PATH + + diff --git a/tests/londiste/conf/fread.ini b/tests/londiste/conf/fread.ini new file mode 100644 index 00000000..f3595d7f --- /dev/null +++ b/tests/londiste/conf/fread.ini @@ -0,0 +1,17 @@ + +[londiste] +job_name = fread_test + +method = file_read + +provider_db = dbname=provider +subscriber_db = dbname=file_subscriber + +# it will be used as sql ident so no dots/spaces +pgq_queue_name = londiste_test_replic + +pidfile = sys/pid.%(job_name)s +logfile = sys/log.%(job_name)s + +file_src = ./file_logs + diff --git a/tests/londiste/conf/fwrite.ini b/tests/londiste/conf/fwrite.ini new file mode 100644 index 00000000..ecba2379 --- /dev/null +++ b/tests/londiste/conf/fwrite.ini @@ -0,0 +1,17 @@ + +[londiste] +job_name = fwrite_test + +method = file_write + +provider_db = dbname=provider +subscriber_db = dbname=subscriber + +# it will be used as sql ident so no dots/spaces +pgq_queue_name = londiste_replic + +pidfile = sys/pid.%(job_name)s +logfile = sys/log.%(job_name)s + +file_dst = ./file_logs + diff --git a/tests/londiste/conf/linkticker.ini b/tests/londiste/conf/linkticker.ini new file mode 100644 index 00000000..a228bb83 --- /dev/null +++ b/tests/londiste/conf/linkticker.ini @@ -0,0 +1,17 @@ +[pgqadm] + +job_name = link_ticker + +db = dbname=subscriber + +# how often to run maintenance [minutes] +maint_delay_min = 1 + +# how often to check fot activity [secs] +loop_delay = 0.1 + +logfile = sys/log.%(job_name)s +pidfile = sys/pid.%(job_name)s + +use_skylog = 0 + diff --git a/tests/londiste/conf/replic.ini b/tests/londiste/conf/replic.ini new file mode 100644 index 00000000..e34e4adf --- /dev/null +++ b/tests/londiste/conf/replic.ini @@ -0,0 +1,15 @@ + +[londiste] +job_name = replic + +provider_db = dbname=provider +subscriber_db = dbname=subscriber + +# it will be used as sql ident so no dots/spaces +pgq_queue_name = londiste_replic + +pidfile = sys/pid.%(job_name)s +logfile = sys/log.%(job_name)s + +loop_delay = 0.3 + diff --git a/tests/londiste/conf/tester.ini b/tests/londiste/conf/tester.ini new file mode 100644 index 00000000..8293ee1b --- /dev/null +++ b/tests/londiste/conf/tester.ini @@ -0,0 +1,16 @@ + +[londiste] +job_name = replic_test + +provider_db = dbname=provider +subscriber_db = dbname=subscriber + +# it will be used as sql ident so no dots/spaces +pgq_queue_name = londiste_test_replic + +pidfile = sys/pid.%(job_name)s +logfile = sys/log.%(job_name)s + +# should be in right sequence +table_list = data1, data2 + diff --git a/tests/londiste/conf/ticker.ini b/tests/londiste/conf/ticker.ini new file mode 100644 index 00000000..5a35c85f --- /dev/null +++ b/tests/londiste/conf/ticker.ini @@ -0,0 +1,17 @@ +[pgqadm] + +job_name = ticker + +db = dbname=provider + +# how often to run maintenance [minutes] +maint_delay_min = 1 + +# how often to check fot activity [secs] +loop_delay = 0.1 + +logfile = sys/log.%(job_name)s +pidfile = sys/pid.%(job_name)s + +use_skylog = 0 + diff --git a/tests/londiste/data.sql b/tests/londiste/data.sql new file mode 100644 index 00000000..9bf6e819 --- /dev/null +++ b/tests/londiste/data.sql @@ -0,0 +1,24 @@ + +set client_min_messages = 'warning'; + +create table data1 ( + id serial primary key, + data text +); + +create unique index idx_data1_uq on data1 (data); + +create index idx_data1_rand on data1 (id, data); + +create table data2 ( + id serial primary key, + data text, + -- old_id integer references data1, + constraint uq_data2 unique (data) +); + +create index idx_data2_rand on data2 (id, data); + + +create sequence test_seq; +select setval('test_seq', 50); diff --git a/tests/londiste/env.sh b/tests/londiste/env.sh new file mode 100644 index 00000000..45c82d89 --- /dev/null +++ b/tests/londiste/env.sh @@ -0,0 +1,7 @@ + +PYTHONPATH=../../python:$PYTHONPATH +PATH=../../python:../../scripts:$PATH +export PYTHONPATH PATH + +#. /opt/apps/pgsql-dev/env + diff --git a/tests/londiste/gendb.sh b/tests/londiste/gendb.sh new file mode 100755 index 00000000..6effbe6d --- /dev/null +++ b/tests/londiste/gendb.sh @@ -0,0 +1,47 @@ +#! /bin/sh + +. ../env.sh + +contrib=/usr/share/postgresql/8.1/contrib +contrib=/opt/apps/pgsql-dev/share/contrib +contrib=/opt/pgsql/share/contrib + +db=provider + + +mkdir -p file_logs sys +./stop.sh +sleep 1 + +rm -rf file_logs sys +mkdir -p file_logs sys + +echo "creating database: $db" +dropdb $db +sleep 1 +createdb $db +pgqadm.py conf/ticker.ini install +psql -q $db -f data.sql + +db=subscriber +echo "creating database: $db" +dropdb $db +sleep 1 +createdb $db +pgqadm.py conf/linkticker.ini install +psql -q $db -f data.sql + +db=file_subscriber +echo "creating database: $db" +dropdb $db +sleep 1 +createdb $db +createlang plpgsql $db +createlang plpythonu $db +psql -q $db -f data.sql + +echo "done, testing" + +#pgqmgr.py -d conf/ticker.ini ticker +#./run-tests.sh + diff --git a/tests/londiste/run-tests.sh b/tests/londiste/run-tests.sh new file mode 100755 index 00000000..2e6fa8a8 --- /dev/null +++ b/tests/londiste/run-tests.sh @@ -0,0 +1,58 @@ +#! /bin/sh + +. ./env.sh + +script=londiste.py + +set -e + +$script conf/replic.ini provider install + +psql -c "update pgq.queue set queue_ticker_idle_period = '3', queue_ticker_max_lag = '2'" provider + +pgqadm.py -d conf/ticker.ini ticker + +$script conf/replic.ini subscriber install + +$script conf/replic.ini subscriber register +$script conf/replic.ini subscriber unregister + +$script -v -d conf/replic.ini replay +$script -v -d conf/fwrite.ini replay + +sleep 2 + +$script conf/replic.ini provider add data1 +$script conf/replic.ini subscriber add data1 + +sleep 2 + +$script conf/replic.ini provider add data2 +$script conf/replic.ini subscriber add data2 + +sleep 2 + +$script conf/replic.ini provider tables +$script conf/replic.ini provider remove data2 + +sleep 2 + +$script conf/replic.ini provider add data2 + +$script conf/replic.ini provider add-seq data1_id_seq +$script conf/replic.ini provider add-seq test_seq +$script conf/replic.ini subscriber add-seq data1_id_seq +$script conf/replic.ini subscriber add-seq test_seq + +sleep 2 + +$script conf/replic.ini subscriber tables +$script conf/replic.ini subscriber missing + +$script conf/replic.ini subscriber remove data2 +sleep 2 +$script conf/replic.ini subscriber add data2 +sleep 2 + +./testing.py conf/tester.ini + diff --git a/tests/londiste/stop.sh b/tests/londiste/stop.sh new file mode 100755 index 00000000..b6b951e4 --- /dev/null +++ b/tests/londiste/stop.sh @@ -0,0 +1,12 @@ +#! /bin/sh + +. ../env.sh +./testing.py -s conf/tester.ini +londiste.py -s conf/fwrite.ini +londiste.py -s conf/replic.ini + +sleep 1 + +pgqadm.py -s conf/ticker.ini +pgqadm.py -s conf/linkticker.ini + diff --git a/tests/londiste/testing.py b/tests/londiste/testing.py new file mode 100755 index 00000000..6f62ae1b --- /dev/null +++ b/tests/londiste/testing.py @@ -0,0 +1,80 @@ +#! /usr/bin/env python + +"""Londiste tester. +""" + +import sys, os, skytools + + +class Tester(skytools.DBScript): + test_pos = 0 + nr = 1 + def __init__(self, args): + skytools.DBScript.__init__(self, 'londiste', args) + self.log.info('start testing') + + def reload(self): + skytools.DBScript.reload(self) + self.loop_delay = 0.1 + + def work(self): + + src_db = self.get_database('provider_db') + dst_db = self.get_database('subscriber_db') + src_curs = src_db.cursor() + dst_curs = dst_db.cursor() + src_curs.execute("insert into data1 (data) values ('foo%d')" % self.nr) + src_curs.execute("insert into data2 (data) values ('foo%d')" % self.nr) + src_db.commit() + self.nr += 1 + + if self.bad_state(dst_db, dst_curs): + return + + if self.test_pos == 0: + self.resync_table(dst_db, dst_curs) + self.test_pos += 1 + elif self.test_pos == 1: + self.run_compare() + self.test_pos += 1 + + def bad_state(self, db, curs): + q = "select * from londiste.subscriber_table" + curs.execute(q) + db.commit() + ok = 0 + bad = 0 + cnt = 0 + for row in curs.dictfetchall(): + cnt += 1 + if row['merge_state'] == 'ok': + ok += 1 + else: + bad += 1 + + if cnt < 2: + return 1 + if bad > 0: + return 1 + + if ok > 0: + return 0 + + return 1 + + def resync_table(self, db, curs): + self.log.info('trying to remove table') + curs.execute("update londiste.subscriber_table"\ + " set merge_state = null" + " where table_name='public.data1'") + db.commit() + + def run_compare(self): + args = ["londiste.py", "conf/replic.ini", "compare"] + err = os.spawnvp(os.P_WAIT, "londiste.py", args) + self.log.info("Compare result=%d" % err) + +if __name__ == '__main__': + script = Tester(sys.argv[1:]) + script.start() + diff --git a/tests/scripts/conf/cube.ini b/tests/scripts/conf/cube.ini new file mode 100644 index 00000000..9b31b41c --- /dev/null +++ b/tests/scripts/conf/cube.ini @@ -0,0 +1,18 @@ +[cube_dispatcher] +job_name = cube_test + +src_db = dbname=scriptsrc +dst_db = dbname=scriptdst + +pgq_queue_name = data.middle + +logfile = sys/%(job_name)s.log +pidfile = sys/%(job_name)s.pid + +# how many rows are kept: keep_latest, keep_all +mode = keep_latest + +part_template = + create table _DEST_TABLE (like _PARENT); + alter table only _DEST_TABLE add primary key (_PKEY); + diff --git a/tests/scripts/conf/mover.ini b/tests/scripts/conf/mover.ini new file mode 100644 index 00000000..5ad7cd7f --- /dev/null +++ b/tests/scripts/conf/mover.ini @@ -0,0 +1,13 @@ +[queue_mover] +job_name = queue_mover_test + +src_db = dbname=scriptsrc +dst_db = dbname=scriptsrc + +pgq_queue_name = data.src + +dst_queue_name = data.middle + +logfile = sys/%(job_name)s.log +pidfile = sys/%(job_name)s.pid + diff --git a/tests/scripts/conf/table.ini b/tests/scripts/conf/table.ini new file mode 100644 index 00000000..c7165ea3 --- /dev/null +++ b/tests/scripts/conf/table.ini @@ -0,0 +1,29 @@ +[table_dispatcher] +job_name = table_test + +src_db = dbname=scriptsrc +dst_db = dbname=scriptdst + +pgq_queue_name = data.middle + +logfile = sys/%(job_name)s.log +pidfile = sys/%(job_name)s.pid + +# where to put data. when partitioning, will be used as base name +dest_table = data2 + +# date field with will be used for partitioning +# special value: _EVTIME - event creation time +part_column = start_date + +#fields = * +#fields = id_pstn_cdr, foo, bar +#fields = id_pstn_cdr:my_id, foo, bar:baz + + +# template used for creating partition tables +# _DEST_TABLE +part_template = + create table _DEST_TABLE () inherits (data2); + + diff --git a/tests/scripts/conf/ticker.ini b/tests/scripts/conf/ticker.ini new file mode 100644 index 00000000..8e2ee9b2 --- /dev/null +++ b/tests/scripts/conf/ticker.ini @@ -0,0 +1,26 @@ +[pgqadm] + +job_name = ticker + +db = dbname=scriptsrc + +# how often to run maintenance [minutes] +maint_delay_min = 1 + +# how often to check fot activity [secs] +loop_delay = 0.1 + +# if there's no events, how often to tick (to show liveness) [secs] +idle_interval = 1 + +# if there's this much events available, tick +max_events = 10 + +# if there is not many events, then don't let them stay more than [secs] +max_lag = 0.5 + +logfile = sys/log.%(job_name)s +pidfile = sys/pid.%(job_name)s + +use_skylog = 0 + diff --git a/tests/scripts/data.sql b/tests/scripts/data.sql new file mode 100644 index 00000000..84572549 --- /dev/null +++ b/tests/scripts/data.sql @@ -0,0 +1,22 @@ + +set client_min_messages = 'warning'; + +create table data1 ( + id serial primary key, + data text +); + +create unique index idx_data1_uq on data1 (data); + +create index idx_data1_rand on data1 (id, data); + +create table data2 ( + id serial primary key, + data text, + -- old_id integer references data1, + constraint uq_data2 unique (data) +); + +create index idx_data2_rand on data2 (id, data); + + diff --git a/tests/scripts/env.sh b/tests/scripts/env.sh new file mode 100644 index 00000000..1ab42fa8 --- /dev/null +++ b/tests/scripts/env.sh @@ -0,0 +1,7 @@ + +PYTHONPATH=../../python:$PYTHONPATH +PATH=../../python:../../scripts:$PATH +export PYTHONPATH PATH + +#. /opt/pgsql/env + diff --git a/tests/scripts/gendb.sh b/tests/scripts/gendb.sh new file mode 100755 index 00000000..8f329980 --- /dev/null +++ b/tests/scripts/gendb.sh @@ -0,0 +1,45 @@ +#! /bin/sh + +. ./env.sh + +contrib=/usr/share/postgresql/8.1/contrib +contrib=/opt/pgsql/share/contrib + +mkdir -p sys +./stop.sh +sleep 1 + +rm -f sys/* + + +db=scriptsrc +echo "creating database: $db" +dropdb $db +sleep 1 +createdb $db + +pgqadm.py conf/ticker.ini install + +#createlang plpgsql $db +#createlang plpythonu $db +#psql -q $db -f $contrib/txid.sql +#psql -q $db -f $contrib/pgq.sql +psql -q $db -f $contrib/pgq_ext.sql +psql -q $db -f $contrib/logutriga.sql +psql -q $db -f data.sql +psql -q $db -f install.sql + +db=scriptdst +echo "creating database: $db" +dropdb $db +sleep 1 +createdb $db +createlang plpgsql $db +psql -q $db -f data.sql +psql -q $db -f $contrib/pgq_ext.sql + +echo "done, testing" + +#pgqmgr.py -d conf/ticker.ini ticker +#./run-tests.sh + diff --git a/tests/scripts/install.sql b/tests/scripts/install.sql new file mode 100644 index 00000000..392980bf --- /dev/null +++ b/tests/scripts/install.sql @@ -0,0 +1,7 @@ + +select pgq.create_queue('data.src'); +select pgq.create_queue('data.middle'); + +create trigger test_logger after insert or update or delete +on data1 for each row execute procedure pgq.logutriga('data.src'); + diff --git a/tests/scripts/run-tests.sh b/tests/scripts/run-tests.sh new file mode 100755 index 00000000..b9223e18 --- /dev/null +++ b/tests/scripts/run-tests.sh @@ -0,0 +1,15 @@ +#! /bin/sh + +. ./env.sh + +pgqadm.py -d conf/ticker.ini ticker +queue_mover.py -d conf/mover.ini +cube_dispatcher.py -d conf/cube.ini +table_dispatcher.py -d conf/table.ini + +sleep 1 +psql scriptsrc <<EOF +insert into data1 (data) values ('data1.1'); +insert into data1 (data) values ('data1.2'); +EOF + diff --git a/tests/scripts/stop.sh b/tests/scripts/stop.sh new file mode 100755 index 00000000..423eb062 --- /dev/null +++ b/tests/scripts/stop.sh @@ -0,0 +1,14 @@ +#! /bin/sh + +. ./env.sh + +cube_dispatcher.py -s conf/cube.ini +table_dispatcher.py -s conf/table.ini +queue_mover.py -s conf/mover.ini + +sleep 1 + +pgqadm.py -s conf/ticker.ini + +#killall python + diff --git a/tests/skylog/logtest.py b/tests/skylog/logtest.py new file mode 100755 index 00000000..8a1df30b --- /dev/null +++ b/tests/skylog/logtest.py @@ -0,0 +1,17 @@ +#! /usr/bin/env python + +import sys, os, skytools + +import skytools.skylog + +class LogTest(skytools.DBScript): + def work(self): + self.log.error('test error') + self.log.warning('test warning') + self.log.info('test info') + self.log.debug('test debug') + +if __name__ == '__main__': + script = LogTest('log_test', sys.argv[1:]) + script.start() + diff --git a/tests/skylog/runtest.sh b/tests/skylog/runtest.sh new file mode 100755 index 00000000..d456a7d0 --- /dev/null +++ b/tests/skylog/runtest.sh @@ -0,0 +1,6 @@ +#! /bin/sh + +. ../env.sh + +exec ./logtest.py test.ini "$@" + diff --git a/tests/skylog/skylog.ini b/tests/skylog/skylog.ini new file mode 100644 index 00000000..d2490044 --- /dev/null +++ b/tests/skylog/skylog.ini @@ -0,0 +1,73 @@ +; notes: +; - 'args' is mandatory in [handler_*] sections +; - in lists there must not be spaces + +; +; top-level config +; + +; list of all loggers +[loggers] +keys=root +; root logger sees everything. there can be per-job configs by +; specifing loggers with job_name of the script + +; list of all handlers +[handlers] +keys=stderr,logdb,logsrv,logfile + +; list of all formatters +[formatters] +keys=short,long,none + +; +; map specific loggers to specifig handlers +; +[logger_root] +level=DEBUG +handlers=stderr,logdb,logsrv,logfile +;,logfile +;logdb,logsrv,logfile + +; +; configure formatters +; +[formatter_short] +format=%(asctime)s %(levelname)s %(message)s +datefmt=%H:%M + +[formatter_long] +format=%(asctime)s %(process)s %(levelname)s %(message)s + +[formatter_none] +format=%(message)s + +; +; configure handlers +; + +; file. args: stream +[handler_stderr] +class=StreamHandler +args=(sys.stderr,) +formatter=short + +; log into db. args: conn_string +[handler_logdb] +class=skylog.LogDBHandler +args=("host=127.0.0.1 port=5432 user=marko dbname=logdb",) +formatter=none +level=INFO + +; JSON messages over UDP. args: host, port +[handler_logsrv] +class=skylog.UdpLogServerHandler +args=('127.0.0.1', 6666) +formatter=none + +; rotating logfile. args: filename, maxsize, maxcount +[handler_logfile] +class=skylog.EasyRotatingFileHandler +args=('~/log/%(job_name)s.log', 100*1024*1024, 3) +formatter=long + diff --git a/tests/skylog/test.ini b/tests/skylog/test.ini new file mode 100644 index 00000000..2c6dda93 --- /dev/null +++ b/tests/skylog/test.ini @@ -0,0 +1,6 @@ +[log_test] + +loop_delay = 5 + +logfile = xtest.log + diff --git a/tests/walmgr/conf.master/pg_hba.conf b/tests/walmgr/conf.master/pg_hba.conf new file mode 100644 index 00000000..c126e645 --- /dev/null +++ b/tests/walmgr/conf.master/pg_hba.conf @@ -0,0 +1,75 @@ +# PostgreSQL Client Authentication Configuration File +# =================================================== +# +# Refer to the PostgreSQL Administrator's Guide, chapter "Client +# Authentication" for a complete description. A short synopsis +# follows. +# +# This file controls: which hosts are allowed to connect, how clients +# are authenticated, which PostgreSQL user names they can use, which +# databases they can access. Records take one of these forms: +# +# local DATABASE USER METHOD [OPTION] +# host DATABASE USER CIDR-ADDRESS METHOD [OPTION] +# hostssl DATABASE USER CIDR-ADDRESS METHOD [OPTION] +# hostnossl DATABASE USER CIDR-ADDRESS METHOD [OPTION] +# +# (The uppercase items must be replaced by actual values.) +# +# The first field is the connection type: "local" is a Unix-domain socket, +# "host" is either a plain or SSL-encrypted TCP/IP socket, "hostssl" is an +# SSL-encrypted TCP/IP socket, and "hostnossl" is a plain TCP/IP socket. +# +# DATABASE can be "all", "sameuser", "samerole", a database name, or +# a comma-separated list thereof. +# +# USER can be "all", a user name, a group name prefixed with "+", or +# a comma-separated list thereof. In both the DATABASE and USER fields +# you can also write a file name prefixed with "@" to include names from +# a separate file. +# +# CIDR-ADDRESS specifies the set of hosts the record matches. +# It is made up of an IP address and a CIDR mask that is an integer +# (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that specifies +# the number of significant bits in the mask. Alternatively, you can write +# an IP address and netmask in separate columns to specify the set of hosts. +# +# METHOD can be "trust", "reject", "md5", "crypt", "password", +# "krb5", "ident", or "pam". Note that "password" sends passwords +# in clear text; "md5" is preferred since it sends encrypted passwords. +# +# OPTION is the ident map or the name of the PAM service, depending on METHOD. +# +# Database and user names containing spaces, commas, quotes and other special +# characters must be quoted. Quoting one of the keywords "all", "sameuser" or +# "samerole" makes the name lose its special character, and just match a +# database or username with that name. +# +# This file is read on server startup and when the postmaster receives +# a SIGHUP signal. If you edit the file on a running system, you have +# to SIGHUP the postmaster for the changes to take effect. You can use +# "pg_ctl reload" to do that. + +# Put your actual configuration here +# ---------------------------------- +# +# If you want to allow non-local connections, you need to add more +# "host" records. In that case you will also need to make PostgreSQL listen +# on a non-local interface via the listen_addresses configuration parameter, +# or via the -i or -h command line switches. +# + +# CAUTION: Configuring the system for local "trust" authentication allows +# any local user to connect as any PostgreSQL user, including the database +# superuser. If you do not trust all your local users, use another +# authentication method. + + +# TYPE DATABASE USER CIDR-ADDRESS METHOD + +# "local" is for Unix domain socket connections only +local all all trust +# IPv4 local connections: +host all all 127.0.0.1/32 trust +# IPv6 local connections: +host all all ::1/128 trust diff --git a/tests/walmgr/conf.master/pg_ident.conf b/tests/walmgr/conf.master/pg_ident.conf new file mode 100644 index 00000000..4019f6fe --- /dev/null +++ b/tests/walmgr/conf.master/pg_ident.conf @@ -0,0 +1,36 @@ +# PostgreSQL Ident Authentication Maps +# ==================================== +# +# Refer to the PostgreSQL Administrator's Guide, chapter "Client +# Authentication" for a complete description. A short synopsis +# follows. +# +# This file controls PostgreSQL ident-based authentication. It maps +# ident user names (typically Unix user names) to their corresponding +# PostgreSQL user names. Records are of the form: +# +# MAPNAME IDENT-USERNAME PG-USERNAME +# +# (The uppercase quantities must be replaced by actual values.) +# +# MAPNAME is the (otherwise freely chosen) map name that was used in +# pg_hba.conf. IDENT-USERNAME is the detected user name of the +# client. PG-USERNAME is the requested PostgreSQL user name. The +# existence of a record specifies that IDENT-USERNAME may connect as +# PG-USERNAME. Multiple maps may be specified in this file and used +# by pg_hba.conf. +# +# This file is read on server startup and when the postmaster receives +# a SIGHUP signal. If you edit the file on a running system, you have +# to SIGHUP the postmaster for the changes to take effect. You can use +# "pg_ctl reload" to do that. + +# Put your actual configuration here +# ---------------------------------- +# +# No map names are defined in the default configuration. If all ident +# user names and PostgreSQL user names are the same, you don't need +# this file. Instead, use the special map name "sameuser" in +# pg_hba.conf. + +# MAPNAME IDENT-USERNAME PG-USERNAME diff --git a/tests/walmgr/conf.master/postgresql.conf b/tests/walmgr/conf.master/postgresql.conf new file mode 100644 index 00000000..ca910c61 --- /dev/null +++ b/tests/walmgr/conf.master/postgresql.conf @@ -0,0 +1,17 @@ +# - Connection Settings - +#port = 5432 +port = 7200 +unix_socket_directory = '/tmp' + +#archive_command = '' # command to use to archive a logfile +archive_command = '/usr/bin/walmgr.py /opt/src/mgrtest/wal-master.ini xarchive %p %f' + # segment + + +# These settings are initialized by initdb -- they might be changed +lc_messages = 'C' # locale for system error message + # strings +lc_monetary = 'C' # locale for monetary formatting +lc_numeric = 'C' # locale for number formatting +lc_time = 'C' # locale for time formatting + diff --git a/tests/walmgr/conf.slave/pg_hba.conf b/tests/walmgr/conf.slave/pg_hba.conf new file mode 100644 index 00000000..c126e645 --- /dev/null +++ b/tests/walmgr/conf.slave/pg_hba.conf @@ -0,0 +1,75 @@ +# PostgreSQL Client Authentication Configuration File +# =================================================== +# +# Refer to the PostgreSQL Administrator's Guide, chapter "Client +# Authentication" for a complete description. A short synopsis +# follows. +# +# This file controls: which hosts are allowed to connect, how clients +# are authenticated, which PostgreSQL user names they can use, which +# databases they can access. Records take one of these forms: +# +# local DATABASE USER METHOD [OPTION] +# host DATABASE USER CIDR-ADDRESS METHOD [OPTION] +# hostssl DATABASE USER CIDR-ADDRESS METHOD [OPTION] +# hostnossl DATABASE USER CIDR-ADDRESS METHOD [OPTION] +# +# (The uppercase items must be replaced by actual values.) +# +# The first field is the connection type: "local" is a Unix-domain socket, +# "host" is either a plain or SSL-encrypted TCP/IP socket, "hostssl" is an +# SSL-encrypted TCP/IP socket, and "hostnossl" is a plain TCP/IP socket. +# +# DATABASE can be "all", "sameuser", "samerole", a database name, or +# a comma-separated list thereof. +# +# USER can be "all", a user name, a group name prefixed with "+", or +# a comma-separated list thereof. In both the DATABASE and USER fields +# you can also write a file name prefixed with "@" to include names from +# a separate file. +# +# CIDR-ADDRESS specifies the set of hosts the record matches. +# It is made up of an IP address and a CIDR mask that is an integer +# (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that specifies +# the number of significant bits in the mask. Alternatively, you can write +# an IP address and netmask in separate columns to specify the set of hosts. +# +# METHOD can be "trust", "reject", "md5", "crypt", "password", +# "krb5", "ident", or "pam". Note that "password" sends passwords +# in clear text; "md5" is preferred since it sends encrypted passwords. +# +# OPTION is the ident map or the name of the PAM service, depending on METHOD. +# +# Database and user names containing spaces, commas, quotes and other special +# characters must be quoted. Quoting one of the keywords "all", "sameuser" or +# "samerole" makes the name lose its special character, and just match a +# database or username with that name. +# +# This file is read on server startup and when the postmaster receives +# a SIGHUP signal. If you edit the file on a running system, you have +# to SIGHUP the postmaster for the changes to take effect. You can use +# "pg_ctl reload" to do that. + +# Put your actual configuration here +# ---------------------------------- +# +# If you want to allow non-local connections, you need to add more +# "host" records. In that case you will also need to make PostgreSQL listen +# on a non-local interface via the listen_addresses configuration parameter, +# or via the -i or -h command line switches. +# + +# CAUTION: Configuring the system for local "trust" authentication allows +# any local user to connect as any PostgreSQL user, including the database +# superuser. If you do not trust all your local users, use another +# authentication method. + + +# TYPE DATABASE USER CIDR-ADDRESS METHOD + +# "local" is for Unix domain socket connections only +local all all trust +# IPv4 local connections: +host all all 127.0.0.1/32 trust +# IPv6 local connections: +host all all ::1/128 trust diff --git a/tests/walmgr/conf.slave/pg_ident.conf b/tests/walmgr/conf.slave/pg_ident.conf new file mode 100644 index 00000000..4019f6fe --- /dev/null +++ b/tests/walmgr/conf.slave/pg_ident.conf @@ -0,0 +1,36 @@ +# PostgreSQL Ident Authentication Maps +# ==================================== +# +# Refer to the PostgreSQL Administrator's Guide, chapter "Client +# Authentication" for a complete description. A short synopsis +# follows. +# +# This file controls PostgreSQL ident-based authentication. It maps +# ident user names (typically Unix user names) to their corresponding +# PostgreSQL user names. Records are of the form: +# +# MAPNAME IDENT-USERNAME PG-USERNAME +# +# (The uppercase quantities must be replaced by actual values.) +# +# MAPNAME is the (otherwise freely chosen) map name that was used in +# pg_hba.conf. IDENT-USERNAME is the detected user name of the +# client. PG-USERNAME is the requested PostgreSQL user name. The +# existence of a record specifies that IDENT-USERNAME may connect as +# PG-USERNAME. Multiple maps may be specified in this file and used +# by pg_hba.conf. +# +# This file is read on server startup and when the postmaster receives +# a SIGHUP signal. If you edit the file on a running system, you have +# to SIGHUP the postmaster for the changes to take effect. You can use +# "pg_ctl reload" to do that. + +# Put your actual configuration here +# ---------------------------------- +# +# No map names are defined in the default configuration. If all ident +# user names and PostgreSQL user names are the same, you don't need +# this file. Instead, use the special map name "sameuser" in +# pg_hba.conf. + +# MAPNAME IDENT-USERNAME PG-USERNAME diff --git a/tests/walmgr/conf.slave/postgresql.conf b/tests/walmgr/conf.slave/postgresql.conf new file mode 100644 index 00000000..e4d5ce67 --- /dev/null +++ b/tests/walmgr/conf.slave/postgresql.conf @@ -0,0 +1,434 @@ +# ----------------------------- +# PostgreSQL configuration file +# ----------------------------- +# +# This file consists of lines of the form: +# +# name = value +# +# (The '=' is optional.) White space may be used. Comments are introduced +# with '#' anywhere on a line. The complete list of option names and +# allowed values can be found in the PostgreSQL documentation. The +# commented-out settings shown in this file represent the default values. +# +# Please note that re-commenting a setting is NOT sufficient to revert it +# to the default value, unless you restart the postmaster. +# +# Any option can also be given as a command line switch to the +# postmaster, e.g. 'postmaster -c log_connections=on'. Some options +# can be changed at run-time with the 'SET' SQL command. +# +# This file is read on postmaster startup and when the postmaster +# receives a SIGHUP. If you edit the file on a running system, you have +# to SIGHUP the postmaster for the changes to take effect, or use +# "pg_ctl reload". Some settings, such as listen_addresses, require +# a postmaster shutdown and restart to take effect. + + +#--------------------------------------------------------------------------- +# FILE LOCATIONS +#--------------------------------------------------------------------------- + +# The default values of these variables are driven from the -D command line +# switch or PGDATA environment variable, represented here as ConfigDir. + +#data_directory = 'ConfigDir' # use data in another directory +#hba_file = 'ConfigDir/pg_hba.conf' # host-based authentication file +#ident_file = 'ConfigDir/pg_ident.conf' # IDENT configuration file + +# If external_pid_file is not explicitly set, no extra pid file is written. +#external_pid_file = '(none)' # write an extra pid file + + +#--------------------------------------------------------------------------- +# CONNECTIONS AND AUTHENTICATION +#--------------------------------------------------------------------------- + +# - Connection Settings - + +#listen_addresses = 'localhost' # what IP address(es) to listen on; + # comma-separated list of addresses; + # defaults to 'localhost', '*' = all +#port = 5432 +port = 7201 +max_connections = 100 +# note: increasing max_connections costs ~400 bytes of shared memory per +# connection slot, plus lock space (see max_locks_per_transaction). You +# might also need to raise shared_buffers to support more connections. +#superuser_reserved_connections = 2 +unix_socket_directory = '/opt/src/mgrtest' +#unix_socket_directory = '' +#unix_socket_group = '' +#unix_socket_permissions = 0777 # octal +#bonjour_name = '' # defaults to the computer name + +# - Security & Authentication - + +#authentication_timeout = 60 # 1-600, in seconds +#ssl = off +#password_encryption = on +#db_user_namespace = off + +# Kerberos +#krb_server_keyfile = '' +#krb_srvname = 'postgres' +#krb_server_hostname = '' # empty string matches any keytab entry +#krb_caseins_users = off + +# - TCP Keepalives - +# see 'man 7 tcp' for details + +#tcp_keepalives_idle = 0 # TCP_KEEPIDLE, in seconds; + # 0 selects the system default +#tcp_keepalives_interval = 0 # TCP_KEEPINTVL, in seconds; + # 0 selects the system default +#tcp_keepalives_count = 0 # TCP_KEEPCNT; + # 0 selects the system default + + +#--------------------------------------------------------------------------- +# RESOURCE USAGE (except WAL) +#--------------------------------------------------------------------------- + +# - Memory - + +shared_buffers = 1000 # min 16 or max_connections*2, 8KB each +#temp_buffers = 1000 # min 100, 8KB each +#max_prepared_transactions = 5 # can be 0 or more +# note: increasing max_prepared_transactions costs ~600 bytes of shared memory +# per transaction slot, plus lock space (see max_locks_per_transaction). +#work_mem = 1024 # min 64, size in KB +#maintenance_work_mem = 16384 # min 1024, size in KB +#max_stack_depth = 2048 # min 100, size in KB + +# - Free Space Map - + +#max_fsm_pages = 20000 # min max_fsm_relations*16, 6 bytes each +#max_fsm_relations = 1000 # min 100, ~70 bytes each + +# - Kernel Resource Usage - + +#max_files_per_process = 1000 # min 25 +#preload_libraries = '' + +# - Cost-Based Vacuum Delay - + +#vacuum_cost_delay = 0 # 0-1000 milliseconds +#vacuum_cost_page_hit = 1 # 0-10000 credits +#vacuum_cost_page_miss = 10 # 0-10000 credits +#vacuum_cost_page_dirty = 20 # 0-10000 credits +#vacuum_cost_limit = 200 # 0-10000 credits + +# - Background writer - + +#bgwriter_delay = 200 # 10-10000 milliseconds between rounds +#bgwriter_lru_percent = 1.0 # 0-100% of LRU buffers scanned/round +#bgwriter_lru_maxpages = 5 # 0-1000 buffers max written/round +#bgwriter_all_percent = 0.333 # 0-100% of all buffers scanned/round +#bgwriter_all_maxpages = 5 # 0-1000 buffers max written/round + + +#--------------------------------------------------------------------------- +# WRITE AHEAD LOG +#--------------------------------------------------------------------------- + +# - Settings - + +#fsync = on # turns forced synchronization on or off +#wal_sync_method = fsync # the default is the first option + # supported by the operating system: + # open_datasync + # fdatasync + # fsync + # fsync_writethrough + # open_sync +#full_page_writes = on # recover from partial page writes +#wal_buffers = 8 # min 4, 8KB each +#commit_delay = 0 # range 0-100000, in microseconds +#commit_siblings = 5 # range 1-1000 + +# - Checkpoints - + +#checkpoint_segments = 3 # in logfile segments, min 1, 16MB each +#checkpoint_timeout = 300 # range 30-3600, in seconds +#checkpoint_warning = 30 # in seconds, 0 is off + +# - Archiving - + +#archive_command = '' # command to use to archive a logfile +archive_command = '/usr/bin/walmgr.py /opt/src/mgrtest/wal-master.ini xarchive %p %f' + # segment + + +#--------------------------------------------------------------------------- +# QUERY TUNING +#--------------------------------------------------------------------------- + +# - Planner Method Configuration - + +#enable_bitmapscan = on +#enable_hashagg = on +#enable_hashjoin = on +#enable_indexscan = on +#enable_mergejoin = on +#enable_nestloop = on +#enable_seqscan = on +#enable_sort = on +#enable_tidscan = on + +# - Planner Cost Constants - + +#effective_cache_size = 1000 # typically 8KB each +#random_page_cost = 4 # units are one sequential page fetch + # cost +#cpu_tuple_cost = 0.01 # (same) +#cpu_index_tuple_cost = 0.001 # (same) +#cpu_operator_cost = 0.0025 # (same) + +# - Genetic Query Optimizer - + +#geqo = on +#geqo_threshold = 12 +#geqo_effort = 5 # range 1-10 +#geqo_pool_size = 0 # selects default based on effort +#geqo_generations = 0 # selects default based on effort +#geqo_selection_bias = 2.0 # range 1.5-2.0 + +# - Other Planner Options - + +#default_statistics_target = 10 # range 1-1000 +#constraint_exclusion = off +#from_collapse_limit = 8 +#join_collapse_limit = 8 # 1 disables collapsing of explicit + # JOINs + + +#--------------------------------------------------------------------------- +# ERROR REPORTING AND LOGGING +#--------------------------------------------------------------------------- + +# - Where to Log - + +#log_destination = 'stderr' # Valid values are combinations of + # stderr, syslog and eventlog, + # depending on platform. + +# This is used when logging to stderr: +#redirect_stderr = off # Enable capturing of stderr into log + # files + +# These are only used if redirect_stderr is on: +#log_directory = 'pg_log' # Directory where log files are written + # Can be absolute or relative to PGDATA +#log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log' # Log file name pattern. + # Can include strftime() escapes +#log_truncate_on_rotation = off # If on, any existing log file of the same + # name as the new log file will be + # truncated rather than appended to. But + # such truncation only occurs on + # time-driven rotation, not on restarts + # or size-driven rotation. Default is + # off, meaning append to existing files + # in all cases. +#log_rotation_age = 1440 # Automatic rotation of logfiles will + # happen after so many minutes. 0 to + # disable. +#log_rotation_size = 10240 # Automatic rotation of logfiles will + # happen after so many kilobytes of log + # output. 0 to disable. + +# These are relevant when logging to syslog: +#syslog_facility = 'LOCAL0' +#syslog_ident = 'postgres' + + +# - When to Log - + +#client_min_messages = notice # Values, in order of decreasing detail: + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # log + # notice + # warning + # error + +#log_min_messages = notice # Values, in order of decreasing detail: + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # info + # notice + # warning + # error + # log + # fatal + # panic + +#log_error_verbosity = default # terse, default, or verbose messages + +#log_min_error_statement = panic # Values in order of increasing severity: + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # info + # notice + # warning + # error + # panic(off) + +#log_min_duration_statement = -1 # -1 is disabled, 0 logs all statements + # and their durations, in milliseconds. + +#silent_mode = off # DO NOT USE without syslog or + # redirect_stderr + +# - What to Log - + +#debug_print_parse = off +#debug_print_rewritten = off +#debug_print_plan = off +#debug_pretty_print = off +#log_connections = off +#log_disconnections = off +#log_duration = off +#log_line_prefix = '' # Special values: + # %u = user name + # %d = database name + # %r = remote host and port + # %h = remote host + # %p = PID + # %t = timestamp (no milliseconds) + # %m = timestamp with milliseconds + # %i = command tag + # %c = session id + # %l = session line number + # %s = session start timestamp + # %x = transaction id + # %q = stop here in non-session + # processes + # %% = '%' + # e.g. '<%u%%%d> ' +#log_statement = 'none' # none, mod, ddl, all +#log_hostname = off + + +#--------------------------------------------------------------------------- +# RUNTIME STATISTICS +#--------------------------------------------------------------------------- + +# - Statistics Monitoring - + +#log_parser_stats = off +#log_planner_stats = off +#log_executor_stats = off +#log_statement_stats = off + +# - Query/Index Statistics Collector - + +#stats_start_collector = on +#stats_command_string = off +#stats_block_level = off +#stats_row_level = off +#stats_reset_on_server_start = off + + +#--------------------------------------------------------------------------- +# AUTOVACUUM PARAMETERS +#--------------------------------------------------------------------------- + +#autovacuum = off # enable autovacuum subprocess? +#autovacuum_naptime = 60 # time between autovacuum runs, in secs +#autovacuum_vacuum_threshold = 1000 # min # of tuple updates before + # vacuum +#autovacuum_analyze_threshold = 500 # min # of tuple updates before + # analyze +#autovacuum_vacuum_scale_factor = 0.4 # fraction of rel size before + # vacuum +#autovacuum_analyze_scale_factor = 0.2 # fraction of rel size before + # analyze +#autovacuum_vacuum_cost_delay = -1 # default vacuum cost delay for + # autovac, -1 means use + # vacuum_cost_delay +#autovacuum_vacuum_cost_limit = -1 # default vacuum cost limit for + # autovac, -1 means use + # vacuum_cost_limit + + +#--------------------------------------------------------------------------- +# CLIENT CONNECTION DEFAULTS +#--------------------------------------------------------------------------- + +# - Statement Behavior - + +#search_path = '$user,public' # schema names +#default_tablespace = '' # a tablespace name, '' uses + # the default +#check_function_bodies = on +#default_transaction_isolation = 'read committed' +#default_transaction_read_only = off +#statement_timeout = 0 # 0 is disabled, in milliseconds + +# - Locale and Formatting - + +#datestyle = 'iso, mdy' +#timezone = unknown # actually, defaults to TZ + # environment setting +#australian_timezones = off +#extra_float_digits = 0 # min -15, max 2 +#client_encoding = sql_ascii # actually, defaults to database + # encoding + +# These settings are initialized by initdb -- they might be changed +lc_messages = 'C' # locale for system error message + # strings +lc_monetary = 'C' # locale for monetary formatting +lc_numeric = 'C' # locale for number formatting +lc_time = 'C' # locale for time formatting + +# - Other Defaults - + +#explain_pretty_print = on +#dynamic_library_path = '$libdir' + + +#--------------------------------------------------------------------------- +# LOCK MANAGEMENT +#--------------------------------------------------------------------------- + +#deadlock_timeout = 1000 # in milliseconds +#max_locks_per_transaction = 64 # min 10 +# note: each lock table slot uses ~220 bytes of shared memory, and there are +# max_locks_per_transaction * (max_connections + max_prepared_transactions) +# lock table slots. + + +#--------------------------------------------------------------------------- +# VERSION/PLATFORM COMPATIBILITY +#--------------------------------------------------------------------------- + +# - Previous Postgres Versions - + +#add_missing_from = off +#backslash_quote = safe_encoding # on, off, or safe_encoding +#default_with_oids = off +#escape_string_warning = off +#regex_flavor = advanced # advanced, extended, or basic +#sql_inheritance = on + +# - Other Platforms & Clients - + +#transform_null_equals = off + + +#--------------------------------------------------------------------------- +# CUSTOMIZED OPTIONS +#--------------------------------------------------------------------------- + +#custom_variable_classes = '' # list of custom variable class names diff --git a/tests/walmgr/run-test.sh b/tests/walmgr/run-test.sh new file mode 100755 index 00000000..555ef282 --- /dev/null +++ b/tests/walmgr/run-test.sh @@ -0,0 +1,85 @@ +#! /bin/sh + +set -e + +tmp=/tmp/waltest +src=$PWD +walmgr=$src/../../python/walmgr.py + +test -f $tmp/data.master/postmaster.pid \ +&& kill `head -1 $tmp/data.master/postmaster.pid` || true +test -f $tmp/data.slave/postmaster.pid \ +&& kill `head -1 $tmp/data.slave/postmaster.pid` || true + +rm -rf $tmp +mkdir -p $tmp +cd $tmp + +LANG=C +PATH=/usr/lib/postgresql/8.1/bin:$PATH +export PATH LANG + +mkdir log slave + +# +# Prepare configs +# + +### wal.master.ini ### +cat > wal.master.ini <<EOF +[wal-master] +logfile = $tmp/log/wal-master.log +master_db = dbname=template1 port=7200 host=127.0.0.1 +master_data = $tmp/data.master +master_config = %(master_data)s/postgresql.conf +slave = localhost:$tmp/slave +completed_wals = %(slave)s/logs.complete +partial_wals = %(slave)s/logs.partial +full_backup = %(slave)s/data.master +# syncdaemon update frequency +loop_delay = 10.0 +EOF + +### wal.slave.ini ### +cat > wal.slave.ini <<EOF +[wal-slave] +logfile = $tmp/log/wal-slave.log +slave_data = $tmp/data.slave +slave_stop_cmd = $tmp/rc.slave stop +slave_start_cmd = $tmp/rc.slave start +slave = $tmp/slave +completed_wals = %(slave)s/logs.complete +partial_wals = %(slave)s/logs.partial +full_backup = %(slave)s/data.master +EOF + +### rc.slave ### +cat > rc.slave <<EOF +#! /bin/sh +cd $tmp +test -f $tmp/data.slave/postgresql.conf \ +|| cp $src/conf.slave/* $tmp/data.slave +pg_ctl -l $tmp/log/pg.slave.log -D $tmp/data.slave "\$1" +EOF +chmod +x rc.slave + +# +# Initialize master db +# +echo "### Running initdb for master ###" +initdb data.master > log/initdb.log 2>&1 +cp $src/conf.master/* data.master/ +pg_ctl -D data.master -l log/pg.master.log start +sleep 4 + +echo '####' $walmgr $tmp/wal.master.ini setup +$walmgr wal.master.ini setup +echo '####' $walmgr $tmp/wal.master.ini backup +$walmgr wal.master.ini backup + +echo '####' $walmgr $tmp/wal.slave.ini restore +$walmgr $tmp/wal.slave.ini restore +sleep 10 +echo '####' $walmgr $tmp/wal.slave.ini boot +$walmgr $tmp/wal.slave.ini boot + |