add new script docs to -head

12 files changed, 775 insertions, 18 deletions

diff --git a/doc/Makefile b/doc/Makefile
index 3854e6d3..e6af9c54 100644
--- a/doc/Makefile
+++ b/doc/Makefile
@@ -11,12 +11,22 @@ EPYARGS = --no-private --url="http://pgfoundry.org/projects/skytools/" \
 
 HTMLS = londiste.cmdline.html londiste.config.html README.html INSTALL.html \
 	londiste.ref.html TODO.html pgq-sql.html pgq-admin.html pgq-nodupes.html \
-	walmgr.html set.notes.html \
-	upgrade.html
+	$(SCRIPT_HTMLS)
+
+SCRIPT_TXTS = walmgr.txt cube_dispatcher.txt table_dispatcher.txt \
+	      queue_mover.txt queue_splitter.txt bulk_loader.txt \
+	      scriptmgr.txt skytools_upgrade.txt
+SCRIPT_MANS = $(SCRIPT_TXTS:.txt=.1)
+SCRIPT_HTMLS = $(SCRIPT_TXTS:.txt=.html)
+
+COMMON = common.switches.txt common.config.txt common.logutriga.txt
+
+GETATTRS = python ./getattrs.py
 
 all: man
 
-man: londiste.1 londiste.5 pgqadm.1 walmgr.1
+man: londiste.1 londiste.5 pgqadm.1 $(SCRIPT_MANS)
+
 html: $(HTMLS)
 
 install: man
@@ -25,7 +35,9 @@ install: man
 	install -m 644 londiste.1 $(DESTDIR)/$(mandir)/man1
 	install -m 644 londiste.5 $(DESTDIR)/$(mandir)/man5
 	install -m 644 pgqadm.1 $(DESTDIR)/$(mandir)/man1
-	install -m 644 walmgr.1 $(DESTDIR)/$(mandir)/man1
+	for m in $(SCRIPT_MANS); do \
+	  install -m 644 $$m $(DESTDIR)/$(mandir)/man1 ; \
+	done
 
 old.wiki.upload:
 	devupload.sh overview.txt $(wiki)
@@ -57,39 +69,48 @@ apiupload: apidoc
 	rsync -rtlz ../sql/pgq/docs/html/* $(web)/pgq/
 
 clean:
-	rm -rf api *.1 *.5 *.html *.xml
+	rm -rf api *.html
+
+distclean: clean
+	rm -rf ../sql/pgq/docs/pgq
 
-distclean:
-	rm -rf ../sql/pgq/docs/pgq api
+realclean: distclean
+	rm -f *.[15] *.xml
 
+ifneq ($(XMLTO),)
 londiste.1: londiste.cmdline.xml
-	xmlto man $<
+	$(XMLTO) man $<
 
 londiste.5: londiste.config.xml
-	xmlto man $<
+	$(XMLTO) man $<
 
 pgqadm.1: pgq-admin.xml
-	xmlto man $<
+	$(XMLTO) man $<
 
 walmgr.1: walmgr.xml
-	xmlto man $<
+	$(XMLTO) man $<
+endif
 
-%.xml: %.txt
-	#asciidoc -b docbook -d manpage -o - $< > $@
-	asciidoc -b docbook -d manpage -o - $< \
+ifneq ($(ASCIIDOC),)
+%.xml: %.txt $(COMMON)
+	$(ASCIIDOC) -b docbook -d manpage `$(GETATTRS) $<` -o - $< \
 	| python fixman.py > $@
 
-%.html: %.txt
-	asciidoc -a toc $<
+%.1: %.xml
+	$(XMLTO) man $<
+
+%.html: %.txt $(COMMON)
+	$(ASCIIDOC) -a toc `$(GETATTRS) $<` $<
 
 README.html: ../README
 	cat $< \
 	| sed -e 's,doc/\([!-~]*\)[.]txt,link:\1.html[],g' \
 	      -e 's,http:[!-~]*,&[],g' \
-	| asciidoc -o $@ -
+	| $(ASCIIDOC) -o $@ -
 
 INSTALL.html: ../INSTALL
-	asciidoc -o $@ $<
+	$(ASCIIDOC) -o $@ $<
+endif
 
 web: $(HTMLS)
 	rsync -avz $(HTMLS) $(web)/doc/
diff --git a/doc/bulk_loader.txt b/doc/bulk_loader.txt
new file mode 100644
index 00000000..45f725ac
--- /dev/null
+++ b/doc/bulk_loader.txt
@@ -0,0 +1,89 @@
+
+= bulk_loader(1) =
+
+== NAME ==
+
+bulk_loader - PgQ consumer that loads urlencoded records to slow databases
+
+== SYNOPSIS ==
+
+  bulk_loader.py [switches] config.ini 
+
+== DESCRIPTION ==
+
+bulk_loader is PgQ consumer that reads url encoded records from source queue
+and writes them into tables according to configuration file.  It is targeted
+to slow databases that cannot handle applying each row as separate statement.
+Originally written for BizgresMPP/greenplumDB which have very high per-statement
+overhead, but can also be used to load regular PostgreSQL database that cannot
+manage regular replication.
+
+Behaviour properties:
+- reads urlencoded "logutriga" records.
+- does not do partitioning, but allows optionally redirect table events.
+- does not keep event order.
+- always loads data with COPY, either directly to main table (INSERTs)
+  or to temp tables (UPDATE/COPY) then applies from there.
+
+Events are usually procuded by `pgq.logutriga()`.  Logutriga adds all the data
+of the record  into the event (also in case of updates and deletes).
+
+== QUICK-START ==
+
+Basic bulk_loader setup and usage can be summarized by the following
+steps:
+
+ 1. pgq and logutriga must be installed in source databases.
+    See pgqadm man page for details.  target database must also
+    have pgq_ext schema.
+
+ 2. edit a bulk_loader configuration file, say bulk_loader_sample.ini
+
+ 3. create source queue 
+
+      $ pgqadm.py ticker.ini create <queue>
+ 
+ 4. Tune source queue to have big batches:
+
+      $ pgqadm.py ticker.ini config <queue> ticker_max_count="10000" ticker_max_lag="10 minutes" ticker_idle_period="10 minutes"
+
+ 5. create target database and tables in it.
+
+ 6. launch bulk_loader in daemon mode
+
+      $ bulk_loader.py -d bulk_loader_sample.ini
+
+ 7. start producing events (create logutriga trggers on tables)
+      CREATE OR REPLACE TRIGGER trig_bulk_replica AFTER INSERT OR UPDATE ON some_table 
+           FOR EACH ROW EXECUTE PROCEDURE pgq.logutriga('<queue>')
+
+== CONFIG ==
+
+include::common.config.txt[]
+
+=== Config options specific to `bulk_loader` ===
+
+  src_db::
+    Connect string for source database where the queue resides.
+ 
+  dst_db::
+    Connect string for target database where the tables should be created.
+
+  remap_tables::
+    Optional parameter for table redirection.  Contains comma-separated
+    list of <oldname>:<newname> pairs.  Eg: `oldtable1:newtable1, oldtable2:newtable2`.
+
+  load_method::
+    Optional parameter for load method selection.  Available options:
+    0:: UPDATE as UPDATE from temp table.  This is default.
+    1:: UPDATE as DELETE+COPY from temp table.
+    2:: merge INSERTs with UPDATEs, then do DELETE+COPY from temp table.
+
+== LOGUTRIGA EVENT FORMAT ==
+
+include::common.logutriga.txt[]
+
+== COMMAND LINE SWITCHES ==
+
+include::common.switches.txt[]
+
diff --git a/doc/common.config.txt b/doc/common.config.txt
new file mode 100644
index 00000000..7a74623f
--- /dev/null
+++ b/doc/common.config.txt
@@ -0,0 +1,38 @@
+
+=== Common configuration parameters ===
+
+  job_name::
+    Name for particulat job the script does.  Script will log under this name
+    to logdb/logserver.  The name is also used as default for PgQ consumer name.
+    It should be unique.
+
+  pidfile::
+    Location for pid file.  If not given, script is disallowed to daemonize.
+
+  logfile::
+    Location for log file.
+
+  loop_delay::
+    If continuisly running process, how long to sleep after each work loop,
+    in seconds.  Default: 1.
+ 
+  connection_lifetime::
+    Close and reconnect older database connections.
+
+  use_skylog::
+     foo.
+
+ifdef::pgq[]
+
+=== Common PgQ consumer parameters ===
+
+  pgq_queue_name::
+    Queue name to attach to.
+    No default.
+
+  pgq_consumer_id::
+    Consumers ID to use when registering.
+    Default: %(job_name)s
+
+endif::pgq[]
+
diff --git a/doc/common.logutriga.txt b/doc/common.logutriga.txt
new file mode 100644
index 00000000..99f02312
--- /dev/null
+++ b/doc/common.logutriga.txt
@@ -0,0 +1,19 @@
+
+PgQ trigger function `pgq.logutriga()` sends table change event into
+queue in following format:
+
+  ev_type::
+    `(op || ":" || pkey_fields)`.  Where op is either "I", "U" or "D",
+    corresponging to insert, update or delete.  And `pkey_fields`
+    is comma-separated list of primary key fields for table.
+    Operation type is always present but pkey_fields list can be empty,
+    if table has no primary keys.  Example: `I:col1,col2`
+
+  ev_data::
+    Urlencoded record of data.  It uses db-specific urlecoding where
+    existence of '=' is meaningful - missing '=' means NULL, present
+    '=' means literal value.  Example: `id=3&name=str&nullvalue&emptyvalue=`
+
+  ev_extra1::
+    Fully qualified table name.
+
diff --git a/doc/common.switches.txt b/doc/common.switches.txt
new file mode 100644
index 00000000..72da7bc3
--- /dev/null
+++ b/doc/common.switches.txt
@@ -0,0 +1,29 @@
+
+Following switches are common to all skytools.DBScript-based
+Python programs.
+
+  -h, --help::
+	show help message and exit
+
+  -q, --quiet::
+	make program silent
+
+  -v, --verbose::
+	make program more verbose
+
+  -d, --daemon::
+	make program go background
+
+Following switches are used to control already running process.
+The pidfile is read from config then signal is sent to process
+id specified there.
+
+  -r, --reload::
+	reload config (send SIGHUP)
+
+  -s, --stop::
+	stop program safely (send SIGINT)
+
+  -k, --kill::
+	kill program immidiately (send SIGTERM)
+
diff --git a/doc/cube_dispatcher.txt b/doc/cube_dispatcher.txt
new file mode 100644
index 00000000..63ab1270
--- /dev/null
+++ b/doc/cube_dispatcher.txt
@@ -0,0 +1,120 @@
+
+= cube_dispatcher(1) =
+
+== NAME ==
+
+cube_dispatcher - PgQ consumer that is used to write source records into partitoned tables
+
+== SYNOPSIS ==
+
+  cube_dispatcher.py [switches] config.ini 
+
+== DESCRIPTION ==
+
+cube_dispatcher is PgQ consumer that reads url encoded records from source queue
+and writes them into partitioned tables according to configuration file.
+Used to prepare data for business intelligence. Name of the table is read from 
+producer field in event. Batch creation time is used for partitioning. All records
+created in same day will go into same table partion. If partiton does not exist
+cube dispatcer will create it according to template.
+
+Events are usually procuded by `pgq.logutriga()`.  Logutriga adds all the data
+of the record  into the event (also in case of updates and deletes).
+
+`cube_dispatcher` can be used in to modes:
+
+keep_all::
+  keeps all the data that comes in. If record is updated several times 
+  during one day then table partiton for that day will contain several instances of 
+  that record.
+keep_latest::
+  only last instance of each record is kept for each day. That also
+  means that all tables must have primary keys so cube dispatcher can delete previous 
+  versions of records before inserting new data.
+
+== QUICK-START ==
+
+Basic cube_dispatcher setup and usage can be summarized by the following
+steps:
+
+ 1. pgq and logutriga must be installed in source databases.
+    See pgqadm man page for details.  target database must also
+    have pgq_ext schema.
+
+ 2. edit a cube_dispatcher configuration file, say cube_dispatcher_sample.ini
+
+ 3. create source queue 
+
+      $ pgqadm.py ticker.ini create <queue>
+
+ 4. create target database and parent tables in it.
+
+ 5. launch cube dispatcher in daemon mode
+
+      $ cube_dispatcher.py cube_dispatcher_sample.ini -d
+
+ 6. start producing events (create logutriga trggers on tables)
+      CREATE OR REPLACE TRIGGER trig_cube_replica AFTER INSERT OR UPDATE ON some_table 
+           FOR EACH ROW EXECUTE PROCEDURE pgq.logutriga('<queue>')
+
+== CONFIG ==
+
+include::common.config.txt[]
+
+=== Config options specific to `cube_dispatcher` ===
+
+  src_db::
+    Connect string for source database where the queue resides.
+ 
+  dst_db::
+    Connect string for target database where the tables should be created.
+
+  mode::
+    Operation mode for cube_dispatcher.  Either `keep_all` or `keep_latest`.
+
+  dateformat::
+    Optional parameter to specify how to suffix data tables.
+    Default is `YYYY_MM_DD` which creates per-day tables.
+    With `YYYY_MM` per-month tables can be created.
+    If explicitly set empty, partitioning is disabled.
+
+  part_template::
+    SQL fragment for table creation.  Various magic replacements are done there:
+       _PKEY:: comma separated list of primery key columns.
+       _PARENT:: schema-qualified parent table name.
+       _DEST_TABLE:: schema-qualified partition table.
+       _SCHEMA_TABLE:: same as _DEST_TABLE but dots replaced with "__", to allow use as index names.
+
+=== Example config file ===
+
+ [cube_dispatcher]
+ job_name          = some_queue_to_cube
+ 
+ src_db            = dbname=sourcedb_test
+ dst_db            = dbname=dataminedb_test
+ 
+ pgq_queue_name    = udata.some_queue
+ 
+ logfile           = ~/log/%(job_name)s.log
+ pidfile           = ~/pid/%(job_name)s.pid
+ 
+ # how many rows are kept: keep_latest, keep_all
+ mode = keep_latest
+ 
+ # to_char() fmt for table suffix
+ #dateformat = YYYY_MM_DD
+ # following disables table suffixes:
+ #dateformat =
+ 
+ part_template =
+      create table _DEST_TABLE (like _PARENT);
+      alter table only _DEST_TABLE add primary key (_PKEY);
+
+== LOGUTRIGA EVENT FORMAT ==
+
+include::common.logutriga.txt[]
+
+== COMMAND LINE SWITCHES ==
+
+include::common.switches.txt[]
+
diff --git a/doc/getattrs.py b/doc/getattrs.py
new file mode 100755
index 00000000..e538b033
--- /dev/null
+++ b/doc/getattrs.py
@@ -0,0 +1,9 @@
+#! /usr/bin/env python
+
+import sys
+
+buf = open(sys.argv[1], "r").read().lower()
+
+if buf.find("pgq consumer") >= 0:
+    print "-a pgq"
+
diff --git a/doc/queue_mover.txt b/doc/queue_mover.txt
new file mode 100644
index 00000000..3f8acbfd
--- /dev/null
+++ b/doc/queue_mover.txt
@@ -0,0 +1,87 @@
+
+= queue_mover(1) =
+
+== NAME ==
+
+queue_mover - PgQ consumer that copies data from one queue to another.
+
+== SYNOPSIS ==
+
+  queue_mover.py [switches] config.ini 
+
+== DESCRIPTION ==
+
+queue_mover is PgQ consumer that transports events from source queue into 
+target queue. One use case is when events are produced in several databases 
+then queue_mover is used to consolidate these events into single queue
+that can then be processed by consumers who need to handle theses events.
+For example in case of patitioned databases it's convenient to move events
+from each partition into one central queue database and then process them there.
+That way configuration and dependancies of partiton databases are 
+simpler and more robust. Another use case is to move events from OLTP 
+database to batch processing server.
+
+Transactionality: events will be inserted as one transaction on target side.
+That means only batch_id needs to be tracked on target side.
+
+== QUICK-START ==
+
+Basic PgQ setup and usage can be summarized by the following
+steps:
+
+  1. PgQ must be installed both in source and target databases.
+     See pgqadm man page for details.
+
+  2. Target database must also have pgq_ext schema installed.
+     It is used to keep sync between two databases.
+
+  3. Create a queue_mover configuration file, say qmover_sourceq_to_targetdb.ini
+
+  4. create source and target queues 
+
+     $ pgqadm.py sourcedb_ticker.ini create <srcqueue>
+     $ pgqadm.py targetdb_ticker.ini create <dstqueue>
+
+  5. launch queue mover in daemon mode
+
+     $ queue_mover.py -d qmover_sourceq_to_targetdb.ini
+
+  6. start producing and consuming events 
+
+
+== CONFIG ==
+
+include::common.config.txt[]
+
+=== queue_mover parameters ===
+
+src_db::
+  Source database.
+
+dst_db::
+  Target database.
+
+dst_queue_name::
+  Target queue name.
+
+=== Example config file ===
+
+  [queue_mover]
+  job_name = eventlog_to_target_mover
+  src_db = dbname=sourcedb
+  dst_db = dbname=targetdb
+  pgq_queue_name = eventlog
+  dst_queue_name = copy_of_eventlog
+  pidfile = log/%(job_name)s.pid
+  logfile = pid/%(job_name)s.log
+  
+== COMMAND LINE SWITCHES ==
+
+include::common.switches.txt[]
+
+== BUGS ==
+
+Event ID is not kept on target side.  If needed is can be kept,
+then event_id seq at target side need to be increased by hand to
+inform ticker about new events.
+
diff --git a/doc/queue_splitter.txt b/doc/queue_splitter.txt
new file mode 100644
index 00000000..0d4cb3f2
--- /dev/null
+++ b/doc/queue_splitter.txt
@@ -0,0 +1,99 @@
+= queue_splitter(1) =
+
+== NAME ==
+
+queue_splitter - PgQ consumer that transports events from one queue into several target queues
+
+== SYNOPSIS ==
+
+  queue_splitter.py [switches] config.ini 
+
+== DESCRIPTION ==
+
+queue_spliter is PgQ consumer that transports events from source queue into 
+several target queues.  `ev_extra1` field in each event shows into which
+target queue it must go.  (`pgq.logutriga()` puts there the table name.)
+
+One use case is to move events from OLTP database to batch processing server.
+By using queue spliter it is possible to move all kinds of events for batch
+processing with one consumer thus keeping OLTP database less crowded.
+
+== QUICK-START ==
+
+Basic queue_splitter setup and usage can be summarized by the following
+steps:
+
+ 1. pgq must be installed both in source and target databases.
+    See pgqadm man page for details.  Target database must also
+    have pgq_ext schema installed.
+
+ 2. edit a queue_splitter configuration file, say queue_splitter_sourcedb_sourceq_targetdb.ini
+
+ 3. create source and target queues 
+
+      $ pgqadm.py ticker.ini create <queue>
+
+ 4. launch queue splitter in daemon mode
+
+      $ queue_splitter.py queue_splitter_sourcedb_sourceq_targetdb.ini -d
+
+ 5. start producing and consuming events 
+
+== CONFIG ==
+
+include::common.config.txt[]
+
+=== queue_splitter parameters ===
+
+src_db::
+  Source database.
+
+dst_db::
+  Target database.
+
+=== Example config file ===
+
+  [queue_splitter]
+  job_name        = queue_spliter_sourcedb_sourceq_targetdb
+
+  src_db          = dbname=sourcedb
+  dst_db          = dbname=targetdb
+
+  pgq_queue_name  = sourceq
+
+  logfile         = ~/log/%(job_name)s.log
+  pidfile         = ~/pid/%(job_name)s.pid
+
+== COMMAND LINE SWITCHES ==
+
+include::common.switches.txt[]
+
+== USECASE ==
+
+How to to process events created in secondary database
+with several queues but have only one queue in primary
+database.  This also shows how to insert events into
+queues with regular SQL easily.
+
+   CREATE SCHEMA queue;
+   CREATE TABLE queue.event1 (
+        -- this should correspond to event internal structure
+	-- here you can put checks that correct data is put into queue
+   	id int4,
+	name text,
+	-- not needed, but good to have:
+	primary key (id)
+   );
+   -- put data into queue in urlencoded format, skip actual insert
+   CREATE TRIGGER redirect_queue1_trg BEFORE INSERT ON queue.event1
+   FOR EACH ROW EXECUTE PROCEDURE pgq.logutriga('singlequeue', 'SKIP');
+   -- repeat the above for event2
+
+   -- now the data can be inserted:
+   INSERT INTO queue.event1 (id, name) VALUES (1, 'user');
+
+If the queue_splitter is put on "singlequeue", it spreads the event
+on target to queues named "queue.event1", "queue.event2", etc.
+This keeps PgQ load on primary database minimal both CPU-wise
+and maintenance-wise.
+
diff --git a/doc/scriptmgr.txt b/doc/scriptmgr.txt
new file mode 100644
index 00000000..ac73ed0f
--- /dev/null
+++ b/doc/scriptmgr.txt
@@ -0,0 +1,113 @@
+= scriptmgr(1) =
+
+== NAME ==
+
+scriptmgr - utility for controlling other skytools scripts.
+
+== SYNOPSIS ==
+
+  scriptmgr.py [switches] config.ini <command> [-a | job_name ... ]
+
+== DESCRIPTION ==
+
+scriptmgr is used to manage several scripts together.  It discovers
+potential jobs based on config file glob expression.  From config
+file it gets both job_name and service type (that is the main section
+name eg [cube_dispatcher]).  For each service type there is subsection
+in the config how to handle it.  Unknown services are ignored.
+
+== COMMANDS ==
+
+=== status ===
+
+  scriptmgr config.ini status
+
+Show status for all known jobs.
+
+=== start ===
+
+  scriptmgr config.ini start -a
+  scriptmgr config.ini start job_name1 job_name2 ...
+
+launch script(s) that are not running.
+
+=== stop ===
+
+  scriptmgr config.ini stop -a
+  scriptmgr config.ini stop job_name1 job_name2 ...
+
+stop script(s) that are running.
+
+=== restart ===
+
+  scriptmgr config.ini restart -a
+  scriptmgr config.ini restart job_name1 job_name2 ...
+
+restart scripts.
+
+=== reload ===
+
+  scriptmgr config.ini reload -a
+  scriptmgr config.ini reload job_name1 job_name2 ...
+
+Send SIGHUP to scripts that are running.
+
+== CONFIG ==
+
+include::common.config.txt[]
+
+=== scriptmgr parameters ===
+
+config_list::
+  List of glob patters for finding config files.  Example:
+  
+    config_list = ~/dbscripts/conf/*.ini, ~/random/conf/*.ini
+
+=== Service section parameters ===
+
+cwd::
+  Working directory for script.
+
+args::
+  Arguments to give to script, in addition to `-d`.
+
+script::
+  Path to script.  Unless script is in PATH, full path should be given.
+
+disabled::
+  If this service should be ignored.
+
+=== Example config file ===
+
+  [scriptmgr]
+  job_name        = scriptmgr_livesrv
+  logfile         = ~/log/%(job_name)s.log
+  pidfile         = ~/pid/%(job_name)s.pid
+
+  config_list     = ~/scripts/conf/*.ini
+
+  # defaults for all service sections
+  [DEFAULT]
+  cwd = ~/scripts
+
+  [table_dispatcher]
+  script = table_dispatcher.py
+  args = -v
+
+  [cube_dispatcher]
+  script = python2.4 cube_dispatcher.py
+  disabled = 1
+
+  [pgqadm]
+  script = ~/scripts/pgqadm.py
+  args = ticker
+
+== COMMAND LINE SWITCHES ==
+
+include::common.switches.txt[]
+
+Options specific to scriptmgr:
+
+-a, --all::
+  Operate on all non-disabled scripts.
+
diff --git a/doc/skytools_upgrade.txt b/doc/skytools_upgrade.txt
new file mode 100644
index 00000000..3189cb78
--- /dev/null
+++ b/doc/skytools_upgrade.txt
@@ -0,0 +1,33 @@
+= skytools_upgrade(1) =
+
+== NAME ==
+
+skytools_upgrade - utility for upgrading Skytools code in databases.
+
+== SYNOPSIS ==
+
+  skytools_upgrade.py connstr [connstr ..]
+
+== DESCRIPTION ==
+
+It connects to given database, then looks for following schemas:
+
+  pgq::
+    Main PgQ code.
+  pgq_ext::
+    PgQ batch/event tracking in remote database.
+  londiste::
+    Londiste replication.
+
+If schema exists, its version is detected by querying .version()
+function under schema.   If the function does not exists, there
+is some heiristics built in to differentiate between 2.1.4 and
+2.1.5 version of ther schemas.
+
+If detected that version is older that current, it is upgraded
+by applying upgrade scripts in order.
+
+== COMMAND LINE SWITCHES ==
+
+include::common.switches.txt[]
+
diff --git a/doc/table_dispatcher.txt b/doc/table_dispatcher.txt
new file mode 100644
index 00000000..34033327
--- /dev/null
+++ b/doc/table_dispatcher.txt
@@ -0,0 +1,100 @@
+= table_dispatcher(1) =
+
+== NAME ==
+
+table_dispatcher - PgQ consumer that is used to write source records into partitoned table.
+
+== SYNOPSIS ==
+
+  table_dispatcher.py [switches] config.ini 
+
+== DESCRIPTION ==
+
+table_dispatcher is PgQ consumer that reads url encoded records from source queue
+and writes them into partitioned tables according to configuration file.
+Used to partiton data. For example change log's that need to kept online only shortly
+can be written to daily tables and then dropped as they become irrelevant.
+Also allows to select which columns have to be written into target database
+Creates target tables according to configuration file as needed.
+
+== QUICK-START ==
+
+Basic table_dispatcher setup and usage can be summarized by the following steps:
+
+ 1. PgQ must be installed in source database. See pgqadm man page for details.
+    Target database must have `pgq_ext` schema installed.
+
+ 2. edit a table_dispatcher configuration file, say table_dispatcher_sample.ini
+
+ 3. create source queue 
+
+      $ pgqadm.py ticker.ini create <queue>
+
+ 4. launch table dispatcher in daemon mode
+
+      $ table_dispatcher.py table_dispatcher_sample.ini -d
+
+ 5. start producing events
+
+== CONFIG ==
+
+include::common.config.txt[]
+
+=== table_dispatcher parameters ===
+
+src_db::
+  Source database.
+
+dst_db::
+  Target database.
+
+dest_table::
+  Where to put data.  when partitioning, will be used as base name
+
+part_field::
+  date field with will be used for partitioning.
+
+part_template::
+  SQL code used to create partition tables.  Various magic replacements are done there:
+      _PKEY:: comma separated list of primery key columns.
+      _PARENT:: schema-qualified parent table name.
+      _DEST_TABLE:: schema-qualified partition table.
+      _SCHEMA_TABLE:: same as _DEST_TABLE but dots replaced with "__", to allow use as index names.
+
+=== Example config ===
+
+
+    [table_dispatcher]
+    job_name          = table_dispatcher_source_table_targetdb
+
+    src_db            = dbname=sourcedb
+    dst_db            = dbname=targetdb
+
+    pgq_queue_name    = sourceq
+
+    logfile           = log/%(job_name)s.log
+    pidfile           = pid/%(job_name)s.pid
+
+    # where to put data.  when partitioning, will be used as base name
+    dest_table        = orders
+
+    # names of the fields that must be read from source records
+    fields            = id, order_date, customer_name
+
+    # date field with will be used for partitioning
+    part_field        = order_date
+
+    # template used for creating partition tables
+    part_template     = 
+         create table _DEST_TABLE () inherits (orders);
+         alter table only _DEST_TABLE add constraint _DEST_TABLE_pkey primary key (id);
+         grant select on _DEST_TABLE to group reporting;
+
+== COMMAND LINE SWITCHES ==
+
+include::common.switches.txt[]
+
+== LOGUTRIGA EVENT FORMAT ==
+
+include::common.logutriga.txt[]
+