summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMarko Kreen2012-11-22 18:21:20 +0000
committerMarko Kreen2012-11-26 06:56:34 +0000
commit32208f0d51d998fbec56f32c777a6feef8f3486e (patch)
tree27977e0581237d5fc9f3669bb5f47aa4c0f2bf2a
parent22af8879b35279317e6a37f54eca190d9ba16895 (diff)
londiste resurrect: Get dead root back into cascade
In some cases the 'takeover --dead-root' will be used although actual data on node is fine. Eg. network downtime, lost power. It would be good to have a way to bring it back to cascade without need for full rebuild. This command sync queue contents and re-subscribes node. Any events found on old root that are not spread to rest of cascade are dumped and deleted. No actual table changes are made, thus there must be external mechanism to avoid data conflicts.
-rw-r--r--doc/londiste3.txt24
-rwxr-xr-xpython/londiste.py2
-rw-r--r--python/londiste/setup.py34
-rw-r--r--python/pgq/cascade/admin.py352
-rwxr-xr-xtests/londiste/test-resurrect.sh257
5 files changed, 665 insertions, 4 deletions
diff --git a/doc/londiste3.txt b/doc/londiste3.txt
index 11ab8d9f..f9a0d592 100644
--- a/doc/londiste3.txt
+++ b/doc/londiste3.txt
@@ -152,6 +152,30 @@ providers (root or branch nodes).
--dead-branch::
Old node was branch
+=== resurrect ===
+
+In case root was down and taken over with `--dead-root`, this command
+fixes queue contents on that old root to match the rest of cascade.
+Events that did not propagate to rest of the cascade before failure
+(lost events) are dumped into file in JSON format and deleted from queue.
+Then the node is registered into cascade and it's worker will
+be paused.
+
+It requires that there is another active root in cascade and
+there is consumer named `NODENAME.gravestone` registered
+on same node, it uses that to get position where rest of the
+cascade moved on.
+
+It does not touch actual tables, which means there must
+be external mechanism to survive unsynced tables. Options:
+
+* Ignore lost events. May need trigger/rule on tables to handle conflicts.
+* Replay the lost events on new root.
+ May need trigger/rule on tables to handle conflicts.
+* Roll back table changes.
+ May need old version of row stored in events.
+ (Achieved with `backup` parameter to `pgq.logutriga`)
+
=== drop-node <node> ===
Remove the node from the Londiste replication.
diff --git a/python/londiste.py b/python/londiste.py
index 66137afc..8c2919c8 100755
--- a/python/londiste.py
+++ b/python/londiste.py
@@ -47,7 +47,7 @@ Internal Commands:
cmd_handlers = (
(('create-root', 'create-branch', 'create-leaf', 'members', 'tag-dead', 'tag-alive',
'change-provider', 'rename-node', 'status', 'pause', 'resume', 'node-info',
- 'drop-node', 'takeover'), londiste.LondisteSetup),
+ 'drop-node', 'takeover', 'resurrect'), londiste.LondisteSetup),
(('add-table', 'remove-table', 'add-seq', 'remove-seq', 'tables', 'seqs',
'missing', 'resync', 'wait-sync', 'wait-root', 'wait-provider',
'check', 'fkeys', 'execute'), londiste.LondisteSetup),
diff --git a/python/londiste/setup.py b/python/londiste/setup.py
index 122b146f..0ddbb852 100644
--- a/python/londiste/setup.py
+++ b/python/londiste/setup.py
@@ -632,3 +632,37 @@ class LondisteSetup(CascadeAdmin):
self.log.info("All done")
+ def resurrect_dump_event(self, ev, stats, batch_info):
+ """Collect per-table stats."""
+
+ super(LondisteSetup, self).resurrect_dump_event(ev, stats, batch_info)
+
+ ROLLBACK = 'can rollback'
+ NO_ROLLBACK = 'cannot rollback'
+
+ if ev.ev_type == 'TRUNCATE':
+ if 'truncated_tables' not in stats:
+ stats['truncated_tables'] = []
+ tlist = stats['truncated_tables']
+ tbl = ev.ev_extra1
+ if tbl not in tlist:
+ tlist.append(tbl)
+ elif ev.ev_type[:2] in ('I:', 'U:', 'D:', 'I', 'U', 'D'):
+ op = ev.ev_type[0]
+ tbl = ev.ev_extra1
+ bak = ev.ev_extra3
+ tblkey = 'table: %s' % tbl
+ if tblkey not in stats:
+ stats[tblkey] = [0,0,0,ROLLBACK]
+ tinfo = stats[tblkey]
+ if op == 'I':
+ tinfo[0] += 1
+ elif op == 'U':
+ tinfo[1] += 1
+ if not bak:
+ tinfo[3] = NO_ROLLBACK
+ elif op == 'D':
+ tinfo[2] += 1
+ if not bak and ev.ev_type == 'D':
+ tinfo[3] = NO_ROLLBACK
+
diff --git a/python/pgq/cascade/admin.py b/python/pgq/cascade/admin.py
index 3779ada6..cc4d1265 100644
--- a/python/pgq/cascade/admin.py
+++ b/python/pgq/cascade/admin.py
@@ -11,13 +11,15 @@ setadm.py INI pause NODE [CONS]
"""
-import sys, time, optparse, skytools
+import sys, time, optparse, skytools, os.path
from skytools import UsageError, DBError
from pgq.cascade.nodeinfo import *
__all__ = ['CascadeAdmin']
+RESURRECT_DUMP_FILE = "resurrect-lost-events.json"
+
command_usage = """\
%prog [options] INI CMD [subcmd args]
@@ -596,13 +598,24 @@ class CascadeAdmin(skytools.AdminScript):
self.demote_node(old_node_name, 1, new_node_name)
last_tick = self.demote_node(old_node_name, 2, new_node_name)
self.wait_for_catchup(new_node_name, last_tick)
+ else:
+ q = "select * from pgq.get_queue_info(%s)"
+ db = self.get_node_database(new_node_name)
+ curs = db.cursor()
+ curs.execute(q, [self.queue_name])
+ row = curs.fetchone()
+ last_tick = row['last_tick_id']
+ db.commit()
self.pause_node(new_node_name)
self.promote_branch(new_node_name)
if self.node_alive(old_node_name):
- q = 'select * from pgq_node.register_subscriber(%s, %s, %s, %s)'
- self.node_cmd(new_node_name, q, [self.queue_name, old_node_name, old_info.worker_name, last_tick])
+ old_worker_name = old_info.worker_name
+ else:
+ old_worker_name = self.failover_consumer_name(old_node_name)
+ q = 'select * from pgq_node.register_subscriber(%s, %s, %s, %s)'
+ self.node_cmd(new_node_name, q, [self.queue_name, old_node_name, old_worker_name, last_tick])
q = "select * from pgq_node.unregister_subscriber(%s, %s)"
self.node_cmd(new_info.provider_node, q, [self.queue_name, new_node_name])
@@ -840,6 +853,339 @@ class CascadeAdmin(skytools.AdminScript):
self.sleep(2)
+ def cmd_resurrect(self):
+ """Convert out-of-sync old root to branch and sync queue contents.
+ """
+ self.load_local_info()
+
+ db = self.get_database(self.initial_db_name)
+ curs = db.cursor()
+
+ # stop if leaf
+ if self.queue_info.local_node.type == 'leaf':
+ self.log.info("Current node is leaf, nothing to do")
+ return
+
+ # stop if dump file exists
+ if os.path.lexists(RESURRECT_DUMP_FILE):
+ self.log.error("Dump file exists, cannot perform resurrection: %s", RESURRECT_DUMP_FILE)
+ sys.exit(1)
+
+ #
+ # Find failover position
+ #
+
+ self.log.info("** Searching for gravestone **")
+
+ # load subscribers
+ sub_list = []
+ q = "select * from pgq_node.get_subscriber_info(%s)"
+ curs.execute(q, [self.queue_name])
+ for row in curs.fetchall():
+ sub_list.append(row['node_name'])
+ db.commit()
+
+ # find backup subscription
+ this_node = self.queue_info.local_node.name
+ failover_cons = self.failover_consumer_name(this_node)
+ full_list = self.queue_info.member_map.keys()
+ done_nodes = { this_node: 1 }
+ prov_node = None
+ root_node = None
+ for node_name in sub_list + full_list:
+ if node_name in done_nodes:
+ continue
+ done_nodes[node_name] = 1
+ if not self.node_alive(node_name):
+ self.log.info('Node %s is dead, skipping', node_name)
+ continue
+ self.log.info('Looking on node %s', node_name)
+ node_db = None
+ try:
+ node_db = self.get_node_database(node_name)
+ node_curs = node_db.cursor()
+ node_curs.execute("select * from pgq.get_consumer_info(%s, %s)", [self.queue_name, failover_cons])
+ cons_rows = node_curs.fetchall()
+ node_curs.execute("select * from pgq_node.get_node_info(%s)", [self.queue_name])
+ node_info = node_curs.fetchone()
+ node_db.commit()
+ if len(cons_rows) == 1:
+ if prov_node:
+ raise Exception('Unexcpeted situation: there are two gravestones - on nodes %s and %s' % (prov_node, node_name))
+ prov_node = node_name
+ failover_tick = cons_rows[0]['last_tick']
+ self.log.info("Found gravestone on node: %s", node_name)
+ if node_info['node_type'] == 'root':
+ self.log.info("Found new root node: %s", node_name)
+ root_node = node_name
+ self.close_node_database(node_name)
+ node_db = None
+ if root_node and prov_node:
+ break
+ except skytools.DBError:
+ self.log.warning("failed to check node %s", node_name)
+ if node_db:
+ self.close_node_database(node_name)
+ node_db = None
+
+ if not root_node:
+ self.log.error("Cannot find new root node", failover_cons)
+ sys.exit(1)
+ if not prov_node:
+ self.log.error("Cannot find failover position (%s)", failover_cons)
+ sys.exit(1)
+
+ # load worker state
+ q = "select * from pgq_node.get_worker_state(%s)"
+ rows = self.exec_cmd(db, q, [self.queue_name])
+ state = rows[0]
+
+ # demote & pause
+ self.log.info("** Demote & pause local node **")
+ if self.queue_info.local_node.type == 'root':
+ self.log.info('Node %s is root, demoting', this_node)
+ q = "select * from pgq_node.demote_root(%s, %s, %s)"
+ self.exec_cmd(db, q, [self.queue_name, 1, prov_node])
+ self.exec_cmd(db, q, [self.queue_name, 2, prov_node])
+
+ # change node type and set worker paused in same TX
+ curs = db.cursor()
+ self.exec_cmd(curs, q, [self.queue_name, 3, prov_node])
+ q = "select * from pgq_node.set_consumer_paused(%s, %s, true)"
+ self.exec_cmd(curs, q, [self.queue_name, state['worker_name']])
+ db.commit()
+ elif not state['paused']:
+ # pause worker, don't wait for reaction, as it may be dead
+ self.log.info('Node %s is branch, pausing worker: %s', this_node, state['worker_name'])
+ q = "select * from pgq_node.set_consumer_paused(%s, %s, true)"
+ self.exec_cmd(db, q, [self.queue_name, state['worker_name']])
+ else:
+ self.log.info('Node %s is branch and worker is paused', this_node)
+
+ #
+ # Drop old consumers and subscribers
+ #
+ self.log.info("** Dropping old subscribers and consumers **")
+
+ # unregister subscriber nodes
+ q = "select pgq_node.unregister_subscriber(%s, %s)"
+ for node_name in sub_list:
+ self.log.info("Dropping old subscriber node: %s", node_name)
+ curs.execute(q, [self.queue_name, node_name])
+
+ # unregister consumers
+ q = "select consumer_name from pgq.get_consumer_info(%s)"
+ curs.execute(q, [self.queue_name])
+ for row in curs.fetchall():
+ cname = row['consumer_name']
+ if cname[0] == '.':
+ self.log.info("Keeping consumer: %s", cname)
+ continue
+ self.log.info("Dropping old consumer: %s", cname)
+ q = "pgq.unregister_consumer(%s, %s)"
+ curs.execute(q, [self.queue_name, cname])
+ db.commit()
+
+ # dump events
+ self.log.info("** Dump & delete lost events **")
+ stats = self.resurrect_process_lost_events(db, failover_tick)
+
+ self.log.info("** Subscribing %s to %s **", this_node, prov_node)
+
+ # set local position
+ self.log.info("Reset local completed pos")
+ q = "select * from pgq_node.set_consumer_completed(%s, %s, %s)"
+ self.exec_cmd(db, q, [self.queue_name, state['worker_name'], failover_tick])
+
+ # rename gravestone
+ self.log.info("Rename gravestone to worker: %s", state['worker_name'])
+ prov_db = self.get_node_database(prov_node)
+ prov_curs = prov_db.cursor()
+ q = "select * from pgq_node.unregister_subscriber(%s, %s)"
+ self.exec_cmd(prov_curs, q, [self.queue_name, this_node], quiet = True)
+ q = "select ret_code, ret_note, global_watermark"\
+ " from pgq_node.register_subscriber(%s, %s, %s, %s)"
+ res = self.exec_cmd(prov_curs, q, [self.queue_name, this_node, state['worker_name'], failover_tick], quiet = True)
+ global_wm = res[0]['global_watermark']
+ prov_db.commit()
+
+ # import new global watermark
+ self.log.info("Reset global watermark")
+ q = "select * from pgq_node.set_global_watermark(%s, %s)"
+ self.exec_cmd(db, q, [self.queue_name, global_wm], quiet = True)
+
+ # show stats
+ if stats:
+ self.log.info("** Statistics **")
+ klist = stats.keys()
+ klist.sort()
+ for k in klist:
+ v = stats[k]
+ self.log.info(" %s: %s", k, str(v))
+ self.log.info("** Resurrection done, worker paused **")
+
+ def resurrect_process_lost_events(self, db, failover_tick):
+ curs = db.cursor()
+ this_node = self.queue_info.local_node.name
+ cons_name = this_node + '.dumper'
+
+ self.log.info("Dumping lost events")
+
+ # register temp consumer on queue
+ q = "select pgq.register_consumer_at(%s, %s, %s)"
+ curs.execute(q, [self.queue_name, cons_name, failover_tick])
+ db.commit()
+
+ # process events as usual
+ total_count = 0
+ final_tick_id = -1
+ stats = {}
+ while 1:
+ q = "select * from pgq.next_batch_info(%s, %s)"
+ curs.execute(q, [self.queue_name, cons_name])
+ b = curs.fetchone()
+ batch_id = b['batch_id']
+ if batch_id is None:
+ break
+ final_tick_id = b['cur_tick_id']
+ q = "select * from pgq.get_batch_events(%s)"
+ curs.execute(q, [batch_id])
+ cnt = 0
+ for ev in curs.fetchall():
+ cnt += 1
+ total_count += 1
+ self.resurrect_dump_event(ev, stats, b)
+
+ q = "select pgq.finish_batch(%s)"
+ curs.execute(q, [batch_id])
+ if cnt > 0:
+ db.commit()
+
+ stats['dumped_count'] = total_count
+
+ self.resurrect_dump_finish()
+
+ self.log.info("%s events dumped", total_count)
+
+ # unregiser consumer
+ q = "select pgq.unregister_consumer(%s, %s)"
+ curs.execute(q, [self.queue_name, cons_name])
+ db.commit()
+
+ if failover_tick == final_tick_id:
+ self.log.info("No batches found")
+ return None
+
+ #
+ # Delete the events from queue
+ #
+ # This is done snapshots, to make sure we delete only events
+ # that were dumped out previously. This uses the long-tx
+ # resustant logic described in pgq.batch_event_sql().
+ #
+
+ # find snapshots
+ q = "select t1.tick_snapshot as s1, t2.tick_snapshot as s2"\
+ " from pgq.tick t1, pgq.tick t2"\
+ " where t1.tick_id = %s"\
+ " and t2.tick_id = %s"
+ curs.execute(q, [failover_tick, final_tick_id])
+ ticks = curs.fetchone()
+ s1 = skytools.Snapshot(ticks['s1'])
+ s2 = skytools.Snapshot(ticks['s2'])
+
+ xlist = []
+ for tx in s1.txid_list:
+ if s2.contains(tx):
+ xlist.append(str(tx))
+
+ # create where clauses
+ W1 = None
+ if len(xlist) > 0:
+ W1 = "ev_txid in (%s)" % (",".join(xlist),)
+ W2 = "ev_txid >= %d AND ev_txid <= %d"\
+ " and not txid_visible_in_snapshot(ev_txid, '%s')"\
+ " and txid_visible_in_snapshot(ev_txid, '%s')" % (
+ s1.xmax, s2.xmax, ticks['s1'], ticks['s2'])
+
+ # loop over all queue data tables
+ q = "select * from pgq.queue where queue_name = %s"
+ curs.execute(q, [self.queue_name])
+ row = curs.fetchone()
+ ntables = row['queue_ntables']
+ tbl_pfx = row['queue_data_pfx']
+ schema, table = tbl_pfx.split('.')
+ total_del_count = 0
+ self.log.info("Deleting lost events")
+ for i in range(ntables):
+ del_count = 0
+ self.log.debug("Deleting events from table %d" % i)
+ qtbl = "%s.%s" % (skytools.quote_ident(schema),
+ skytools.quote_ident(table + '_' + str(i)))
+ q = "delete from " + qtbl + " where "
+ if W1:
+ self.log.debug(q + W1)
+ curs.execute(q + W1)
+ if curs.rowcount and curs.rowcount > 0:
+ del_count += curs.rowcount
+ self.log.debug(q + W2)
+ curs.execute(q + W2)
+ if curs.rowcount and curs.rowcount > 0:
+ del_count += curs.rowcount
+ total_del_count += del_count
+ self.log.debug('%d events deleted', del_count)
+ self.log.info('%d events deleted', total_del_count)
+ stats['deleted_count'] = total_del_count
+
+ # delete new ticks
+ q = "delete from pgq.tick t using pgq.queue q"\
+ " where q.queue_name = %s"\
+ " and t.tick_queue = q.queue_id"\
+ " and t.tick_id > %s"\
+ " and t.tick_id <= %s"
+ curs.execute(q, [self.queue_name, failover_tick, final_tick_id])
+ self.log.info("%s ticks deleted", curs.rowcount)
+
+ db.commit()
+
+ return stats
+
+ _json_dump_file = None
+ def resurrect_dump_event(self, ev, stats, batch_info):
+ if self._json_dump_file is None:
+ self._json_dump_file = open(RESURRECT_DUMP_FILE, 'w')
+ sep = '['
+ else:
+ sep = ','
+
+ # create orinary dict to avoid problems with row class and datetime
+ d = {
+ 'ev_id': ev.ev_id,
+ 'ev_type': ev.ev_type,
+ 'ev_data': ev.ev_data,
+ 'ev_extra1': ev.ev_extra1,
+ 'ev_extra2': ev.ev_extra2,
+ 'ev_extra3': ev.ev_extra3,
+ 'ev_extra4': ev.ev_extra4,
+ 'ev_time': ev.ev_time.isoformat(),
+ 'ev_txid': ev.ev_txid,
+ 'ev_retry': ev.ev_retry,
+ 'tick_id': batch_info['cur_tick_id'],
+ 'prev_tick_id': batch_info['prev_tick_id'],
+ }
+ jsev = skytools.json_encode(d)
+ s = sep + '\n' + jsev
+ self._json_dump_file.write(s)
+
+ def resurrect_dump_finish(self):
+ if self._json_dump_file:
+ self._json_dump_file.write('\n]\n')
+ self._json_dump_file.close()
+ self._json_dump_file = None
+
+ def failover_consumer_name(self, node_name):
+ return node_name + ".gravestone"
+
#
# Shortcuts for operating on nodes.
#
diff --git a/tests/londiste/test-resurrect.sh b/tests/londiste/test-resurrect.sh
new file mode 100755
index 00000000..05c62813
--- /dev/null
+++ b/tests/londiste/test-resurrect.sh
@@ -0,0 +1,257 @@
+#! /bin/bash
+
+. ../testlib.sh
+
+../zstop.sh
+
+rm -f resurrect-lost-events.json
+
+v='-q'
+v=''
+nocheck=1
+
+db_list="db1 db2 db3 db4 db5"
+
+kdb_list=`echo $db_list | sed 's/ /,/g'`
+
+#( cd ../..; make -s install )
+
+do_check() {
+ test $nocheck = 1 || ../zcheck.sh
+}
+
+title Resurrect test
+
+# create ticker conf
+cat > conf/pgqd.ini <<EOF
+[pgqd]
+database_list = $kdb_list
+logfile = log/pgqd.log
+pidfile = pid/pgqd.pid
+EOF
+
+# londiste3 configs
+for db in $db_list; do
+cat > conf/londiste_$db.ini <<EOF
+[londiste3]
+job_name = londiste_$db
+db = dbname=$db
+queue_name = replika
+logfile = log/%(job_name)s.log
+pidfile = pid/%(job_name)s.pid
+
+pgq_autocommit = 1
+pgq_lazy_fetch = 0
+EOF
+done
+
+for n in 1 2 3; do
+cat > conf/gen$n.ini <<EOF
+[loadgen]
+job_name = loadgen$n
+db = dbname=db$n
+logfile = log/%(job_name)s.log
+pidfile = pid/%(job_name)s.pid
+EOF
+done
+
+psql -d template1 -c 'drop database if exists db1x'
+psql -d template1 -c 'drop database if exists db2x'
+createdb db1
+createdb db2
+
+for db in $db_list; do
+ cleardb $db
+done
+
+clearlogs
+
+set -e
+
+msg "Basic config"
+run cat conf/pgqd.ini
+run cat conf/londiste_db1.ini
+
+msg "Install londiste3 and initialize nodes"
+run londiste3 $v conf/londiste_db1.ini create-root node1 'dbname=db1'
+run londiste3 $v conf/londiste_db2.ini create-branch node2 'dbname=db2' --provider='dbname=db1'
+run londiste3 $v conf/londiste_db3.ini create-branch node3 'dbname=db3' --provider='dbname=db2'
+run londiste3 $v conf/londiste_db4.ini create-branch node4 'dbname=db4' --provider='dbname=db2'
+run londiste3 $v conf/londiste_db5.ini create-branch node5 'dbname=db5' --provider='dbname=db2'
+
+msg "Run ticker"
+run pgqd $v -d conf/pgqd.ini
+run sleep 5
+
+msg "See topology"
+run londiste3 $v conf/londiste_db1.ini status
+
+msg "Run londiste3 daemon for each node"
+for db in $db_list; do
+ run psql -d $db -c "update pgq.queue set queue_ticker_idle_period='2 secs'"
+ run londiste3 $v -d conf/londiste_$db.ini worker
+done
+
+msg "Create table on root node and fill couple of rows"
+run psql -d db1 -c "create table mytable (id serial primary key, data text)"
+for n in 1 2 3 4; do
+ run psql -d db1 -c "insert into mytable (data) values ('row$n')"
+done
+
+msg "Run loadgen on table"
+run ./loadgen.py -d conf/gen1.ini
+
+msg "Register table on root node"
+run londiste3 $v conf/londiste_db1.ini add-table mytable
+run londiste3 $v conf/londiste_db1.ini add-seq mytable_id_seq
+
+msg "Register table on other node with creation"
+for db in db2 db3 db4 db5; do
+ run psql -d $db -c "create sequence mytable_id_seq"
+ run londiste3 $v conf/londiste_$db.ini add-seq mytable_id_seq
+ run londiste3 $v conf/londiste_$db.ini add-table mytable --create-full
+done
+
+msg "Wait until tables are in sync on db3"
+
+run londiste3 conf/londiste_db2.ini wait-sync
+run londiste3 conf/londiste_db3.ini wait-sync
+
+run londiste3 conf/londiste_db3.ini status
+
+###################
+
+#msg "Stop Londiste on Node2"
+#run londiste3 conf/londiste_db2.ini worker -s
+#sleep 1
+
+#msg "Wait a bit"
+#run sleep 10
+#############################
+msg "Force lag on db2"
+run londiste3 $v conf/londiste_db2.ini worker -s
+run sleep 20
+
+msg "Stop Londiste on Node1"
+run londiste3 conf/londiste_db1.ini worker -s
+
+msg "Stop loadgen"
+run sleep 5
+run ./loadgen.py -s conf/gen1.ini
+
+#msg "Kill old root"
+#ps aux | grep 'postgres[:].* db1 ' | awk '{print $2}' | xargs -r kill
+#sleep 3
+#ps aux | grep 'postgres[:].* db1 ' | awk '{print $2}' | xargs -r kill -9
+#sleep 3
+
+run londiste3 $v conf/londiste_db2.ini status --dead=node1
+
+#msg "Stop Ticker"
+run pgqd -s conf/pgqd.ini
+run psql -d db2 -c 'alter database db1 rename to db1x'
+run pgqd -d conf/pgqd.ini
+#run londiste3 $v conf/londiste_db2.ini tag-dead node1
+run londiste3 $v conf/londiste_db2.ini worker -d
+
+
+msg "Take over root role"
+run londiste3 $v conf/londiste_db2.ini takeover node1 --dead-root
+run londiste3 $v conf/londiste_db2.ini tag-dead node1
+run londiste3 $v conf/londiste_db2.ini status
+
+msg "Move database back"
+run psql -d db2 -c 'alter database db1x rename to db1'
+
+msg "Do resurrection ritual"
+run londiste3 conf/londiste_db1.ini resurrect
+
+exit 0
+
+
+
+
+##
+## basic setup done
+##
+
+# test lagged takeover
+if true; then
+
+msg "Force lag on db2"
+run londiste3 $v conf/londiste_db2.ini worker -s
+run sleep 20
+
+msg "Kill old root"
+ps aux | grep 'postgres[:].* db1 ' | awk '{print $2}' | xargs -r kill
+sleep 3
+ps aux | grep 'postgres[:].* db1 ' | awk '{print $2}' | xargs -r kill -9
+sleep 3
+run psql -d db2 -c 'drop database db1'
+run psql -d db2 -c 'create database db1'
+run londiste3 $v conf/londiste_db2.ini status --dead=node1
+
+msg "Change db2 to read from db3"
+run londiste3 $v conf/londiste_db2.ini worker -d
+run londiste3 $v conf/londiste_db2.ini change-provider --provider=node3 --dead=node1
+
+msg "Wait until catchup"
+run londiste3 $v conf/londiste_db2.ini wait-provider
+
+msg "Promoting db2 to root"
+run londiste3 $v conf/londiste_db2.ini takeover node1 --dead-root
+run londiste3 $v conf/londiste_db2.ini tag-dead node1
+run londiste3 $v conf/londiste_db2.ini status
+
+run sleep 5
+
+msg "Done"
+
+do_check
+
+exit 0
+fi
+
+
+
+msg "Change provider"
+run londiste3 $v conf/londiste_db4.ini status
+run londiste3 $v conf/londiste_db4.ini change-provider --provider=node3
+run londiste3 $v conf/londiste_db4.ini status
+run londiste3 $v conf/londiste_db5.ini change-provider --provider=node2
+run londiste3 $v conf/londiste_db5.ini status
+
+msg "Change topology"
+run londiste3 $v conf/londiste_db1.ini status
+run londiste3 $v conf/londiste_db3.ini takeover node2
+run londiste3 $v conf/londiste_db2.ini status
+run londiste3 $v conf/londiste_db2.ini takeover node1
+run londiste3 $v conf/londiste_db2.ini status
+
+msg "Restart loadgen"
+run ./loadgen.py -s conf/gen1.ini
+run ./loadgen.py -d conf/gen2.ini
+
+run sleep 10
+do_check
+
+msg "Change topology / failover"
+ps aux | grep 'postgres[:].* db2 ' | awk '{print $2}' | xargs -r kill
+sleep 3
+ps aux | grep 'postgres[:].* db2 ' | awk '{print $2}' | xargs -r kill -9
+sleep 3
+run psql -d db1 -c 'alter database db2 rename to db2x'
+run londiste3 $v conf/londiste_db1.ini status --dead=node2
+run londiste3 $v conf/londiste_db3.ini takeover db2 --dead-root || true
+run londiste3 $v conf/londiste_db3.ini takeover node2 --dead-root
+run londiste3 $v conf/londiste_db1.ini status
+
+msg "Restart loadgen"
+run ./loadgen.py -s conf/gen2.ini
+run ./loadgen.py -d conf/gen3.ini
+
+
+run sleep 10
+do_check
+
+msg Done