Two merge related tests created.

merge_all tests simple merge from two partitions into one target
          Also tests adding columns through cascade
merge_qnode tests same thing as merge all but uses also qnodes

8 files changed, 375 insertions, 0 deletions

diff --git a/tests/merge_all/addcol-data2.sql b/tests/merge_all/addcol-data2.sql
new file mode 100644
index 00000000..fe5de5a7
--- /dev/null
+++ b/tests/merge_all/addcol-data2.sql
@@ -0,0 +1,5 @@
+
+--*-- Local-Table: mydata
+
+alter table @mydata@ add column data2 text;
+
diff --git a/tests/merge_all/init.sh b/tests/merge_all/init.sh
new file mode 100755
index 00000000..d44d63ee
--- /dev/null
+++ b/tests/merge_all/init.sh
@@ -0,0 +1,16 @@
+#! /bin/sh
+
+. ../env.sh
+
+lst="part1 part2 full"
+
+../zstop.sh
+
+for db in $lst; do
+  echo dropdb $db
+  dropdb $db
+done
+for db in $lst; do
+  echo createdb $db
+  createdb $db
+done
diff --git a/tests/merge_all/overview.sh b/tests/merge_all/overview.sh
new file mode 100755
index 00000000..f25e1b39
--- /dev/null
+++ b/tests/merge_all/overview.sh
@@ -0,0 +1,7 @@
+#! /bin/sh
+
+for db in part1 full1 full2; do
+  echo "==== $db ==="
+  psql -d $db -c "select * from pgq.get_consumer_info() where not consumer_name like '.%'"
+  psql -d $db -c "select * from pgq_node.local_state order by 1,2"
+done
diff --git a/tests/merge_all/regen.sh b/tests/merge_all/regen.sh
new file mode 100755
index 00000000..1fe48989
--- /dev/null
+++ b/tests/merge_all/regen.sh
@@ -0,0 +1,162 @@
+#! /bin/sh
+
+. ../testlib.sh
+
+title "Merge several shards into one database"
+
+part_list="part1 part2"
+full_list="full"
+merge_list=""
+for dst in $full_list; do
+  for src in $part_list; do
+    merge_list="$merge_list ${src}_${dst}"
+  done
+done
+all_list="$part_list $full_list"
+kdb_list="`echo $all_list|sed 's/ /,/g'`"
+
+for db in $part_list $full_list; do
+  cleardb $db
+done
+
+clearlogs
+
+msg "Create configs"
+
+# create ticker conf
+cat > conf/pgqd.ini << EOF
+[pgqd]
+database_list = $kdb_list
+logfile = log/pgqd.log
+pidfile = pid/pgqd.pid
+EOF
+
+# partition replicas
+for db in $part_list; do
+
+# londiste on part node
+cat > conf/londiste_$db.ini << EOF
+[londiste3]
+job_name = londiste_$db
+db = dbname=$db
+queue_name = replika_$db
+logfile = log/%(job_name)s.log
+pidfile = pid/%(job_name)s.pid
+EOF
+
+cat > conf/londiste_${db}_full.ini << EOF
+[londiste3]
+job_name = londiste_${db}_full
+db = dbname=full
+queue_name = replika_$db
+logfile = log/%(job_name)s.log
+pidfile = pid/%(job_name)s.pid
+EOF
+
+done
+
+for f in conf/*.ini; do
+  echo "$f"
+  echo "------------------"
+  cat "$f"
+  echo "------------------"
+  echo
+done
+
+set -e
+
+msg "Create nodes for partition queues"
+
+for src in $part_list; do
+  run londiste3 $v conf/londiste_${src}.ini create-root ${src}_root "dbname=${src}"
+done
+
+msg "Create merge nodes for partition queues"
+
+for src in $part_list; do
+  run londiste3 $v conf/londiste_${src}_full.ini \
+    create-leaf merge_${src}_full "dbname=full" --provider="dbname=$src" 
+done
+
+msg "Optimize pgq for testing to handle empty ticks faster"
+for db in $all_list; do
+  run_sql $db "update pgq.queue set queue_ticker_idle_period='3 secs'"
+done
+
+msg "Launch ticker"
+run pgqd $v -d conf/pgqd.ini
+
+msg "Launch londiste worker"
+for db in $part_list; do
+  run londiste3 $v -d conf/londiste_$db.ini worker
+done
+
+msg "Launch merge londiste"
+for src in $part_list; do
+  run londiste3 $v -d conf/londiste_${src}_full.ini worker
+done
+
+msg "Create table in partition nodes"
+for db in $part_list; do
+  run_sql "$db" "create table mydata (id int4 primary key, data text)"
+done
+
+msg "Register table in partition nodes"
+for db in $part_list; do
+  run londiste3 $v conf/londiste_$db.ini add-table mydata
+done
+
+msg "Wait for cascade sync (root->leaf)"
+for src in $part_list; do
+  run londiste3 $v conf/londiste_${src}_full.ini wait-root
+done
+
+msg "Insert few rows"
+for n in 1 2; do
+  run_sql part$n "insert into mydata values ($n, 'part$n')"
+done
+
+msg "Create table and register it in merge nodes"
+run_sql full "create table mydata (id int4 primary key, data text)"
+run londiste3 $v conf/londiste_part1_full.ini add-table mydata --merge-all
+
+msg "Wait for replica to cach up"
+for src in $part_list; do
+  run londiste3 $v conf/londiste_${src}_full.ini wait-sync
+done
+
+msg "Insert few rows"
+for n in 1 2; do
+  run_sql part$n "insert into mydata values (2 + $n, 'part$n')"
+done
+
+msg "Now check if data apprered"
+run_sql full "select * from mydata order by id"
+run_sql full "select table_name, local, merge_state, table_attrs, dest_table from londiste.get_table_list('replika_part1')"
+run_sql full "select table_name, local, merge_state, table_attrs, dest_table from londiste.get_table_list('replika_part2')"
+
+../zcheck.sh
+
+msg "Test EXECUTE through cascade"
+
+for db in $part_list; do
+  run londiste3 $v conf/londiste_$db.ini execute addcol-data2.sql
+  # do one by one to avoid deadlock on merge side when several ddl's are received simultaneously
+  run londiste3 $v conf/londiste_${src}_full.ini wait-root
+done
+
+msg "Insert more rows with more columns"
+for n in 1 2; do
+  run_sql part$n "insert into mydata values (4 + $n, 'part$n', 'x')"
+done
+
+msg "Wait for cascade sync (root->leaf)"
+for src in $part_list; do
+  run londiste3 $v conf/londiste_${src}_full.ini wait-root
+done
+
+psql -d part1 -c 'select * from mydata order by 1;'
+psql -d part2 -c 'select * from mydata order by 1;'
+psql -d full -c 'select * from mydata order by 1;'
+
+../zcheck.sh
diff --git a/tests/merge_qnode/addcol-data2.sql b/tests/merge_qnode/addcol-data2.sql
new file mode 100644
index 00000000..fe5de5a7
--- /dev/null
+++ b/tests/merge_qnode/addcol-data2.sql
@@ -0,0 +1,5 @@
+
+--*-- Local-Table: mydata
+
+alter table @mydata@ add column data2 text;
+
diff --git a/tests/merge_qnode/init.sh b/tests/merge_qnode/init.sh
new file mode 100755
index 00000000..95c74fb7
--- /dev/null
+++ b/tests/merge_qnode/init.sh
@@ -0,0 +1,16 @@
+#! /bin/sh
+
+. ../env.sh
+
+lst="part1 part2 qn1 qn2 full"
+
+../zstop.sh
+
+for db in $lst; do
+  echo dropdb $db
+  psql postgres -q -c "drop database if exists \"$db\";"
+done
+for db in $lst; do
+  echo createdb $db
+  createdb $db
+done
diff --git a/tests/merge_qnode/overview.sh b/tests/merge_qnode/overview.sh
new file mode 100755
index 00000000..f25e1b39
--- /dev/null
+++ b/tests/merge_qnode/overview.sh
@@ -0,0 +1,7 @@
+#! /bin/sh
+
+for db in part1 full1 full2; do
+  echo "==== $db ==="
+  psql -d $db -c "select * from pgq.get_consumer_info() where not consumer_name like '.%'"
+  psql -d $db -c "select * from pgq_node.local_state order by 1,2"
+done
diff --git a/tests/merge_qnode/regen.sh b/tests/merge_qnode/regen.sh
new file mode 100755
index 00000000..e1b0fb21
--- /dev/null
+++ b/tests/merge_qnode/regen.sh
@@ -0,0 +1,157 @@
+#! /bin/sh
+
+. ../testlib.sh
+
+title "Merge several shards into one database"
+
+part_list="part1 part2"
+qn_list='qn1 qn2'
+full_list="full"
+all_list="$part_list $qn_list $full_list"
+kdb_list="`echo $all_list|sed 's/ /,/g'`"
+
+for db in $all_list; do
+  cleardb $db
+done
+
+clearlogs
+
+msg "Create configs for pgqd and londiste processes"
+
+# create ticker conf
+cat > conf/pgqd.ini << EOF
+[pgqd]
+database_list = $kdb_list
+logfile = log/pgqd.log
+pidfile = pid/pgqd.pid
+EOF
+
+# partition replicas
+for n in 1 2; do
+
+# londiste on part node
+cat > conf/londiste_part$n.ini << EOF
+[londiste3]
+job_name = londiste_part$n
+db = dbname=part$n
+queue_name = replika_part$n
+logfile = log/%(job_name)s.log
+pidfile = pid/%(job_name)s.pid
+EOF
+
+cat > conf/londiste_part${n}_qn$n.ini << EOF
+[londiste3]
+job_name = londiste_qn$n
+db = dbname=qn$n
+queue_name = replika_part$n
+logfile = log/%(job_name)s.log
+pidfile = pid/%(job_name)s.pid
+EOF
+
+cat > conf/londiste_qn${n}_full.ini << EOF
+[londiste3]
+job_name = londiste_qn${n}_full
+db = dbname=full
+queue_name = replika_part$n
+logfile = log/%(job_name)s.log
+pidfile = pid/%(job_name)s.pid
+EOF
+
+done
+
+for f in conf/*.ini; do
+  echo "$f"
+  echo "------------------"
+  cat "$f"
+  echo "------------------"
+  echo
+done
+
+set -e
+
+msg "Create both cascades root (shard) -> branch (qnode) -> leaf (merge). Also installs pgq and londiste into db modules"
+
+for n in 1 2; do
+  run londiste3 $v conf/londiste_part${n}.ini create-root part${n}_root "dbname=part${n}"
+  run londiste3 $v conf/londiste_qn${n}.ini \
+    create-branch qn${n} "dbname=qn${n}" --provider="dbname=part${n}"
+  run londiste3 $v conf/londiste_qn${n}_full.ini \
+    create-leaf merge_qn${n}_full "dbname=full" --provider="dbname=qn${n}"
+done
+
+msg "Optimize pgq for testing to handle empty ticks faster"
+for db in $all_list; do
+  run_sql $db "update pgq.queue set queue_ticker_idle_period='3 secs'"
+done
+
+msg "Launch ticker"
+run pgqd $v -d conf/pgqd.ini
+
+msg "Launch workers"
+
+for n in 1 2; do
+  run londiste3 $v -d conf/londiste_part${n}.ini worker
+  run londiste3 $v -d conf/londiste_qn${n}.ini worker
+  run londiste3 $v -d conf/londiste_qn${n}_full.ini worker
+done
+
+msg "Create table in partition nodes and in target database"
+for db in $part_list; do
+  run_sql "$db" "create table mydata (id int4 primary key, data text)"
+done
+run_sql full "create table mydata (id int4 primary key, data text)"
+
+msg "Register table in partition nodes"
+for db in $part_list; do
+  run londiste3 $v conf/londiste_$db.ini add-table mydata
+done
+
+msg "Wait for cascade sync (root->leaf). Leaf must know that we have this table in root"
+for src in $qn_list; do
+  run londiste3 $v conf/londiste_${src}_full.ini wait-root
+done
+
+msg "Add table into merge node"
+run londiste3 -q conf/londiste_qn1_full.ini add-table  public.mydata --find-copy-node --merge-all
+
+msg "Insert few rows"
+for n in 1 2; do
+  run_sql part$n "insert into mydata values ($n, 'part$n')"
+  run_sql part$n "insert into mydata values (2 + $n, 'part$n')"
+done
+
+msg "Wait for replica to cach up"
+for src in $qn_list; do
+  run londiste3 $v conf/londiste_${src}_full.ini wait-sync
+done
+
+msg "Now check if data apprered"
+run_sql full "select * from mydata order by id"
+run_sql full "select table_name, local, merge_state, table_attrs, dest_table from londiste.get_table_list('replika_part1')"
+run_sql full "select table_name, local, merge_state, table_attrs, dest_table from londiste.get_table_list('replika_part2')"
+
+../zcheck.sh
+
+msg "Test EXECUTE through cascade"
+
+for n in 1 2; do
+  run londiste3 $v conf/londiste_part$n.ini execute addcol-data2.sql
+  # do one by one to avoid deadlock on merge side when several ddl's are received simultaneously
+  run londiste3 $v conf/londiste_qn${n}_full.ini wait-root
+done
+
+msg "Insert more rows with more columns"
+for n in 1 2; do
+  run_sql part$n "insert into mydata values (4 + $n, 'part$n', 'x')"
+done
+
+msg "Wait for cascade sync (root->leaf)"
+for src in $qn_list; do
+  run londiste3 $v conf/londiste_${src}_full.ini wait-root
+done
+
+psql -d part1 -c 'select * from mydata order by 1;'
+psql -d part2 -c 'select * from mydata order by 1;'
+psql -d full -c 'select * from mydata order by 1;'
+
+../zcheck.sh