Skip to content

Commit 145aa5e

Browse files
author
Your Name
committed
Scripts for deploy 3-node local configuration of FDW+partitioning
1 parent 1f1dac8 commit 145aa5e

File tree

7 files changed

+193
-0
lines changed

7 files changed

+193
-0
lines changed

‎cln‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
make clean > /dev/null
2+
make -C contrib clean > /dev/null

‎cmp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
export CFLAGS="-O0"
2+
./configure --prefix=`pwd`/tmp_install --enable-tap-tests --enable-debug \
3+
--enable-cassert --enable-nls --with-openssl --with-perl --with-python

‎mk‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
make -C contrib > /dev/null
2+
make -C contrib install > /dev/null
3+
make > /dev/null
4+
make install > /dev/null

‎n0_pgb_init.sql‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
CREATE SERVER remote1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (port '5433', use_remote_estimate 'on');
2+
CREATE USER MAPPING FOR PUBLIC SERVER remote1;
3+
CREATE SERVER remote2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (port '5434', use_remote_estimate 'on');
4+
CREATE USER MAPPING FOR PUBLIC SERVER remote2;
5+
6+
DROP TABLE pgbench_accounts_2 CASCADE;
7+
DROP TABLE pgbench_accounts_3 CASCADE;
8+
ALTER TABLE pgbench_accounts DROP CONSTRAINT pgbench_accounts_pkey;
9+
10+
DROP TABLE pgbench_branches_2 CASCADE;
11+
DROP TABLE pgbench_branches_3 CASCADE;
12+
ALTER TABLE pgbench_branches DROP CONSTRAINT pgbench_branches_pkey;
13+
14+
DROP TABLE pgbench_tellers_2 CASCADE;
15+
DROP TABLE pgbench_tellers_3 CASCADE;
16+
ALTER TABLE pgbench_tellers DROP CONSTRAINT pgbench_tellers_pkey;
17+
18+
CREATE FOREIGN TABLE pgbench_accounts_2 PARTITION OF pgbench_accounts FOR VALUES WITH (modulus 3, remainder 1) SERVER remote1;
19+
CREATE FOREIGN TABLE pgbench_accounts_3 PARTITION OF pgbench_accounts FOR VALUES WITH (modulus 3, remainder 2) SERVER remote2;
20+
ALTER TABLE pgbench_accounts ADD PRIMARY KEY (aid);
21+
22+
CREATE FOREIGN TABLE pgbench_branches_2 PARTITION OF pgbench_branches FOR VALUES WITH (modulus 3, remainder 1) SERVER remote1;
23+
CREATE FOREIGN TABLE pgbench_branches_3 PARTITION OF pgbench_branches FOR VALUES WITH (modulus 3, remainder 2) SERVER remote2;
24+
ALTER TABLE pgbench_branches ADD PRIMARY KEY (bid);
25+
26+
CREATE FOREIGN TABLE pgbench_tellers_2 PARTITION OF pgbench_tellers FOR VALUES WITH (modulus 3, remainder 1) SERVER remote1;
27+
CREATE FOREIGN TABLE pgbench_tellers_3 PARTITION OF pgbench_tellers FOR VALUES WITH (modulus 3, remainder 2) SERVER remote2;
28+
ALTER TABLE pgbench_tellers ADD PRIMARY KEY (tid);
29+

‎n1_pgb_init.sql‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
CREATE SERVER remote1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (port '5432', use_remote_estimate 'on');
2+
CREATE USER MAPPING FOR PUBLIC SERVER remote1;
3+
CREATE SERVER remote2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (port '5434', use_remote_estimate 'on');
4+
CREATE USER MAPPING FOR PUBLIC SERVER remote2;
5+
6+
DROP TABLE pgbench_accounts_1 CASCADE;
7+
DROP TABLE pgbench_accounts_3 CASCADE;
8+
ALTER TABLE pgbench_accounts DROP CONSTRAINT pgbench_accounts_pkey;
9+
10+
DROP TABLE pgbench_branches_1 CASCADE;
11+
DROP TABLE pgbench_branches_3 CASCADE;
12+
ALTER TABLE pgbench_branches DROP CONSTRAINT pgbench_branches_pkey;
13+
14+
DROP TABLE pgbench_tellers_1 CASCADE;
15+
DROP TABLE pgbench_tellers_3 CASCADE;
16+
ALTER TABLE pgbench_tellers DROP CONSTRAINT pgbench_tellers_pkey;
17+
18+
CREATE FOREIGN TABLE pgbench_accounts_1 PARTITION OF pgbench_accounts FOR VALUES WITH (modulus 3, remainder 0) SERVER remote1;
19+
CREATE FOREIGN TABLE pgbench_accounts_3 PARTITION OF pgbench_accounts FOR VALUES WITH (modulus 3, remainder 2) SERVER remote2;
20+
ALTER TABLE pgbench_accounts ADD PRIMARY KEY (aid);
21+
22+
CREATE FOREIGN TABLE pgbench_branches_1 PARTITION OF pgbench_branches FOR VALUES WITH (modulus 3, remainder 0) SERVER remote1;
23+
CREATE FOREIGN TABLE pgbench_branches_3 PARTITION OF pgbench_branches FOR VALUES WITH (modulus 3, remainder 2) SERVER remote2;
24+
ALTER TABLE pgbench_branches ADD PRIMARY KEY (bid);
25+
26+
CREATE FOREIGN TABLE pgbench_tellers_1 PARTITION OF pgbench_tellers FOR VALUES WITH (modulus 3, remainder 0) SERVER remote1;
27+
CREATE FOREIGN TABLE pgbench_tellers_3 PARTITION OF pgbench_tellers FOR VALUES WITH (modulus 3, remainder 2) SERVER remote2;
28+
ALTER TABLE pgbench_tellers ADD PRIMARY KEY (tid);
29+

‎n2_pgb_init.sql‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
CREATE SERVER remote1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (port '5432', use_remote_estimate 'on');
2+
CREATE USER MAPPING FOR PUBLIC SERVER remote1;
3+
CREATE SERVER remote2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (port '5433', use_remote_estimate 'on');
4+
CREATE USER MAPPING FOR PUBLIC SERVER remote2;
5+
6+
DROP TABLE pgbench_accounts_1 CASCADE;
7+
DROP TABLE pgbench_accounts_2 CASCADE;
8+
ALTER TABLE pgbench_accounts DROP CONSTRAINT pgbench_accounts_pkey;
9+
10+
DROP TABLE pgbench_branches_1 CASCADE;
11+
DROP TABLE pgbench_branches_2 CASCADE;
12+
ALTER TABLE pgbench_branches DROP CONSTRAINT pgbench_branches_pkey;
13+
14+
DROP TABLE pgbench_tellers_1 CASCADE;
15+
DROP TABLE pgbench_tellers_2 CASCADE;
16+
ALTER TABLE pgbench_tellers DROP CONSTRAINT pgbench_tellers_pkey;
17+
18+
CREATE FOREIGN TABLE pgbench_accounts_1 PARTITION OF pgbench_accounts FOR VALUES WITH (modulus 3, remainder 0) SERVER remote1;
19+
CREATE FOREIGN TABLE pgbench_accounts_2 PARTITION OF pgbench_accounts FOR VALUES WITH (modulus 3, remainder 1) SERVER remote2;
20+
ALTER TABLE pgbench_accounts ADD PRIMARY KEY (aid);
21+
22+
CREATE FOREIGN TABLE pgbench_branches_1 PARTITION OF pgbench_branches FOR VALUES WITH (modulus 3, remainder 0) SERVER remote1;
23+
CREATE FOREIGN TABLE pgbench_branches_2 PARTITION OF pgbench_branches FOR VALUES WITH (modulus 3, remainder 1) SERVER remote2;
24+
ALTER TABLE pgbench_branches ADD PRIMARY KEY (bid);
25+
26+
CREATE FOREIGN TABLE pgbench_tellers_1 PARTITION OF pgbench_tellers FOR VALUES WITH (modulus 3, remainder 0) SERVER remote1;
27+
CREATE FOREIGN TABLE pgbench_tellers_2 PARTITION OF pgbench_tellers FOR VALUES WITH (modulus 3, remainder 1) SERVER remote2;
28+
ALTER TABLE pgbench_tellers ADD PRIMARY KEY (tid);
29+

‎shdep.sh‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
#!/bin/bash
2+
3+
# ##############################################################################
4+
#
5+
# Deploy local 3-node configuration of sharded postgres.
6+
#
7+
# This script performs initialization of fdw+partitioning infrastructure for
8+
# parallel (distributed) query execution purposes.
9+
#
10+
# ##############################################################################
11+
12+
PGINSTALL=`pwd`/tmp_install/
13+
DEPLOY_SCRIPTS_PATH=`pwd`
14+
SCALE=100
15+
16+
export LD_LIBRARY_PATH=$PGINSTALL/lib:$LD_LIBRARY_PATH
17+
export PATH=$PGINSTALL/bin:$PATH
18+
export LC_ALL=C
19+
export LANGUAGE="en_US:en"
20+
export PGPORT=5432 #default head
21+
export PGDATABASE=shardman
22+
export PGHOST=localhost
23+
export PGUSER=`whoami`
24+
25+
pkill -U `whoami` -9 -e postgres
26+
pkill -U `whoami` -9 -e pgbench
27+
28+
D1=`pwd`/PGDATA1
29+
D2=`pwd`/PGDATA2
30+
D3=`pwd`/PGDATA3
31+
32+
rm -rf $D1 && mkdir $D1 && rm -rf $D2 && mkdir $D2 && rm -rf $D3 && mkdir $D3
33+
rm -rf $PGINSTALL && rm n0.log && rm n1.log && rm n2.log
34+
35+
# Building project
36+
make > /dev/null
37+
make -C contrib > /dev/null
38+
make install > /dev/null
39+
make -C contrib install > /dev/null
40+
41+
remoteSrvName=fdwremote
42+
43+
initdb -D $D1 -E UTF8 --locale=C
44+
initdb -D $D2 -E UTF8 --locale=C
45+
initdb -D $D3 -E UTF8 --locale=C
46+
47+
echo "shared_preload_libraries = 'postgres_fdw'" >> $D1/postgresql.conf
48+
echo "shared_preload_libraries = 'postgres_fdw'" >> $D2/postgresql.conf
49+
echo "shared_preload_libraries = 'postgres_fdw'" >> $D3/postgresql.conf
50+
echo "shared_buffers = 10GB" >> $D1/postgresql.conf
51+
echo "shared_buffers = 10GB" >> $D2/postgresql.conf
52+
echo "shared_buffers = 10GB" >> $D3/postgresql.conf
53+
echo "listen_addresses = '*'" >> $D1/postgresql.conf
54+
echo "listen_addresses = '*'" >> $D2/postgresql.conf
55+
echo "listen_addresses = '*'" >> $D3/postgresql.conf
56+
echo "host all all 0.0.0.0/0 trust" >> $D1/pg_hba.conf
57+
echo "host all all 0.0.0.0/0 trust" >> $D2/pg_hba.conf
58+
echo "host all all 0.0.0.0/0 trust" >> $D3/pg_hba.conf
59+
60+
# Takeoff
61+
pg_ctl -w -c -o "-p 5434" -D $D3 -l n2.log start
62+
pg_ctl -w -c -o "-p 5433" -D $D2 -l n1.log start
63+
pg_ctl -w -c -o "-p 5432" -D $D1 -l n0.log start
64+
createdb -p 5432
65+
createdb -p 5433
66+
createdb -p 5434
67+
68+
# Init foreign tables
69+
psql -p 5434 -c "CREATE EXTENSION postgres_fdw;"
70+
psql -p 5433 -c "CREATE EXTENSION postgres_fdw;"
71+
psql -p 5432 -c "CREATE EXTENSION postgres_fdw;"
72+
73+
# Create pgbench partitions
74+
pgbench -p 5432 -i -s $SCALE --partitions=3 --partition-method=hash
75+
pgbench -p 5433 -i -s $SCALE --partitions=3 --partition-method=hash
76+
pgbench -p 5434 -i -s $SCALE --partitions=3 --partition-method=hash
77+
78+
# Drop unneeded local partitions and init foreign partitions
79+
psql -p 5432 -f "$DEPLOY_SCRIPTS_PATH/n0_pgb_init.sql"
80+
psql -p 5433 -f "$DEPLOY_SCRIPTS_PATH/n1_pgb_init.sql"
81+
psql -p 5434 -f "$DEPLOY_SCRIPTS_PATH/n2_pgb_init.sql"
82+
83+
# The end of deploy
84+
85+
86+
# Create tables pt,rt,st partitioned by hash on id column.
87+
#psql -p 5432 -f "$DEPLOY_SCRIPTS_PATH/init_node0.sql"
88+
#psql -p 5433 -f "$DEPLOY_SCRIPTS_PATH/init_node1.sql"
89+
#psql -p 5434 -f "$DEPLOY_SCRIPTS_PATH/init_node2.sql"
90+
91+
# Fill the pt relation
92+
#psql -p 5432 -c \
93+
# "INSERT INTO pt (id, payload, test)
94+
# (
95+
# SELECT a.*, b.*,0
96+
# );"
97+

0 commit comments

Comments
 (0)