From 74cdc8ca5e789265679d58be400b64d8e6eed98a Mon Sep 17 00:00:00 2001 From: akartasov Date: Wed, 14 Sep 2022 08:52:19 +0700 Subject: [PATCH 1/8] chore: support newer Postgres versions: 13, 14, 15(beta) (#508) --- .gitlab-ci.yml | 20 +++++++++++++++- pghrep/internal/checkup/cfg/cfg.go | 37 ++++++++++++++++++++---------- 2 files changed, 44 insertions(+), 13 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8cc85e7..54d6428 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -187,9 +187,27 @@ test-check-12: services: - postgres:12 +test-check-13: + extends: ".test-check" + stage: "test" + services: + - postgres:13 + +test-check-14: + extends: ".test-check" + stage: "test" + services: + - postgres:14 + +test-check-15: + extends: ".test-check" + stage: "test" + services: + - postgres:15beta4 + test-check-cli: services: - - postgres:11 + - postgres:14 stage: "test" extends: ".prepare" script: | diff --git a/pghrep/internal/checkup/cfg/cfg.go b/pghrep/internal/checkup/cfg/cfg.go index 4841f1f..2949361 100644 --- a/pghrep/internal/checkup/cfg/cfg.go +++ b/pghrep/internal/checkup/cfg/cfg.go @@ -30,36 +30,49 @@ const POSTGRES_RELEASES_URL string = "https://git.postgresql.org/gitweb/?p=postg const RELEASE_CODE = "REL" var versionsDefault map[string]Version = map[string]Version{ + "14": { + FirstRelease: "2021-09-30", + FinalRelease: "2026-11-12", + MinorVersions: []int{0, 1, 2, 3, 4, 5}, + }, + "13": { + FirstRelease: "2020-09-24", + FinalRelease: "2025-11-13", + MinorVersions: []int{0, 1, 2, 3, 4, 5, 6, 7, 8}, + }, "12": Version{ FirstRelease: "2019-10-03", FinalRelease: "2024-11-14", - MinorVersions: []int{0}, + MinorVersions: []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}, }, "11": Version{ FirstRelease: "2018-10-18", FinalRelease: "2023-11-09", - MinorVersions: []int{0, 1, 2, 3, 4}, + MinorVersions: []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}, }, "10": Version{ - FirstRelease: "2017-10-05", - FinalRelease: "2022-11-10", - MinorVersions: []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, + FirstRelease: "2017-10-05", + FinalRelease: "2022-11-10", + MinorVersions: []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22}, }, "9.6": Version{ - FirstRelease: "2016-09-29", - FinalRelease: "2021-11-11", - MinorVersions: []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, + FirstRelease: "2016-09-29", + FinalRelease: "2021-11-11", + MinorVersions: []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24}, }, "9.5": Version{ - FirstRelease: "2016-01-07", - FinalRelease: "2021-02-11", - MinorVersions: []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18}, + FirstRelease: "2016-01-07", + FinalRelease: "2021-02-11", + MinorVersions: []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25}, }, "9.4": Version{ FirstRelease: "2014-12-18", FinalRelease: "2020-02-13", MinorVersions: []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, - 20, 21, 22, 23}, + 20, 21, 22, 23, 24, 25, 26}, }, "9.3": Version{ FirstRelease: "2013-09-09", -- GitLab From 0998ef6d53cc658bb2ffc6c1fe55bcf24f701a48 Mon Sep 17 00:00:00 2001 From: akartasov Date: Wed, 14 Sep 2022 09:00:36 +0700 Subject: [PATCH 2/8] chore: use Postgres 14 in CI tests by default --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 54d6428..b41f8d6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,7 +27,7 @@ test-general: - echo "export PATH=\$PATH:/usr/local/go/bin" >> ~/.profile - source ~/.profile - cd ./pghrep && make main && cd .. - - sudo .ci/prepare_cluster.sh "11" + - sudo .ci/prepare_cluster.sh "14" - ps ax | grep postgres - psql -U postgres -p 5432 -c 'show data_directory;' - psql -U postgres -p 5433 -c 'show data_directory;' @@ -93,7 +93,7 @@ test-general: - wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - - apt-get update - apt-get -y upgrade - - apt-get -y install postgresql-client-11 + - apt-get -y install postgresql-client-14 - psql --version - wget --quiet https://golang.org/dl/go1.17.2.linux-amd64.tar.gz - sudo rm -rf /usr/local/go && sudo tar -C /usr/local -xzf go1.17.2.linux-amd64.tar.gz -- GitLab From 4f20918e08fd538d76155278587b2476e7ec619a Mon Sep 17 00:00:00 2001 From: akartasov Date: Wed, 14 Sep 2022 09:08:23 +0700 Subject: [PATCH 3/8] chore: use Ubuntu 20.04 in CI tests by default --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b41f8d6..864a6f3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: ubuntu:16.04 +image: ubuntu:20.04 stages: - test -- GitLab From 2cf9d926e39d7243c654c267c4d939e003b36e81 Mon Sep 17 00:00:00 2001 From: akartasov Date: Wed, 14 Sep 2022 09:24:53 +0700 Subject: [PATCH 4/8] chore: prepare cluster for Postrges 14 in CI --- .ci/prepare_cluster.sh | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/.ci/prepare_cluster.sh b/.ci/prepare_cluster.sh index 5b0111d..d098b4b 100755 --- a/.ci/prepare_cluster.sh +++ b/.ci/prepare_cluster.sh @@ -5,7 +5,7 @@ echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - apt-get update apt-get -y upgrade -apt-get -y install postgresql-${PG_VER} postgresql-contrib-${PG_VER} postgresql-client-${PG_VER} postgresql-server-dev-${PG_VER} && apt-get install -y postgresql-${PG_VER}-pg-stat-kcache +apt-get -y install postgresql-${PG_VER} postgresql-contrib-${PG_VER} postgresql-client-${PG_VER} postgresql-server-dev-${PG_VER} && apt-get install -y postgresql-${PG_VER}-pg-stat-kcache bc psql --version source ~/.profile echo "127.0.0.2 postgres.test1.node" >> /etc/hosts # replica 1 @@ -30,10 +30,14 @@ echo "shared_preload_libraries = 'pg_stat_statements,auto_explain,pg_stat_kcache ## Configure general postgres master node params echo "wal_level = hot_standby" >> /etc/postgresql/${PG_VER}/main/postgresql.conf echo "max_wal_senders = 5" >> /etc/postgresql/${PG_VER}/main/postgresql.conf -echo "wal_keep_segments = 32" >> /etc/postgresql/${PG_VER}/main/postgresql.conf echo "archive_mode = on" >> /etc/postgresql/${PG_VER}/main/postgresql.conf echo "archive_command = 'cp %p /path_to/archive/%f'" >> /etc/postgresql/${PG_VER}/main/postgresql.conf +if [ $(echo "$PG_VER < 13" | /usr/bin/bc) = "1" ]; then + # wal_keep_segments is deprecated in Postgres 12 + echo "wal_keep_segments = 32" >> /etc/postgresql/${PG_VER}/main/postgresql.conf +fi + ## Start postgres master node /etc/init.d/postgresql start psql -U postgres -c "create role replication with replication password 'rEpLpAssw' login" @@ -77,10 +81,18 @@ function add_replica() { ## Configure replica postgres settings echo "hot_standby = on" >> /var/lib/postgresql/${PG_VER}/data${num}/postgresql.conf - echo "standby_mode = 'on'" > /var/lib/postgresql/${PG_VER}/data${num}/recovery.conf - echo "primary_conninfo = 'host=127.0.0.4 port=5432 user=replication password=rEpLpAssw'" >> /var/lib/postgresql/${PG_VER}/data${num}/recovery.conf - echo "trigger_file = '/var/lib/postgresql/${PG_VER}/data${num}/trigger'" >> /var/lib/postgresql/${PG_VER}/data${num}/recovery.conf - echo "restore_command = 'cp /path_to/archive/%f "%p"'" >> /var/lib/postgresql/${PG_VER}/data${num}/recovery.conf + + if [ $(echo "$PG_VER < 13" | /usr/bin/bc) = "1" ]; then + echo "standby_mode = 'on'" > /var/lib/postgresql/${PG_VER}/data${num}/recovery.conf + echo "primary_conninfo = 'host=127.0.0.4 port=5432 user=replication password=rEpLpAssw'" >> /var/lib/postgresql/${PG_VER}/data${num}/recovery.conf + echo "trigger_file = '/var/lib/postgresql/${PG_VER}/data${num}/trigger'" >> /var/lib/postgresql/${PG_VER}/data${num}/recovery.conf + echo "restore_command = 'cp /path_to/archive/%f "%p"'" >> /var/lib/postgresql/${PG_VER}/data${num}/recovery.conf + else + touch "/var/lib/postgresql/${PG_VER}/data${num}/standby.signal" + echo "primary_conninfo = 'host=127.0.0.4 port=5432 user=replication password=rEpLpAssw'" >> /var/lib/postgresql/${PG_VER}/data${num}/postgresql.conf + echo "promote_trigger_file = '/var/lib/postgresql/${PG_VER}/data${num}/trigger'" >> /var/lib/postgresql/${PG_VER}/data${num}/postgresql.conf + echo "restore_command = 'cp /path_to/archive/%f "%p"'" >> /var/lib/postgresql/${PG_VER}/data${num}/postgresql.conf + fi ## Start replica sudo -u postgres /usr/lib/postgresql/${PG_VER}/bin/pg_ctl -D /var/lib/postgresql/${PG_VER}/data${num} -l /var/log/postgresql/secondary1.log start || cat /var/log/postgresql/replica${num}.log -- GitLab From e7051f64cef8a1a1c40d6478083376e890361fc0 Mon Sep 17 00:00:00 2001 From: akartasov Date: Wed, 14 Sep 2022 09:36:27 +0700 Subject: [PATCH 5/8] chore: run with Postgres 12 because of 'total_time' does not exist --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 864a6f3..70ebc6c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,7 +27,7 @@ test-general: - echo "export PATH=\$PATH:/usr/local/go/bin" >> ~/.profile - source ~/.profile - cd ./pghrep && make main && cd .. - - sudo .ci/prepare_cluster.sh "14" + - sudo .ci/prepare_cluster.sh "12" - ps ax | grep postgres - psql -U postgres -p 5432 -c 'show data_directory;' - psql -U postgres -p 5433 -c 'show data_directory;' -- GitLab From 6eb13b0df3ca3c9e1b2a950c42abd67d879e1409 Mon Sep 17 00:00:00 2001 From: akartasov Date: Wed, 14 Sep 2022 09:44:50 +0700 Subject: [PATCH 6/8] chore: roll back cluster tests to Postgres 11 --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 70ebc6c..41b703c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,7 +27,7 @@ test-general: - echo "export PATH=\$PATH:/usr/local/go/bin" >> ~/.profile - source ~/.profile - cd ./pghrep && make main && cd .. - - sudo .ci/prepare_cluster.sh "12" + - sudo .ci/prepare_cluster.sh "11" - ps ax | grep postgres - psql -U postgres -p 5432 -c 'show data_directory;' - psql -U postgres -p 5433 -c 'show data_directory;' -- GitLab From c30ea5b5fa7cf762a5848d600a5246771416cf66 Mon Sep 17 00:00:00 2001 From: akartasov Date: Wed, 14 Sep 2022 10:01:37 +0700 Subject: [PATCH 7/8] chore: roll back tests to Postgres 11 --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 41b703c..9e01e5e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -93,7 +93,7 @@ test-general: - wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - - apt-get update - apt-get -y upgrade - - apt-get -y install postgresql-client-14 + - apt-get -y install postgresql-client-11 - psql --version - wget --quiet https://golang.org/dl/go1.17.2.linux-amd64.tar.gz - sudo rm -rf /usr/local/go && sudo tar -C /usr/local -xzf go1.17.2.linux-amd64.tar.gz -- GitLab From 2e08cddd94eb9862bb3813631a420a6531ff2d60 Mon Sep 17 00:00:00 2001 From: akartasov Date: Wed, 14 Sep 2022 10:09:12 +0700 Subject: [PATCH 8/8] chore: roll back tests to Ubuntu 16.04 --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9e01e5e..54d6428 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: ubuntu:20.04 +image: ubuntu:16.04 stages: - test -- GitLab