From e019db30172f48e885f1d4cedd4a5a85b4b07637 Mon Sep 17 00:00:00 2001 From: Dmitry Udalov Date: Wed, 6 Nov 2019 14:09:49 +0300 Subject: [PATCH 1/4] fix: f004, f005 process only one master, g001 - gluing of text in recommendations --- pghrep/src/checkup/f004/f004.go | 1 + pghrep/src/checkup/f005/f005.go | 1 + pghrep/src/checkup/g001/g001.go | 2 +- pghrep/src/checkup/g001/g001messages.go | 2 +- 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pghrep/src/checkup/f004/f004.go b/pghrep/src/checkup/f004/f004.go index 58e3691..c764000 100644 --- a/pghrep/src/checkup/f004/f004.go +++ b/pghrep/src/checkup/f004/f004.go @@ -62,6 +62,7 @@ func F004Process(report F004Report) checkup.ReportResult { bloatedTables = append(bloatedTables, "`"+heapBloatData.TableName+"`") } } + break } if totalBloatIsCritical { result.AppendConclusion(F004_TOTAL_BLOAT_EXCESS, MSG_TOTAL_BLOAT_EXCESS_CONCLUSION, diff --git a/pghrep/src/checkup/f005/f005.go b/pghrep/src/checkup/f005/f005.go index ba013c8..f7a796f 100644 --- a/pghrep/src/checkup/f005/f005.go +++ b/pghrep/src/checkup/f005/f005.go @@ -87,6 +87,7 @@ func F005Process(report F005Report, bloatedTables []string) checkup.ReportResult } } } + break } if totalBloatIsCritical { result.AppendConclusion(F005_TOTAL_BLOAT_EXCESS, MSG_TOTAL_BLOAT_EXCESS_CONCLUSION, diff --git a/pghrep/src/checkup/g001/g001.go b/pghrep/src/checkup/g001/g001.go index a7d42f5..1d4188d 100644 --- a/pghrep/src/checkup/g001/g001.go +++ b/pghrep/src/checkup/g001/g001.go @@ -218,7 +218,7 @@ func G001CheckOOMRisk(report G001Report, a001 a001.A001Report, result checkup.Re } if recommendation != "" { - recommendation = MSG_OOM_BASE_RECOMMENDATION + MSG_OOM_BASE_RECOMMENDATION_DETAIL + recommendation + recommendation = MSG_OOM_BASE_RECOMMENDATION + " " + MSG_OOM_BASE_RECOMMENDATION_DETAIL + recommendation } result.AppendConclusion(G001_OOM, conclusion) diff --git a/pghrep/src/checkup/g001/g001messages.go b/pghrep/src/checkup/g001/g001messages.go index 0469cbd..fc213d3 100644 --- a/pghrep/src/checkup/g001/g001messages.go +++ b/pghrep/src/checkup/g001/g001messages.go @@ -31,4 +31,4 @@ const MSG_OOM_AUTIVACUUM_WORKMEM_BEGIN string = "`autovacuum_work_mem` is %s" const MSG_OOM_AUTIVACUUM_WORKMEM_NOTSET string = "(it's set to `-1` so the actual value is inherited from `maintenance_work_mem`)" const MSG_OOM_AUTIVACUUM_WORKMEM_END string = "and maximum %d autovacuum workers may work simultaneously, so together they may consume up to %s, " + "or %.2f%%%% of RAM. It makes `autovacuum_work_mem/autovacuum_max_workers` pair a major contributor to the overall memory consumption.\n" -const MSG_OOM_BASE_RECOMMENDATION_DETAIL string = "First of all, pay attention to the following settings:\n" +const MSG_OOM_BASE_RECOMMENDATION_DETAIL string = "First of all, pay attention to the following settings: \n" -- GitLab From e288a9ae198687668a47f4813e8a5a88f68af869 Mon Sep 17 00:00:00 2001 From: dmius Date: Wed, 6 Nov 2019 17:54:31 +0300 Subject: [PATCH 2/4] Save host's status to nodes.json --- checkup | 20 +++++++------------- pghrep/src/checkup/f004/f004.go | 14 +++++++++++++- pghrep/src/checkup/f005/f005.go | 14 +++++++++++++- 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/checkup b/checkup index 4dcb539..216dfe4 100755 --- a/checkup +++ b/checkup @@ -734,7 +734,7 @@ generate_report_json() { # Do not use this function before 'host_pre_start_checks()' # # Globals: -# HOST +# HOST, ROLE # Arguments: # None # Returns: @@ -750,6 +750,8 @@ is_in_recovery() { return 0 else msg "ERROR: Cannot connect to the host: ${HOST}" + export ROLE="unavailable" + update_nodes_json exit 1 fi return 13 @@ -1114,14 +1116,6 @@ update_nodes_json() { local host_is_new="false" ALIAS_NAME="" ALIAS_INDEX="" - ROLE="" - - # check 'is role has been changed?" - if is_in_recovery; then - ROLE="standby" - else - ROLE="master" - fi # if file 'nodes.json' does not exist generate alias for a first host if [[ ! -f "${PROJECT_DIR}/nodes.json" ]]; then @@ -1243,6 +1237,10 @@ run_checks() { export PROJECT_DIR="${SCRIPT_DIR}/artifacts/${PROJECT}" test -d "${PROJECT_DIR}" || mkdir -p "${PROJECT_DIR}" + # fix timestamp before check + export TIMESTAMP_DIR=$(date +'%Y_%m_%dT%H_%M_%S_%z') # for use in directories names + export TIMESTAMPTZ=$(date +'%Y-%m-%d %H:%M:%S.0%z') + # perform all checks from './resources/checks/' directory if is_in_recovery; then ROLE="standby" @@ -1294,10 +1292,6 @@ run_checks() { local check_is_failed="false" - # fix timestamp before check - export TIMESTAMP_DIR=$(date +'%Y_%m_%dT%H_%M_%S_%z') # for use in directories names - export TIMESTAMPTZ=$(date +'%Y-%m-%d %H:%M:%S.0%z') - # alot of magic is here update_nodes_json diff --git a/pghrep/src/checkup/f004/f004.go b/pghrep/src/checkup/f004/f004.go index c764000..613edd8 100644 --- a/pghrep/src/checkup/f004/f004.go +++ b/pghrep/src/checkup/f004/f004.go @@ -40,7 +40,19 @@ func F004Process(report F004Report) checkup.ReportResult { totalBloatIsCritical := false var totalData F004HeapBloatTotal var databaseSize int64 - for _, hostData := range report.Results { + var masterHost string + + for host, hostData := range report.LastNodesJson.Hosts { + if hostData.Role == "master" { + masterHost = host + break + } + } + + for host, hostData := range report.Results { + if host != masterHost { + continue + } sortedTables := checkup.GetItemsSortedByNum(hostData.Data.HeapBloat) databaseSize = hostData.Data.DatabaseSizeBytes totalData = hostData.Data.HeapBloatTotal diff --git a/pghrep/src/checkup/f005/f005.go b/pghrep/src/checkup/f005/f005.go index f7a796f..d2d6faf 100644 --- a/pghrep/src/checkup/f005/f005.go +++ b/pghrep/src/checkup/f005/f005.go @@ -51,8 +51,20 @@ func F005Process(report F005Report, bloatedTables []string) checkup.ReportResult totalBloatIsCritical := false var totalData F005IndexBloatTotal var databaseSize int64 + var masterHost string + + for host, hostData := range report.LastNodesJson.Hosts { + if hostData.Role == "master" { + masterHost = host + break + } + } + i := 0 - for _, hostData := range report.Results { + for host, hostData := range report.Results { + if host != masterHost { + continue + } sortedIndexes := checkup.GetItemsSortedByNum(hostData.Data.IndexBloat) databaseSize = hostData.Data.DatabaseSizeBytes totalData = hostData.Data.IndexBloatTotal -- GitLab From 25bb8974cbf279736b3274a8fc2727232f208795 Mon Sep 17 00:00:00 2001 From: dmius Date: Thu, 7 Nov 2019 12:30:09 +0300 Subject: [PATCH 3/4] fix: tests fixed --- pghrep/src/checkup/f004/f004_test.go | 12 ++++++++++++ pghrep/src/checkup/f005/f005_test.go | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/pghrep/src/checkup/f004/f004_test.go b/pghrep/src/checkup/f004/f004_test.go index c5d93db..18f9ef9 100644 --- a/pghrep/src/checkup/f004/f004_test.go +++ b/pghrep/src/checkup/f004/f004_test.go @@ -7,6 +7,14 @@ import ( checkup ".." ) +var TestLastNodesJson checkup.ReportLastNodes = checkup.ReportLastNodes{ + Hosts: checkup.ReportHosts{ + "test-host": { + Role: "master", + }, + }, +} + func TestF004Success(t *testing.T) { fmt.Println(t.Name()) var report F004Report @@ -55,6 +63,7 @@ func TestF004Success(t *testing.T) { } report.Results = F004ReportHostsResults{"test-host": hostResult} + report.LastNodesJson = TestLastNodesJson result := F004Process(report) if result.P1 || result.P2 || @@ -83,6 +92,7 @@ func TestF004TotalExcess(t *testing.T) { } hostResult.Data.HeapBloat = map[string]F004HeapBloat{} report.Results = F004ReportHostsResults{"test-host": hostResult} + report.LastNodesJson = TestLastNodesJson result := F004Process(report) if !result.P1 || !checkup.ResultInList(result.Conclusions, F004_TOTAL_BLOAT_EXCESS) { @@ -199,6 +209,7 @@ func TestF004Warnig(t *testing.T) { }, } report.Results = F004ReportHostsResults{"test-host": hostResult} + report.LastNodesJson = TestLastNodesJson result := F004Process(report) if !result.P2 || !checkup.ResultInList(result.Conclusions, F004_BLOAT_WARNING) || @@ -257,6 +268,7 @@ func TestF004Critical(t *testing.T) { } report.Results = F004ReportHostsResults{"test-host": hostResult} + report.LastNodesJson = TestLastNodesJson result := F004Process(report) if !result.P1 || !checkup.ResultInList(result.Conclusions, F004_BLOAT_CRITICAL) || diff --git a/pghrep/src/checkup/f005/f005_test.go b/pghrep/src/checkup/f005/f005_test.go index 66eda20..ccac7b0 100644 --- a/pghrep/src/checkup/f005/f005_test.go +++ b/pghrep/src/checkup/f005/f005_test.go @@ -7,6 +7,14 @@ import ( checkup ".." ) +var TestLastNodesJson checkup.ReportLastNodes = checkup.ReportLastNodes{ + Hosts: checkup.ReportHosts{ + "test-host": { + Role: "master", + }, + }, +} + func TestF005Success(t *testing.T) { fmt.Println(t.Name()) var report F005Report @@ -56,6 +64,7 @@ func TestF005Success(t *testing.T) { }, } report.Results = F005ReportHostsResults{"test-host": hostResult} + report.LastNodesJson = TestLastNodesJson result := F005Process(report, []string{}) if result.P1 || result.P2 || result.P3 || !checkup.ResultInList(result.Conclusions, F005_TOTAL_BLOAT_LOW) { @@ -82,6 +91,7 @@ func TestF005TotalExcess(t *testing.T) { } hostResult.Data.IndexBloat = map[string]F005IndexBloat{} report.Results = F005ReportHostsResults{"test-host": hostResult} + report.LastNodesJson = TestLastNodesJson result := F005Process(report, []string{}) if !result.P1 || !checkup.ResultInList(result.Conclusions, F005_TOTAL_BLOAT_EXCESS) || @@ -142,6 +152,7 @@ func TestF005Warnig(t *testing.T) { } report.Results = F005ReportHostsResults{"test-host": hostResult} + report.LastNodesJson = TestLastNodesJson result := F005Process(report, []string{}) if !result.P2 || !checkup.ResultInList(result.Conclusions, F005_BLOAT_WARNING) || @@ -202,6 +213,7 @@ func TestF005Critical(t *testing.T) { } report.Results = F005ReportHostsResults{"test-host": hostResult} + report.LastNodesJson = TestLastNodesJson result := F005Process(report, []string{}) if !result.P1 || !checkup.ResultInList(result.Conclusions, F005_BLOAT_CRITICAL) || -- GitLab From 671cbd572430264f907098f94b748f5c7016ea9b Mon Sep 17 00:00:00 2001 From: dmius Date: Wed, 13 Nov 2019 10:54:31 +0300 Subject: [PATCH 4/4] fix: do not save not available host on first start --- checkup | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/checkup b/checkup index 216dfe4..3d52f75 100755 --- a/checkup +++ b/checkup @@ -1174,12 +1174,21 @@ update_nodes_json() { mkdir -p "${JSON_REPORTS_DIR}" local role_changed_at="never" - if [[ "${prev_role}" != "${ROLE}" ]] && [[ "$prev_role" != "null" ]]; then - msg "WARNING: important difference between checks detected:" - msg "WARNING: host's role has been changed from '${prev_role}' to '${ROLE}'" - role_changed_at="${TIMESTAMPTZ}" + + if [[ "$prev_role" != "null" ]]; then + if [[ "${prev_role}" != "${ROLE}" ]]; then + msg "WARNING: important difference between checks detected:" + msg "WARNING: host's role has been changed from '${prev_role}' to '${ROLE}'" + role_changed_at="${TIMESTAMPTZ}" + fi + else + # if we checking host first time and it does not available do not store it to nodes.json + if [[ "$ROLE" == "unavailable" ]]; then + return + fi fi + # generate new ALIAS_INDEX and ALIAS_NAME for a new host if [[ "$host_is_new" = "true" ]]; then # get maximum index -- GitLab