diff --git a/checkup b/checkup index 4dcb5397c2d6ceb7dc300b1c6e6d8922c1dfccbd..3d52f75eaf005bf80724923da8d2f884da6e5d59 100755 --- a/checkup +++ b/checkup @@ -734,7 +734,7 @@ generate_report_json() { # Do not use this function before 'host_pre_start_checks()' # # Globals: -# HOST +# HOST, ROLE # Arguments: # None # Returns: @@ -750,6 +750,8 @@ is_in_recovery() { return 0 else msg "ERROR: Cannot connect to the host: ${HOST}" + export ROLE="unavailable" + update_nodes_json exit 1 fi return 13 @@ -1114,14 +1116,6 @@ update_nodes_json() { local host_is_new="false" ALIAS_NAME="" ALIAS_INDEX="" - ROLE="" - - # check 'is role has been changed?" - if is_in_recovery; then - ROLE="standby" - else - ROLE="master" - fi # if file 'nodes.json' does not exist generate alias for a first host if [[ ! -f "${PROJECT_DIR}/nodes.json" ]]; then @@ -1180,12 +1174,21 @@ update_nodes_json() { mkdir -p "${JSON_REPORTS_DIR}" local role_changed_at="never" - if [[ "${prev_role}" != "${ROLE}" ]] && [[ "$prev_role" != "null" ]]; then - msg "WARNING: important difference between checks detected:" - msg "WARNING: host's role has been changed from '${prev_role}' to '${ROLE}'" - role_changed_at="${TIMESTAMPTZ}" + + if [[ "$prev_role" != "null" ]]; then + if [[ "${prev_role}" != "${ROLE}" ]]; then + msg "WARNING: important difference between checks detected:" + msg "WARNING: host's role has been changed from '${prev_role}' to '${ROLE}'" + role_changed_at="${TIMESTAMPTZ}" + fi + else + # if we checking host first time and it does not available do not store it to nodes.json + if [[ "$ROLE" == "unavailable" ]]; then + return + fi fi + # generate new ALIAS_INDEX and ALIAS_NAME for a new host if [[ "$host_is_new" = "true" ]]; then # get maximum index @@ -1243,6 +1246,10 @@ run_checks() { export PROJECT_DIR="${SCRIPT_DIR}/artifacts/${PROJECT}" test -d "${PROJECT_DIR}" || mkdir -p "${PROJECT_DIR}" + # fix timestamp before check + export TIMESTAMP_DIR=$(date +'%Y_%m_%dT%H_%M_%S_%z') # for use in directories names + export TIMESTAMPTZ=$(date +'%Y-%m-%d %H:%M:%S.0%z') + # perform all checks from './resources/checks/' directory if is_in_recovery; then ROLE="standby" @@ -1294,10 +1301,6 @@ run_checks() { local check_is_failed="false" - # fix timestamp before check - export TIMESTAMP_DIR=$(date +'%Y_%m_%dT%H_%M_%S_%z') # for use in directories names - export TIMESTAMPTZ=$(date +'%Y-%m-%d %H:%M:%S.0%z') - # alot of magic is here update_nodes_json diff --git a/pghrep/src/checkup/f004/f004.go b/pghrep/src/checkup/f004/f004.go index 58e369158f08f391ab064ae46ccd9bf4ffb21504..613edd819bd754830116f6d1d9d427396b8d0599 100644 --- a/pghrep/src/checkup/f004/f004.go +++ b/pghrep/src/checkup/f004/f004.go @@ -40,7 +40,19 @@ func F004Process(report F004Report) checkup.ReportResult { totalBloatIsCritical := false var totalData F004HeapBloatTotal var databaseSize int64 - for _, hostData := range report.Results { + var masterHost string + + for host, hostData := range report.LastNodesJson.Hosts { + if hostData.Role == "master" { + masterHost = host + break + } + } + + for host, hostData := range report.Results { + if host != masterHost { + continue + } sortedTables := checkup.GetItemsSortedByNum(hostData.Data.HeapBloat) databaseSize = hostData.Data.DatabaseSizeBytes totalData = hostData.Data.HeapBloatTotal @@ -62,6 +74,7 @@ func F004Process(report F004Report) checkup.ReportResult { bloatedTables = append(bloatedTables, "`"+heapBloatData.TableName+"`") } } + break } if totalBloatIsCritical { result.AppendConclusion(F004_TOTAL_BLOAT_EXCESS, MSG_TOTAL_BLOAT_EXCESS_CONCLUSION, diff --git a/pghrep/src/checkup/f004/f004_test.go b/pghrep/src/checkup/f004/f004_test.go index c5d93db963592eb537bb4a1bf3bc369d41cb4e7f..18f9ef92872541d1c2624173237d27bb50e713c5 100644 --- a/pghrep/src/checkup/f004/f004_test.go +++ b/pghrep/src/checkup/f004/f004_test.go @@ -7,6 +7,14 @@ import ( checkup ".." ) +var TestLastNodesJson checkup.ReportLastNodes = checkup.ReportLastNodes{ + Hosts: checkup.ReportHosts{ + "test-host": { + Role: "master", + }, + }, +} + func TestF004Success(t *testing.T) { fmt.Println(t.Name()) var report F004Report @@ -55,6 +63,7 @@ func TestF004Success(t *testing.T) { } report.Results = F004ReportHostsResults{"test-host": hostResult} + report.LastNodesJson = TestLastNodesJson result := F004Process(report) if result.P1 || result.P2 || @@ -83,6 +92,7 @@ func TestF004TotalExcess(t *testing.T) { } hostResult.Data.HeapBloat = map[string]F004HeapBloat{} report.Results = F004ReportHostsResults{"test-host": hostResult} + report.LastNodesJson = TestLastNodesJson result := F004Process(report) if !result.P1 || !checkup.ResultInList(result.Conclusions, F004_TOTAL_BLOAT_EXCESS) { @@ -199,6 +209,7 @@ func TestF004Warnig(t *testing.T) { }, } report.Results = F004ReportHostsResults{"test-host": hostResult} + report.LastNodesJson = TestLastNodesJson result := F004Process(report) if !result.P2 || !checkup.ResultInList(result.Conclusions, F004_BLOAT_WARNING) || @@ -257,6 +268,7 @@ func TestF004Critical(t *testing.T) { } report.Results = F004ReportHostsResults{"test-host": hostResult} + report.LastNodesJson = TestLastNodesJson result := F004Process(report) if !result.P1 || !checkup.ResultInList(result.Conclusions, F004_BLOAT_CRITICAL) || diff --git a/pghrep/src/checkup/f005/f005.go b/pghrep/src/checkup/f005/f005.go index ba013c86515c6b20d5a1acb0a71ba77bb0af81b3..d2d6faf12d203ea5cd80927becc86c0a25dd02d8 100644 --- a/pghrep/src/checkup/f005/f005.go +++ b/pghrep/src/checkup/f005/f005.go @@ -51,8 +51,20 @@ func F005Process(report F005Report, bloatedTables []string) checkup.ReportResult totalBloatIsCritical := false var totalData F005IndexBloatTotal var databaseSize int64 + var masterHost string + + for host, hostData := range report.LastNodesJson.Hosts { + if hostData.Role == "master" { + masterHost = host + break + } + } + i := 0 - for _, hostData := range report.Results { + for host, hostData := range report.Results { + if host != masterHost { + continue + } sortedIndexes := checkup.GetItemsSortedByNum(hostData.Data.IndexBloat) databaseSize = hostData.Data.DatabaseSizeBytes totalData = hostData.Data.IndexBloatTotal @@ -87,6 +99,7 @@ func F005Process(report F005Report, bloatedTables []string) checkup.ReportResult } } } + break } if totalBloatIsCritical { result.AppendConclusion(F005_TOTAL_BLOAT_EXCESS, MSG_TOTAL_BLOAT_EXCESS_CONCLUSION, diff --git a/pghrep/src/checkup/f005/f005_test.go b/pghrep/src/checkup/f005/f005_test.go index 66eda20c89e2daaa7b0ad766a2773708bd6c9e38..ccac7b0745d6ca3926713302a3a38154667045ec 100644 --- a/pghrep/src/checkup/f005/f005_test.go +++ b/pghrep/src/checkup/f005/f005_test.go @@ -7,6 +7,14 @@ import ( checkup ".." ) +var TestLastNodesJson checkup.ReportLastNodes = checkup.ReportLastNodes{ + Hosts: checkup.ReportHosts{ + "test-host": { + Role: "master", + }, + }, +} + func TestF005Success(t *testing.T) { fmt.Println(t.Name()) var report F005Report @@ -56,6 +64,7 @@ func TestF005Success(t *testing.T) { }, } report.Results = F005ReportHostsResults{"test-host": hostResult} + report.LastNodesJson = TestLastNodesJson result := F005Process(report, []string{}) if result.P1 || result.P2 || result.P3 || !checkup.ResultInList(result.Conclusions, F005_TOTAL_BLOAT_LOW) { @@ -82,6 +91,7 @@ func TestF005TotalExcess(t *testing.T) { } hostResult.Data.IndexBloat = map[string]F005IndexBloat{} report.Results = F005ReportHostsResults{"test-host": hostResult} + report.LastNodesJson = TestLastNodesJson result := F005Process(report, []string{}) if !result.P1 || !checkup.ResultInList(result.Conclusions, F005_TOTAL_BLOAT_EXCESS) || @@ -142,6 +152,7 @@ func TestF005Warnig(t *testing.T) { } report.Results = F005ReportHostsResults{"test-host": hostResult} + report.LastNodesJson = TestLastNodesJson result := F005Process(report, []string{}) if !result.P2 || !checkup.ResultInList(result.Conclusions, F005_BLOAT_WARNING) || @@ -202,6 +213,7 @@ func TestF005Critical(t *testing.T) { } report.Results = F005ReportHostsResults{"test-host": hostResult} + report.LastNodesJson = TestLastNodesJson result := F005Process(report, []string{}) if !result.P1 || !checkup.ResultInList(result.Conclusions, F005_BLOAT_CRITICAL) || diff --git a/pghrep/src/checkup/g001/g001.go b/pghrep/src/checkup/g001/g001.go index a7d42f5344045fddc97932a742d793f5a1d5147d..1d4188df86cf374a13801454dd106b2c0fca0799 100644 --- a/pghrep/src/checkup/g001/g001.go +++ b/pghrep/src/checkup/g001/g001.go @@ -218,7 +218,7 @@ func G001CheckOOMRisk(report G001Report, a001 a001.A001Report, result checkup.Re } if recommendation != "" { - recommendation = MSG_OOM_BASE_RECOMMENDATION + MSG_OOM_BASE_RECOMMENDATION_DETAIL + recommendation + recommendation = MSG_OOM_BASE_RECOMMENDATION + " " + MSG_OOM_BASE_RECOMMENDATION_DETAIL + recommendation } result.AppendConclusion(G001_OOM, conclusion) diff --git a/pghrep/src/checkup/g001/g001messages.go b/pghrep/src/checkup/g001/g001messages.go index 0469cbd847fa8a4e8d46b61f5d5e7fbb2fb9c1b7..fc213d3ff426b59779b29809eab597742813af6e 100644 --- a/pghrep/src/checkup/g001/g001messages.go +++ b/pghrep/src/checkup/g001/g001messages.go @@ -31,4 +31,4 @@ const MSG_OOM_AUTIVACUUM_WORKMEM_BEGIN string = "`autovacuum_work_mem` is %s" const MSG_OOM_AUTIVACUUM_WORKMEM_NOTSET string = "(it's set to `-1` so the actual value is inherited from `maintenance_work_mem`)" const MSG_OOM_AUTIVACUUM_WORKMEM_END string = "and maximum %d autovacuum workers may work simultaneously, so together they may consume up to %s, " + "or %.2f%%%% of RAM. It makes `autovacuum_work_mem/autovacuum_max_workers` pair a major contributor to the overall memory consumption.\n" -const MSG_OOM_BASE_RECOMMENDATION_DETAIL string = "First of all, pay attention to the following settings:\n" +const MSG_OOM_BASE_RECOMMENDATION_DETAIL string = "First of all, pay attention to the following settings: \n"