From 60b215814733685011686d1b81e285ff1853ec08 Mon Sep 17 00:00:00 2001 From: Bogdan Tsechoev Date: Mon, 29 Sep 2025 18:07:28 +0700 Subject: [PATCH 1/8] feat: Add initial Coolify Docker Compose setup --- docker-compose.coolify.yml | 190 +++++++++++++++++++++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 docker-compose.coolify.yml diff --git a/docker-compose.coolify.yml b/docker-compose.coolify.yml new file mode 100644 index 0000000..cee0812 --- /dev/null +++ b/docker-compose.coolify.yml @@ -0,0 +1,190 @@ +services: + # Sources Generator - Generates sources.yml files from instances.yaml template + sources-generator: + image: alpine:3.22.0 + container_name: sources-generator + working_dir: /app + volumes: + - ./instances.yml:/app/instances.yaml + - ./config:/app/config + command: > + sh -c " + mkdir -p /app/config/pgwatch-postgres /app/config/pgwatch-prometheus && + echo '# PGWatch Sources Configuration - PostgreSQL Instance' > /app/config/pgwatch-postgres/sources.yml && + sed 's/~sink_type~/postgresql/g' /app/instances.yaml >> /app/config/pgwatch-postgres/sources.yml && + echo '# PGWatch Sources Configuration - Prometheus Instance' > /app/config/pgwatch-prometheus/sources.yml && + echo '' >> /app/config/pgwatch-prometheus/sources.yml && + sed 's/~sink_type~/prometheus/g' /app/instances.yaml >> /app/config/pgwatch-prometheus/sources.yml && + echo 'Generated sources.yml files for both postgres and prometheus' + " + + # Target Database - The PostgreSQL database being monitored + target-db: + image: postgres:15 + container_name: target-db + environment: + POSTGRES_DB: target_database + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + command: + [ + "postgres", + "-c", + "shared_preload_libraries=pg_stat_statements", + "-c", + "pg_stat_statements.track=all", + ] + ports: + - "55432:5432" + volumes: + - target_db_data:/var/lib/postgresql/data + - ./config/target-db/init.sql:/docker-entrypoint-initdb.d/init.sql + + # Postgres Sink - Storage for metrics in PostgreSQL format + sink-postgres: + image: postgres:15 + container_name: sink-postgres + environment: + POSTGRES_DB: postgres + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + ports: + - "55433:5432" + volumes: + - sink_postgres_data:/var/lib/postgresql/data + - ./config/sink-postgres/init.sql:/docker-entrypoint-initdb.d/init.sql + + # Prometheus Sink - Storage for metrics in Prometheus format + sink-prometheus: + image: prom/prometheus:v3.4.2 + container_name: sink-prometheus + ports: + - "59090:9090" + volumes: + - ./config/prometheus:/etc/prometheus + - prometheus_data:/prometheus + command: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + - "--web.console.libraries=/etc/prometheus/console_libraries" + - "--web.console.templates=/etc/prometheus/consoles" + - "--storage.tsdb.retention.time=200h" + - "--web.enable-lifecycle" + + # PGWatch Instance 1 - Monitoring service (Postgres sink) + pgwatch-postgres: + image: cybertecpostgresql/pgwatch:3 + container_name: pgwatch-postgres + command: + [ + "--sources=/etc/pgwatch/sources.yml", + "--metrics=/etc/pgwatch/metrics.yml", + "--sink=postgresql://pgwatch:pgwatchadmin@sink-postgres:5432/measurements", + "--web-addr=:8080", + ] + ports: + - "58080:8080" + depends_on: + - sources-generator + - sink-postgres + volumes: + - ./config/pgwatch-postgres/sources.yml:/etc/pgwatch/sources.yml + - ./config/pgwatch-postgres/metrics.yml:/etc/pgwatch/metrics.yml + + # PGWatch Instance 2 - Monitoring service (Prometheus sink) + pgwatch-prometheus: + image: cybertecpostgresql/pgwatch:3 + container_name: pgwatch-prometheus + command: + [ + "--sources=/etc/pgwatch/sources.yml", + "--metrics=/etc/pgwatch/metrics.yml", + "--sink=prometheus://0.0.0.0:9091/pgwatch", + "--web-addr=:8089", + ] + ports: + - "58089:8089" + - "59091:9091" + depends_on: + - sources-generator + - sink-prometheus + volumes: + - ./config/pgwatch-prometheus/sources.yml:/etc/pgwatch/sources.yml + - ./config/pgwatch-prometheus/metrics.yml:/etc/pgwatch/metrics.yml + + # Grafana with datasources - Visualization layer + grafana: + image: grafana/grafana:12.0.2 + container_name: grafana-with-datasources + environment: + GF_SECURITY_ADMIN_USER: monitor + GF_SECURITY_ADMIN_PASSWORD: demo + GF_INSTALL_PLUGINS: yesoreyeram-infinity-datasource + ports: + - "3000:3000" + volumes: + - grafana_data:/var/lib/grafana + - ./config/grafana/provisioning:/etc/grafana/provisioning + - ./config/grafana/dashboards:/var/lib/grafana/dashboards + depends_on: + - sink-postgres + - sink-prometheus + flask-backend: + build: + context: ./flask-backend + dockerfile: Dockerfile + container_name: flask-pgss-api + environment: + - FLASK_ENV=production + - PROMETHEUS_URL=http://sink-prometheus:9090 + depends_on: + - sink-prometheus + ports: + - "55000:5000" + + # PostgreSQL Reports Generator - Runs reports after 1 hour + postgres-reports: + image: python:3.11-slim + container_name: postgres-reports + working_dir: /app + volumes: + - ./reporter/postgres_reports.py:/app/postgres_reports.py + - ./reporter/requirements.txt:/app/requirements.txt + - ./.pgwatch-config:/app/.pgwatch-config + - ./instances.yml:/app/instances.yml + environment: + - PROMETHEUS_URL=http://sink-prometheus:9090 + depends_on: + - sink-prometheus + - pgwatch-prometheus + command: > + sh -c " + echo 'Installing Python dependencies...' && + pip install -r requirements.txt && + pip install pyyaml && + echo 'Waiting 30 minutes before generating reports...' && + sleep 1800 && + echo 'Starting PostgreSQL reports generation...' && + while true; do + echo 'Extracting cluster and node name from instances.yml...' && + CLUSTER=$$(python3 -c \"import yaml; data=yaml.safe_load(open('instances.yml')); print(data[0]['custom_tags']['cluster'])\") && + NODE_NAME=$$(python3 -c \"import yaml; data=yaml.safe_load(open('instances.yml')); print(data[0]['custom_tags']['node_name'])\") && + echo \"Using cluster: $$CLUSTER, node: $$NODE_NAME\" && + echo 'Generating PostgreSQL reports...' && + if [ -f /app/.pgwatch-config ] && grep -q '^api_key=' /app/.pgwatch-config; then + API_KEY=$$(grep '^api_key=' /app/.pgwatch-config | cut -d'=' -f2-) && + python postgres_reports.py --prometheus-url http://sink-prometheus:9090 --cluster \"$$CLUSTER\" --node-name \"$$NODE_NAME\" --output /app/all_reports_$$(date +%Y%m%d_%H%M%S).json --token $$API_KEY --project postgres-ai-monitoring + else + echo 'No API key configured, generating reports without upload...' && + python postgres_reports.py --prometheus-url http://sink-prometheus:9090 --cluster \"$$CLUSTER\" --node-name \"$$NODE_NAME\" --output /app/all_reports_$$(date +%Y%m%d_%H%M%S).json --no-upload + fi && + echo 'Reports generated. Sleeping for 24 hours...' && + sleep 86400 + done + " + +volumes: + target_db_data: + sink_postgres_data: + prometheus_data: + grafana_data: From f260afd81997e71e85e45a5155b896a8bb4c6ee0 Mon Sep 17 00:00:00 2001 From: Bogdan Tsechoev Date: Mon, 29 Sep 2025 18:39:06 +0700 Subject: [PATCH 2/8] Fixes Prometheus config generation Ensures the Prometheus configuration is always a file, not a directory, to prevent errors. Adds a dependency on the sources generator to ensure Prometheus starts after the configuration is ready. --- docker-compose.coolify.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker-compose.coolify.yml b/docker-compose.coolify.yml index cee0812..7a9194e 100644 --- a/docker-compose.coolify.yml +++ b/docker-compose.coolify.yml @@ -16,6 +16,10 @@ services: echo '' >> /app/config/pgwatch-prometheus/sources.yml && sed 's/~sink_type~/prometheus/g' /app/instances.yaml >> /app/config/pgwatch-prometheus/sources.yml && echo 'Generated sources.yml files for both postgres and prometheus' + + # --- PROMETHEUS CONFIG: always ensure it's a FILE, not a dir --- + rm -rf /app/config/prometheus/prometheus.yml && + cat > /app/config/prometheus/prometheus.yml <<'YAML' " # Target Database - The PostgreSQL database being monitored @@ -70,6 +74,8 @@ services: - "--web.console.templates=/etc/prometheus/consoles" - "--storage.tsdb.retention.time=200h" - "--web.enable-lifecycle" + depends_on: + - sources-generator # PGWatch Instance 1 - Monitoring service (Postgres sink) pgwatch-postgres: From 5000dd8dc18d0c2266796c97091fd3c53666b2f7 Mon Sep 17 00:00:00 2001 From: Bogdan Tsechoev Date: Mon, 29 Sep 2025 18:53:46 +0700 Subject: [PATCH 3/8] Fixes Prometheus config directory issue Ensures Prometheus configuration uses a file, not a directory. This prevents potential startup failures if the configuration is incorrectly set up as a directory. --- docker-compose.coolify.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker-compose.coolify.yml b/docker-compose.coolify.yml index 7a9194e..162df35 100644 --- a/docker-compose.coolify.yml +++ b/docker-compose.coolify.yml @@ -68,6 +68,12 @@ services: - ./config/prometheus:/etc/prometheus - prometheus_data:/prometheus command: + - > + if [ -d /etc/prometheus/prometheus.yml ]; then + echo "prometheus.yml is a directory — fixing..."; + rm -rf /etc/prometheus/prometheus.yml; + fi; + exec /bin/prometheus - "--config.file=/etc/prometheus/prometheus.yml" - "--storage.tsdb.path=/prometheus" - "--web.console.libraries=/etc/prometheus/console_libraries" From 770ccc16b8222ad09d38a08d04f7957158abe8a0 Mon Sep 17 00:00:00 2001 From: Bogdan Tsechoev Date: Mon, 29 Sep 2025 19:00:46 +0700 Subject: [PATCH 4/8] Revert "Fixes Prometheus config directory issue" This reverts commit 5000dd8dc18d0c2266796c97091fd3c53666b2f7. --- docker-compose.coolify.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docker-compose.coolify.yml b/docker-compose.coolify.yml index 162df35..7a9194e 100644 --- a/docker-compose.coolify.yml +++ b/docker-compose.coolify.yml @@ -68,12 +68,6 @@ services: - ./config/prometheus:/etc/prometheus - prometheus_data:/prometheus command: - - > - if [ -d /etc/prometheus/prometheus.yml ]; then - echo "prometheus.yml is a directory — fixing..."; - rm -rf /etc/prometheus/prometheus.yml; - fi; - exec /bin/prometheus - "--config.file=/etc/prometheus/prometheus.yml" - "--storage.tsdb.path=/prometheus" - "--web.console.libraries=/etc/prometheus/console_libraries" From c575152546f4b879f1a3b5ac0bbc477f34119172 Mon Sep 17 00:00:00 2001 From: Bogdan Tsechoev Date: Mon, 29 Sep 2025 19:20:39 +0700 Subject: [PATCH 5/8] Configures Prometheus with bind mounts Updates Prometheus to use bind mounts for configuration. This simplifies configuration management and makes it consistent with other services. Adds grafana ini to provisioning folder --- docker-compose.coolify.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/docker-compose.coolify.yml b/docker-compose.coolify.yml index 7a9194e..42b69d5 100644 --- a/docker-compose.coolify.yml +++ b/docker-compose.coolify.yml @@ -16,10 +16,6 @@ services: echo '' >> /app/config/pgwatch-prometheus/sources.yml && sed 's/~sink_type~/prometheus/g' /app/instances.yaml >> /app/config/pgwatch-prometheus/sources.yml && echo 'Generated sources.yml files for both postgres and prometheus' - - # --- PROMETHEUS CONFIG: always ensure it's a FILE, not a dir --- - rm -rf /app/config/prometheus/prometheus.yml && - cat > /app/config/prometheus/prometheus.yml <<'YAML' " # Target Database - The PostgreSQL database being monitored @@ -61,11 +57,13 @@ services: # Prometheus Sink - Storage for metrics in Prometheus format sink-prometheus: image: prom/prometheus:v3.4.2 - container_name: sink-prometheus ports: - "59090:9090" volumes: - - ./config/prometheus:/etc/prometheus + - type: bind + source: ./config/prometheus/prometheus.yml + target: /etc/prometheus/prometheus.yml + read_only: true - prometheus_data:/prometheus command: - "--config.file=/etc/prometheus/prometheus.yml" @@ -74,8 +72,6 @@ services: - "--web.console.templates=/etc/prometheus/consoles" - "--storage.tsdb.retention.time=200h" - "--web.enable-lifecycle" - depends_on: - - sources-generator # PGWatch Instance 1 - Monitoring service (Postgres sink) pgwatch-postgres: @@ -132,6 +128,7 @@ services: - grafana_data:/var/lib/grafana - ./config/grafana/provisioning:/etc/grafana/provisioning - ./config/grafana/dashboards:/var/lib/grafana/dashboards + - ./config/grafana/provisioning/grafana.ini:/etc/grafana/grafana.ini depends_on: - sink-postgres - sink-prometheus From e4fd6f58b3bdc199fc8ea38fe1285ee81db852c6 Mon Sep 17 00:00:00 2001 From: Bogdan Tsechoev Date: Mon, 29 Sep 2025 19:41:49 +0700 Subject: [PATCH 6/8] Ensures Prometheus config is correctly handled Adds parameters to ensure that Prometheus configuration is correctly handled by Docker Compose. Prevents potential errors due to incorrect file handling. --- docker-compose.coolify.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.coolify.yml b/docker-compose.coolify.yml index 42b69d5..944cc9f 100644 --- a/docker-compose.coolify.yml +++ b/docker-compose.coolify.yml @@ -64,6 +64,8 @@ services: source: ./config/prometheus/prometheus.yml target: /etc/prometheus/prometheus.yml read_only: true + is_directory: false + content: "" - prometheus_data:/prometheus command: - "--config.file=/etc/prometheus/prometheus.yml" From 7cdbe95471cb7b3ff2a5e70c5750b5efd3b43727 Mon Sep 17 00:00:00 2001 From: Bogdan Tsechoev Date: Mon, 29 Sep 2025 20:02:16 +0700 Subject: [PATCH 7/8] Specifies volume mount types and attributes Ensures that volume mounts are explicitly defined as 'bind' types with appropriate attributes such as read-only access and indication of whether they are directories. This change enhances the clarity and control over volume configurations, making the deployment more robust and predictable. --- docker-compose.coolify.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/docker-compose.coolify.yml b/docker-compose.coolify.yml index 944cc9f..2ab5133 100644 --- a/docker-compose.coolify.yml +++ b/docker-compose.coolify.yml @@ -113,8 +113,18 @@ services: - sources-generator - sink-prometheus volumes: - - ./config/pgwatch-prometheus/sources.yml:/etc/pgwatch/sources.yml - - ./config/pgwatch-prometheus/metrics.yml:/etc/pgwatch/metrics.yml + - type: bind + source: ./config/pgwatch-prometheus/sources.yml + target: /etc/pgwatch/sources.yml + read_only: true + is_directory: false + content: "" + - type: bind + source: ./config/pgwatch-prometheus/metrics.yml + target: /etc/pgwatch/metrics.yml + read_only: true + is_directory: false + content: "" # Grafana with datasources - Visualization layer grafana: From b851d3cc91f00d533c3e0a112ac82e8bf5a8c3b9 Mon Sep 17 00:00:00 2001 From: Bogdan Tsechoev Date: Tue, 30 Sep 2025 16:21:02 +0700 Subject: [PATCH 8/8] Specifies volume mount types and attributes Adds explicit volume types and attributes for enhanced configuration and clarity. This ensures proper volume handling and read-only access where applicable. --- docker-compose.coolify.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/docker-compose.coolify.yml b/docker-compose.coolify.yml index 2ab5133..0d50db4 100644 --- a/docker-compose.coolify.yml +++ b/docker-compose.coolify.yml @@ -92,8 +92,18 @@ services: - sources-generator - sink-postgres volumes: - - ./config/pgwatch-postgres/sources.yml:/etc/pgwatch/sources.yml - - ./config/pgwatch-postgres/metrics.yml:/etc/pgwatch/metrics.yml + - type: bind + source: ./config/pgwatch-postgres/sources.yml + target: /etc/pgwatch/sources.yml + read_only: true + is_directory: false + content: "" + - type: bind + source: ./config/pgwatch-postgres/metrics.yml + target: /etc/pgwatch/metrics.yml + read_only: true + is_directory: false + content: "" # PGWatch Instance 2 - Monitoring service (Prometheus sink) pgwatch-prometheus: