From f8ebe79b81e74273738e3b3cdf6aaef88680c2bc Mon Sep 17 00:00:00 2001 From: "dementii.priadko" <45518657+DEMNERD@users.noreply.github.com> Date: Tue, 4 Nov 2025 02:05:56 +0200 Subject: [PATCH 1/3] Added index_definition metric and added uptime time of the db as a backup in case stats weren't reset --- config/pgwatch-postgres/metrics.yml | 15 ++++- reporter/postgres_reports.py | 98 +++++++++++++++++++++++++++-- reporter/requirements.txt | 3 +- 3 files changed, 108 insertions(+), 8 deletions(-) diff --git a/config/pgwatch-postgres/metrics.yml b/config/pgwatch-postgres/metrics.yml index 7ff6f83..b222df9 100644 --- a/config/pgwatch-postgres/metrics.yml +++ b/config/pgwatch-postgres/metrics.yml @@ -14,8 +14,21 @@ metrics: gauges: - '*' + index_definitions: + description: "Index definitions from all databases" + sqls: + 11: |- + select /* pgwatch_generated */ + indexname, + pg_get_indexdef(indexrelid) as index_definition + from pg_stat_all_indexes + order by schemaname, tablename, indexname; + gauges: + - '*' + presets: full: description: "Full metrics for PostgreSQL storage" metrics: - pgss_queryid_queries: 300 \ No newline at end of file + pgss_queryid_queries: 300 + index_definitions: 3600 \ No newline at end of file diff --git a/reporter/postgres_reports.py b/reporter/postgres_reports.py index d8d262b..0243d04 100644 --- a/reporter/postgres_reports.py +++ b/reporter/postgres_reports.py @@ -14,18 +14,25 @@ import argparse import sys import os +import psycopg2 +import psycopg2.extras class PostgresReportGenerator: - def __init__(self, prometheus_url: str = "http://localhost:9090"): + def __init__(self, prometheus_url: str = "http://sink-prometheus:9090", + postgres_sink_url: str = "postgresql://pgwatch:pgwatchadmin@sink-postgres:5432/measurements"): """ Initialize the PostgreSQL report generator. Args: - prometheus_url: URL of the Prometheus instance + prometheus_url: URL of the Prometheus instance (default: http://sink-prometheus:9090) + postgres_sink_url: Connection string for the Postgres sink database + (default: postgresql://pgwatch:pgwatchadmin@sink-postgres:5432/measurements) """ self.prometheus_url = prometheus_url self.base_url = f"{prometheus_url}/api/v1" + self.postgres_sink_url = postgres_sink_url + self.pg_conn = None def test_connection(self) -> bool: """Test connection to Prometheus.""" @@ -36,6 +43,59 @@ def test_connection(self) -> bool: print(f"Connection failed: {e}") return False + def connect_postgres_sink(self) -> bool: + """Connect to Postgres sink database.""" + if not self.postgres_sink_url: + return False + + try: + self.pg_conn = psycopg2.connect(self.postgres_sink_url) + return True + except Exception as e: + print(f"Postgres sink connection failed: {e}") + return False + + def close_postgres_sink(self): + """Close Postgres sink connection.""" + if self.pg_conn: + self.pg_conn.close() + self.pg_conn = None + + def get_index_definitions_from_sink(self) -> Dict[str, str]: + """ + Get index definitions from the Postgres sink database. + + Returns: + Dictionary mapping index names to their definitions + """ + if not self.pg_conn: + if not self.connect_postgres_sink(): + return {} + + index_definitions = {} + + try: + with self.pg_conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cursor: + # Query the index_definitions table for the most recent data + query = """ + SELECT DISTINCT ON (data->>'indexname') + data->>'indexname' as indexname, + data->>'index_definition' as index_definition + FROM public.index_definitions + WHERE data ? 'indexname' AND data ? 'index_definition' + ORDER BY data->>'indexname', time DESC + """ + cursor.execute(query) + + for row in cursor.fetchall(): + if row['indexname']: + index_definitions[row['indexname']] = row['index_definition'] + + except Exception as e: + print(f"Error fetching index definitions from Postgres sink: {e}") + + return index_definitions + def query_instant(self, query: str) -> Dict[str, Any]: """ Execute an instant PromQL query. @@ -315,6 +375,21 @@ def generate_h002_unused_indexes_report(self, cluster: str = "local", node_name: # Get all databases databases = self.get_all_databases(cluster, node_name) + # Query postmaster uptime to get startup time + postmaster_uptime_query = f'last_over_time(pgwatch_db_stats_postmaster_uptime_s{{cluster="{cluster}", node_name="{node_name}"}}[10h])' + postmaster_uptime_result = self.query_instant(postmaster_uptime_query) + + postmaster_startup_time = None + postmaster_startup_epoch = None + if postmaster_uptime_result.get('status') == 'success' and postmaster_uptime_result.get('data', {}).get('result'): + uptime_seconds = float(postmaster_uptime_result['data']['result'][0]['value'][1]) if postmaster_uptime_result['data']['result'] else None + if uptime_seconds: + postmaster_startup_epoch = datetime.now().timestamp() - uptime_seconds + postmaster_startup_time = datetime.fromtimestamp(postmaster_startup_epoch).isoformat() + + # Get index definitions from Postgres sink database + index_definitions = self.get_index_definitions_from_sink() + unused_indexes_by_db = {} for db_name in databases: # Query stats_reset timestamp for this database @@ -353,10 +428,14 @@ def generate_h002_unused_indexes_report(self, cluster: str = "local", node_name: {}).get( 'result') else 0 + # Get index definition from collected metrics + index_definition = index_definitions.get(index_name, 'Definition not available') + index_data = { "schema_name": schema_name, "table_name": table_name, "index_name": index_name, + "index_definition": index_definition, "reason": reason, "idx_scan": idx_scan, "index_size_bytes": index_size_bytes, @@ -381,7 +460,9 @@ def generate_h002_unused_indexes_report(self, cluster: str = "local", node_name: "stats_reset": { "stats_reset_epoch": stats_reset_epoch, "stats_reset_time": stats_reset_time, - "days_since_reset": days_since_reset + "days_since_reset": days_since_reset, + "postmaster_startup_epoch": postmaster_startup_epoch, + "postmaster_startup_time": postmaster_startup_time } } @@ -1764,8 +1845,10 @@ def make_request(api_url, endpoint, request_data): def main(): parser = argparse.ArgumentParser(description='Generate PostgreSQL reports using PromQL') - parser.add_argument('--prometheus-url', default='http://localhost:9090', - help='Prometheus URL (default: http://localhost:9090)') + parser.add_argument('--prometheus-url', default='http://sink-prometheus:9090', + help='Prometheus URL (default: http://sink-prometheus:9090 for Docker, use http://localhost:59090 for external access)') + parser.add_argument('--postgres-sink-url', default='postgresql://pgwatch:pgwatchadmin@sink-postgres:5432/measurements', + help='Postgres sink connection string (default: postgresql://pgwatch:pgwatchadmin@sink-postgres:5432/measurements for Docker, use postgresql://pgwatch:pgwatchadmin@localhost:55433/measurements for external access)') parser.add_argument('--cluster', default='local', help='Cluster name (default: local)') parser.add_argument('--node-name', default='node-01', @@ -1785,7 +1868,7 @@ def main(): args = parser.parse_args() - generator = PostgresReportGenerator(args.prometheus_url) + generator = PostgresReportGenerator(args.prometheus_url, args.postgres_sink_url) # Test connection if not generator.test_connection(): @@ -1852,6 +1935,9 @@ def main(): print(f"Error generating reports: {e}") raise e sys.exit(1) + finally: + # Clean up postgres connection + generator.close_postgres_sink() if __name__ == "__main__": diff --git a/reporter/requirements.txt b/reporter/requirements.txt index 659c37c..6813242 100644 --- a/reporter/requirements.txt +++ b/reporter/requirements.txt @@ -1 +1,2 @@ -requests>=2.31.0 \ No newline at end of file +requests>=2.31.0 +psycopg2-binary>=2.9.9 \ No newline at end of file From f069a2f5908985bd0437ff92a0614d262b6b8cd5 Mon Sep 17 00:00:00 2001 From: "dementii.priadko" <45518657+DEMNERD@users.noreply.github.com> Date: Fri, 7 Nov 2025 15:41:55 +0200 Subject: [PATCH 2/3] Added passwordless access to sink-postgres, removed unnecessary port mappings, fixed index definition metric, added a trigger that prevents duplicates in index definitions table, made init.sql for sink-postgres lowercase --- config/pgwatch-postgres/metrics.yml | 7 +- config/sink-postgres/00-configure-pg-hba.sh | 34 +++++ config/sink-postgres/init.sql | 145 ++++++++++++++------ docker-compose.yml | 25 +--- reporter/postgres_reports.py | 18 +-- 5 files changed, 157 insertions(+), 72 deletions(-) create mode 100644 config/sink-postgres/00-configure-pg-hba.sh diff --git a/config/pgwatch-postgres/metrics.yml b/config/pgwatch-postgres/metrics.yml index b222df9..510abb0 100644 --- a/config/pgwatch-postgres/metrics.yml +++ b/config/pgwatch-postgres/metrics.yml @@ -19,10 +19,13 @@ metrics: sqls: 11: |- select /* pgwatch_generated */ - indexname, + indexrelname, + schemaname, + relname, pg_get_indexdef(indexrelid) as index_definition from pg_stat_all_indexes - order by schemaname, tablename, indexname; + order by schemaname, relname, indexrelname + limit 10000; gauges: - '*' diff --git a/config/sink-postgres/00-configure-pg-hba.sh b/config/sink-postgres/00-configure-pg-hba.sh new file mode 100644 index 0000000..0cce1f0 --- /dev/null +++ b/config/sink-postgres/00-configure-pg-hba.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Configure pg_hba.conf to allow trust authentication from Docker networks + +cat > ${PGDATA}/pg_hba.conf <>'queryid'; + queryid_value := new.data->>'queryid'; -- Allow NULL queryids through - IF queryid_value IS NULL THEN - RETURN NEW; - END IF; + if queryid_value is null then + return new; + end if; -- Silently skip if duplicate exists - IF EXISTS ( - SELECT 1 - FROM pgss_queryid_queries - WHERE dbname = NEW.dbname - AND data->>'queryid' = queryid_value - LIMIT 1 - ) THEN - RETURN NULL; -- Cancels INSERT silently - END IF; + if exists ( + select 1 + from pgss_queryid_queries + where dbname = new.dbname + and data->>'queryid' = queryid_value + limit 1 + ) then + return null; -- Cancels INSERT silently + end if; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; + return new; +end; +$$ language plpgsql; -CREATE OR REPLACE TRIGGER enforce_queryid_uniqueness_trigger - BEFORE INSERT - ON pgss_queryid_queries - FOR EACH ROW - EXECUTE FUNCTION enforce_queryid_uniqueness(); +create or replace trigger enforce_queryid_uniqueness_trigger + before insert + on pgss_queryid_queries + for each row + execute function enforce_queryid_uniqueness(); + +-- Create a partitioned table for index definitions with LIST partitioning by dbname +create table if not exists public.index_definitions ( + time timestamptz not null, + dbname text not null, + data jsonb not null, + tag_data jsonb +) partition by list (dbname); + +-- Create indexes for efficient lookups +create index if not exists index_definitions_dbname_time_idx on public.index_definitions (dbname, time); + +-- Set ownership and grant permissions to pgwatch +alter table public.index_definitions owner to pgwatch; +grant all privileges on table public.index_definitions to pgwatch; + +-- Create function to enforce index definition uniqueness +create or replace function enforce_index_definition_uniqueness() +returns trigger as $$ +declare + index_name text; + schema_name text; + table_name text; + index_definition text; +begin + -- Extract index information from the data JSONB + index_name := new.data->>'indexrelname'; + schema_name := new.data->>'schemaname'; + table_name := new.data->>'relname'; + index_definition := new.data->>'index_definition'; + + -- Allow NULL index names through + if index_name is null then + return new; + end if; + + -- Silently skip if duplicate exists + if exists ( + select 1 + from index_definitions + where dbname = new.dbname + and data->>'indexrelname' = index_name + and data->>'schemaname' = schema_name + and data->>'relname' = table_name + and data->>'index_definition' = index_definition + limit 1 + ) then + return null; -- Cancels INSERT silently + end if; + + return new; +end; +$$ language plpgsql; + +create or replace trigger enforce_index_definition_uniqueness_trigger + before insert + on index_definitions + for each row + execute function enforce_index_definition_uniqueness(); diff --git a/docker-compose.yml b/docker-compose.yml index d60be9e..5b69dda 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,13 +34,15 @@ services: "-c", "pg_stat_statements.track=all", ] - ports: - - "${BIND_HOST:-}55432:5432" volumes: - target_db_data:/var/lib/postgresql/data - ./config/target-db/init.sql:/docker-entrypoint-initdb.d/init.sql # Postgres Sink - Storage for metrics in PostgreSQL format + # Note: pg_hba.conf is configured to allow passwordless connections (trust) + # for local connections within the Docker network. This simplifies pgwatch + # and postgres-exporter connectivity without compromising security since + # the database is not exposed externally. sink-postgres: image: postgres:15 container_name: sink-postgres @@ -48,10 +50,10 @@ services: POSTGRES_DB: postgres POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres - ports: - - "${BIND_HOST:-}55433:5432" + POSTGRES_HOST_AUTH_METHOD: trust volumes: - sink_postgres_data:/var/lib/postgresql/data + - ./config/sink-postgres/00-configure-pg-hba.sh:/docker-entrypoint-initdb.d/00-configure-pg-hba.sh - ./config/sink-postgres/init.sql:/docker-entrypoint-initdb.d/init.sql # VictoriaMetrics Sink - Storage for metrics in Prometheus format @@ -79,11 +81,9 @@ services: [ "--sources=/etc/pgwatch/sources.yml", "--metrics=/etc/pgwatch/metrics.yml", - "--sink=postgresql://pgwatch:pgwatchadmin@sink-postgres:5432/measurements", + "--sink=postgresql://pgwatch@sink-postgres:5432/measurements?sslmode=disable", "--web-addr=:8080", ] - ports: - - "${BIND_HOST:-}58080:8080" depends_on: - sources-generator - sink-postgres @@ -103,9 +103,6 @@ services: "--sink=prometheus://0.0.0.0:9091/pgwatch", "--web-addr=:8089", ] - ports: - - "${BIND_HOST:-}58089:8089" - - "${BIND_HOST:-}59091:9091" depends_on: - sources-generator - sink-prometheus @@ -143,8 +140,6 @@ services: - PROMETHEUS_URL=http://sink-prometheus:9090 depends_on: - sink-prometheus - ports: - - "${BIND_HOST:-}55000:5000" restart: unless-stopped # PostgreSQL Reports Generator - Runs reports after 1 hour postgres-reports: @@ -194,8 +189,6 @@ services: image: gcr.io/cadvisor/cadvisor:v0.51.0 container_name: cadvisor privileged: true - ports: - - "58081:8080" volumes: - /:/rootfs:ro - /var/run:/var/run:ro @@ -212,8 +205,6 @@ services: node-exporter: image: prom/node-exporter:v1.8.2 container_name: node-exporter - ports: - - "59100:9100" command: - '--path.rootfs=/host' - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)' @@ -227,8 +218,6 @@ services: container_name: postgres-exporter-sink environment: DATA_SOURCE_NAME: "postgresql://postgres:postgres@sink-postgres:5432/measurements?sslmode=disable" - ports: - - "59187:9187" depends_on: - sink-postgres restart: unless-stopped diff --git a/reporter/postgres_reports.py b/reporter/postgres_reports.py index 0243d04..8fdf071 100644 --- a/reporter/postgres_reports.py +++ b/reporter/postgres_reports.py @@ -78,18 +78,18 @@ def get_index_definitions_from_sink(self) -> Dict[str, str]: with self.pg_conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cursor: # Query the index_definitions table for the most recent data query = """ - SELECT DISTINCT ON (data->>'indexname') - data->>'indexname' as indexname, + SELECT DISTINCT ON (data->>'indexrelname') + data->>'indexrelname' as indexrelname, data->>'index_definition' as index_definition FROM public.index_definitions - WHERE data ? 'indexname' AND data ? 'index_definition' - ORDER BY data->>'indexname', time DESC + WHERE data ? 'indexrelname' AND data ? 'index_definition' + ORDER BY data->>'indexrelname', time DESC """ cursor.execute(query) for row in cursor.fetchall(): - if row['indexname']: - index_definitions[row['indexname']] = row['index_definition'] + if row['indexrelname']: + index_definitions[row['indexrelname']] = row['index_definition'] except Exception as e: print(f"Error fetching index definitions from Postgres sink: {e}") @@ -1846,9 +1846,9 @@ def make_request(api_url, endpoint, request_data): def main(): parser = argparse.ArgumentParser(description='Generate PostgreSQL reports using PromQL') parser.add_argument('--prometheus-url', default='http://sink-prometheus:9090', - help='Prometheus URL (default: http://sink-prometheus:9090 for Docker, use http://localhost:59090 for external access)') - parser.add_argument('--postgres-sink-url', default='postgresql://pgwatch:pgwatchadmin@sink-postgres:5432/measurements', - help='Postgres sink connection string (default: postgresql://pgwatch:pgwatchadmin@sink-postgres:5432/measurements for Docker, use postgresql://pgwatch:pgwatchadmin@localhost:55433/measurements for external access)') + help='Prometheus URL (default: http://sink-prometheus:9090)') + parser.add_argument('--postgres-sink-url', default='postgresql://pgwatch@sink-postgres:5432/measurements', + help='Postgres sink connection string (default: postgresql://pgwatch@sink-postgres:5432/measurements)') parser.add_argument('--cluster', default='local', help='Cluster name (default: local)') parser.add_argument('--node-name', default='node-01', From e448d8406a0a06e26e3375b3eafd12ff64a78070 Mon Sep 17 00:00:00 2001 From: "dementii.priadko" <45518657+DEMNERD@users.noreply.github.com> Date: Fri, 7 Nov 2025 15:44:39 +0200 Subject: [PATCH 3/3] Changed sql in postgres_reports.py to lowercase --- reporter/postgres_reports.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/reporter/postgres_reports.py b/reporter/postgres_reports.py index 8fdf071..ba79a44 100644 --- a/reporter/postgres_reports.py +++ b/reporter/postgres_reports.py @@ -20,14 +20,14 @@ class PostgresReportGenerator: def __init__(self, prometheus_url: str = "http://sink-prometheus:9090", - postgres_sink_url: str = "postgresql://pgwatch:pgwatchadmin@sink-postgres:5432/measurements"): + postgres_sink_url: str = "postgresql://pgwatch@sink-postgres:5432/measurements"): """ Initialize the PostgreSQL report generator. Args: prometheus_url: URL of the Prometheus instance (default: http://sink-prometheus:9090) postgres_sink_url: Connection string for the Postgres sink database - (default: postgresql://pgwatch:pgwatchadmin@sink-postgres:5432/measurements) + (default: postgresql://pgwatch@sink-postgres:5432/measurements) """ self.prometheus_url = prometheus_url self.base_url = f"{prometheus_url}/api/v1" @@ -78,12 +78,12 @@ def get_index_definitions_from_sink(self) -> Dict[str, str]: with self.pg_conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cursor: # Query the index_definitions table for the most recent data query = """ - SELECT DISTINCT ON (data->>'indexrelname') + select distinct on (data->>'indexrelname') data->>'indexrelname' as indexrelname, data->>'index_definition' as index_definition - FROM public.index_definitions - WHERE data ? 'indexrelname' AND data ? 'index_definition' - ORDER BY data->>'indexrelname', time DESC + from public.index_definitions + where data ? 'indexrelname' and data ? 'index_definition' + order by data->>'indexrelname', time desc """ cursor.execute(query)