diff --git a/config/pgwatch-postgres/metrics.yml b/config/pgwatch-postgres/metrics.yml index 7ff6f83..510abb0 100644 --- a/config/pgwatch-postgres/metrics.yml +++ b/config/pgwatch-postgres/metrics.yml @@ -14,8 +14,24 @@ metrics: gauges: - '*' + index_definitions: + description: "Index definitions from all databases" + sqls: + 11: |- + select /* pgwatch_generated */ + indexrelname, + schemaname, + relname, + pg_get_indexdef(indexrelid) as index_definition + from pg_stat_all_indexes + order by schemaname, relname, indexrelname + limit 10000; + gauges: + - '*' + presets: full: description: "Full metrics for PostgreSQL storage" metrics: - pgss_queryid_queries: 300 \ No newline at end of file + pgss_queryid_queries: 300 + index_definitions: 3600 \ No newline at end of file diff --git a/config/sink-postgres/00-configure-pg-hba.sh b/config/sink-postgres/00-configure-pg-hba.sh new file mode 100644 index 0000000..0cce1f0 --- /dev/null +++ b/config/sink-postgres/00-configure-pg-hba.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Configure pg_hba.conf to allow trust authentication from Docker networks + +cat > ${PGDATA}/pg_hba.conf <>'queryid'; + queryid_value := new.data->>'queryid'; -- Allow NULL queryids through - IF queryid_value IS NULL THEN - RETURN NEW; - END IF; + if queryid_value is null then + return new; + end if; -- Silently skip if duplicate exists - IF EXISTS ( - SELECT 1 - FROM pgss_queryid_queries - WHERE dbname = NEW.dbname - AND data->>'queryid' = queryid_value - LIMIT 1 - ) THEN - RETURN NULL; -- Cancels INSERT silently - END IF; + if exists ( + select 1 + from pgss_queryid_queries + where dbname = new.dbname + and data->>'queryid' = queryid_value + limit 1 + ) then + return null; -- Cancels INSERT silently + end if; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; + return new; +end; +$$ language plpgsql; -CREATE OR REPLACE TRIGGER enforce_queryid_uniqueness_trigger - BEFORE INSERT - ON pgss_queryid_queries - FOR EACH ROW - EXECUTE FUNCTION enforce_queryid_uniqueness(); +create or replace trigger enforce_queryid_uniqueness_trigger + before insert + on pgss_queryid_queries + for each row + execute function enforce_queryid_uniqueness(); + +-- Create a partitioned table for index definitions with LIST partitioning by dbname +create table if not exists public.index_definitions ( + time timestamptz not null, + dbname text not null, + data jsonb not null, + tag_data jsonb +) partition by list (dbname); + +-- Create indexes for efficient lookups +create index if not exists index_definitions_dbname_time_idx on public.index_definitions (dbname, time); + +-- Set ownership and grant permissions to pgwatch +alter table public.index_definitions owner to pgwatch; +grant all privileges on table public.index_definitions to pgwatch; + +-- Create function to enforce index definition uniqueness +create or replace function enforce_index_definition_uniqueness() +returns trigger as $$ +declare + index_name text; + schema_name text; + table_name text; + index_definition text; +begin + -- Extract index information from the data JSONB + index_name := new.data->>'indexrelname'; + schema_name := new.data->>'schemaname'; + table_name := new.data->>'relname'; + index_definition := new.data->>'index_definition'; + + -- Allow NULL index names through + if index_name is null then + return new; + end if; + + -- Silently skip if duplicate exists + if exists ( + select 1 + from index_definitions + where dbname = new.dbname + and data->>'indexrelname' = index_name + and data->>'schemaname' = schema_name + and data->>'relname' = table_name + and data->>'index_definition' = index_definition + limit 1 + ) then + return null; -- Cancels INSERT silently + end if; + + return new; +end; +$$ language plpgsql; + +create or replace trigger enforce_index_definition_uniqueness_trigger + before insert + on index_definitions + for each row + execute function enforce_index_definition_uniqueness(); diff --git a/docker-compose.yml b/docker-compose.yml index d60be9e..5b69dda 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,13 +34,15 @@ services: "-c", "pg_stat_statements.track=all", ] - ports: - - "${BIND_HOST:-}55432:5432" volumes: - target_db_data:/var/lib/postgresql/data - ./config/target-db/init.sql:/docker-entrypoint-initdb.d/init.sql # Postgres Sink - Storage for metrics in PostgreSQL format + # Note: pg_hba.conf is configured to allow passwordless connections (trust) + # for local connections within the Docker network. This simplifies pgwatch + # and postgres-exporter connectivity without compromising security since + # the database is not exposed externally. sink-postgres: image: postgres:15 container_name: sink-postgres @@ -48,10 +50,10 @@ services: POSTGRES_DB: postgres POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres - ports: - - "${BIND_HOST:-}55433:5432" + POSTGRES_HOST_AUTH_METHOD: trust volumes: - sink_postgres_data:/var/lib/postgresql/data + - ./config/sink-postgres/00-configure-pg-hba.sh:/docker-entrypoint-initdb.d/00-configure-pg-hba.sh - ./config/sink-postgres/init.sql:/docker-entrypoint-initdb.d/init.sql # VictoriaMetrics Sink - Storage for metrics in Prometheus format @@ -79,11 +81,9 @@ services: [ "--sources=/etc/pgwatch/sources.yml", "--metrics=/etc/pgwatch/metrics.yml", - "--sink=postgresql://pgwatch:pgwatchadmin@sink-postgres:5432/measurements", + "--sink=postgresql://pgwatch@sink-postgres:5432/measurements?sslmode=disable", "--web-addr=:8080", ] - ports: - - "${BIND_HOST:-}58080:8080" depends_on: - sources-generator - sink-postgres @@ -103,9 +103,6 @@ services: "--sink=prometheus://0.0.0.0:9091/pgwatch", "--web-addr=:8089", ] - ports: - - "${BIND_HOST:-}58089:8089" - - "${BIND_HOST:-}59091:9091" depends_on: - sources-generator - sink-prometheus @@ -143,8 +140,6 @@ services: - PROMETHEUS_URL=http://sink-prometheus:9090 depends_on: - sink-prometheus - ports: - - "${BIND_HOST:-}55000:5000" restart: unless-stopped # PostgreSQL Reports Generator - Runs reports after 1 hour postgres-reports: @@ -194,8 +189,6 @@ services: image: gcr.io/cadvisor/cadvisor:v0.51.0 container_name: cadvisor privileged: true - ports: - - "58081:8080" volumes: - /:/rootfs:ro - /var/run:/var/run:ro @@ -212,8 +205,6 @@ services: node-exporter: image: prom/node-exporter:v1.8.2 container_name: node-exporter - ports: - - "59100:9100" command: - '--path.rootfs=/host' - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)' @@ -227,8 +218,6 @@ services: container_name: postgres-exporter-sink environment: DATA_SOURCE_NAME: "postgresql://postgres:postgres@sink-postgres:5432/measurements?sslmode=disable" - ports: - - "59187:9187" depends_on: - sink-postgres restart: unless-stopped diff --git a/reporter/postgres_reports.py b/reporter/postgres_reports.py index d8d262b..ba79a44 100644 --- a/reporter/postgres_reports.py +++ b/reporter/postgres_reports.py @@ -14,18 +14,25 @@ import argparse import sys import os +import psycopg2 +import psycopg2.extras class PostgresReportGenerator: - def __init__(self, prometheus_url: str = "http://localhost:9090"): + def __init__(self, prometheus_url: str = "http://sink-prometheus:9090", + postgres_sink_url: str = "postgresql://pgwatch@sink-postgres:5432/measurements"): """ Initialize the PostgreSQL report generator. Args: - prometheus_url: URL of the Prometheus instance + prometheus_url: URL of the Prometheus instance (default: http://sink-prometheus:9090) + postgres_sink_url: Connection string for the Postgres sink database + (default: postgresql://pgwatch@sink-postgres:5432/measurements) """ self.prometheus_url = prometheus_url self.base_url = f"{prometheus_url}/api/v1" + self.postgres_sink_url = postgres_sink_url + self.pg_conn = None def test_connection(self) -> bool: """Test connection to Prometheus.""" @@ -36,6 +43,59 @@ def test_connection(self) -> bool: print(f"Connection failed: {e}") return False + def connect_postgres_sink(self) -> bool: + """Connect to Postgres sink database.""" + if not self.postgres_sink_url: + return False + + try: + self.pg_conn = psycopg2.connect(self.postgres_sink_url) + return True + except Exception as e: + print(f"Postgres sink connection failed: {e}") + return False + + def close_postgres_sink(self): + """Close Postgres sink connection.""" + if self.pg_conn: + self.pg_conn.close() + self.pg_conn = None + + def get_index_definitions_from_sink(self) -> Dict[str, str]: + """ + Get index definitions from the Postgres sink database. + + Returns: + Dictionary mapping index names to their definitions + """ + if not self.pg_conn: + if not self.connect_postgres_sink(): + return {} + + index_definitions = {} + + try: + with self.pg_conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cursor: + # Query the index_definitions table for the most recent data + query = """ + select distinct on (data->>'indexrelname') + data->>'indexrelname' as indexrelname, + data->>'index_definition' as index_definition + from public.index_definitions + where data ? 'indexrelname' and data ? 'index_definition' + order by data->>'indexrelname', time desc + """ + cursor.execute(query) + + for row in cursor.fetchall(): + if row['indexrelname']: + index_definitions[row['indexrelname']] = row['index_definition'] + + except Exception as e: + print(f"Error fetching index definitions from Postgres sink: {e}") + + return index_definitions + def query_instant(self, query: str) -> Dict[str, Any]: """ Execute an instant PromQL query. @@ -315,6 +375,21 @@ def generate_h002_unused_indexes_report(self, cluster: str = "local", node_name: # Get all databases databases = self.get_all_databases(cluster, node_name) + # Query postmaster uptime to get startup time + postmaster_uptime_query = f'last_over_time(pgwatch_db_stats_postmaster_uptime_s{{cluster="{cluster}", node_name="{node_name}"}}[10h])' + postmaster_uptime_result = self.query_instant(postmaster_uptime_query) + + postmaster_startup_time = None + postmaster_startup_epoch = None + if postmaster_uptime_result.get('status') == 'success' and postmaster_uptime_result.get('data', {}).get('result'): + uptime_seconds = float(postmaster_uptime_result['data']['result'][0]['value'][1]) if postmaster_uptime_result['data']['result'] else None + if uptime_seconds: + postmaster_startup_epoch = datetime.now().timestamp() - uptime_seconds + postmaster_startup_time = datetime.fromtimestamp(postmaster_startup_epoch).isoformat() + + # Get index definitions from Postgres sink database + index_definitions = self.get_index_definitions_from_sink() + unused_indexes_by_db = {} for db_name in databases: # Query stats_reset timestamp for this database @@ -353,10 +428,14 @@ def generate_h002_unused_indexes_report(self, cluster: str = "local", node_name: {}).get( 'result') else 0 + # Get index definition from collected metrics + index_definition = index_definitions.get(index_name, 'Definition not available') + index_data = { "schema_name": schema_name, "table_name": table_name, "index_name": index_name, + "index_definition": index_definition, "reason": reason, "idx_scan": idx_scan, "index_size_bytes": index_size_bytes, @@ -381,7 +460,9 @@ def generate_h002_unused_indexes_report(self, cluster: str = "local", node_name: "stats_reset": { "stats_reset_epoch": stats_reset_epoch, "stats_reset_time": stats_reset_time, - "days_since_reset": days_since_reset + "days_since_reset": days_since_reset, + "postmaster_startup_epoch": postmaster_startup_epoch, + "postmaster_startup_time": postmaster_startup_time } } @@ -1764,8 +1845,10 @@ def make_request(api_url, endpoint, request_data): def main(): parser = argparse.ArgumentParser(description='Generate PostgreSQL reports using PromQL') - parser.add_argument('--prometheus-url', default='http://localhost:9090', - help='Prometheus URL (default: http://localhost:9090)') + parser.add_argument('--prometheus-url', default='http://sink-prometheus:9090', + help='Prometheus URL (default: http://sink-prometheus:9090)') + parser.add_argument('--postgres-sink-url', default='postgresql://pgwatch@sink-postgres:5432/measurements', + help='Postgres sink connection string (default: postgresql://pgwatch@sink-postgres:5432/measurements)') parser.add_argument('--cluster', default='local', help='Cluster name (default: local)') parser.add_argument('--node-name', default='node-01', @@ -1785,7 +1868,7 @@ def main(): args = parser.parse_args() - generator = PostgresReportGenerator(args.prometheus_url) + generator = PostgresReportGenerator(args.prometheus_url, args.postgres_sink_url) # Test connection if not generator.test_connection(): @@ -1852,6 +1935,9 @@ def main(): print(f"Error generating reports: {e}") raise e sys.exit(1) + finally: + # Clean up postgres connection + generator.close_postgres_sink() if __name__ == "__main__": diff --git a/reporter/requirements.txt b/reporter/requirements.txt index 659c37c..6813242 100644 --- a/reporter/requirements.txt +++ b/reporter/requirements.txt @@ -1 +1,2 @@ -requests>=2.31.0 \ No newline at end of file +requests>=2.31.0 +psycopg2-binary>=2.9.9 \ No newline at end of file