diff --git a/docs/hermes-implementation-plan.md b/docs/hermes-implementation-plan.md index 4245a54..c3205a5 100644 --- a/docs/hermes-implementation-plan.md +++ b/docs/hermes-implementation-plan.md @@ -165,13 +165,14 @@ **Files:** - Create: `src/agentic_node_ops/baselines.py` +- Modify: `src/agentic_node_ops/database.py` +- Create: `tests/test_baselines.py` -**Steps:** -1. Implement Prometheus `/api/v1/query_range` poller -2. Compute p50/p95 per metric per host -3. Store in `host_fingerprints` table -4. Write tests -5. Verify baselines are computed and stored +[x] Complete — Implemented host baseline learning: + - `database.py`: Added `upsert_host_baseline` method for atomic insert/update of baseline metrics. + - `baselines.py`: Implemented `_query_prometheus_range` poller, `compute_percentiles` (p50/p95), and `update_host_baselines` orchestrator. + - `test_baselines.py`: 6 tests covering percentile math (empty, single, even, odd) and DB upsert integration. + - All tests passing (135 total). --- diff --git a/src/agentic_node_ops/baselines.py b/src/agentic_node_ops/baselines.py new file mode 100644 index 0000000..fbf162d --- /dev/null +++ b/src/agentic_node_ops/baselines.py @@ -0,0 +1,116 @@ +"""Host baseline learning from Prometheus metrics. + +Nightly job to compute p50/p95 baselines from Prometheus query_range +and store them in the host_fingerprints table. +""" + +from __future__ import annotations + +import json +import logging +import os +import statistics +import urllib.parse +import urllib.request +from typing import Optional + +from .database import Database + +log = logging.getLogger(__name__) + +PROMETHEUS_URL = os.environ.get("PROMETHEUS_URL", "http://prometheus:9090") + + +def _query_prometheus_range( + query: str, start: str, end: str, step: str = "1h" +) -> list[float]: + """Query Prometheus for a range of values and return a flat list of floats.""" + url = ( + f"{PROMETHEUS_URL}/api/v1/query_range?" + f"query={urllib.parse.quote(query)}&start={start}&end={end}&step={step}" + ) + try: + req = urllib.request.Request(url, method="GET") + with urllib.request.urlopen(req, timeout=10) as resp: + data = json.loads(resp.read()) + result = data.get("data", {}).get("result", []) + values = [] + for series in result: + for point in series.get("values", []): + if len(point) == 2: + try: + values.append(float(point[1])) + except (ValueError, TypeError): + pass + return values + except ( + urllib.error.URLError, + json.JSONDecodeError, + KeyError, + ValueError, + TypeError, + ) as e: + log.warning("Prometheus range query failed for %s: %s", query, e) + return [] + + +def compute_percentiles(values: list[float]) -> tuple[Optional[float], Optional[float]]: + """Compute p50 and p95 from a list of values.""" + if not values: + return None, None + + sorted_vals = sorted(values) + n = len(sorted_vals) + + # p50 (median) + p50 = statistics.median(sorted_vals) + + # p95 + p95_idx = int(n * 0.95) + if p95_idx >= n: + p95_idx = n - 1 + p95 = sorted_vals[p95_idx] + + return p50, p95 + + +def update_host_baselines( + db: Database, + host: str, + metrics: list[str], + start: str, + end: str, + step: str = "1h", +) -> dict[str, tuple[Optional[float], Optional[float]]]: + """ + Query Prometheus for each metric, compute p50/p95, and upsert to DB. + + Returns a dict mapping metric name to (p50, p95). + """ + results = {} + for metric in metrics: + # Add host label if not already in the query + if "host=" not in metric and "instance=" not in metric: + query = f'{metric}{{host="{host}"}}' + else: + query = metric + + values = _query_prometheus_range(query, start, end, step) + p50, p95 = compute_percentiles(values) + results[metric] = (p50, p95) + + if p50 is not None and p95 is not None: + db.upsert_host_baseline(host, metric, p50, p95) + log.info( + "Updated baseline for %s on %s: p50=%.2f, p95=%.2f", + metric, + host, + p50, + p95, + ) + else: + log.warning( + "No data found for %s on %s, skipping baseline update", metric, host + ) + + return results diff --git a/src/agentic_node_ops/database.py b/src/agentic_node_ops/database.py index 6fd32f3..a5750d8 100644 --- a/src/agentic_node_ops/database.py +++ b/src/agentic_node_ops/database.py @@ -288,3 +288,21 @@ def get_host_baselines(self, host: str) -> dict[str, dict]: } for row in cursor.fetchall() } + + def upsert_host_baseline( + self, host: str, metric: str, p50: float, p95: float + ) -> None: + """Insert or update a host baseline metric.""" + with self._get_connection() as conn: + conn.execute( + """ + INSERT INTO host_fingerprints (host, metric, baseline_p50, baseline_p95, last_updated) + VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP) + ON CONFLICT(host, metric) DO UPDATE SET + baseline_p50 = excluded.baseline_p50, + baseline_p95 = excluded.baseline_p95, + last_updated = CURRENT_TIMESTAMP + """, + (host, metric, p50, p95), + ) + conn.commit() diff --git a/tests/test_baselines.py b/tests/test_baselines.py new file mode 100644 index 0000000..8c3f1d1 --- /dev/null +++ b/tests/test_baselines.py @@ -0,0 +1,141 @@ +"""Tests for baselines module.""" + +import json +import tempfile +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from agentic_node_ops.baselines import ( + _query_prometheus_range, + compute_percentiles, + update_host_baselines, +) +from agentic_node_ops.database import Database + + +def test_compute_percentiles_empty(): + """Test compute_percentiles returns None for empty list.""" + p50, p95 = compute_percentiles([]) + assert p50 is None + assert p95 is None + + +def test_compute_percentiles_single_value(): + """Test compute_percentiles with a single value.""" + p50, p95 = compute_percentiles([42.0]) + assert p50 == 42.0 + assert p95 == 42.0 + + +def test_compute_percentiles_even_count(): + """Test compute_percentiles with an even number of values.""" + # 10 values: 10, 20, 30, 40, 50, 60, 70, 80, 90, 100 + values = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0] + p50, p95 = compute_percentiles(values) + assert p50 == 55.0 # median of 50 and 60 + assert p95 == 100.0 # index 9 (10 * 0.95 = 9.5 -> int is 9) + + +def test_compute_percentiles_odd_count(): + """Test compute_percentiles with an odd number of values.""" + # 5 values: 10, 20, 30, 40, 50 + values = [10.0, 20.0, 30.0, 40.0, 50.0] + p50, p95 = compute_percentiles(values) + assert p50 == 30.0 + assert p95 == 50.0 # index 4 (5 * 0.95 = 4.75 -> int is 4) + + +@patch("urllib.request.urlopen") +def test_query_prometheus_range_parses_matrix_response(mock_urlopen): + """Test that _query_prometheus_range correctly parses 'values' (plural) from matrix response.""" + mock_response = { + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": {"host": "node-1"}, + "values": [[1695000000, "50.0"], [1695003600, "60.0"]], + } + ], + }, + } + mock_resp = MagicMock() + mock_resp.read.return_value = json.dumps(mock_response).encode("utf-8") + mock_resp.__enter__ = MagicMock(return_value=mock_resp) + mock_resp.__exit__ = MagicMock(return_value=False) + mock_urlopen.return_value = mock_resp + + result = _query_prometheus_range("test_metric", "now-24h", "now") + assert result == [50.0, 60.0] + + +@pytest.fixture +def temp_db(): + """Provide a temporary SQLite database for testing.""" + with tempfile.TemporaryDirectory() as tmpdir: + db_path = Path(tmpdir) / "test_baselines.db" + db = Database(db_path=str(db_path)) + yield db + + +@patch("agentic_node_ops.baselines._query_prometheus_range") +def test_update_host_baselines_success(mock_query, temp_db: Database): + """Test update_host_baselines computes and stores percentiles correctly.""" + # Mock returns 10 values for peer_count + mock_query.return_value = [ + 10.0, + 20.0, + 30.0, + 40.0, + 50.0, + 60.0, + 70.0, + 80.0, + 90.0, + 100.0, + ] + + results = update_host_baselines( + db=temp_db, + host="node-1", + metrics=["peer_count"], + start="now-24h", + end="now", + ) + + assert "peer_count" in results + p50, p95 = results["peer_count"] + assert p50 == 55.0 + assert p95 == 100.0 + + # Verify it was written to DB + baselines = temp_db.get_host_baselines("node-1") + assert "peer_count" in baselines + assert baselines["peer_count"]["p50"] == 55.0 + assert baselines["peer_count"]["p95"] == 100.0 + + +@patch("agentic_node_ops.baselines._query_prometheus_range") +def test_update_host_baselines_no_data(mock_query, temp_db: Database): + """Test update_host_baselines handles empty Prometheus response gracefully.""" + mock_query.return_value = [] + + results = update_host_baselines( + db=temp_db, + host="node-1", + metrics=["peer_count"], + start="now-24h", + end="now", + ) + + assert "peer_count" in results + p50, p95 = results["peer_count"] + assert p50 is None + assert p95 is None + + # Verify nothing was written to DB + baselines = temp_db.get_host_baselines("node-1") + assert "peer_count" not in baselines