[Router] Count routing fallbacks per routing logic

src/tests/test_kvaware_router.py

@@ -3,6 +3,7 @@
 from types import SimpleNamespace
 
 import pytest
+from prometheus_client import REGISTRY
 
 import vllm_router.routers.routing_logic as routing_logic
 from vllm_router.routers.routing_logic import KvawareRouter
@@ -32,6 +33,11 @@ def encode(self, prompt):
         return list(range(1000))
 
 
+def fallback_count(reason):
+    labels = {"routing_logic": "kvaware", "reason": reason}
+    return REGISTRY.get_sample_value("vllm:routing_fallbacks_total", labels) or 0
+
+
 @pytest.mark.parametrize(
     "threshold",
     [
@@ -188,6 +194,57 @@ async def query_manager(_msg):
     assert selected == url_b
 
 
+@pytest.mark.parametrize(
+    "layout_info, reason",
+    [
+        pytest.param({}, "no_cache_hit", id="nothing-cached"),
+        pytest.param(
+            {"dead-instance": ("LocalCPUBackend", 1000)},
+            "no_live_holder",
+            id="only-dead-holder",
+        ),
+        pytest.param(
+            {"instance-b": ("LocalCPUBackend", 100)},
+            "below_threshold",
+            id="short-match",
+        ),
+        pytest.param({"instance-b": ("LocalCPUBackend", 1000)}, None, id="cache-hit"),
+    ],
+)
+@pytest.mark.asyncio
+async def test_kvaware_counts_fallbacks_by_reason(layout_info, reason):
+    url_a = "http://10.0.0.1:8000"
+    url_b = "http://10.0.0.2:8000"
+    router = KvawareRouter.__new__(KvawareRouter)
+    router.tokenizers = {"test-model": Tokenizer()}
+    router.threshold = 0
+    router.session_key = "x-session-id"
+    router.hash_ring = routing_logic.HashRing()
+    router.instance_id_to_ip = {
+        "dead-instance": "http://10.0.0.9:8000",
+        "instance-a": url_a,
+        "instance-b": url_b,
+    }
+
+    async def query_manager(_msg):
+        return SimpleNamespace(layout_info=layout_info)
+
+    router.query_manager = query_manager
+    reasons = ("no_cache_hit", "no_live_holder", "below_threshold")
+    before = {r: fallback_count(r) for r in reasons}
+
+    await router.route_request(
+        [EndpointInfo(url_a), EndpointInfo(url_b)],
+        {},
+        {},
+        SimpleNamespace(headers={}),
+        {"prompt": "test"},
+    )
+
+    counted = {r: fallback_count(r) - before[r] for r in reasons}
+    assert counted == {r: 1 if r == reason else 0 for r in reasons}
+
+
 @pytest.mark.asyncio
 async def test_kvaware_refreshes_unknown_dead_holder_and_uses_live_match():
     url_a = "http://10.0.0.1:8000"

src/tests/test_loadaware_router.py

@@ -14,8 +14,10 @@
 """
 
 from typing import Any, Dict
+from unittest.mock import AsyncMock
 
 import pytest
+from prometheus_client import REGISTRY
 
 import vllm_router.routers.routing_logic as routing_logic
 from vllm_router.routers.routing_logic import (
@@ -809,6 +811,57 @@ async def query_manager(msg):
     assert url == URL_B
 
 
+# --- routing fallback metric --------------------------------------------------
+
+
+def fallback_count(reason):
+    labels = {"routing_logic": "loadaware", "reason": reason}
+    return REGISTRY.get_sample_value("vllm:routing_fallbacks_total", labels) or 0
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "query_manager, reason",
+    [
+        pytest.param(
+            AsyncMock(return_value=LookupRet({INST_B: (LOCAL, 3)})),
+            None,
+            id="cache-hit",
+        ),
+        pytest.param(
+            AsyncMock(return_value=LookupRet({})),
+            "no_cache_hit",
+            id="nothing-cached",
+        ),
+        pytest.param(
+            AsyncMock(side_effect=TimeoutError("controller unavailable")),
+            "lookup_failed",
+            id="lookup-failed",
+        ),
+    ],
+)
+async def test_route_request_counts_fallbacks_by_reason(
+    tokenize_calls, query_manager, reason
+):
+    """Each fallback is counted once, under its reason; a request scored
+    against a cache hit is not a fallback."""
+    router = make_fallback_router()
+    router.query_manager = query_manager
+    reasons = ("no_cache_hit", "lookup_failed")
+    before = {r: fallback_count(r) for r in reasons}
+
+    await router.route_request(
+        endpoints(URL_A, URL_B),
+        {},
+        {},
+        QpsFallbackRequest(),
+        {"messages": [{"role": "user", "content": "x"}]},
+    )
+
+    counted = {r: fallback_count(r) - before[r] for r in reasons}
+    assert counted == {r: 1 if r == reason else 0 for r in reasons}
+
+
 # --- configuration ------------------------------------------------------------
 
 

src/tests/test_prefixaware_router.py

@@ -3,6 +3,7 @@
 from unittest.mock import AsyncMock, patch
 
 import pytest
+from prometheus_client import REGISTRY
 
 from vllm_router.routers.routing_logic import (
     PrefixAwareRouter,
@@ -348,6 +349,44 @@ async def test_route_uses_matched_endpoint_when_match_above_threshold():
     fake_hashtrie.insert.assert_awaited_once()
 
 
+@pytest.mark.parametrize(
+    "match_length, fallbacks",
+    [
+        pytest.param(0, 1, id="below-threshold"),
+        pytest.param(4096, 0, id="above-threshold"),
+    ],
+)
+@pytest.mark.asyncio
+async def test_below_threshold_match_is_counted_as_fallback(match_length, fallbacks):
+    """
+    Only a match shorter than prefix_min_match_length leaves prefix affinity,
+    so only that request is counted as a routing fallback.
+    """
+
+    endpoints = [
+        EndpointInfo(url="http://engine1.com"),
+        EndpointInfo(url="http://engine2.com"),
+    ]
+    request = Request(headers={})
+    request_json = {"prompt": "some prompt text"}
+    labels = {"routing_logic": "prefixaware", "reason": "below_threshold"}
+
+    router = PrefixAwareRouter(prefix_min_match_length=128)
+
+    fake_hashtrie = AsyncMock()
+    fake_hashtrie.longest_prefix_match.return_value = (
+        match_length,
+        {"http://engine1.com"},
+    )
+    router.hashtrie = fake_hashtrie
+
+    before = REGISTRY.get_sample_value("vllm:routing_fallbacks_total", labels) or 0
+    await router.route_request(endpoints, None, {}, request, request_json)
+    after = REGISTRY.get_sample_value("vllm:routing_fallbacks_total", labels) or 0
+
+    assert after - before == fallbacks
+
+
 @pytest.mark.asyncio
 async def test_default_threshold_zero_preserves_original_behavior():
     """

src/tests/test_session_router.py

@@ -1,6 +1,7 @@
 from typing import Any, Dict
 
 import pytest
+from prometheus_client import REGISTRY
 
 from vllm_router.routers.routing_logic import SessionRouter
 
@@ -70,6 +71,37 @@ async def test_route_request_without_session_id():
     assert url == "http://engine2.com"
 
 
+@pytest.mark.parametrize(
+    "headers, fallbacks",
+    [
+        pytest.param({}, 1, id="no-session-id"),
+        pytest.param({"session_id": "abc123"}, 0, id="session-id"),
+    ],
+)
+@pytest.mark.asyncio
+async def test_route_request_counts_missing_session_id_as_fallback(headers, fallbacks):
+    """
+    Test that only a request without a session ID is counted as a fallback.
+    """
+    endpoints = [
+        EndpointInfo(url="http://engine1.com"),
+        EndpointInfo(url="http://engine2.com"),
+    ]
+    request_stats = {
+        "http://engine1.com": RequestStats(qps=10),
+        "http://engine2.com": RequestStats(qps=5),
+    }
+    request = Request(headers=headers)
+    labels = {"routing_logic": "session", "reason": "no_session_id"}
+
+    router = SessionRouter(session_key="session_id")
+    before = REGISTRY.get_sample_value("vllm:routing_fallbacks_total", labels) or 0
+    await router.route_request(endpoints, None, request_stats, request, {})
+    after = REGISTRY.get_sample_value("vllm:routing_fallbacks_total", labels) or 0
+
+    assert after - before == fallbacks
+
+
 @pytest.mark.asyncio
 async def test_route_request_with_dynamic_endpoints():
     """

src/vllm_router/routers/routing_logic.py

@@ -45,6 +45,7 @@
 
 from vllm_router.log import init_logger
 from vllm_router.service_discovery import EndpointInfo
+from vllm_router.services.metrics_service import routing_fallbacks_total
 from vllm_router.stats.engine_stats import EngineStats
 from vllm_router.stats.request_stats import RequestStats
 from vllm_router.utils import SingletonABCMeta
@@ -273,6 +274,9 @@ async def route_request(
         self._update_hash_ring(endpoints)
 
         if session_id is None:
+            routing_fallbacks_total.labels(
+                routing_logic="session", reason="no_session_id"
+            ).inc()
             # Route based on QPS if no session ID is present
             url = self._qps_routing(endpoints, request_stats)
         else:
@@ -464,6 +468,13 @@ async def route_request(
             or matched_instance_id is None
             or matched_tokens < max(len(token_ids) - self.threshold, 0)
         ):
+            if not layout_info:
+                reason = "no_cache_hit"
+            elif matched_instance_id is None:
+                reason = "no_live_holder"
+            else:
+                reason = "below_threshold"
+            routing_fallbacks_total.labels(routing_logic="kvaware", reason=reason).inc()
             session_id = self.extract_session_id(request, request_json)
             logger.debug(f"Fallback to using session id: {session_id}")
             # Update the hash ring with the current list of endpoints
@@ -828,12 +839,18 @@ async def route_request(
             # or the controller lookup failed: route without cache information
             # instead of failing the request with an HTTP 500.
             logger.warning(f"loadaware lookup failed, using fallback route: {e!r}")
+            routing_fallbacks_total.labels(
+                routing_logic="loadaware", reason="lookup_failed"
+            ).inc()
             return self.fallback_url(endpoints, request_stats, request, request_json)
         logger.debug(f"Lookup return message: {lookup_ret}")
         layout_info = getattr(lookup_ret, "layout_info", None) or {}
 
         if not layout_info:
             # Nothing cached anywhere - no benefit term to weigh.
+            routing_fallbacks_total.labels(
+                routing_logic="loadaware", reason="no_cache_hit"
+            ).inc()
             return self.fallback_url(endpoints, request_stats, request, request_json)
 
         await self.refresh_instance_map(endpoints, layout_info)
@@ -925,6 +942,9 @@ async def route_request(
         )
 
         if match_length < self.prefix_min_match_length:
+            routing_fallbacks_total.labels(
+                routing_logic="prefixaware", reason="below_threshold"
+            ).inc()
             # sub-threshold: no useful prefix affinity yet. least-inflight
             # via request_stats (same signal as LoadAwareRouter.load_penalty).
             # missing url => load 0. random among exact min. no engine_stats

src/vllm_router/services/metrics_service/__init__.py

@@ -38,6 +38,11 @@
     "Total valid incoming requests to router (including when no backends available).",
     ["model"],
 )
+routing_fallbacks_total = Counter(
+    "vllm:routing_fallbacks_total",
+    "Requests routed by a fallback path instead of the routing logic's primary policy",
+    ["routing_logic", "reason"],
+)
 
 # New metrics per dashboard update
 healthy_pods_total = Gauge(