feat(routing): automatic prompt caching + a per-session routing budget

Dockerfile

@@ -47,10 +47,13 @@ FROM docker.io/envoyproxy/envoy:${ENVOY_VERSION} AS envoy
 
 FROM python:3.14-slim AS arch
 
+# Install runtime deps first, then upgrade — so security patches also land on
+# packages pulled in transitively here (e.g. curl -> libssh2-1t64), not just those
+# already present in the base image.
 RUN set -eux; \
   apt-get update; \
-  apt-get upgrade -y; \
   apt-get install -y --no-install-recommends gettext-base curl procps; \
+  apt-get upgrade -y; \
   apt-get clean; rm -rf /var/lib/apt/lists/*
 
 RUN pip install --no-cache-dir supervisor

config/envoy.template.yaml

@@ -269,6 +269,13 @@ static_resources:
                             auto_host_rewrite: true
                             cluster: {{ cluster_name }}
                             timeout: 300s
+                            {% if cluster.prefix_affinity %}
+                            # Feeds the cluster's RING_HASH lb_policy: requests with the
+                            # same prompt-prefix hash stick to the same replica.
+                            hash_policy:
+                              - header:
+                                  header_name: x-plano-prefix-hash
+                            {% endif %}
                         {% endfor %}
                 http_filters:
                   - name: envoy.filters.http.router
@@ -977,9 +984,18 @@ static_resources:
       {% else -%}
       connect_timeout: {{ upstream_connect_timeout | default('5s') }}
       {% endif -%}
+      {% if cluster.prefix_affinity -%}
+      # KV-aware replica stickiness: resolve every replica address and consistent-hash
+      # requests (by x-plano-prefix-hash, see the route-level hash_policy) so the same
+      # prompt prefix lands on the replica holding its warm KV cache.
+      type: STRICT_DNS
+      dns_lookup_family: V4_ONLY
+      lb_policy: RING_HASH
+      {% else -%}
       type: LOGICAL_DNS
       dns_lookup_family: V4_ONLY
       lb_policy: ROUND_ROBIN
+      {% endif -%}
       load_assignment:
         cluster_name: {{ cluster_name }}
         endpoints:

config/plano_config_schema.yaml

@@ -155,6 +155,13 @@ properties:
               - https
           http_host:
             type: string
+          prefix_affinity:
+            type: boolean
+            description: >
+              For self-hosted multi-replica backends (e.g. vLLM): consistent-hash requests
+              to replicas by the x-plano-prefix-hash header so the same prompt prefix lands
+              on the same replica and reuses its KV cache. Uses ring-hash load balancing
+              across all resolved endpoint addresses. Default false.
         additionalProperties: false
         required:
           - endpoint
@@ -316,6 +323,33 @@ properties:
       orchestrator_model_context_length:
         type: integer
         description: "Maximum token length for the orchestrator/routing model context window. Default is 8192."
+  prompt_caching:
+    type: object
+    description: >
+      Automatic provider prompt caching, configured once for the whole Plano instance.
+      Disabled by default; set enabled: true to opt in. Prompt caching never changes
+      which model routing selects — it only keeps a conversation on the same
+      model/provider so the upstream prompt cache stays warm across turns, and
+      auto-injects provider cache-control markers where supported.
+    properties:
+      enabled:
+        type: boolean
+        description: "Master switch. Default false (opt-in). Applies across the entire instance."
+      session_affinity:
+        type: boolean
+        description: "Auto-derive a session key from the prompt prefix (system + tools + first user message) when X-Model-Affinity is absent, so follow-up turns reuse the same warm cache. Default true when enabled."
+      inject_cache_control:
+        type: boolean
+        description: "Auto-insert ephemeral cache_control breakpoints for providers that need explicit markers (Anthropic). Default true when enabled."
+      min_prefix_tokens:
+        type: integer
+        minimum: 1
+        description: "Skip breakpoint injection when the estimated stable prefix is below this many tokens. Default 1024."
+      session_ttl_seconds:
+        type: integer
+        minimum: 1
+        description: "Pin lifetime for implicit/explicit sessions; align with the provider cache window (e.g. 3600 for Anthropic 1h caching). Defaults to routing.session_ttl_seconds."
+    additionalProperties: false
   system_prompt:
     type: string
   prompt_targets:
@@ -510,6 +544,51 @@ properties:
               Optional HTTP header name whose value is used as a tenant prefix in the cache key.
               When set, keys are scoped as plano:affinity:{tenant_id}:{session_id}.
         additionalProperties: false
+      routing_budget:
+        type: object
+        description: >
+          Per-session cost gate on model switching. Independent of prompt caching — it
+          applies whenever configured (presence of this block turns it on). The default
+          posture is to stick to the model a session is warm on. When routing proposes a
+          different model while that session's provider cache is plausibly still warm
+          (inferred from the idle gap vs. the provider's cache window), the actual
+          input-token cost of abandoning the cache —
+          context_tokens x (candidate_uncached_input_rate - anchor_cached_input_rate),
+          output cost deliberately excluded — is drawn from the session's budget: a paid
+          switch is allowed only while budget remains, and an outright-cheaper switch is
+          free and can credit the budget back. Requires a cost source in
+          model_metrics_sources.
+        properties:
+          seed_usd:
+            type: number
+            minimum: 0
+            description: >
+              Initial cumulative budget (USD) granted to a session. 0 means never pay to
+              switch (only outright-cheaper switches are allowed); larger values buy more
+              quality-driven switches over the session.
+          replenish_on_rebind:
+            type: boolean
+            description: "Re-seed the budget to seed_usd when a cold session re-binds. Default true."
+          credit_negative:
+            type: boolean
+            description: "Credit the budget back when a switch is outright cheaper (negative cost). Default true."
+          cache_read_discount:
+            type: number
+            minimum: 0
+            maximum: 1
+            description: >
+              Fallback used to estimate a model's cached input rate when the pricing feed
+              doesn't publish one (cached_rate = input_rate x discount). A pricing detail,
+              not a cost policy. Default 0.1.
+          record_counterfactual:
+            type: boolean
+            description: >
+              When true, a vetoed switch records the route the gate would have taken had
+              the switch been allowed, as the plano.switch.counterfactual_route span
+              attribute. Telemetry only — the counterfactual model is never dispatched.
+              Useful for evals/benchmarks. Default false.
+        required: ["seed_usd"]
+        additionalProperties: false
     additionalProperties: false
   state_storage:
     type: object

crates/brightstaff/src/affinity.rs

@@ -0,0 +1,181 @@
+//! Implicit session-affinity key derivation.
+//!
+//! When a request carries no explicit `X-Model-Affinity` header, Plano derives a
+//! stable session key from the parts of the prompt that repeat verbatim at the head
+//! of every turn — the same bytes the provider's prompt cache is keyed on:
+//!
+//! ```text
+//! session_key  = hash(system + tools + first_user_message)
+//! prefix_hash  = hash(system + tools)
+//! ```
+//!
+//! The session key is constant for the life of a conversation (history grows at the
+//! tail, not the head), so turns 2+ reuse the same pin without any client changes.
+//! The prefix hash covers only the fully-stable segment and is stored with the pin
+//! for drift detection: if it changes, the provider cache is already lost and
+//! re-routing fresh is safe.
+//!
+//! Only salted hashes are ever stored — never prompt content.
+
+use hermesllm::apis::openai::{Message, Role};
+
+/// Salt folded into every hash so stored keys can't be trivially correlated with
+/// prompt content across systems. Deterministic across processes/replicas so a
+/// shared Redis session cache keys consistently.
+const HASH_SALT: &str = "plano-affinity-v1";
+
+/// Prefix distinguishing derived keys from client-supplied `X-Model-Affinity` ids.
+const IMPLICIT_KEY_PREFIX: &str = "implicit:";
+
+/// Derived affinity identifiers for one request.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ImplicitAffinity {
+    /// Session-cache key: `implicit:{hex}` over system + tools + first user message.
+    pub session_key: String,
+    /// Hash of the stable prefix only (system + tools), for drift detection.
+    pub prefix_hash: u64,
+}
+
+/// FNV-1a 64-bit — stable across processes and Rust versions (unlike `DefaultHasher`),
+/// dependency-free, and plenty for cache keying (collisions merely over-pin, which is
+/// cache-friendly).
+fn fnv1a64(chunks: &[&str]) -> u64 {
+    const OFFSET_BASIS: u64 = 0xcbf2_9ce4_8422_2325;
+    const PRIME: u64 = 0x0000_0100_0000_01b3;
+    let mut hash = OFFSET_BASIS;
+    let mut feed = |bytes: &[u8]| {
+        for &b in bytes {
+            hash ^= b as u64;
+            hash = hash.wrapping_mul(PRIME);
+        }
+        // Field separator so ("ab","c") and ("a","bc") hash differently.
+        hash ^= 0x1f;
+        hash = hash.wrapping_mul(PRIME);
+    };
+    feed(HASH_SALT.as_bytes());
+    for chunk in chunks {
+        feed(chunk.as_bytes());
+    }
+    hash
+}
+
+/// Derive the implicit affinity key from parsed request messages and tool names.
+///
+/// Returns `None` when there is no user message to anchor on (nothing distinguishes
+/// the conversation, so pinning would be meaningless).
+pub fn derive_implicit_affinity(
+    messages: &[Message],
+    tool_names: Option<&[String]>,
+    tenant_id: Option<&str>,
+) -> Option<ImplicitAffinity> {
+    let system_text: String = messages
+        .iter()
+        .filter(|m| matches!(m.role, Role::System | Role::Developer))
+        .filter_map(|m| m.content.as_ref().map(|c| c.to_string()))
+        .collect::<Vec<_>>()
+        .join("\n");
+
+    let first_user = messages
+        .iter()
+        .find(|m| matches!(m.role, Role::User))
+        .and_then(|m| m.content.as_ref().map(|c| c.to_string()))?;
+
+    let tools_text = tool_names.map(|names| names.join(",")).unwrap_or_default();
+    let tenant = tenant_id.unwrap_or_default();
+
+    let prefix_hash = fnv1a64(&[tenant, &system_text, &tools_text]);
+    let session_hash = fnv1a64(&[tenant, &system_text, &tools_text, &first_user]);
+
+    Some(ImplicitAffinity {
+        session_key: format!("{IMPLICIT_KEY_PREFIX}{session_hash:016x}"),
+        prefix_hash,
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use hermesllm::apis::openai::MessageContent;
+
+    fn msg(role: Role, text: &str) -> Message {
+        Message {
+            role,
+            content: Some(MessageContent::Text(text.to_string())),
+            name: None,
+            tool_call_id: None,
+            tool_calls: None,
+        }
+    }
+
+    #[test]
+    fn key_is_stable_as_history_grows() {
+        let turn1 = vec![
+            msg(Role::System, "You are a coding agent."),
+            msg(Role::User, "Fix the bug in main.rs"),
+        ];
+        let turn5 = vec![
+            msg(Role::System, "You are a coding agent."),
+            msg(Role::User, "Fix the bug in main.rs"),
+            msg(Role::Assistant, "Done. Anything else?"),
+            msg(Role::User, "Now add tests"),
+            msg(Role::Assistant, "Added."),
+        ];
+
+        let a1 = derive_implicit_affinity(&turn1, None, None).unwrap();
+        let a5 = derive_implicit_affinity(&turn5, None, None).unwrap();
+        assert_eq!(a1.session_key, a5.session_key);
+        assert_eq!(a1.prefix_hash, a5.prefix_hash);
+        assert!(a1.session_key.starts_with("implicit:"));
+    }
+
+    #[test]
+    fn different_first_user_message_yields_different_session() {
+        let base = msg(Role::System, "You are a coding agent.");
+        let a = derive_implicit_affinity(
+            &[base.clone(), msg(Role::User, "conversation A")],
+            None,
+            None,
+        )
+        .unwrap();
+        let b = derive_implicit_affinity(&[base, msg(Role::User, "conversation B")], None, None)
+            .unwrap();
+        // Same stable prefix, different conversations.
+        assert_eq!(a.prefix_hash, b.prefix_hash);
+        assert_ne!(a.session_key, b.session_key);
+    }
+
+    #[test]
+    fn changed_system_prompt_changes_prefix_hash() {
+        let a = derive_implicit_affinity(
+            &[msg(Role::System, "v1 prompt"), msg(Role::User, "hi")],
+            None,
+            None,
+        )
+        .unwrap();
+        let b = derive_implicit_affinity(
+            &[msg(Role::System, "v2 prompt"), msg(Role::User, "hi")],
+            None,
+            None,
+        )
+        .unwrap();
+        assert_ne!(a.prefix_hash, b.prefix_hash);
+        assert_ne!(a.session_key, b.session_key);
+    }
+
+    #[test]
+    fn tools_and_tenant_are_part_of_the_key() {
+        let messages = [msg(Role::System, "s"), msg(Role::User, "u")];
+        let plain = derive_implicit_affinity(&messages, None, None).unwrap();
+        let with_tools =
+            derive_implicit_affinity(&messages, Some(&["get_weather".to_string()]), None).unwrap();
+        let with_tenant = derive_implicit_affinity(&messages, None, Some("acme")).unwrap();
+        assert_ne!(plain.session_key, with_tools.session_key);
+        assert_ne!(plain.session_key, with_tenant.session_key);
+    }
+
+    #[test]
+    fn no_user_message_yields_none() {
+        assert!(derive_implicit_affinity(&[msg(Role::System, "s")], None, None).is_none());
+        assert!(derive_implicit_affinity(&[], None, None).is_none());
+    }
+}

crates/brightstaff/src/app_state.rs

@@ -1,7 +1,10 @@
 use std::collections::HashMap;
 use std::sync::Arc;
 
-use common::configuration::{Agent, FilterPipeline, Listener, ModelAlias, SpanAttributes};
+use common::configuration::{
+    Agent, EffectivePromptCaching, EffectiveRoutingBudget, FilterPipeline, Listener, ModelAlias,
+    SpanAttributes,
+};
 use common::llm_providers::LlmProviders;
 use tokio::sync::RwLock;
 
@@ -31,4 +34,10 @@ pub struct AppState {
     /// When false, agentic signal analysis is skipped on LLM responses to save CPU.
     /// Controlled by `overrides.disable_signals` in plano config.
     pub signals_enabled: bool,
+    /// Instance-wide automatic prompt-caching settings, resolved once from the
+    /// top-level `prompt_caching` config. Disabled by default (opt-in).
+    pub prompt_caching: EffectivePromptCaching,
+    /// Per-session model-switch cost gate, resolved from `routing.routing_budget`.
+    /// Independent of prompt caching; `None` when not configured (off by default).
+    pub routing_budget: Option<EffectiveRoutingBudget>,
 }

crates/brightstaff/src/handlers/function_calling.rs

@@ -1239,6 +1239,7 @@ impl ArchFunctionHandler {
                 total_tokens: 0,
                 prompt_tokens_details: None,
                 completion_tokens_details: None,
+                ..Default::default()
             },
             system_fingerprint: None,
             service_tier: None,