Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions crates/libsy/src/algorithms/util/classifier_contract.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,13 @@
//! Prompt and structured-output contracts shared by LLM classifiers.

use jsonschema::Validator;
use serde::Deserialize;
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};

use crate::{LibsyError, Result};

/// Provider-side structured-output mode used by a classifier judge.
#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq)]
#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum ClassifierResponseFormat {
/// Send the verdict schema through the provider's strict JSON Schema wrapper.
Expand Down
4 changes: 2 additions & 2 deletions crates/libsy/src/algorithms/util/escalation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
//! [`build_judge`] is the whole surface; the confirmation policy that consumes its verdicts
//! lives with the assembled algorithm in [`crate::algorithms::escalation`].

use serde::Deserialize;
use serde::{Deserialize, Serialize};
use switchyard_protocol::{ContentBlock, Message, ModelId, Role};

use super::classifier_contract::{ClassifierContract, ClassifierContractConfig};
Expand Down Expand Up @@ -43,7 +43,7 @@ const MAX_REQUEST_CHARS: usize = 18_000;
///
/// The routing settings retain their benchmarked defaults. Everything else is a fixed invariant
/// (the constants above).
#[derive(Clone, Debug, Deserialize)]
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct EscalationJudgeConfig {
/// Consecutive escalate verdicts required before a turn moves to the capable tier, which
Expand Down
6 changes: 3 additions & 3 deletions crates/libsy/src/algorithms/util/stage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

use async_trait::async_trait;
use opentelemetry::KeyValue;
use serde::Deserialize;
use serde::{Deserialize, Serialize};

use super::prompts;
use super::tool_signals::ToolSignals;
Expand Down Expand Up @@ -131,7 +131,7 @@ impl StageTargets {
}

/// Which tier to default to when the scorer is not confident.
#[derive(Clone, Copy, Debug, Eq, PartialEq, Deserialize)]
#[derive(Clone, Copy, Debug, Eq, PartialEq, Deserialize, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum PickerMode {
/// Default to capable unless the scorer confidently picks efficient.
Expand Down Expand Up @@ -436,7 +436,7 @@ fn ratio(numerator: u32, denominator: u32) -> f64 {
/// Stateless: a note describes the turn's own signals, so every turn they drive
/// carries one. It rides in the forwarded request only, never in the caller's
/// conversation, so notes cannot accumulate across turns.
#[derive(Clone, Debug, Deserialize)]
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct HandoffNoteConfig {
/// Note handed to the capable tier on a signal-driven escalation.
escalation_note: String,
Expand Down
3 changes: 3 additions & 0 deletions crates/switchyard-server/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -129,12 +129,15 @@ documented in [Stage-Router Routing](../../docs/routing_algorithms/stage_router_
| `POST` | `/v1/messages` | Anthropic Messages |
| `POST` | `/v1/responses` | OpenAI Responses |
| `POST` | `/v1/messages/count_tokens` | Token count from a route's Anthropic target |
| `GET` | `/v1/config` | Loaded TOML deployment represented as JSON |
| `GET` | `/v1/models` | Routes served by this deployment |
| `GET` | `/v1/stats` | Per-model usage plus curated algorithm stats |
| `POST` | `/v1/stats/reset` | Clear accumulated stats |
| `GET` | `/metrics` | Prometheus text, see [Metrics](#metrics) |
| `GET` | `/health` | Liveness |

`GET /v1/config` omits the environment variable name configured by `api_key_env`.

Requests name a route by its `id`, so `POST /v1/chat/completions` with `"model": "switchyard/general"`
routes through the `[routes.general]` entry above. Any of the three request formats can address any
route, and the server translates between them.
Expand Down
25 changes: 15 additions & 10 deletions crates/switchyard-server/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ use libsy::{
LlmFallback, LlmTaskClassifier, Noop, Passthrough, PickerMode, Random, StageRouter,
StageRouterConfig, TargetPrompts, TaskClassifierConfig,
};
use serde::Deserialize;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use switchyard_llm_client::{
Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig,
Expand Down Expand Up @@ -46,10 +46,14 @@ pub fn load_server_state(path: impl AsRef<Path>) -> ServerResult<ServerState> {
fn server_state_from_toml(toml: &str) -> ServerResult<ServerState> {
let config: ServerConfig = toml::from_str(toml)
.map_err(|error| ServerError::new(format!("failed to parse TOML: {error}")))?;
config.build()
let deployment_config = serde_json::to_value(&config)
.map_err(|error| ServerError::new(format!("failed to encode config as JSON: {error}")))?;
config
.build()
.map(|state| state.with_deployment_config(deployment_config))
}

#[derive(Debug, Deserialize)]
#[derive(Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
struct ServerConfig {
schema_version: u32,
Expand Down Expand Up @@ -244,11 +248,12 @@ fn count_tokens_priority(target_name: &str, model_id: &ModelId) -> usize {
.unwrap_or(3)
}

#[derive(Debug, Deserialize)]
#[derive(Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
struct LlmClientConfig {
format: ClientFormat,
base_url: String,
#[serde(skip_serializing)]
api_key_env: Option<String>,
#[serde(default)]
forward_auth: bool,
Expand All @@ -258,7 +263,7 @@ struct LlmClientConfig {
max_retries: u32,
}

#[derive(Debug, Deserialize)]
#[derive(Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
struct TargetConfig {
id: ModelId,
Expand All @@ -267,7 +272,7 @@ struct TargetConfig {
extra_body: BTreeMap<String, Value>,
}

#[derive(Clone, Copy, Debug, Deserialize)]
#[derive(Clone, Copy, Debug, Deserialize, Serialize)]
enum ClientFormat {
#[serde(rename = "openai_chat")]
OpenAiChat,
Expand All @@ -286,13 +291,13 @@ impl ClientFormat {
}
}

#[derive(Clone, Debug, Deserialize)]
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
enum ClassifierPolicyConfig {
TargetSelector { selector: String },
}

#[derive(Clone, Copy, Debug, Deserialize)]
#[derive(Clone, Copy, Debug, Deserialize, Serialize)]
#[serde(rename_all = "snake_case")]
enum ClassifierMode {
Capability,
Expand Down Expand Up @@ -352,7 +357,7 @@ struct CustomClassifierRouteConfig {
max_output_tokens: u64,
}

#[derive(Debug, Deserialize)]
#[derive(Debug, Deserialize, Serialize)]
#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
enum RouteConfig {
Noop {
Expand Down Expand Up @@ -459,7 +464,7 @@ enum RouteConfig {
}

/// The judge a `stage_router` route falls through to, and how it routes.
#[derive(Debug, Deserialize)]
#[derive(Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
struct StageClassifierConfig {
/// Target the judge is called through. Not a routing destination.
Expand Down
23 changes: 23 additions & 0 deletions crates/switchyard-server/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,7 @@ impl CountTokensTarget {
#[derive(Clone)]
pub struct ServerState {
routes: Arc<BTreeMap<ModelId, RouteEntry>>,
deployment_config: Option<Arc<Value>>,
metrics: prometheus::Registry,
stats: StatsAccumulator,
routing_log: Option<SharedRoutingLog>,
Expand Down Expand Up @@ -257,13 +258,20 @@ impl ServerState {
);
Ok(Self {
routes: Arc::new(entries),
deployment_config: None,
metrics,
stats,
routing_log: None,
track_cache_eligibility: tracking_enabled_from_env(),
})
}

// Retains the validated source deployment for the config discovery endpoint.
fn with_deployment_config(mut self, config: Value) -> Self {
self.deployment_config = Some(Arc::new(config));
self
}

/// Enables durable per-request routing records at `path`.
pub fn with_routing_log(mut self, path: impl Into<PathBuf>) -> ServerResult<Self> {
self.routing_log = Some(SharedRoutingLog::new(path.into())?);
Expand Down Expand Up @@ -502,6 +510,7 @@ pub fn build_switchyard_router(state: ServerState) -> Router {
.route("/v1/messages", post(anthropic_messages))
.route("/v1/responses", post(openai_responses))
.route("/v1/messages/count_tokens", post(anthropic_count_tokens))
.route("/v1/config", get(get_config))
.route("/v1/models", get(models))
.route("/v1/stats", get(get_stats))
.route("/v1/stats/reset", post(reset_stats))
Expand Down Expand Up @@ -1066,6 +1075,19 @@ async fn models(State(state): State<ServerState>) -> Json<Value> {
))
}

// Returns the source TOML deployment represented as JSON.
async fn get_config(State(state): State<ServerState>) -> Response {
match state.deployment_config {
Some(config) => Json((*config).clone()).into_response(),
None => error_response(
StatusCode::NOT_FOUND,
"server was not loaded from a TOML deployment",
"not_found_error",
"config_not_found",
),
}
}

async fn get_stats(State(state): State<ServerState>) -> Json<StatsSnapshot> {
Json(state.stats.snapshot())
}
Expand Down Expand Up @@ -1339,6 +1361,7 @@ fn endpoint_listing(has_routing_log: bool) -> String {
" POST /v1/messages Anthropic Messages",
" POST /v1/responses OpenAI Responses",
" POST /v1/messages/count_tokens",
" GET /v1/config deployment config",
" GET /v1/models configured routes",
" GET /v1/stats routing stats",
" POST /v1/stats/reset",
Expand Down
39 changes: 39 additions & 0 deletions crates/switchyard-server/tests/server.rs
Original file line number Diff line number Diff line change
Expand Up @@ -680,6 +680,45 @@ fn load_test_config(toml: &str) -> TestResult<ServerState> {
Ok(load_server_state(config.path())?)
}

#[tokio::test]
async fn config_endpoint_returns_loaded_toml_as_json() -> TestResult {
let state = load_test_config(
r#"
schema_version = 1

[llm_clients.provider]
format = "openai_chat"
base_url = "https://example.test/v1"
api_key_env = "PATH"

[targets.fast]
id = "model/fast"
llm_client = "provider"

[routes.default]
id = "switchyard/default"
type = "passthrough"
target = "fast"
"#,
)?;
let response = send(&build_switchyard_router(state), "GET", "/v1/config", None).await?;

assert_eq!(response.status, StatusCode::OK);
let config = response.json()?;
assert_eq!(config["schema_version"], 1);
assert_eq!(config["llm_clients"]["provider"]["format"], "openai_chat");
assert!(
config["llm_clients"]["provider"]
.get("api_key_env")
.is_none()
);
assert_eq!(config["targets"]["fast"]["id"], "model/fast");
assert_eq!(config["targets"]["fast"]["llm_client"], "provider");
assert_eq!(config["routes"]["default"]["type"], "passthrough");
assert_eq!(config["routes"]["default"]["target"], "fast");
Ok(())
}

/// A `random` route that selects `first` before any request-local fallback.
fn fallback_state(base_url: &str) -> TestResult<ServerState> {
load_test_config(&format!(
Expand Down
Loading