From 69eadb57c78f0e45d7676c12955539fa164bfd18 Mon Sep 17 00:00:00 2001 From: "Dr Shivakumara B.V." Date: Mon, 7 Sep 2026 03:40:47 +0530 Subject: [PATCH] fix(inference): honour max_tokens / max_completion_tokens in local server (Refs #5498) --- src/openhuman/inference/http/http_tests.rs | 36 ++++++++++++++++++++++ src/openhuman/inference/http/server.rs | 5 ++- src/openhuman/inference/http/types.rs | 9 ++++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/src/openhuman/inference/http/http_tests.rs b/src/openhuman/inference/http/http_tests.rs index 2fbea3b763..c269b57ca0 100644 --- a/src/openhuman/inference/http/http_tests.rs +++ b/src/openhuman/inference/http/http_tests.rs @@ -140,3 +140,39 @@ fn strip_temperature_suffix_only_removes_numeric_suffixes() { assert_eq!(strip_temperature_suffix("llama3.1:8b@1"), "llama3.1:8b"); assert_eq!(strip_temperature_suffix("gpt@beta"), "gpt@beta"); } + +/// Asserts that `ChatCompletionRequest` parses `max_completion_tokens` and respects +/// fallback precedence over `max_tokens` (Refs #5498). +#[test] +fn test_chat_completion_request_deserializes_max_completion_tokens() { + use crate::openhuman::inference::http::types::ChatCompletionRequest; + + let json_both = serde_json::json!({ + "model": "gpt-5", + "messages": [{ "role": "user", "content": "hello" }], + "max_tokens": 100, + "max_completion_tokens": 200 + }); + let parsed: ChatCompletionRequest = serde_json::from_value(json_both).unwrap(); + assert_eq!(parsed.max_tokens, Some(100)); + assert_eq!(parsed.max_completion_tokens, Some(200)); + assert_eq!( + parsed.max_completion_tokens.or(parsed.max_tokens), + Some(200) + ); + + let json_legacy = serde_json::json!({ + "model": "gpt-4o", + "messages": [{ "role": "user", "content": "hello" }], + "max_tokens": 150 + }); + let parsed_legacy: ChatCompletionRequest = serde_json::from_value(json_legacy).unwrap(); + assert_eq!(parsed_legacy.max_tokens, Some(150)); + assert_eq!(parsed_legacy.max_completion_tokens, None); + assert_eq!( + parsed_legacy + .max_completion_tokens + .or(parsed_legacy.max_tokens), + Some(150) + ); +} diff --git a/src/openhuman/inference/http/server.rs b/src/openhuman/inference/http/server.rs index 272798c97b..985359ef71 100644 --- a/src/openhuman/inference/http/server.rs +++ b/src/openhuman/inference/http/server.rs @@ -145,9 +145,12 @@ async fn chat_completions_handler( let completion_id = format!("chatcmpl-{}", uuid::Uuid::new_v4()); let created = chrono::Utc::now().timestamp(); let model_name = req.model.clone(); - let model_request = ModelRequest::new(messages) + let mut model_request = ModelRequest::new(messages) .with_model(model_id.clone()) .with_temperature(temperature); + if let Some(tokens) = req.max_completion_tokens.or(req.max_tokens) { + model_request = model_request.with_max_tokens(tokens); + } if req.stream { let model_stream = match chat_model.stream(&(), model_request).await { diff --git a/src/openhuman/inference/http/types.rs b/src/openhuman/inference/http/types.rs index 6bf58fe518..206e046986 100644 --- a/src/openhuman/inference/http/types.rs +++ b/src/openhuman/inference/http/types.rs @@ -4,16 +4,25 @@ use serde::{Deserialize, Serialize}; // ── Chat Completions ────────────────────────────────────────────────────────── +/// Request payload for OpenAI-compatible chat completions (`POST /v1/chat/completions`). #[derive(Debug, Deserialize)] pub struct ChatCompletionRequest { + /// Identifier of the model to query. pub model: String, + /// List of input messages in conversation order. pub messages: Vec, + /// Whether to stream back partial progress via SSE chunks. #[serde(default)] pub stream: bool, + /// Sampling temperature between 0.0 and 2.0. #[serde(default)] pub temperature: Option, + /// Legacy maximum tokens limit (superseded by `max_completion_tokens`). #[serde(default)] pub max_tokens: Option, + /// Maximum number of output tokens for newer OpenAI / reasoning models (#5498). + #[serde(default)] + pub max_completion_tokens: Option, /// Optional tool definitions (ignored if the provider doesn't support them). #[serde(default)] pub tools: Option,