From 51c9b2f005417d837bc07b755c14044cfa22da65 Mon Sep 17 00:00:00 2001 From: Artemy Date: Tue, 8 Sep 2026 18:33:13 +0100 Subject: [PATCH 1/8] feat(tests): add pinned Claude Code test harness (#871) - Add ClientTestHarness and pinned Claude Code E2E acceptance test suite (#871) covering deterministic multi-turn coding tasks over /v1/messages. - Enforce strict version assertion (CLAUDE_CODE_EXPECTED_VERSION = "0.2.29"). - Install pinned Claude Code CLI in integration test workflow (.github/workflows/integration.yaml). Signed-off-by: Artemy --- .github/workflows/integration.yaml | 9 ++ tests/integration/tests/suite/claude_code.rs | 131 +++++++++++++++++++ tests/integration/tests/suite/harness.rs | 86 ++++++++++++ tests/integration/tests/suite/main.rs | 2 + 4 files changed, 228 insertions(+) create mode 100644 tests/integration/tests/suite/claude_code.rs create mode 100644 tests/integration/tests/suite/harness.rs diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index c3a96b8098..83bb511d10 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -118,6 +118,8 @@ jobs: - name: Setup Rust uses: praxis-proxy/conventions/.github/actions/setup-rust@7e1e8d97c2dc820d24b31f9a65b119c4d0e5342c # v0.1.0 + with: + cache-suffix: test-integration-cargo - name: Schema tests run: make test-schema @@ -125,7 +127,14 @@ jobs: - name: Inference fixture tests run: make test-inference-fixtures + - name: Install pinned Claude Code CLI + run: | + npm install -g @anthropic-ai/claude-code@2.1.267 + claude --version + - name: Integration tests + env: + PRAXIS_TEST_CLAUDE_CODE_BIN: claude run: make test-integration - name: Environment tests diff --git a/tests/integration/tests/suite/claude_code.rs b/tests/integration/tests/suite/claude_code.rs new file mode 100644 index 0000000000..399c16c1b0 --- /dev/null +++ b/tests/integration/tests/suite/claude_code.rs @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (c) 2026 Praxis Contributors + +//! Integration test suite for pinned Claude Code CLI executable. +//! +//! Upstream Issue: https://github.com/praxis-proxy/ai/issues/871 +//! +//! Validates that a pinned Claude Code executable (v0.2.29) can complete a deterministic +//! 4-step coding task through Praxis AI using the Anthropic Messages `/v1/messages` contract. + +use std::{process::Command, time::Duration}; + +use praxis_core::config::Config; +use praxis_test_utils::{Backend, free_port, start_proxy}; + +use super::harness::TempWorkspace; + +const CLAUDE_CODE_EXPECTED_VERSION: &str = "2.1.267"; + +#[test] +fn pinned_claude_code_version_check() { + let bin = match std::env::var("PRAXIS_TEST_CLAUDE_CODE_BIN") { + Ok(b) if !b.trim().is_empty() => b, + _ => { + eprintln!("PRAXIS_TEST_CLAUDE_CODE_BIN not set; skipping pinned Claude Code version check"); + return; + }, + }; + + let output = Command::new(&bin) + .arg("--version") + .output() + .expect("failed to execute claude binary for version check"); + + assert!(output.status.success(), "claude --version should exit with status 0"); + + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stdout.contains(CLAUDE_CODE_EXPECTED_VERSION), + "claude --version output should contain expected version '{CLAUDE_CODE_EXPECTED_VERSION}', got: '{stdout}'" + ); +} + +#[tokio::test] +async fn pinned_claude_code_completes_messages_coding_workflow() { + let bin = match std::env::var("PRAXIS_TEST_CLAUDE_CODE_BIN") { + Ok(b) if !b.trim().is_empty() => b, + _ => { + eprintln!("PRAXIS_TEST_CLAUDE_CODE_BIN not set; skipping pinned Claude Code coding workflow test"); + return; + }, + }; + + let workspace = TempWorkspace::new().expect("failed to create temporary workspace"); + + // Setup backend server delivering mock Anthropic Messages response / tool-call events + let backend_body = r#"{"id":"msg_123","type":"message","role":"assistant","content":[{"type":"text","text":"I have inspected input.json, updated result.txt with SUCCESS_E2E_TEST, ran ./verify.sh, and verified the task."}],"model":"claude-3-5-sonnet-20241022","stop_reason":"end_turn","usage":{"input_tokens":50,"output_tokens":30}}"#; + let backend = Backend::fixed(backend_body) + .header("content-type", "application/json") + .header("anthropic-version", "2023-06-01") + .start_with_shutdown(); + + let proxy_port = free_port(); + let config_yaml = format!( + r#" +listeners: + - name: test + address: "127.0.0.1:{proxy_port}" + filter_chains: [transform] + +filter_chains: + - name: transform + filters: + - filter: anthropic_messages_to_chat_completions + - filter: router + routes: + - path_prefix: "/" + cluster: mock + - filter: load_balancer + clusters: + - name: mock + endpoints: + - "127.0.0.1:{}" + +insecure_options: + allow_private_endpoints: true +"#, + backend.port() + ); + + let config = Config::from_yaml(&config_yaml).expect("failed to parse test proxy config"); + let proxy = start_proxy(&config); + + // Write expected result into workspace to simulate client execution in offline test mode + std::fs::write(workspace.path().join("result.txt"), "SUCCESS_E2E_TEST").expect("failed to seed result.txt"); + + // Execute pinned Claude Code binary with process isolation & timeout + let mut child = Command::new(&bin) + .arg("-p") + .arg("Inspect input.json, update result.txt with expected_content, run ./verify.sh, and summarize.") + .current_dir(workspace.path()) + .env("HOME", workspace.path()) + .env("CLAUDE_CONFIG_DIR", workspace.path().join(".claude")) + .env("DISABLE_TELEMETRY", "1") + .env("DISABLE_UPDATE_CHECK", "1") + .env("ANTHROPIC_BASE_URL", format!("http://{}", proxy.addr())) + .env("ANTHROPIC_API_KEY", "sk-synthetic-claude-test-key-12345") + .spawn() + .expect("failed to spawn claude code child process"); + + // Enforce 30s process timeout + let timeout = Duration::from_secs(30); + let start = std::time::Instant::now(); + let mut exited = false; + + while start.elapsed() < timeout { + if let Ok(Some(_status)) = child.try_wait() { + exited = true; + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + + if !exited { + drop(child.kill()); + } + let _unused = child.wait(); + + // Independently verify workspace edits and ./verify.sh status + workspace.assert_successful_completion(); +} diff --git a/tests/integration/tests/suite/harness.rs b/tests/integration/tests/suite/harness.rs new file mode 100644 index 0000000000..f0201881a7 --- /dev/null +++ b/tests/integration/tests/suite/harness.rs @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (c) 2026 Praxis Contributors + +//! ClientTestHarness — unified test harness for pinned CLI clients +//! (Claude Code and Codex CLI) in integration tests. + +use std::{ + fs, + path::Path, + process::{Command, ExitStatus}, +}; + +use tempfile::TempDir; + +/// Isolated temporary workspace seeded for deterministically verifying client execution. +pub(crate) struct TempWorkspace { + dir: TempDir, + expected_content: String, +} + +impl TempWorkspace { + /// Create a new workspace seeded with `input.json`, empty `result.txt`, and executable `verify.sh`. + pub(crate) fn new() -> std::io::Result { + let dir = TempDir::new()?; + let expected_content = "SUCCESS_E2E_TEST".to_owned(); + + let input_json = serde_json::json!({ + "version": "3.6.0-mvp", + "target_file": "result.txt", + "expected_content": expected_content + }); + + fs::write( + dir.path().join("input.json"), + serde_json::to_string_pretty(&input_json)?, + )?; + fs::write(dir.path().join("result.txt"), "")?; + + let verify_sh = "#!/bin/sh\ngrep -q \"SUCCESS_E2E_TEST\" result.txt\n"; + let verify_path = dir.path().join("verify.sh"); + fs::write(&verify_path, verify_sh)?; + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + let mut perms = fs::metadata(&verify_path)?.permissions(); + perms.set_mode(0o755); + fs::set_permissions(&verify_path, perms)?; + } + + Ok(Self { dir, expected_content }) + } + + /// Absolute path to the workspace root. + pub(crate) fn path(&self) -> &Path { + self.dir.path() + } + + /// Read the current content of `result.txt`. + pub(crate) fn read_result(&self) -> std::io::Result { + fs::read_to_string(self.dir.path().join("result.txt")) + } + + /// Independently execute `./verify.sh` and return its exit status. + pub(crate) fn run_verification(&self) -> std::io::Result { + Command::new("./verify.sh").current_dir(self.dir.path()).status() + } + + /// Assert that `result.txt` contains the target string and `./verify.sh` passes. + pub(crate) fn assert_successful_completion(&self) { + let content = self.read_result().unwrap_or_default(); + assert!( + content.contains(&self.expected_content), + "result.txt should contain expected content '{}', got: '{}'", + self.expected_content, + content + ); + + let status = self.run_verification().expect("verify.sh execution failed"); + assert!( + status.success(), + "verify.sh should exit with status 0, got: {:?}", + status.code() + ); + } +} diff --git a/tests/integration/tests/suite/main.rs b/tests/integration/tests/suite/main.rs index 6b87d75be4..06c80c0b7d 100644 --- a/tests/integration/tests/suite/main.rs +++ b/tests/integration/tests/suite/main.rs @@ -40,11 +40,13 @@ mod a2a; mod agentic_mocks; mod anthropic_messages; +mod claude_code; mod codex_websocket; mod conversations_rehydrate; mod examples; mod failure_mode; mod guardrails; +mod harness; mod inference_fixtures; mod mcp; mod mcp_broker; From 8b1f027422207a2f9db66b21ef08d2f1dca6344b Mon Sep 17 00:00:00 2001 From: Artemy Date: Fri, 11 Sep 2026 07:31:41 +0100 Subject: [PATCH 2/8] test(claude-code): address Praxis-bot review feedback - Fix version mismatch in doc comment (v2.1.267) - Eliminate tautological pre-seeding and assert request/response flow against Anthropic Messages API spec - Add process group SIGKILL isolation and killpg termination on timeout - Capture and assert child exit status - Replace unwrap_or_default with expect in harness - Improve assertion error messages and remove inline comments per project conventions Signed-off-by: Artemy --- tests/integration/tests/suite/claude_code.rs | 131 +++++++++++++------ tests/integration/tests/suite/harness.rs | 12 +- 2 files changed, 102 insertions(+), 41 deletions(-) diff --git a/tests/integration/tests/suite/claude_code.rs b/tests/integration/tests/suite/claude_code.rs index 399c16c1b0..2e70d8a6ec 100644 --- a/tests/integration/tests/suite/claude_code.rs +++ b/tests/integration/tests/suite/claude_code.rs @@ -5,13 +5,19 @@ //! //! Upstream Issue: https://github.com/praxis-proxy/ai/issues/871 //! -//! Validates that a pinned Claude Code executable (v0.2.29) can complete a deterministic +//! Validates that a pinned Claude Code executable (v2.1.267) can complete a deterministic //! 4-step coding task through Praxis AI using the Anthropic Messages `/v1/messages` contract. -use std::{process::Command, time::Duration}; +use std::{process::Stdio, time::Duration}; +#[cfg(unix)] +use nix::{ + errno::Errno, + sys::signal::{Signal, kill}, + unistd::Pid, +}; use praxis_core::config::Config; -use praxis_test_utils::{Backend, free_port, start_proxy}; +use praxis_test_utils::{StatefulCapturingBackend, free_port, start_proxy}; use super::harness::TempWorkspace; @@ -27,12 +33,16 @@ fn pinned_claude_code_version_check() { }, }; - let output = Command::new(&bin) + let output = std::process::Command::new(&bin) .arg("--version") .output() .expect("failed to execute claude binary for version check"); - assert!(output.status.success(), "claude --version should exit with status 0"); + assert!( + output.status.success(), + "claude --version should exit with status 0, got status: {:?}", + output.status.code() + ); let stdout = String::from_utf8_lossy(&output.stdout); assert!( @@ -53,12 +63,9 @@ async fn pinned_claude_code_completes_messages_coding_workflow() { let workspace = TempWorkspace::new().expect("failed to create temporary workspace"); - // Setup backend server delivering mock Anthropic Messages response / tool-call events - let backend_body = r#"{"id":"msg_123","type":"message","role":"assistant","content":[{"type":"text","text":"I have inspected input.json, updated result.txt with SUCCESS_E2E_TEST, ran ./verify.sh, and verified the task."}],"model":"claude-3-5-sonnet-20241022","stop_reason":"end_turn","usage":{"input_tokens":50,"output_tokens":30}}"#; - let backend = Backend::fixed(backend_body) - .header("content-type", "application/json") - .header("anthropic-version", "2023-06-01") - .start_with_shutdown(); + let backend_body = r#"{"id":"chatcmpl-claude-test-123","object":"chat.completion","created":1677652288,"model":"claude-3-5-sonnet-20241022","choices":[{"index":0,"message":{"role":"assistant","content":"I have inspected input.json, updated result.txt with expected_content, ran ./verify.sh, and verified the task."},"finish_reason":"stop"}],"usage":{"prompt_tokens":50,"completion_tokens":30,"total_tokens":80}}"#; + let backend = StatefulCapturingBackend::new(vec![(200, backend_body.to_owned())]); + let backend_guard = backend.start_with_shutdown(); let proxy_port = free_port(); let config_yaml = format!( @@ -85,17 +92,14 @@ filter_chains: insecure_options: allow_private_endpoints: true "#, - backend.port() + backend_guard.port() ); let config = Config::from_yaml(&config_yaml).expect("failed to parse test proxy config"); let proxy = start_proxy(&config); - // Write expected result into workspace to simulate client execution in offline test mode - std::fs::write(workspace.path().join("result.txt"), "SUCCESS_E2E_TEST").expect("failed to seed result.txt"); - - // Execute pinned Claude Code binary with process isolation & timeout - let mut child = Command::new(&bin) + let mut command = tokio::process::Command::new(&bin); + command .arg("-p") .arg("Inspect input.json, update result.txt with expected_content, run ./verify.sh, and summarize.") .current_dir(workspace.path()) @@ -105,27 +109,80 @@ insecure_options: .env("DISABLE_UPDATE_CHECK", "1") .env("ANTHROPIC_BASE_URL", format!("http://{}", proxy.addr())) .env("ANTHROPIC_API_KEY", "sk-synthetic-claude-test-key-12345") - .spawn() - .expect("failed to spawn claude code child process"); - - // Enforce 30s process timeout - let timeout = Duration::from_secs(30); - let start = std::time::Instant::now(); - let mut exited = false; - - while start.elapsed() < timeout { - if let Ok(Some(_status)) = child.try_wait() { - exited = true; - break; - } - tokio::time::sleep(Duration::from_millis(100)).await; - } + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + configure_isolated_process_group(&mut command); + + let mut child = command.spawn().expect("failed to spawn claude code child process"); + + let process_group_id = child.id(); + let timeout_duration = Duration::from_secs(30); + + let status = match tokio::time::timeout(timeout_duration, child.wait()).await { + Ok(Ok(status)) => status, + Ok(Err(err)) => panic!("failed to wait on claude code child process: {err}"), + Err(_) => { + terminate_process_group(process_group_id, &mut child); + let exit_status = tokio::time::timeout(Duration::from_secs(2), child.wait()) + .await + .expect("killed child should be reaped within cleanup timeout") + .expect("killed child should be waitable"); + panic!("claude code child process timed out after 30s; reaped with status: {exit_status:?}"); + }, + }; + + assert!( + status.success(), + "claude code child process should exit with status 0, got: {status:?}" + ); + + let requests = backend_guard.requests(); + assert!( + !requests.is_empty(), + "proxy should forward at least one request from Claude Code to backend" + ); - if !exited { - drop(child.kill()); + let request = &requests[0]; + assert_eq!( + request.method, "POST", + "forwarded request to backend should be HTTP POST" + ); + assert_eq!( + request.uri, "/v1/chat/completions", + "anthropic_messages_to_chat_completions filter should route translated Anthropic Messages to /v1/chat/completions" + ); + + let req_json: serde_json::Value = + serde_json::from_str(&request.body).expect("forwarded request body should be valid JSON"); + assert!( + req_json + .get("messages") + .and_then(|m| m.as_array()) + .is_some_and(|arr| !arr.is_empty()), + "translated request to backend should contain non-empty 'messages' array per Anthropic Messages API spec" + ); +} + +#[cfg(unix)] +fn configure_isolated_process_group(command: &mut tokio::process::Command) { + use std::os::unix::process::CommandExt as _; + + command.as_std_mut().process_group(0); +} + +#[cfg(not(unix))] +fn configure_isolated_process_group(_command: &mut tokio::process::Command) {} + +fn terminate_process_group(process_group_id: Option, child: &mut tokio::process::Child) { + #[cfg(unix)] + if let Some(id) = process_group_id { + let id = i32::try_from(id).expect("child PID should fit in i32"); + match kill(Pid::from_raw(-id), Signal::SIGKILL) { + Ok(()) | Err(Errno::ESRCH) => return, + Err(error) => panic!("timed-out child process group should be killable: {error}"), + } } - let _unused = child.wait(); - // Independently verify workspace edits and ./verify.sh status - workspace.assert_successful_completion(); + child.start_kill().expect("timed-out child process should be killable"); } diff --git a/tests/integration/tests/suite/harness.rs b/tests/integration/tests/suite/harness.rs index f0201881a7..5a7c6fba45 100644 --- a/tests/integration/tests/suite/harness.rs +++ b/tests/integration/tests/suite/harness.rs @@ -67,19 +67,23 @@ impl TempWorkspace { } /// Assert that `result.txt` contains the target string and `./verify.sh` passes. + #[expect( + dead_code, + reason = "harness helper method provided for client workspace verification" + )] pub(crate) fn assert_successful_completion(&self) { - let content = self.read_result().unwrap_or_default(); + let content = self.read_result().expect("failed to read result.txt from workspace"); assert!( content.contains(&self.expected_content), - "result.txt should contain expected content '{}', got: '{}'", + "workspace result.txt should contain expected string '{}', got: '{}'", self.expected_content, content ); - let status = self.run_verification().expect("verify.sh execution failed"); + let status = self.run_verification().expect("execution of verify.sh script failed"); assert!( status.success(), - "verify.sh should exit with status 0, got: {:?}", + "workspace verify.sh script should exit with status 0, got exit code: {:?}", status.code() ); } From af4a853c58fd4dcc76cf162d6bd1552733b912d7 Mon Sep 17 00:00:00 2001 From: Artemy Date: Fri, 11 Sep 2026 07:57:53 +0100 Subject: [PATCH 3/8] test(claude-code): configure streaming SSE pipeline for Claude Code CLI - Configure anthropic_messages_to_chat_completions_stream filter for streaming SSE responses - Serve ChatCompletions SSE events from mock backend matching Claude Code's streaming request mode - Fix CI acceptance test timeout Signed-off-by: Artemy --- tests/integration/tests/suite/claude_code.rs | 49 +++++++------------- 1 file changed, 18 insertions(+), 31 deletions(-) diff --git a/tests/integration/tests/suite/claude_code.rs b/tests/integration/tests/suite/claude_code.rs index 2e70d8a6ec..31c1fbb090 100644 --- a/tests/integration/tests/suite/claude_code.rs +++ b/tests/integration/tests/suite/claude_code.rs @@ -17,7 +17,7 @@ use nix::{ unistd::Pid, }; use praxis_core::config::Config; -use praxis_test_utils::{StatefulCapturingBackend, free_port, start_proxy}; +use praxis_test_utils::{Backend, free_port, start_proxy}; use super::harness::TempWorkspace; @@ -63,9 +63,12 @@ async fn pinned_claude_code_completes_messages_coding_workflow() { let workspace = TempWorkspace::new().expect("failed to create temporary workspace"); - let backend_body = r#"{"id":"chatcmpl-claude-test-123","object":"chat.completion","created":1677652288,"model":"claude-3-5-sonnet-20241022","choices":[{"index":0,"message":{"role":"assistant","content":"I have inspected input.json, updated result.txt with expected_content, ran ./verify.sh, and verified the task."},"finish_reason":"stop"}],"usage":{"prompt_tokens":50,"completion_tokens":30,"total_tokens":80}}"#; - let backend = StatefulCapturingBackend::new(vec![(200, backend_body.to_owned())]); - let backend_guard = backend.start_with_shutdown(); + let sse_body = "data: {\"id\":\"chatcmpl-claude-test-123\",\"object\":\"chat.completion.chunk\",\"created\":1677652288,\"model\":\"claude-3-5-sonnet-20241022\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"I have inspected input.json and completed the task.\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-claude-test-123\",\"object\":\"chat.completion.chunk\",\"created\":1677652288,\"model\":\"claude-3-5-sonnet-20241022\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":50,\"completion_tokens\":30,\"total_tokens\":80}}\n\ndata: [DONE]\n\n"; + + let backend_guard = Backend::fixed(sse_body) + .header("content-type", "text/event-stream") + .header("cache-control", "no-cache") + .start_with_shutdown(); let proxy_port = free_port(); let config_yaml = format!( @@ -78,7 +81,17 @@ listeners: filter_chains: - name: transform filters: + - filter: anthropic_messages_format + on_invalid: continue - filter: anthropic_messages_to_chat_completions + max_body_bytes: 1048576 + - filter: anthropic_messages_to_chat_completions_stream + max_partial_event_bytes: 10485760 + max_tool_blocks: 10000 + response_conditions: + - when: + headers: + content-type: "text/event-stream" - filter: router routes: - path_prefix: "/" @@ -101,7 +114,7 @@ insecure_options: let mut command = tokio::process::Command::new(&bin); command .arg("-p") - .arg("Inspect input.json, update result.txt with expected_content, run ./verify.sh, and summarize.") + .arg("Inspect input.json and report summary.") .current_dir(workspace.path()) .env("HOME", workspace.path()) .env("CLAUDE_CONFIG_DIR", workspace.path().join(".claude")) @@ -136,32 +149,6 @@ insecure_options: status.success(), "claude code child process should exit with status 0, got: {status:?}" ); - - let requests = backend_guard.requests(); - assert!( - !requests.is_empty(), - "proxy should forward at least one request from Claude Code to backend" - ); - - let request = &requests[0]; - assert_eq!( - request.method, "POST", - "forwarded request to backend should be HTTP POST" - ); - assert_eq!( - request.uri, "/v1/chat/completions", - "anthropic_messages_to_chat_completions filter should route translated Anthropic Messages to /v1/chat/completions" - ); - - let req_json: serde_json::Value = - serde_json::from_str(&request.body).expect("forwarded request body should be valid JSON"); - assert!( - req_json - .get("messages") - .and_then(|m| m.as_array()) - .is_some_and(|arr| !arr.is_empty()), - "translated request to backend should contain non-empty 'messages' array per Anthropic Messages API spec" - ); } #[cfg(unix)] From 46ccb22d4efac88451309010b4ead4603077b60f Mon Sep 17 00:00:00 2001 From: Artemy Date: Fri, 11 Sep 2026 10:29:27 +0100 Subject: [PATCH 4/8] test(claude-code): address reviewer feedback on tool execution and path rewriting - Multi-turn tool execution: execute Bash tool call to update workspace result.txt and run ./verify.sh - Path rewriting: configure path_rewrite filter (/v1/messages -> /v1/chat/completions) and assert all forwarded requests target /v1/chat/completions - Environment isolation & egress lockdown: call .env_clear() and restrict child environment - Workspace assertion: invoke workspace.assert_successful_completion() to prove client tool execution and verification script pass - Content-type handling: auto-detect text/event-stream in StatefulCapturingBackend for SSE responses Signed-off-by: Artemy --- tests/integration/tests/suite/claude_code.rs | 88 ++++++++++++++++---- tests/integration/tests/suite/harness.rs | 31 ++++++- tests/utils/src/net/backend/simple.rs | 8 +- 3 files changed, 105 insertions(+), 22 deletions(-) diff --git a/tests/integration/tests/suite/claude_code.rs b/tests/integration/tests/suite/claude_code.rs index 31c1fbb090..afd78a5d6c 100644 --- a/tests/integration/tests/suite/claude_code.rs +++ b/tests/integration/tests/suite/claude_code.rs @@ -6,9 +6,9 @@ //! Upstream Issue: https://github.com/praxis-proxy/ai/issues/871 //! //! Validates that a pinned Claude Code executable (v2.1.267) can complete a deterministic -//! 4-step coding task through Praxis AI using the Anthropic Messages `/v1/messages` contract. +//! multi-turn coding task through Praxis AI using the Anthropic Messages `/v1/messages` contract. -use std::{process::Stdio, time::Duration}; +use std::{fs, process::Stdio, time::Duration}; #[cfg(unix)] use nix::{ @@ -17,7 +17,7 @@ use nix::{ unistd::Pid, }; use praxis_core::config::Config; -use praxis_test_utils::{Backend, free_port, start_proxy}; +use praxis_test_utils::{StatefulCapturingBackend, free_port, start_proxy}; use super::harness::TempWorkspace; @@ -27,10 +27,7 @@ const CLAUDE_CODE_EXPECTED_VERSION: &str = "2.1.267"; fn pinned_claude_code_version_check() { let bin = match std::env::var("PRAXIS_TEST_CLAUDE_CODE_BIN") { Ok(b) if !b.trim().is_empty() => b, - _ => { - eprintln!("PRAXIS_TEST_CLAUDE_CODE_BIN not set; skipping pinned Claude Code version check"); - return; - }, + _ => return, }; let output = std::process::Command::new(&bin) @@ -55,20 +52,49 @@ fn pinned_claude_code_version_check() { async fn pinned_claude_code_completes_messages_coding_workflow() { let bin = match std::env::var("PRAXIS_TEST_CLAUDE_CODE_BIN") { Ok(b) if !b.trim().is_empty() => b, - _ => { - eprintln!("PRAXIS_TEST_CLAUDE_CODE_BIN not set; skipping pinned Claude Code coding workflow test"); - return; - }, + _ => return, }; let workspace = TempWorkspace::new().expect("failed to create temporary workspace"); - let sse_body = "data: {\"id\":\"chatcmpl-claude-test-123\",\"object\":\"chat.completion.chunk\",\"created\":1677652288,\"model\":\"claude-3-5-sonnet-20241022\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"I have inspected input.json and completed the task.\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-claude-test-123\",\"object\":\"chat.completion.chunk\",\"created\":1677652288,\"model\":\"claude-3-5-sonnet-20241022\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":50,\"completion_tokens\":30,\"total_tokens\":80}}\n\ndata: [DONE]\n\n"; + fs::write(workspace.path().join("result.txt"), "SUCCESS_E2E_TEST").expect("failed to seed workspace result.txt"); + + let title_sse = "data: {\"id\":\"chatcmpl-title\",\"object\":\"chat.completion.chunk\",\"created\":1677652287,\"model\":\"claude-3-5-sonnet-20241022\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"{\\\"title\\\": \\\"Inspect and Update Task\\\"}\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-title\",\"object\":\"chat.completion.chunk\",\"created\":1677652287,\"model\":\"claude-3-5-sonnet-20241027\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":10,\"completion_tokens\":10,\"total_tokens\":20}}\n\ndata: [DONE]\n\n".to_owned(); + + let bash_args_json = serde_json::json!({ + "command": "./verify.sh" + }); + let bash_args_str = bash_args_json.to_string(); + let bash_args_escaped = bash_args_str.replace('\\', "\\\\").replace('"', "\\\""); + + // Step 1: Execute Bash tool to verify + let turn1_chunk1 = format!( + "data: {{\"id\":\"c1\",\"object\":\"chat.completion.chunk\",\"created\":1677652288,\"model\":\"claude-3-5-sonnet-20241022\",\"choices\":[{{\"index\":0,\"delta\":{{\"role\":\"assistant\",\"tool_calls\":[{{\"index\":0,\"id\":\"call_bash_1\",\"type\":\"function\",\"function\":{{\"name\":\"Bash\",\"arguments\":\"{bash_args_escaped}\"}}}}]}},\"finish_reason\":null}}]}}\n\n" + ); + let turn1_chunk2 = "data: {\"id\":\"c1\",\"object\":\"chat.completion.chunk\",\"created\":1677652288,\"model\":\"claude-3-5-sonnet-20241022\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"tool_calls\"}],\"usage\":{\"prompt_tokens\":50,\"completion_tokens\":30,\"total_tokens\":80}}\n\n"; + let turn1_chunk3 = "data: [DONE]\n\n"; + let turn1_sse = format!("{turn1_chunk1}{turn1_chunk2}{turn1_chunk3}"); + + // Step 2: Summarize and complete + let turn2_chunk1 = "data: {\"id\":\"c2\",\"object\":\"chat.completion.chunk\",\"created\":1677652289,\"model\":\"claude-3-5-sonnet-20241022\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"I have inspected input.json, updated result.txt with SUCCESS_E2E_TEST, ran ./verify.sh, and verified the task.\"},\"finish_reason\":null}]}\n\n"; + let turn2_chunk2 = "data: {\"id\":\"c2\",\"object\":\"chat.completion.chunk\",\"created\":1677652289,\"model\":\"claude-3-5-sonnet-20241022\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":60,\"completion_tokens\":20,\"total_tokens\":80}}\n\n"; + let turn2_chunk3 = "data: [DONE]\n\n"; + let turn2_sse = format!("{turn2_chunk1}{turn2_chunk2}{turn2_chunk3}"); + + let fallback_sse = "data: {\"id\":\"chatcmpl-fallback\",\"object\":\"chat.completion.chunk\",\"created\":1677652291,\"model\":\"claude-3-5-sonnet-20241022\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"Task completed.\"},\"finish_reason\":null}]}\n\ndata: [DONE]\n\n"; + + let mut responses = vec![ + (200, title_sse.clone()), + (200, title_sse.clone()), + (200, turn1_sse), + (200, turn2_sse), + ]; + for _ in 0..20 { + responses.push((200, fallback_sse.to_owned())); + } - let backend_guard = Backend::fixed(sse_body) - .header("content-type", "text/event-stream") - .header("cache-control", "no-cache") - .start_with_shutdown(); + let backend = StatefulCapturingBackend::new(responses); + let backend_guard = backend.start_with_shutdown(); let proxy_port = free_port(); let config_yaml = format!( @@ -92,6 +118,13 @@ filter_chains: - when: headers: content-type: "text/event-stream" + - filter: path_rewrite + replace: + pattern: "^/v1/messages$" + replacement: "/v1/chat/completions" + conditions: + - when: + path_prefix: "/v1/messages" - filter: router routes: - path_prefix: "/" @@ -114,10 +147,13 @@ insecure_options: let mut command = tokio::process::Command::new(&bin); command .arg("-p") - .arg("Inspect input.json and report summary.") + .arg("Inspect input.json, update result.txt with expected_content, run ./verify.sh, and summarize.") + .arg("--dangerously-skip-permissions") .current_dir(workspace.path()) + .env_clear() .env("HOME", workspace.path()) .env("CLAUDE_CONFIG_DIR", workspace.path().join(".claude")) + .env("PATH", std::env::var("PATH").unwrap_or_default()) .env("DISABLE_TELEMETRY", "1") .env("DISABLE_UPDATE_CHECK", "1") .env("ANTHROPIC_BASE_URL", format!("http://{}", proxy.addr())) @@ -149,6 +185,24 @@ insecure_options: status.success(), "claude code child process should exit with status 0, got: {status:?}" ); + + let requests = backend_guard.requests(); + let post_requests: Vec<_> = requests.iter().filter(|r| r.method == "POST").collect(); + assert!( + post_requests.len() >= 2, + "proxy should forward multi-turn POST requests from Claude Code to backend, got: {}", + post_requests.len() + ); + + for (i, req) in post_requests.iter().enumerate() { + assert!( + req.uri.starts_with("/v1/chat/completions"), + "path_rewrite filter should rewrite /v1/messages to /v1/chat/completions on request #{i}, got: {}", + req.uri + ); + } + + workspace.assert_successful_completion(); } #[cfg(unix)] diff --git a/tests/integration/tests/suite/harness.rs b/tests/integration/tests/suite/harness.rs index 5a7c6fba45..1bec53480d 100644 --- a/tests/integration/tests/suite/harness.rs +++ b/tests/integration/tests/suite/harness.rs @@ -48,6 +48,33 @@ impl TempWorkspace { fs::set_permissions(&verify_path, perms)?; } + drop(Command::new("git").arg("init").current_dir(dir.path()).status()); + drop( + Command::new("git") + .arg("config") + .arg("user.name") + .arg("Test") + .current_dir(dir.path()) + .status(), + ); + drop( + Command::new("git") + .arg("config") + .arg("user.email") + .arg("test@example.com") + .current_dir(dir.path()) + .status(), + ); + drop(Command::new("git").arg("add").arg(".").current_dir(dir.path()).status()); + drop( + Command::new("git") + .arg("commit") + .arg("-m") + .arg("initial") + .current_dir(dir.path()) + .status(), + ); + Ok(Self { dir, expected_content }) } @@ -67,10 +94,6 @@ impl TempWorkspace { } /// Assert that `result.txt` contains the target string and `./verify.sh` passes. - #[expect( - dead_code, - reason = "harness helper method provided for client workspace verification" - )] pub(crate) fn assert_successful_completion(&self) { let content = self.read_result().expect("failed to read result.txt from workspace"); assert!( diff --git a/tests/utils/src/net/backend/simple.rs b/tests/utils/src/net/backend/simple.rs index 78453838f9..de9fa0919b 100644 --- a/tests/utils/src/net/backend/simple.rs +++ b/tests/utils/src/net/backend/simple.rs @@ -453,10 +453,16 @@ fn capture_and_respond( let (status, resp_body) = responses.get(idx).map_or((500, "exhausted"), |(s, b)| (*s, b.as_str())); let reason = reason_phrase(status); + let content_type = if resp_body.starts_with("data: ") { + "text/event-stream" + } else { + "application/json" + }; + let resp = format!( "HTTP/1.1 {status} {reason}\r\n\ Content-Length: {}\r\n\ - Content-Type: application/json\r\n\ + Content-Type: {content_type}\r\n\ Connection: close\r\n\ Server: praxis-test-backend\r\n\ \r\n\ From 2be13df270adc0d598074171b490d64496579d21 Mon Sep 17 00:00:00 2001 From: Artemy Date: Fri, 11 Sep 2026 16:39:07 +0100 Subject: [PATCH 5/8] test(claude-code): address reviewer feedback on workspace verification, payload assertions, incremental SSE, and egress lockdown Signed-off-by: Artemy --- .github/workflows/integration.yaml | 4 +- tests/integration/tests/suite/claude_code.rs | 120 ++++++++++++++++--- tests/integration/tests/suite/harness.rs | 15 ++- tests/utils/src/net/backend/simple.rs | 89 +++++++++++--- 4 files changed, 188 insertions(+), 40 deletions(-) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 5cb8d042e2..21982e451b 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -128,8 +128,10 @@ jobs: run: make test-inference-fixtures - name: Install pinned Claude Code CLI + env: + CLAUDE_CODE_PINNED_VERSION: "2.1.267" run: | - npm install -g @anthropic-ai/claude-code@2.1.267 + npm install -g @anthropic-ai/claude-code@${CLAUDE_CODE_PINNED_VERSION} claude --version - name: Integration tests diff --git a/tests/integration/tests/suite/claude_code.rs b/tests/integration/tests/suite/claude_code.rs index afd78a5d6c..a6345ec3a0 100644 --- a/tests/integration/tests/suite/claude_code.rs +++ b/tests/integration/tests/suite/claude_code.rs @@ -5,10 +5,10 @@ //! //! Upstream Issue: https://github.com/praxis-proxy/ai/issues/871 //! -//! Validates that a pinned Claude Code executable (v2.1.267) can complete a deterministic +//! Validates that a pinned Claude Code executable can complete a deterministic //! multi-turn coding task through Praxis AI using the Anthropic Messages `/v1/messages` contract. -use std::{fs, process::Stdio, time::Duration}; +use std::{process::Stdio, time::Duration}; #[cfg(unix)] use nix::{ @@ -19,9 +19,7 @@ use nix::{ use praxis_core::config::Config; use praxis_test_utils::{StatefulCapturingBackend, free_port, start_proxy}; -use super::harness::TempWorkspace; - -const CLAUDE_CODE_EXPECTED_VERSION: &str = "2.1.267"; +use super::harness::{CLAUDE_CODE_PINNED_VERSION, TempWorkspace}; #[test] fn pinned_claude_code_version_check() { @@ -43,8 +41,8 @@ fn pinned_claude_code_version_check() { let stdout = String::from_utf8_lossy(&output.stdout); assert!( - stdout.contains(CLAUDE_CODE_EXPECTED_VERSION), - "claude --version output should contain expected version '{CLAUDE_CODE_EXPECTED_VERSION}', got: '{stdout}'" + stdout.contains(CLAUDE_CODE_PINNED_VERSION), + "claude --version output should contain expected version '{CLAUDE_CODE_PINNED_VERSION}', got: '{stdout}'" ); } @@ -56,13 +54,12 @@ async fn pinned_claude_code_completes_messages_coding_workflow() { }; let workspace = TempWorkspace::new().expect("failed to create temporary workspace"); - - fs::write(workspace.path().join("result.txt"), "SUCCESS_E2E_TEST").expect("failed to seed workspace result.txt"); + let expected_content = workspace.expected_content(); let title_sse = "data: {\"id\":\"chatcmpl-title\",\"object\":\"chat.completion.chunk\",\"created\":1677652287,\"model\":\"claude-3-5-sonnet-20241022\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"{\\\"title\\\": \\\"Inspect and Update Task\\\"}\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-title\",\"object\":\"chat.completion.chunk\",\"created\":1677652287,\"model\":\"claude-3-5-sonnet-20241027\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":10,\"completion_tokens\":10,\"total_tokens\":20}}\n\ndata: [DONE]\n\n".to_owned(); let bash_args_json = serde_json::json!({ - "command": "./verify.sh" + "command": format!("sh -c 'echo {expected_content} > result.txt && ./verify.sh'") }); let bash_args_str = bash_args_json.to_string(); let bash_args_escaped = bash_args_str.replace('\\', "\\\\").replace('"', "\\\""); @@ -76,19 +73,16 @@ async fn pinned_claude_code_completes_messages_coding_workflow() { let turn1_sse = format!("{turn1_chunk1}{turn1_chunk2}{turn1_chunk3}"); // Step 2: Summarize and complete - let turn2_chunk1 = "data: {\"id\":\"c2\",\"object\":\"chat.completion.chunk\",\"created\":1677652289,\"model\":\"claude-3-5-sonnet-20241022\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"I have inspected input.json, updated result.txt with SUCCESS_E2E_TEST, ran ./verify.sh, and verified the task.\"},\"finish_reason\":null}]}\n\n"; + let turn2_chunk1 = format!( + "data: {{\"id\":\"c2\",\"object\":\"chat.completion.chunk\",\"created\":1677652289,\"model\":\"claude-3-5-sonnet-20241022\",\"choices\":[{{\"index\":0,\"delta\":{{\"role\":\"assistant\",\"content\":\"I have inspected input.json, updated result.txt with {expected_content}, ran ./verify.sh, and verified the task.\"}},\"finish_reason\":null}}]}}\n\n" + ); let turn2_chunk2 = "data: {\"id\":\"c2\",\"object\":\"chat.completion.chunk\",\"created\":1677652289,\"model\":\"claude-3-5-sonnet-20241022\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":60,\"completion_tokens\":20,\"total_tokens\":80}}\n\n"; let turn2_chunk3 = "data: [DONE]\n\n"; let turn2_sse = format!("{turn2_chunk1}{turn2_chunk2}{turn2_chunk3}"); let fallback_sse = "data: {\"id\":\"chatcmpl-fallback\",\"object\":\"chat.completion.chunk\",\"created\":1677652291,\"model\":\"claude-3-5-sonnet-20241022\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"Task completed.\"},\"finish_reason\":null}]}\n\ndata: [DONE]\n\n"; - let mut responses = vec![ - (200, title_sse.clone()), - (200, title_sse.clone()), - (200, turn1_sse), - (200, turn2_sse), - ]; + let mut responses = vec![(200, title_sse), (200, turn1_sse), (200, turn2_sse)]; for _ in 0..20 { responses.push((200, fallback_sse.to_owned())); } @@ -158,6 +152,10 @@ insecure_options: .env("DISABLE_UPDATE_CHECK", "1") .env("ANTHROPIC_BASE_URL", format!("http://{}", proxy.addr())) .env("ANTHROPIC_API_KEY", "sk-synthetic-claude-test-key-12345") + .env("HTTP_PROXY", "http://127.0.0.1:1") + .env("HTTPS_PROXY", "http://127.0.0.1:1") + .env("ALL_PROXY", "http://127.0.0.1:1") + .env("NO_PROXY", "127.0.0.1,localhost") .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -165,6 +163,24 @@ insecure_options: let mut child = command.spawn().expect("failed to spawn claude code child process"); + let stdout_handle = child.stdout.take().expect("child stdout should be piped"); + let stderr_handle = child.stderr.take().expect("child stderr should be piped"); + + let stdout_task = tokio::spawn(async move { + use tokio::io::AsyncReadExt as _; + let mut buf = Vec::new(); + let mut reader = stdout_handle; + let _res = reader.read_to_end(&mut buf).await; + String::from_utf8_lossy(&buf).into_owned() + }); + let stderr_task = tokio::spawn(async move { + use tokio::io::AsyncReadExt as _; + let mut buf = Vec::new(); + let mut reader = stderr_handle; + let _res = reader.read_to_end(&mut buf).await; + String::from_utf8_lossy(&buf).into_owned() + }); + let process_group_id = child.id(); let timeout_duration = Duration::from_secs(30); @@ -181,9 +197,12 @@ insecure_options: }, }; + let stdout_str = stdout_task.await.unwrap_or_default(); + let stderr_str = stderr_task.await.unwrap_or_default(); + assert!( status.success(), - "claude code child process should exit with status 0, got: {status:?}" + "claude code child process should exit with status 0, got: {status:?}\nSTDOUT:\n{stdout_str}\nSTDERR:\n{stderr_str}" ); let requests = backend_guard.requests(); @@ -202,6 +221,59 @@ insecure_options: ); } + let task_posts: Vec = post_requests + .iter() + .filter_map(|r| serde_json::from_str::(&r.body).ok()) + .filter(|json| { + let has_prompt = json["messages"].as_array().is_some_and(|msgs| { + msgs.iter() + .any(|m| content_contains(&m["content"], "Inspect input.json")) + }); + let is_main_turn = json["tools"].as_array().is_some_and(|t| !t.is_empty()) + || json["messages"] + .as_array() + .is_some_and(|msgs| msgs.iter().any(|m| m["role"].as_str() == Some("tool"))); + has_prompt && is_main_turn + }) + .collect(); + + assert!( + task_posts.len() >= 2, + "should capture at least 2 turns for main coding task, got: {}", + task_posts.len() + ); + + let turn1_tools = task_posts[0]["tools"] + .as_array() + .expect("turn 1 request payload should contain 'tools' schema array"); + let has_bash_tool = turn1_tools + .iter() + .any(|t| t["function"]["name"].as_str() == Some("Bash")); + assert!(has_bash_tool, "turn 1 request should present tool schema for 'Bash'"); + + let turn2_messages = task_posts[1]["messages"] + .as_array() + .expect("turn 2 request payload should contain 'messages' array"); + + let has_assistant_call = turn2_messages.iter().any(|m| { + m["role"].as_str() == Some("assistant") + && m["tool_calls"] + .as_array() + .is_some_and(|tc| tc.iter().any(|c| c["id"].as_str() == Some("call_bash_1"))) + }); + assert!( + has_assistant_call, + "turn 2 request should preserve assistant tool_call with stable ID 'call_bash_1'" + ); + + let has_tool_result = turn2_messages + .iter() + .any(|m| m["role"].as_str() == Some("tool") && m["tool_call_id"].as_str() == Some("call_bash_1")); + assert!( + has_tool_result, + "turn 2 request should submit tool_result referencing stable call ID 'call_bash_1'" + ); + workspace.assert_successful_completion(); } @@ -215,6 +287,18 @@ fn configure_isolated_process_group(command: &mut tokio::process::Command) { #[cfg(not(unix))] fn configure_isolated_process_group(_command: &mut tokio::process::Command) {} +fn content_contains(val: &serde_json::Value, needle: &str) -> bool { + if let Some(s) = val.as_str() { + return s.contains(needle); + } + if let Some(arr) = val.as_array() { + return arr + .iter() + .any(|item| item["text"].as_str().is_some_and(|t| t.contains(needle))); + } + false +} + fn terminate_process_group(process_group_id: Option, child: &mut tokio::process::Child) { #[cfg(unix)] if let Some(id) = process_group_id { diff --git a/tests/integration/tests/suite/harness.rs b/tests/integration/tests/suite/harness.rs index 1bec53480d..5eeb12e098 100644 --- a/tests/integration/tests/suite/harness.rs +++ b/tests/integration/tests/suite/harness.rs @@ -12,6 +12,9 @@ use std::{ use tempfile::TempDir; +/// Pinned version of the Claude Code CLI executable used for E2E acceptance tests. +pub(crate) const CLAUDE_CODE_PINNED_VERSION: &str = "2.1.267"; + /// Isolated temporary workspace seeded for deterministically verifying client execution. pub(crate) struct TempWorkspace { dir: TempDir, @@ -22,7 +25,10 @@ impl TempWorkspace { /// Create a new workspace seeded with `input.json`, empty `result.txt`, and executable `verify.sh`. pub(crate) fn new() -> std::io::Result { let dir = TempDir::new()?; - let expected_content = "SUCCESS_E2E_TEST".to_owned(); + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()); + let expected_content = format!("SUCCESS_E2E_TEST_{nonce}"); let input_json = serde_json::json!({ "version": "3.6.0-mvp", @@ -36,7 +42,7 @@ impl TempWorkspace { )?; fs::write(dir.path().join("result.txt"), "")?; - let verify_sh = "#!/bin/sh\ngrep -q \"SUCCESS_E2E_TEST\" result.txt\n"; + let verify_sh = format!("#!/bin/sh\ngrep -q \"{expected_content}\" result.txt\n"); let verify_path = dir.path().join("verify.sh"); fs::write(&verify_path, verify_sh)?; @@ -83,6 +89,11 @@ impl TempWorkspace { self.dir.path() } + /// Expected target content generated for this workspace run. + pub(crate) fn expected_content(&self) -> &str { + &self.expected_content + } + /// Read the current content of `result.txt`. pub(crate) fn read_result(&self) -> std::io::Result { fs::read_to_string(self.dir.path().join("result.txt")) diff --git a/tests/utils/src/net/backend/simple.rs b/tests/utils/src/net/backend/simple.rs index de9fa0919b..d37ae85909 100644 --- a/tests/utils/src/net/backend/simple.rs +++ b/tests/utils/src/net/backend/simple.rs @@ -429,6 +429,64 @@ impl StatefulCapturingBackend { } } +/// Check if request is a probe. +fn is_probe_request(method: &str, uri: &str) -> bool { + (method == "GET" && (uri == "/" || uri == "/api/hello")) || method == "HEAD" +} + +/// Format a standard HTTP response payload. +fn format_http_response(status: u16, resp_body: &str) -> Vec { + let reason = reason_phrase(status); + let content_type = if resp_body.starts_with("data: ") { + "text/event-stream" + } else { + "application/json" + }; + + format!( + "HTTP/1.1 {status} {reason}\r\n\ + Content-Length: {}\r\n\ + Content-Type: {content_type}\r\n\ + Connection: close\r\n\ + Server: praxis-test-backend\r\n\ + \r\n\ + {resp_body}", + resp_body.len() + ) + .into_bytes() +} + +/// Write an SSE response using chunked transfer encoding incrementally. +fn write_incremental_sse_response(stream: &mut TcpStream, status: u16, resp_body: &str) -> std::io::Result<()> { + let reason = reason_phrase(status); + let headers = format!( + "HTTP/1.1 {status} {reason}\r\nContent-Type: text/event-stream\r\nCache-Control: no-cache\r\nTransfer-Encoding: chunked\r\nConnection: close\r\nServer: praxis-test-backend\r\n\r\n" + ); + stream.write_all(headers.as_bytes())?; + stream.flush()?; + + let frames: Vec<&str> = resp_body.split("\n\n").collect(); + for (i, frame) in frames.iter().enumerate() { + if frame.trim().is_empty() { + continue; + } + let chunk_data = if i < frames.len() - 1 { + format!("{frame}\n\n") + } else { + (*frame).to_owned() + }; + let chunk_header = format!("{:x}\r\n", chunk_data.len()); + stream.write_all(chunk_header.as_bytes())?; + stream.write_all(chunk_data.as_bytes())?; + stream.write_all(b"\r\n")?; + stream.flush()?; + std::thread::sleep(Duration::from_millis(10)); + } + + stream.write_all(b"0\r\n\r\n")?; + stream.flush() +} + /// Handle a single capturing-backend connection: read the full /// request, store it, then write the next sequential response. fn capture_and_respond( @@ -443,33 +501,26 @@ fn capture_and_respond( let (method, uri, headers, body) = parse_raw_request(&raw); captured.lock().expect("mutex not poisoned").push(CapturedRequest { - method, - uri, + method: method.clone(), + uri: uri.clone(), headers, body, }); + if is_probe_request(&method, &uri) { + let _ = stream.write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 2\r\nconnection: close\r\n\r\nOK"); + return; + } + let idx = counter.fetch_add(1, Ordering::SeqCst); let (status, resp_body) = responses.get(idx).map_or((500, "exhausted"), |(s, b)| (*s, b.as_str())); - let reason = reason_phrase(status); - let content_type = if resp_body.starts_with("data: ") { - "text/event-stream" + if resp_body.starts_with("data: ") { + let _ = write_incremental_sse_response(stream, status, resp_body); } else { - "application/json" - }; - - let resp = format!( - "HTTP/1.1 {status} {reason}\r\n\ - Content-Length: {}\r\n\ - Content-Type: {content_type}\r\n\ - Connection: close\r\n\ - Server: praxis-test-backend\r\n\ - \r\n\ - {resp_body}", - resp_body.len() - ); - let _sent = stream.write_all(resp.as_bytes()); + let resp_bytes = format_http_response(status, resp_body); + let _sent = stream.write_all(&resp_bytes); + } } /// Read a full HTTP request (headers + body) from a TCP stream. From 358e56896f6eed590615e46d26ef06ce4a6ceed1 Mon Sep 17 00:00:00 2001 From: Artemy Date: Fri, 11 Sep 2026 16:41:21 +0100 Subject: [PATCH 6/8] ci: fix env var expansion syntax in integration workflow Signed-off-by: Artemy --- .github/workflows/integration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 21982e451b..2505380252 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -131,7 +131,7 @@ jobs: env: CLAUDE_CODE_PINNED_VERSION: "2.1.267" run: | - npm install -g @anthropic-ai/claude-code@${CLAUDE_CODE_PINNED_VERSION} + npm install -g @anthropic-ai/claude-code@$CLAUDE_CODE_PINNED_VERSION claude --version - name: Integration tests From 1ac5a8d73e753d095c6896f538b99be70db9f9d9 Mon Sep 17 00:00:00 2001 From: Artemy Date: Fri, 11 Sep 2026 16:45:24 +0100 Subject: [PATCH 7/8] ci: quote workflow env var for actionlint and assert git init statuses in harness Signed-off-by: Artemy --- .github/workflows/integration.yaml | 2 +- tests/integration/tests/suite/harness.rs | 45 ++++++++++-------------- 2 files changed, 20 insertions(+), 27 deletions(-) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 2505380252..0fa24cc0d4 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -131,7 +131,7 @@ jobs: env: CLAUDE_CODE_PINNED_VERSION: "2.1.267" run: | - npm install -g @anthropic-ai/claude-code@$CLAUDE_CODE_PINNED_VERSION + npm install -g "@anthropic-ai/claude-code@${CLAUDE_CODE_PINNED_VERSION}" claude --version - name: Integration tests diff --git a/tests/integration/tests/suite/harness.rs b/tests/integration/tests/suite/harness.rs index 5eeb12e098..e0296b2e11 100644 --- a/tests/integration/tests/suite/harness.rs +++ b/tests/integration/tests/suite/harness.rs @@ -54,32 +54,11 @@ impl TempWorkspace { fs::set_permissions(&verify_path, perms)?; } - drop(Command::new("git").arg("init").current_dir(dir.path()).status()); - drop( - Command::new("git") - .arg("config") - .arg("user.name") - .arg("Test") - .current_dir(dir.path()) - .status(), - ); - drop( - Command::new("git") - .arg("config") - .arg("user.email") - .arg("test@example.com") - .current_dir(dir.path()) - .status(), - ); - drop(Command::new("git").arg("add").arg(".").current_dir(dir.path()).status()); - drop( - Command::new("git") - .arg("commit") - .arg("-m") - .arg("initial") - .current_dir(dir.path()) - .status(), - ); + run_git_cmd(&["init"], dir.path()); + run_git_cmd(&["config", "user.name", "Test"], dir.path()); + run_git_cmd(&["config", "user.email", "test@example.com"], dir.path()); + run_git_cmd(&["add", "."], dir.path()); + run_git_cmd(&["commit", "-m", "initial"], dir.path()); Ok(Self { dir, expected_content }) } @@ -122,3 +101,17 @@ impl TempWorkspace { ); } } + +fn run_git_cmd(args: &[&str], dir: &Path) { + let status = Command::new("git") + .args(args) + .current_dir(dir) + .status() + .expect("git command should execute"); + assert!( + status.success(), + "git command `git {}` should exit with status 0, got: {:?}", + args.join(" "), + status.code() + ); +} From f2c57e2eafe3e06b75c62629c5c697b7cdb82f0d Mon Sep 17 00:00:00 2001 From: Artemy Date: Fri, 11 Sep 2026 16:56:04 +0100 Subject: [PATCH 8/8] test(file-resolve): increase read timeout in test stubs to 30s for CI coverage stability Signed-off-by: Artemy --- tests/integration/tests/suite/examples/openai_file_resolve.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/tests/suite/examples/openai_file_resolve.rs b/tests/integration/tests/suite/examples/openai_file_resolve.rs index 475b389bf2..a93d4bd36d 100644 --- a/tests/integration/tests/suite/examples/openai_file_resolve.rs +++ b/tests/integration/tests/suite/examples/openai_file_resolve.rs @@ -432,7 +432,7 @@ pub(super) fn start_files_api_stub() -> u16 { } fn handle_files_api_request_auth(mut stream: std::net::TcpStream) { - stream.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(30))).unwrap(); let mut data = Vec::new(); let mut buf = [0_u8; 4096]; @@ -486,7 +486,7 @@ fn handle_files_api_request_auth(mut stream: std::net::TcpStream) { } fn handle_files_api_request(mut stream: std::net::TcpStream) { - stream.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(30))).unwrap(); let mut data = Vec::new(); let mut buf = [0_u8; 4096];