From 1834e018bba83774ae49afd835c6a1b4db5fd268 Mon Sep 17 00:00:00 2001 From: are-ces <195810094+are-ces@users.noreply.github.com> Date: Thu, 2 Jul 2026 08:54:30 +0200 Subject: [PATCH 1/5] LCORE-2860: fix RHOAI e2e pipeline build and vector store conflict Remove --image flag from oc new-build that was overriding the containerfile's FROM (ubi9/python-312) with ubi-minimal, which lacks dnf. Remove duplicate vector_stores and embedding model registration from run.yaml since the lightspeed-stack enrichment script handles both. Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/e2e-prow/rhoai/configs/run.yaml | 12 +----------- tests/e2e-prow/rhoai/pipeline.sh | 1 - 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/tests/e2e-prow/rhoai/configs/run.yaml b/tests/e2e-prow/rhoai/configs/run.yaml index 935ae206f..9348af6b9 100644 --- a/tests/e2e-prow/rhoai/configs/run.yaml +++ b/tests/e2e-prow/rhoai/configs/run.yaml @@ -139,17 +139,7 @@ registered_resources: provider_id: vllm model_type: llm provider_model_id: null - - model_id: all-mpnet-base-v2 - model_type: embedding - provider_id: sentence-transformers - provider_model_id: all-mpnet-base-v2 - metadata: - embedding_dimension: 768 - vector_stores: - - embedding_dimension: 768 - embedding_model: sentence-transformers/all-mpnet-base-v2 - provider_id: faiss - vector_store_id: ${env.FAISS_VECTOR_STORE_ID} + vector_stores: [] shields: - shield_id: llama-guard provider_id: llama-guard diff --git a/tests/e2e-prow/rhoai/pipeline.sh b/tests/e2e-prow/rhoai/pipeline.sh index 6f1ba0807..b57835b7c 100755 --- a/tests/e2e-prow/rhoai/pipeline.sh +++ b/tests/e2e-prow/rhoai/pipeline.sh @@ -225,7 +225,6 @@ export LLAMA_STACK_IMAGE oc new-build --name=llama-stack-e2e \ --binary \ --strategy=docker \ - --image="registry.access.redhat.com/ubi9/ubi-minimal" \ --to="llama-stack-e2e:latest" \ -n "$NAMESPACE" 2>/dev/null || echo "BuildConfig llama-stack-e2e already exists" From 89115a7d49e4cc33cbc4bb4cba76b1404fa33c83 Mon Sep 17 00:00:00 2001 From: are-ces <195810094+are-ces@users.noreply.github.com> Date: Thu, 2 Jul 2026 13:07:06 +0200 Subject: [PATCH 2/5] LCORE-2860: add RHOAI-specific lightspeed-stack config with vLLM defaults The generic lightspeed-stack.yaml uses default_model=gpt-4o-mini and default_provider=openai, which don't exist in the RHOAI environment. This caused 500/404 errors in tests that rely on default model resolution. Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/e2e-prow/rhoai/pipeline.sh | 2 +- .../server-mode/lightspeed-stack-rhoai.yaml | 35 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 tests/e2e/configuration/server-mode/lightspeed-stack-rhoai.yaml diff --git a/tests/e2e-prow/rhoai/pipeline.sh b/tests/e2e-prow/rhoai/pipeline.sh index b57835b7c..f15398b3d 100755 --- a/tests/e2e-prow/rhoai/pipeline.sh +++ b/tests/e2e-prow/rhoai/pipeline.sh @@ -255,7 +255,7 @@ create_secret api-url-secret --from-literal=key="$KSVC_URL" oc create configmap llama-stack-config -n "$NAMESPACE" \ --from-file="$REPO_ROOT/tests/e2e-prow/rhoai/configs/run.yaml" oc create configmap lightspeed-stack-config -n "$NAMESPACE" \ - --from-file="$REPO_ROOT/tests/e2e/configuration/server-mode/lightspeed-stack.yaml" + --from-file=lightspeed-stack.yaml="$REPO_ROOT/tests/e2e/configuration/server-mode/lightspeed-stack-rhoai.yaml" # Create RAG data ConfigMap from the e2e test RAG data echo "Creating RAG data ConfigMap..." diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-rhoai.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-rhoai.yaml new file mode 100644 index 000000000..ea67aa6b1 --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-rhoai.yaml @@ -0,0 +1,35 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to separate llama-stack service + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: vllm + default_model: ${env.INFERENCE_MODEL} +byok_rag: + - rag_id: e2e-test-docs + rag_type: inline::faiss + embedding_model: sentence-transformers/all-mpnet-base-v2 + embedding_dimension: 768 + vector_db_id: ${env.FAISS_VECTOR_STORE_ID} + db_path: ${env.KV_RAG_PATH:=~/.llama/storage/rag/kv_store.db} + score_multiplier: 1.0 + +rag: + tool: + - e2e-test-docs From bb4f55906738ef882508dad3f10f3a723343597c Mon Sep 17 00:00:00 2001 From: are-ces <195810094+are-ces@users.noreply.github.com> Date: Fri, 3 Jul 2026 10:43:54 +0200 Subject: [PATCH 3/5] LCORE-2860: parametrize model name via vllm-model-secret Create vllm-model-secret from MODEL_NAME in pipeline.sh so both llama-stack and lightspeed-stack pods read the model from a single source instead of hardcoding it. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../rhoai/manifests/lightspeed/llama-stack-prow.yaml | 5 ++++- tests/e2e-prow/rhoai/pipeline.sh | 1 + .../configuration/server-mode/lightspeed-stack-rhoai.yaml | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/e2e-prow/rhoai/manifests/lightspeed/llama-stack-prow.yaml b/tests/e2e-prow/rhoai/manifests/lightspeed/llama-stack-prow.yaml index d81932df8..f33f9519b 100644 --- a/tests/e2e-prow/rhoai/manifests/lightspeed/llama-stack-prow.yaml +++ b/tests/e2e-prow/rhoai/manifests/lightspeed/llama-stack-prow.yaml @@ -102,7 +102,10 @@ spec: name: vllm-api-key-secret key: key - name: INFERENCE_MODEL - value: "meta-llama/Llama-3.1-8B-Instruct" + valueFrom: + secretKeyRef: + name: vllm-model-secret + key: key - name: OPENAI_API_KEY valueFrom: secretKeyRef: diff --git a/tests/e2e-prow/rhoai/pipeline.sh b/tests/e2e-prow/rhoai/pipeline.sh index f15398b3d..4ae604e1c 100755 --- a/tests/e2e-prow/rhoai/pipeline.sh +++ b/tests/e2e-prow/rhoai/pipeline.sh @@ -64,6 +64,7 @@ create_secret() { create_secret hf-token-secret --from-literal=token="$HUGGING_FACE_HUB_TOKEN" create_secret vllm-api-key-secret --from-literal=key="$VLLM_API_KEY" create_secret openai-api-key-secret --from-literal=key="" +create_secret vllm-model-secret --from-literal=key="$MODEL_NAME" # MCP token secrets for lightspeed-stack REPO_ROOT="$(cd "$PIPELINE_DIR/../../.." && pwd)" diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-rhoai.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-rhoai.yaml index ea67aa6b1..4313c7605 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-rhoai.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-rhoai.yaml @@ -20,7 +20,7 @@ authentication: module: "noop" inference: default_provider: vllm - default_model: ${env.INFERENCE_MODEL} + default_model: ${env.VLLM_MODEL} byok_rag: - rag_id: e2e-test-docs rag_type: inline::faiss From 543c3b7af48e133b4e4258f757c1053d73f6a9a8 Mon Sep 17 00:00:00 2001 From: are-ces <195810094+are-ces@users.noreply.github.com> Date: Fri, 3 Jul 2026 15:37:15 +0200 Subject: [PATCH 4/5] LCORE-2860: override inference defaults in Prow config swaps When E2E_DEFAULT_MODEL_OVERRIDE or E2E_DEFAULT_PROVIDER_OVERRIDE env vars are set, patch default_model/default_provider in every config applied via update_config_configmap. This ensures tests that swap to configs like lightspeed-stack-auth-noop-token.yaml use the correct model for the environment instead of hardcoded gpt-4o-mini. Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/e2e/utils/prow_utils.py | 47 ++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/tests/e2e/utils/prow_utils.py b/tests/e2e/utils/prow_utils.py index 48785e29f..9659ae00a 100644 --- a/tests/e2e/utils/prow_utils.py +++ b/tests/e2e/utils/prow_utils.py @@ -5,6 +5,7 @@ """ import os +import re import subprocess import tempfile from typing import Optional @@ -234,6 +235,28 @@ def remove_configmap_backup(backup_key: str) -> None: print(f"ConfigMap backup {backup_key} removed from memory") +def _apply_inference_overrides(content: str) -> str: + """Override inference defaults from E2E env vars if set. + + When E2E_DEFAULT_MODEL_OVERRIDE or E2E_DEFAULT_PROVIDER_OVERRIDE are set, + patch the YAML content so every config swap uses the correct model/provider + for the current environment (e.g. vLLM instead of OpenAI in RHOAI Prow). + """ + model = os.getenv("E2E_DEFAULT_MODEL_OVERRIDE") + provider = os.getenv("E2E_DEFAULT_PROVIDER_OVERRIDE") + if not model and not provider: + return content + if model: + content = re.sub( + r"(default_model:\s*).*", rf"\g<1>{model}", content + ) + if provider: + content = re.sub( + r"(default_provider:\s*).*", rf"\g<1>{provider}", content + ) + return content + + def _recreate_configmap( configmap_name: str, source_file: str, @@ -270,7 +293,7 @@ def update_config_configmap( """ # Check if source is a backup key (restore from memory) if source in _configmap_backups: - config_content = _configmap_backups[source] + config_content = _apply_inference_overrides(_configmap_backups[source]) print(f"Restoring ConfigMap {configmap_name} from memory backup...") # Write content to temp file (oc create configmap requires a file) @@ -289,12 +312,30 @@ def update_config_configmap( os.remove(temp_path) return - # Otherwise, source is a file path + # Otherwise, source is a file path — apply inference overrides if needed print(f"Updating ConfigMap {configmap_name} with config from {source}...") + source_to_apply = source + temp_path = None try: - _recreate_configmap(configmap_name, source, configmap_key) + with open(source) as f: + patched = _apply_inference_overrides(f.read()) + with open(source) as f: + original = f.read() + if patched != original: + tmp = tempfile.NamedTemporaryFile( + mode="w", suffix=".yaml", delete=False + ) + tmp.write(patched) + tmp.close() + source_to_apply = tmp.name + temp_path = tmp.name + + _recreate_configmap(configmap_name, source_to_apply, configmap_key) print(f"ConfigMap {configmap_name} updated successfully") except subprocess.CalledProcessError as e: print(f"Failed to update ConfigMap: {e}") raise + finally: + if temp_path and os.path.exists(temp_path): + os.remove(temp_path) From 2d36036de17d420b0628c668f9a582c3f46c98ab Mon Sep 17 00:00:00 2001 From: are-ces <195810094+are-ces@users.noreply.github.com> Date: Tue, 7 Jul 2026 14:21:07 +0200 Subject: [PATCH 5/5] LCORE-2860: create /tmp/data before starting lightspeed-stack pod The conversation cache config uses db_path=/tmp/data/conversation-cache.db but the directory doesn't exist in a fresh pod. SQLite can't create files in nonexistent directories, causing 500 errors on any conversation endpoint. Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/e2e-prow/rhoai/manifests/lightspeed/lightspeed-stack.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/e2e-prow/rhoai/manifests/lightspeed/lightspeed-stack.yaml b/tests/e2e-prow/rhoai/manifests/lightspeed/lightspeed-stack.yaml index 04fae492a..83e2f173a 100644 --- a/tests/e2e-prow/rhoai/manifests/lightspeed/lightspeed-stack.yaml +++ b/tests/e2e-prow/rhoai/manifests/lightspeed/lightspeed-stack.yaml @@ -37,6 +37,7 @@ spec: key: key optional: true image: ${LIGHTSPEED_STACK_IMAGE} + command: ["/bin/sh", "-c", "mkdir -p /tmp/data && exec python3.12 src/lightspeed_stack.py"] ports: - containerPort: 8080 # TCP probes avoid HTTP/auth. LCS + Llama handshake and large images can take 60–120s before :8080 listens;